diff --git a/.agents/skills/ship/SKILL.md b/.agents/skills/ship/SKILL.md index c4c6372ff73..300118c485e 100644 --- a/.agents/skills/ship/SKILL.md +++ b/.agents/skills/ship/SKILL.md @@ -42,6 +42,7 @@ When the user runs `/ship`: - If the diff modifies UI code (any non-test `.tsx` file, or anything under `apps/sim/components/`, `apps/sim/hooks/`, or `apps/sim/stores/`), run `/cleanup`. It fans out the React/UI passes (effects, memo, callbacks, state, React Query, emcn, url-state), the comment pass, and the test-audit pass, and applies fixes so they land in this commit. - Otherwise, if the diff adds or changes tests (`*.test.ts(x)`, `*.integration.ts`, `**/e2e/**`, `apps/sim/scripts/test-*-e2e.ts`), run `/test-audit audit ` on its own. Every new or changed test must pass the authoring gate; delete the ones that don't rather than shipping them. - Then run the test files the diff adds or changes, plus the existing tests beside changed source files, with `bun run --cwd test ` (`bun run --cwd apps/sim test ` for the app; `*.integration.ts` needs the setup in `.claude/rules/sim-testing.md`). A failing test aborts ship. + - Then run root `bun run test` from the repo root. It chains `test:scripts` (the `scripts/*.test.ts` suite CI runs) before every workspace suite; workspace-scoped runs skip it, which is how a `scripts/check-*.test.ts` failure has reached CI. A failing test aborts ship. 5. **Run migration safety** — only if the diff touches `packages/db/migrations/**` or `packages/db/schema.ts`: - Run `/db-migrate` to review the migration for zero-downtime safety (expand/contract phasing, backward-compatibility with the deployed app version). - `bun run check:migrations origin/staging` must pass (staging is the PR base). Do not silence a flagged statement with a `-- migration-safe:` annotation unless `/db-migrate` confirmed the old code no longer depends on it; otherwise split the destructive change into a later deploy. diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 9e2d86607e9..753ea7865a9 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -354,9 +354,8 @@ In addition, you will need to update the registries: // apps/sim/blocks/blocks/pinecone.ts import { PineconeIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' - import type { PineconeResponse } from '@/tools/pinecone/types' - export const PineconeBlock: BlockConfig = { + export const PineconeBlock: BlockConfig = { type: 'pinecone', name: 'Pinecone', description: 'Use Pinecone vector database', diff --git a/.github/actions/setup-workspace/action.yml b/.github/actions/setup-workspace/action.yml new file mode 100644 index 00000000000..b6e2e48fdf2 --- /dev/null +++ b/.github/actions/setup-workspace/action.yml @@ -0,0 +1,61 @@ +name: Setup Workspace +description: Install the pinned Bun and Node toolchain, mount the dependency (and optionally Turbo) caches, and install workspace dependencies. + +inputs: + provider: + description: The CI_PROVIDER repo variable, forwarded to cache-mount. + required: false + default: '' + turbo-cache-key: + description: Suffix for a Turbo cache mounted at ./.turbo. Empty skips the mount. Jobs that write Turbo entries need distinct suffixes, or last-writer-wins commits evict each other's entries. + required: false + default: '' + +# Cache keys are scoped by event name, and fork PRs get their own namespace on +# top: untrusted fork runs must never share a cache with push runs (whose caches +# feed production image builds) or with trusted internal-PR runs. +# +# node_modules also keys on the lockfile hash: a sticky disk is a mutable volume, +# and `bun install --frozen-lockfile` adds what the lockfile needs without +# pruning what it dropped, so branches on different lockfiles were contaminating +# each other (a stale @next/swc 16.2.6 outlived the 16.2.11 bump). The bun and +# Turbo caches are content/hash-addressed, so they stay shared — that is what +# keeps a fresh node_modules disk cheap to fill. +runs: + using: composite + steps: + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.4.2 + + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: 24 + + - name: Mount Bun cache + uses: ./.github/actions/cache-mount + with: + provider: ${{ inputs.provider }} + key: ${{ github.repository }}-bun-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} + path: ~/.bun/install/cache + + - name: Mount node_modules + uses: ./.github/actions/cache-mount + with: + provider: ${{ inputs.provider }} + key: ${{ github.repository }}-node-modules-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}-${{ hashFiles('bun.lock') }} + path: ./node_modules + + - name: Mount Turbo cache + if: inputs.turbo-cache-key != '' + uses: ./.github/actions/cache-mount + with: + provider: ${{ inputs.provider }} + key: ${{ github.repository }}-${{ inputs.turbo-cache-key }}-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} + path: ./.turbo + + - name: Install dependencies + shell: bash + run: bun install --frozen-lockfile --ignore-scripts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a30f3f9eec..547db6f3459 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -222,7 +222,6 @@ jobs: platforms: linux/amd64 tags: ${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}:${{ github.sha }}-dev build-args: | - SIM_SEARCH_LIVE_DEFAULT=true MSHIP_PLAN_MODE_DEFAULT=true max-cache-size-mb: ${{ matrix.cache_mb }} diff --git a/.github/workflows/desktop-e2e.yml b/.github/workflows/desktop-e2e.yml index c9d4f92a539..a86ea559a71 100644 --- a/.github/workflows/desktop-e2e.yml +++ b/.github/workflows/desktop-e2e.yml @@ -11,6 +11,14 @@ on: - '.github/workflows/desktop-release.yml' - 'apps/desktop/**' - 'apps/sim/app/_shell/desktop-update-*.tsx' + - 'apps/sim/app/workspace/**/home/hooks/use-mothership-resize.ts' + - 'apps/sim/app/workspace/**/home/hooks/use-resource-panel.ts' + - 'apps/sim/app/workspace/**/home/components/chat-panel-layout.tsx' + - 'apps/sim/stores/chat-panel/**' + - 'apps/sim/stores/constants.ts' + - 'apps/sim/lib/browser-agent/transport.ts' + - 'apps/sim/lib/core/utils/separator-keys.ts' + - 'apps/sim/scripts/fixtures/chat-panel.tsx' - 'apps/sim/app/layout.tsx' - 'apps/sim/hooks/use-desktop-update-state.ts' - 'apps/sim/lib/desktop/**' @@ -20,7 +28,6 @@ on: - 'apps/sim/hooks/queries/personal-search-integrations.ts' - 'apps/sim/hooks/use-search-integration-connection.ts' - 'apps/sim/hooks/use-github-installation-setup.ts' - - 'apps/sim/app/o/**/integrations/indexed/use-member-enrollment.ts' - 'apps/sim/lib/api/contracts/desktop-source-connect.ts' - 'apps/sim/scripts/fixtures/desktop-source-connect.tsx' - 'apps/sim/app/workspace/**/browser-session/**' diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index e62e49a6676..e6ef55b6ae2 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -8,7 +8,7 @@ permissions: contents: read jobs: - oauth-postgres: + postgres-integration: # Runs the real-infrastructure test layer: every `*.integration.ts` in packages/db and # apps/sim, discovered by glob (`vitest run --mode integration`), against the database each # provisioning path produces. A new integration suite needs no workflow change. @@ -67,25 +67,10 @@ jobs: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - - name: Setup Bun - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 - with: - bun-version: 1.4.2 - - - name: Setup Node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 - with: - node-version: 24 - - - name: Mount Bun cache - uses: ./.github/actions/cache-mount + - name: Setup workspace + uses: ./.github/actions/setup-workspace with: provider: ${{ vars.CI_PROVIDER }} - key: ${{ github.repository }}-bun-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} - path: ~/.bun/install/cache - - - name: Install dependencies - run: bun install --frozen-lockfile --ignore-scripts - name: Provision a fresh database through the supported command working-directory: packages/db @@ -129,79 +114,88 @@ jobs: if-no-files-found: warn retention-days: 14 + # Acceptance suites that cross a real HTTP boundary. Its own job, off the + # integration legs' critical path and on its own database: the SCIM app boots + # hosted, which starts background usage replay against DATABASE_URL, so it must + # never share a database with suites asserting on billing rows. The suites + # exercise HTTP behavior rather than a provisioning path, so they run once, + # against the production (migrate) path. + http-e2e: + name: End-to-end over real HTTP + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }} + timeout-minutes: 20 + services: + postgres: + image: pgvector/pgvector:pg17 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: sim_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d sim_test" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_test + BETTER_AUTH_SECRET: http-e2e-ci-secret-at-least-32-characters + ENCRYPTION_KEY: '0000000000000000000000000000000000000000000000000000000000000000' + + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Setup workspace + uses: ./.github/actions/setup-workspace + with: + provider: ${{ vars.CI_PROVIDER }} + + # Migrations create their own extensions, as on a fresh self-hosted install. + - name: Provision the database through migrations + working-directory: packages/db + run: bun run db:migrate + - name: Verify Google document reads over real HTTP - if: matrix.provision == 'push' working-directory: apps/sim env: NEXT_PUBLIC_APP_URL: http://127.0.0.1:3040 NEXT_PUBLIC_FORCE_HOSTED: 'false' - SEARCH_GOOGLE_CONTENT_REPORT_PATH: ${{ runner.temp }}/search-google-content.json + SEARCH_GOOGLE_CONTENT_REPORT_PATH: ${{ runner.temp }}/e2e/search-google-content.json run: bun scripts/test-search-google-content-e2e.ts - - name: Upload Google content acceptance report - if: failure() && matrix.provision == 'push' - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: search-google-content - path: ${{ runner.temp }}/search-google-content.json - if-no-files-found: ignore - retention-days: 7 - - name: Verify Lucid MCP search and complete diagram reads over real HTTP - if: matrix.provision == 'push' working-directory: apps/sim env: NEXT_PUBLIC_APP_URL: http://127.0.0.1:3040 NEXT_PUBLIC_FORCE_HOSTED: 'false' - SEARCH_LUCID_REPORT_PATH: ${{ runner.temp }}/search-lucid.json + SEARCH_LUCID_REPORT_PATH: ${{ runner.temp }}/e2e/search-lucid.json run: bun scripts/test-search-lucid-e2e.ts - - name: Upload Lucid acceptance report - if: failure() && matrix.provision == 'push' - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: search-lucid - path: ${{ runner.temp }}/search-lucid.json - if-no-files-found: ignore - retention-days: 7 - - name: Verify Zoom search over real HTTP - if: matrix.provision == 'push' working-directory: apps/sim env: NEXT_PUBLIC_APP_URL: http://127.0.0.1:3040 NEXT_PUBLIC_FORCE_HOSTED: 'false' - SEARCH_ZOOM_REPORT_PATH: ${{ runner.temp }}/search-zoom.json + SEARCH_ZOOM_REPORT_PATH: ${{ runner.temp }}/e2e/search-zoom.json run: bun scripts/test-search-zoom-e2e.ts - - name: Upload Zoom acceptance report - if: failure() && matrix.provision == 'push' - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: search-zoom - path: ${{ runner.temp }}/search-zoom.json - if-no-files-found: ignore - retention-days: 7 - - name: Verify Google Meet search over real HTTP - if: matrix.provision == 'push' working-directory: apps/sim env: NEXT_PUBLIC_APP_URL: http://127.0.0.1:3040 NEXT_PUBLIC_FORCE_HOSTED: 'false' - SEARCH_GOOGLE_MEET_REPORT_PATH: ${{ runner.temp }}/search-google-meet.json + SEARCH_GOOGLE_MEET_REPORT_PATH: ${{ runner.temp }}/e2e/search-google-meet.json run: bun scripts/test-search-google-meet-e2e.ts - - name: Upload Google Meet acceptance report - if: failure() && matrix.provision == 'push' - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: search-google-meet - path: ${{ runner.temp }}/search-google-meet.json - if-no-files-found: ignore - retention-days: 7 - - - name: Verify SCIM and administration over real HTTP + # The first request cold-compiles the app under Turbopack, which took 42-150s + # on this runner class: a fixed 120s readiness deadline failed on the slow + # tail. The deadline only has to catch a hung boot; an exited server fails + # immediately, and either way the server log tail lands in the job log. No + # step timeout: the job's bound covers a hang without cutting a slow but + # healthy suite short of writing its report. + - name: Verify SCIM, administration and workflow comparisons over real HTTP working-directory: apps/sim env: NEXT_PUBLIC_APP_URL: http://127.0.0.1:3017 @@ -222,42 +216,49 @@ jobs: DISABLE_TELEMETRY: 'true' NEXT_TELEMETRY_DISABLED: '1' NEXT_PUBLIC_CHAT_DISABLED: 'true' + READY_TIMEOUT_SECONDS: 300 run: | - server_log="$RUNNER_TEMP/scim-next.log" + report_dir="$RUNNER_TEMP/e2e" + server_log="$report_dir/scim-next.log" + mkdir -p "$report_dir" node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 --port 3017 > "$server_log" 2>&1 & server_pid=$! finish() { kill "$server_pid" 2>/dev/null || true wait "$server_pid" 2>/dev/null || true - awk '/^ (GET|POST|PUT|PATCH|DELETE|HEAD) \/api\// { print }' "$server_log" > "$RUNNER_TEMP/scim-http-status.log" + awk '/^ (GET|POST|PUT|PATCH|DELETE|HEAD) \/api\// { print }' "$server_log" > "$report_dir/scim-http-status.log" } trap finish EXIT - deadline=$((SECONDS + 120)) - until curl --fail --silent --max-time 3 http://127.0.0.1:3017/api/health > /dev/null; do - if ! kill -0 "$server_pid" 2>/dev/null; then - echo 'Local SCIM app exited during startup.' - exit 1 - fi - if [ "$SECONDS" -ge "$deadline" ]; then - echo 'Local SCIM app did not become ready within 120 seconds.' - exit 1 - fi + fail_startup() { + echo "::error::$1" + tail -n 200 "$server_log" + exit 1 + } + started=$SECONDS + until curl --fail --silent --max-time 10 http://127.0.0.1:3017/api/health > /dev/null; do + kill -0 "$server_pid" 2>/dev/null || fail_startup 'Local SCIM app exited during startup.' + [ $((SECONDS - started)) -lt "$READY_TIMEOUT_SECONDS" ] || + fail_startup "Local SCIM app did not become ready within $READY_TIMEOUT_SECONDS seconds." sleep 2 done + echo "Local SCIM app ready after $((SECONDS - started))s" SCIM_E2E_BASE_URL="$NEXT_PUBLIC_APP_URL" \ SCIM_E2E_DATABASE_URL="$DATABASE_URL" \ SCIM_E2E_AUTH_SECRET="$BETTER_AUTH_SECRET" \ - SCIM_E2E_REPORT_PATH="$RUNNER_TEMP/scim-e2e-report.json" \ + SCIM_E2E_REPORT_PATH="$report_dir/scim-e2e-report.json" \ bun run test:scim:e2e + VERSION_COMPARE_E2E_BASE_URL="$NEXT_PUBLIC_APP_URL" \ + VERSION_COMPARE_E2E_DATABASE_URL="$DATABASE_URL" \ + VERSION_COMPARE_E2E_AUTH_SECRET="$BETTER_AUTH_SECRET" \ + VERSION_COMPARE_E2E_REPORT_PATH="$report_dir/version-compare-http-report.json" \ + bun run test:workflow-version-compare:e2e - - name: Upload SCIM failure report and HTTP status log + - name: Upload end-to-end reports and server logs if: failure() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: scim-failure-${{ matrix.provision }} - path: | - ${{ runner.temp }}/scim-e2e-report.json - ${{ runner.temp }}/scim-http-status.log + name: http-e2e-reports + path: ${{ runner.temp }}/e2e/ if-no-files-found: ignore retention-days: 7 @@ -280,50 +281,11 @@ jobs: with: fetch-depth: 2 - - name: Setup Bun - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 - with: - bun-version: 1.4.2 - - - name: Setup Node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 - with: - node-version: 24 - - # Cache keys are scoped by event name, and fork PRs get their own - # namespace on top: untrusted fork runs must never share a cache with - # push runs (whose caches feed production image builds) or with trusted - # internal-PR runs. - # - # node_modules also keys on the lockfile hash: a sticky disk is a mutable - # volume, and `bun install --frozen-lockfile` adds what the lockfile needs - # without pruning what it dropped, so branches on different lockfiles were - # contaminating each other (a stale @next/swc 16.2.6 outlived the 16.2.11 - # bump). The bun and Turbo caches are content/hash-addressed, so they stay - # shared — that is what keeps a fresh node_modules disk cheap to fill. - - name: Mount Bun cache - uses: ./.github/actions/cache-mount - with: - provider: ${{ vars.CI_PROVIDER }} - key: ${{ github.repository }}-bun-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} - path: ~/.bun/install/cache - - - name: Mount node_modules - uses: ./.github/actions/cache-mount + - name: Setup workspace + uses: ./.github/actions/setup-workspace with: provider: ${{ vars.CI_PROVIDER }} - key: ${{ github.repository }}-node-modules-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}-${{ hashFiles('bun.lock') }} - path: ./node_modules - - - name: Mount Turbo cache - uses: ./.github/actions/cache-mount - with: - provider: ${{ vars.CI_PROVIDER }} - key: ${{ github.repository }}-turbo-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} - path: ./.turbo - - - name: Install dependencies - run: bun install --frozen-lockfile --ignore-scripts + turbo-cache-key: turbo-cache # Surfaces known CVEs in the dependency tree. Non-blocking until the # existing advisory backlog is triaged, then flip to a required gate by @@ -375,9 +337,6 @@ jobs: # # Depth stays at 1 — without a merge-base the migration audit diffs the two # tips, which under `--diff-filter=AM` is exactly the migrations new here. - # Resolved once for both diff-based audits, and never with `|| true`: a - # swallowed fetch leaves the base absent, which neither audit can tell apart - # from a branch that changed nothing. # # On push the base is `github.event.before`, the tip the branch had before # this push — not `HEAD~1`, which names only the last commit and would let a @@ -481,36 +440,11 @@ jobs: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - - name: Setup Bun - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 - with: - bun-version: 1.4.2 - - - name: Setup Node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 - with: - node-version: 24 - - - name: Mount Bun cache - uses: ./.github/actions/cache-mount - with: - provider: ${{ vars.CI_PROVIDER }} - key: ${{ github.repository }}-bun-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} - path: ~/.bun/install/cache - - - name: Mount node_modules - uses: ./.github/actions/cache-mount + - name: Setup workspace + uses: ./.github/actions/setup-workspace with: provider: ${{ vars.CI_PROVIDER }} - key: ${{ github.repository }}-node-modules-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}-${{ hashFiles('bun.lock') }} - path: ./node_modules - - - name: Mount Turbo cache - uses: ./.github/actions/cache-mount - with: - provider: ${{ vars.CI_PROVIDER }} - key: ${{ github.repository }}-turbo-cache-build-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} - path: ./.turbo + turbo-cache-key: turbo-cache-build # No `.next/cache` mount: the Turbopack persistent build cache is off. A # controlled A/B on one branch (PR #6078) with a byte-identical module graph @@ -534,9 +468,6 @@ jobs: echo "::warning::Runner has ${TOTAL_GB} GB. A cold-cache build peaks ~51 GB, so this run may be OOM-killed (reported only as 'the runner has received a shutdown signal'). Warm/partial builds should still fit." fi - - name: Install dependencies - run: bun install --frozen-lockfile --ignore-scripts - - name: Build application env: NODE_OPTIONS: '--no-warnings --max-old-space-size=8192' diff --git a/CLAUDE.md b/CLAUDE.md index 70309e5dfce..1b8c8272095 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -85,9 +85,10 @@ The `'use client'` server boundary, the app/worker runtime env split, and featur ## Code Conventions -- **Naming**: components PascalCase (`WorkflowList`); hooks `use*`; files kebab-case (`workflow-list.tsx`); constants SCREAMING_SNAKE_CASE; interfaces PascalCase with a suffix (`WorkflowListProps`); stores `stores//store.ts`. +- **Naming**: components PascalCase (`WorkflowList`); hooks `use*`; files kebab-case (`workflow-list.tsx`); constants SCREAMING_SNAKE_CASE; interfaces PascalCase with a suffix (`WorkflowListProps`); stores `stores//store.ts`. A file never repeats its folder's name (`lib/logs/views.ts`, not `lib/logs/log-views.ts`; `utils/date.ts`, not `utils/date-utils.ts`); `check:file-names` enforces this. - **Imports**: absolute (`@/...`) only, never relative. A folder with 3+ exports gets an `index.ts` barrel; never re-export from a non-barrel file. `import type` for type-only imports. Order and lazy-loading through barrels: `.claude/rules/sim-imports.md`. -- **TypeScript**: no `any` (use precise types or `unknown` with guards); a props interface for every component; `as const` for constant objects/arrays; explicit ref types (`useRef(null)`). +- **TypeScript**: no `any` and no non-null `!` (use precise types or `unknown` with guards; `check:explicit-any` ratchets both); no export nothing imports (`check:unused-exports`); a props interface for every component; `as const` for constant objects/arrays; explicit ref types (`useRef(null)`). +- **Unused bindings** fail lint (biome `noUnusedVariables`, `noUnusedFunctionParameters`): delete the dead variable, import, or parameter and update callers; write `catch {}` when the error is unused. Prefix `_` only for a parameter that must hold its position because a later one is used. `const { a, ...rest } = obj` to omit keys is allowed. The rules carry no autofix, so `bun run lint` will not rename anything for you. - **Components**: `'use client'` only for hooks or browser APIs. Structure order, extraction thresholds, and list-render rules: `.claude/rules/sim-components.md`. Render-performance idioms (lazy-init refs, hoisting, `Map` pre-indexing, `[...arr].sort()` never `toSorted()` on client paths): `.claude/rules/sim-react-performance.md`. For effect/state/memo/callback anti-patterns use the `/you-might-not-need-*` skills and verify against the running UI. - **State ownership**: React Query owns server data — never `useState` + `fetch`; shareable client view-state (tabs, filters, search, pagination, selected id) lives in the URL via `nuqs`; Zustand owns global client state; `useState` owns UI-only state. Hooks: `.claude/rules/sim-hooks.md`. Stores (`devtools`, `persist` only with an explicit `partialize` whitelist, workflow value invariants): `.claude/rules/sim-stores.md`. URL state: `.claude/rules/sim-url-state.md`. - **Utils**: inline a helper with one consumer; create `utils.ts` when 2+ files share it — in `lib/` (app-wide) or `feature/utils/` (feature-scoped). Check `lib/` before writing a new one. diff --git a/apps/desktop/e2e/chat-panel.spec.ts b/apps/desktop/e2e/chat-panel.spec.ts new file mode 100644 index 00000000000..920be2ce387 --- /dev/null +++ b/apps/desktop/e2e/chat-panel.spec.ts @@ -0,0 +1,429 @@ +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { createServer } from 'node:http' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { _electron as electron, expect, test } from '@playwright/test' +import type { SimDesktopApi } from '@sim/desktop-bridge' +import { getErrorMessage } from '@sim/utils/errors' +import { build } from 'esbuild' +import postcss from 'postcss' +import loadPostcssConfig from 'postcss-load-config' + +const DESKTOP_DIR = fileURLToPath(new URL('..', import.meta.url)) +const SIM_DIR = fileURLToPath(new URL('../../sim/', import.meta.url)) +const FIXTURE = fileURLToPath(new URL('../../sim/scripts/fixtures/chat-panel.tsx', import.meta.url)) + +test('chat panel sizes survive navigation, chat switches, collapse, and layout constraints', async () => { + const reportPath = process.env.CHAT_PANEL_REPORT_PATH ?? test.info().outputPath('chat-panel.json') + const checks: { + name: string + status: 'passed' | 'failed' + durationMs: number + error?: string + }[] = [] + const check = async (name: string, run: () => Promise) => { + const started = Date.now() + try { + await test.step(name, run) + checks.push({ name, status: 'passed', durationMs: Date.now() - started }) + } catch (error) { + checks.push({ + name, + status: 'failed', + durationMs: Date.now() - started, + error: getErrorMessage(error), + }) + throw error + } + } + const userData = mkdtempSync(join(tmpdir(), 'sim-chat-panel-e2e-')) + let app: Awaited> | undefined + const errors: string[] = [] + let desktopExit: { code: number | null; signal: string | null } | null = null + let rendererCrashed = false + let passed = false + let javascript = '' + let stylesheet = '' + const server = createServer((request, response) => { + const path = new URL(request.url ?? '/', 'http://localhost').pathname + if (path === '/page') { + response.setHeader('Content-Type', 'text/html') + response.end('Native browser resize fixture') + } else if (path === '/fixture.js' || path === '/fixture.css') { + response.setHeader('Content-Type', path.endsWith('.js') ? 'text/javascript' : 'text/css') + response.end(path.endsWith('.js') ? javascript : stylesheet) + } else if (path.startsWith('/api/')) { + response.setHeader('Content-Type', 'application/json') + response.end( + path === '/api/auth/get-session' + ? JSON.stringify({ user: { id: 'fixture-user' }, session: { id: 'fixture-session' } }) + : '{}' + ) + } else { + response.setHeader('Content-Type', 'text/html') + response.setHeader( + 'Set-Cookie', + 'better-auth.session_token=fixture; HttpOnly; SameSite=Lax; Path=/' + ) + response.end( + '
' + ) + } + }) + + try { + await check('load the production resource panel resize hook in Electron', async () => { + const config = await loadPostcssConfig({}, SIM_DIR) + const cssPath = join(SIM_DIR, 'app/_styles/globals.css') + const css = await postcss(config.plugins).process( + `${readFileSync(cssPath, 'utf8')}\n@source ${JSON.stringify(FIXTURE)};`, + { from: cssPath } + ) + const bundle = await build({ + entryPoints: [FIXTURE], + bundle: true, + write: false, + outfile: test.info().outputPath('fixture.js'), + external: ['node:async_hooks'], + banner: { js: 'var process={env:{NODE_ENV:"development"},browser:true};' }, + format: 'iife', + platform: 'browser', + tsconfig: join(SIM_DIR, 'tsconfig.json'), + define: { 'process.env.NODE_ENV': '"development"' }, + }) + javascript = bundle.outputFiles.find((file) => file.path.endsWith('.js'))?.text ?? '' + stylesheet = `${css.css}\n${bundle.outputFiles.find((file) => file.path.endsWith('.css'))?.text ?? ''}` + await new Promise((resolve) => server.listen(0, resolve)) + const address = server.address() + if (!address || typeof address === 'string') throw new Error('Missing fixture address') + app = await electron.launch({ + args: [process.env.SIM_DESKTOP_E2E_MAIN ?? '.'], + cwd: DESKTOP_DIR, + env: { + ...process.env, + SIM_DESKTOP_ORIGIN: `http://127.0.0.1:${address.port}`, + SIM_DESKTOP_USER_DATA: userData, + }, + }) + }) + if (!app) throw new Error('Electron did not launch') + app.process().once('exit', (code, signal) => { + desktopExit = { code, signal } + }) + const shell = app + const page = await shell.firstWindow() + page.on('pageerror', (error) => errors.push(error.message)) + page.on('crash', () => { + rendererCrashed = true + }) + await shell.evaluate(({ app, BrowserWindow }) => { + const window = BrowserWindow.getAllWindows()[0] + // Keep the physical window inside the small displays used by macOS CI. + window.setMinimumSize(0, 0) + window.setContentSize(720, 400) + window.webContents.setBackgroundThrottling(false) + app.focus({ steal: true }) + window.focus() + }) + await page.reload() + await shell.evaluate(({ app, BrowserWindow }) => { + const window = BrowserWindow.getAllWindows()[0] + window.webContents.setZoomFactor(0.5) + app.focus({ steal: true }) + window.focus() + }) + expect(errors).toEqual([]) + await expect + .poll(() => + shell.evaluate(({ BrowserWindow }) => BrowserWindow.getAllWindows()[0].isFocused()) + ) + .toBe(true) + await expect.poll(() => page.evaluate(() => window.innerWidth)).toBe(1440) + const panel = page.locator('[data-mothership-panel]') + const divider = page.getByRole('separator', { name: 'Resize resource view' }) + const width = () => panel.evaluate((element) => element.getBoundingClientRect().width) + const expectWidth = async (expected: number) => { + await expect.poll(width).toBeCloseTo(expected, 0) + } + const beginDrag = async () => { + await shell.evaluate(({ app, BrowserWindow }) => { + app.focus({ steal: true }) + BrowserWindow.getAllWindows()[0].focus() + }) + await expect + .poll(() => + shell.evaluate(({ BrowserWindow }) => BrowserWindow.getAllWindows()[0].isFocused()) + ) + .toBe(true) + await divider.hover({ position: { x: 4, y: 100 } }) + const rect = await panel.boundingBox() + if (!rect) throw new Error('Missing panel bounds') + await page.mouse.down() + await expect + .poll(() => divider.evaluate((element) => element.hasPointerCapture(1))) + .toBe(true) + return rect + } + const dragTo = async (target: number) => { + const rect = await beginDrag() + await page.mouse.move(rect.x + rect.width - target, rect.y + 100, { steps: 12 }) + await page.mouse.up() + await expectWidth(target) + } + + await check('a chosen width survives a settings round trip and reload', async () => { + await expectWidth(720) + await dragTo(620) + await page.getByRole('button', { name: 'Settings', exact: true }).click() + await expect(panel).toHaveCount(0) + await page.getByRole('button', { name: 'Back', exact: true }).click() + await expectWidth(620) + await page.reload() + await expectWidth(620) + }) + + await check( + 'workspace and organization chats restore independent widths without remounting', + async () => { + await page.getByRole('button', { name: 'workspace-chat-b', exact: true }).click() + await expectWidth(720) + await dragTo(830) + await page.getByRole('button', { name: 'organization-chat-a', exact: true }).click() + await expectWidth(720) + await dragTo(560) + await page.getByRole('button', { name: 'Settings', exact: true }).click() + await page.getByRole('button', { name: 'Back', exact: true }).click() + await expectWidth(560) + await page.getByRole('button', { name: 'workspace-chat-a', exact: true }).click() + await expectWidth(620) + await page.getByRole('button', { name: 'workspace-chat-b', exact: true }).click() + await expectWidth(830) + } + ) + + await check( + 'collapse preserves the expanded preference, including keyboard changes', + async () => { + await page.getByRole('button', { name: 'Collapse resource view' }).click() + await expectWidth(0) + await page.getByRole('button', { name: 'Expand resource view' }).click() + await expectWidth(830) + await divider.focus() + await page.keyboard.press('ArrowRight') + await expectWidth(798) + await page.getByRole('button', { name: 'Settings', exact: true }).click() + await page.getByRole('button', { name: 'Back', exact: true }).click() + await expectWidth(798) + } + ) + + await check('container and window clamps do not overwrite the preferred width', async () => { + await page.getByRole('button', { name: 'Resize container' }).click() + await expectWidth(520) + await page.getByRole('button', { name: 'Resize container' }).click() + await expectWidth(798) + await shell.evaluate(({ BrowserWindow }) => + BrowserWindow.getAllWindows()[0].setContentSize(525, 400) + ) + await expectWidth(570) + await page.getByRole('button', { name: 'Settings', exact: true }).click() + await page.getByRole('button', { name: 'Back', exact: true }).click() + await expectWidth(570) + await shell.evaluate(({ BrowserWindow }) => + BrowserWindow.getAllWindows()[0].setContentSize(720, 400) + ) + await expectWidth(798) + }) + + await check('resizing writes storage only when the gesture ends', async () => { + const before = await page.evaluate(() => JSON.stringify(localStorage)) + const rect = await beginDrag() + await page.mouse.move(rect.x + 100, rect.y + 100, { steps: 12 }) + await expectWidth(698) + expect(await page.evaluate(() => JSON.stringify(localStorage))).toBe(before) + await page.mouse.up() + expect(await page.evaluate(() => JSON.stringify(localStorage))).not.toBe(before) + }) + + for (const interruption of [ + 'pointercancel', + 'capture loss', + 'blur', + 'detach', + 'chat switch', + ] as const) { + await check(`${interruption} keeps the previous saved width`, async () => { + const before = await page.evaluate(() => JSON.stringify(localStorage)) + const rect = await beginDrag() + await page.mouse.move(rect.x + 100, rect.y + 100, { steps: 12 }) + await expectWidth(598) + if (interruption === 'pointercancel') { + await divider.dispatchEvent('pointercancel', { pointerId: 1 }) + } else if (interruption === 'capture loss') { + await divider.evaluate((element) => element.releasePointerCapture(1)) + await page.mouse.move(rect.x + 101, rect.y + 100) + } else if (interruption === 'blur') { + await page.evaluate(() => window.dispatchEvent(new Event('blur'))) + } else if (interruption === 'chat switch') { + await page + .getByRole('button', { name: 'organization-chat-a', exact: true }) + .evaluate((element: HTMLButtonElement) => element.click()) + await expectWidth(560) + } else { + await page + .getByRole('button', { name: 'Settings', exact: true }) + .evaluate((element: HTMLButtonElement) => element.click()) + await expect(panel).toHaveCount(0) + } + await page.mouse.up() + if (interruption === 'chat switch') { + await page.getByRole('button', { name: 'workspace-chat-b', exact: true }).click() + } + if (interruption === 'detach') { + await page.getByRole('button', { name: 'Back', exact: true }).click() + } + await expectWidth(698) + expect(await page.evaluate(() => JSON.stringify(localStorage))).toBe(before) + }) + } + + await check('a viewport change during a drag clamps the committed display width', async () => { + await page.getByRole('button', { name: 'Resize container' }).click() + await expectWidth(520) + const rect = await beginDrag() + await page.mouse.move(rect.x + 20, rect.y + 100, { steps: 12 }) + await expectWidth(500) + await shell.evaluate(({ BrowserWindow }) => + BrowserWindow.getAllWindows()[0].setContentSize(300, 400) + ) + await expect.poll(() => page.evaluate(() => window.innerWidth)).toBe(600) + await page.mouse.up() + await expectWidth(480) + await shell.evaluate(({ BrowserWindow }) => + BrowserWindow.getAllWindows()[0].setContentSize(720, 400) + ) + await expectWidth(500) + await page.getByRole('button', { name: 'Resize container' }).click() + await dragTo(698) + }) + + await check('another account cannot inherit the current chat width', async () => { + await page.getByRole('button', { name: 'Switch account' }).click() + await expectWidth(720) + await dragTo(580) + await page.getByRole('button', { name: 'Switch account' }).click() + await expectWidth(698) + }) + await check('assigning a permanent chat ID lets an active drag finish', async () => { + await page.getByRole('button', { name: 'pending:chat', exact: true }).click() + await dragTo(620) + const rect = await beginDrag() + await page.mouse.move(rect.x + 100, rect.y + 100, { steps: 12 }) + await expectWidth(520) + await page + .getByRole('button', { name: 'Assign chat ID', exact: true }) + .evaluate((element: HTMLButtonElement) => element.click()) + await expect(page.getByRole('button', { name: 'Assign chat ID', exact: true })).toBeDisabled() + expect(await divider.evaluate((element) => element.hasPointerCapture(1))).toBe(true) + await page.mouse.up() + await expectWidth(520) + await page.getByRole('button', { name: 'workspace-chat-b', exact: true }).click() + await expectWidth(698) + await page.getByRole('button', { name: 'assigned-chat', exact: true }).click() + await expectWidth(520) + await page.getByRole('button', { name: 'workspace-chat-b', exact: true }).click() + await expectWidth(698) + }) + + await check( + 'cancelling before the first animation frame restores native browser bounds', + async () => { + await page.getByRole('button', { name: 'pending:native', exact: true }).click() + await dragTo(698) + await shell.evaluate(({ app, BrowserWindow }) => { + app.focus({ steal: true }) + BrowserWindow.getAllWindows()[0].focus() + }) + await expect + .poll(() => + shell.evaluate(({ BrowserWindow }) => BrowserWindow.getAllWindows()[0].isFocused()) + ) + .toBe(true) + await page.getByRole('button', { name: 'Start browser', exact: true }).click() + const nativeBounds = () => + shell.evaluate(({ BrowserWindow, WebContentsView }) => { + const view = BrowserWindow.getAllWindows()[0].contentView.children.find( + (child) => + child instanceof WebContentsView && child.webContents.getURL().endsWith('/page') + ) + return view?.getVisible() ? view.getBounds() : null + }) + await expect.poll(nativeBounds).not.toBeNull() + const before = await nativeBounds() + await beginDrag() + await divider.evaluate((element) => { + const rect = element.getBoundingClientRect() + element.dispatchEvent( + new PointerEvent('pointermove', { pointerId: 1, clientX: rect.x + 104, bubbles: true }) + ) + element.dispatchEvent(new PointerEvent('pointercancel', { pointerId: 1, bubbles: true })) + }) + // The bridge round trip observes main-process geometry after the queued bounds messages. + await page.evaluate(() => + ( + globalThis as typeof globalThis & { simDesktop: SimDesktopApi } + ).simDesktop.browserAgent.capturePanelSnapshot('pending:native') + ) + await page.mouse.up() + await expectWidth(698) + expect(await nativeBounds()).toEqual(before) + + await check('native predictions follow a chat ID assigned during a drag', async () => { + if (!before) throw new Error('Missing native browser bounds') + await beginDrag() + await page + .getByRole('button', { name: 'Assign chat ID', exact: true }) + .evaluate((element: HTMLButtonElement) => element.click()) + await expect( + page.getByRole('button', { name: 'Assign chat ID', exact: true }) + ).toBeDisabled() + expect(await divider.evaluate((element) => element.hasPointerCapture(1))).toBe(true) + await divider.evaluate((element) => { + const rect = element.getBoundingClientRect() + element.dispatchEvent( + new PointerEvent('pointermove', { + pointerId: 1, + clientX: rect.x + 104, + bubbles: true, + }) + ) + }) + await page.evaluate(() => + ( + globalThis as typeof globalThis & { simDesktop: SimDesktopApi } + ).simDesktop.browserAgent.capturePanelSnapshot('assigned-chat') + ) + expect(await nativeBounds()).toEqual({ + ...before, + x: before.x + 50, + width: before.width - 50, + }) + await page.mouse.up() + await expectWidth(598) + }) + } + ) + expect(errors).toEqual([]) + passed = true + } finally { + mkdirSync(dirname(reportPath), { recursive: true }) + writeFileSync( + reportPath, + JSON.stringify({ passed, checks, errors, desktopExit, rendererCrashed }, null, 2) + ) + await app?.close() + await new Promise((resolve) => server.close(() => resolve())) + rmSync(userData, { recursive: true, force: true }) + } +}) diff --git a/apps/desktop/src/main/browser-agent/driver.ts b/apps/desktop/src/main/browser-agent/driver.ts index ba968d28051..c624a00beca 100644 --- a/apps/desktop/src/main/browser-agent/driver.ts +++ b/apps/desktop/src/main/browser-agent/driver.ts @@ -1738,7 +1738,7 @@ function requireSnapshotForElementAction(): void { ) } -function pageTargetForElement(contents: WebContents, elementId: number): PageExecutionTarget { +function pageTargetForElement(elementId: number): PageExecutionTarget { requireSnapshotForElementAction() const target = driverScopeState().snapshotTargets.get(elementId) if (!target || ('isDestroyed' in target && target.isDestroyed())) { @@ -2351,7 +2351,7 @@ async function captureSnapshot( if (tab.view.webContents !== contents) { throw new ToolError('The active tab changed before the snapshot started. Try again.') } - if (elementId !== undefined && pageTargetForElement(contents, elementId) !== contents) { + if (elementId !== undefined && pageTargetForElement(elementId) !== contents) { throw new ToolError( 'Scoped snapshots require a top-page element. Omit elementId to capture framed content.' ) @@ -2745,7 +2745,7 @@ async function executeToolInner( const contents = session.requireAutomationTab().view.webContents const elementId = requireNum(params, 'elementId') const paths = uploadPaths(params) - const target = pageTargetForElement(contents, elementId) + const target = pageTargetForElement(elementId) assertCurrentExecution() const frame = 'getURL' in target ? target.mainFrame : target const expression = `(${String(resolveFileInputTarget)})(${elementId})` @@ -2816,8 +2816,7 @@ async function executeToolInner( } const waitedTab = session.requireAutomationTab() const contents = waitedTab.view.webContents - const elementTarget = - elementId === undefined ? undefined : pageTargetForElement(contents, elementId) + const elementTarget = elementId === undefined ? undefined : pageTargetForElement(elementId) if (elementTarget && elementTarget !== contents) { throw new ToolError( 'Element-state waits are limited to the top page. Use a text or URL condition for framed content.' @@ -2946,7 +2945,7 @@ async function executeToolInner( const contents = session.requireAutomationTab().view.webContents const elementId = num(params, 'elementId') if (elementId === undefined) return await readWholePageText(contents, executionDeadline) - const target = pageTargetForElement(contents, elementId) + const target = pageTargetForElement(elementId) return unwrapPageResult( await execInPage(target, readPageText, [elementId], false, executionDeadline) ) @@ -2961,7 +2960,7 @@ async function executeToolInner( const elementId = num(params, 'elementId') let elementClip: Record | undefined if (elementId !== undefined) { - const target = pageTargetForElement(contents, elementId) + const target = pageTargetForElement(elementId) if (target !== contents) { throw new ToolError( 'Element screenshots are limited to the top page. Use browser_screenshot without elementId for framed content.' @@ -3119,7 +3118,7 @@ async function executeToolInner( const contents = clickedTab.view.webContents const elementId = requireNum(params, 'elementId') const click = pointerClick(params) - const target = pageTargetForElement(contents, elementId) + const target = pageTargetForElement(elementId) const targetFrame = frameExecutionTarget(target, contents) let trusted = false let activation = 'synthetic-pointer' @@ -3596,7 +3595,7 @@ async function executeToolInner( let stoppedIndex = 0 let dispatchStarted = false const readField = async (field: FormField) => { - const target = pageTargetForElement(contents, field.elementId) + const target = pageTargetForElement(field.elementId) if (target !== contents) throw new ToolError( 'Form batches require top-page fields; use individual tools for framed fields.' @@ -3757,7 +3756,7 @@ async function executeToolInner( if (typeof text !== 'string') throw new ToolError('Missing required parameter "text"') const submit = params.submit === true const contents = session.requireAutomationTab().view.webContents - const target = pageTargetForElement(contents, elementId) + const target = pageTargetForElement(elementId) const targetFrame = frameExecutionTarget(target, contents) // Native path: focus + select current content, then insert through the @@ -4307,9 +4306,7 @@ async function executeToolInner( const contents = session.requireAutomationTab().view.webContents const elementId = num(params, 'elementId') const target = - elementId !== undefined - ? pageTargetForElement(contents, elementId) - : focusedPageTarget(contents) + elementId !== undefined ? pageTargetForElement(elementId) : focusedPageTarget(contents) const targetFrame = frameExecutionTarget(target, contents) assertCurrentExecution() if (elementId !== undefined) assertElementActionCurrent(contents, elementId, target) @@ -4355,7 +4352,7 @@ async function executeToolInner( const selection = values === undefined ? requireStr(params, 'value') : (values as string[]) const contents = session.requireAutomationTab().view.webContents const elementId = requireNum(params, 'elementId') - const target = pageTargetForElement(contents, elementId) + const target = pageTargetForElement(elementId) const targetFrame = frameExecutionTarget(target, contents) assertCurrentExecution() assertElementActionCurrent(contents, elementId, target) @@ -4434,8 +4431,7 @@ async function executeToolInner( throw new ToolError('Missing required boolean parameter "checked"') } const checked = params.checked - const contents = session.requireAutomationTab().view.webContents - const target = pageTargetForElement(contents, elementId) + const target = pageTargetForElement(elementId) const before = toRecord( unwrapPageResult( await execInPage(target, readCheckableElementState, [elementId], false, executionDeadline) @@ -4562,7 +4558,7 @@ async function executeToolInner( ) } const elementId = requireNum(params, 'elementId') - const target = pageTargetForElement(contents, elementId) + const target = pageTargetForElement(elementId) const targetFrame = frameExecutionTarget(target, contents) let beforePage = await pageActionState(target, true, elementId) let beforeElement = await activeElementState(target) @@ -4939,7 +4935,7 @@ async function executeToolInner( ): Promise<{ x: number; y: number; element?: string }> => { const elementId = num(params, `${which}ElementId`) if (elementId !== undefined) { - const target = pageTargetForElement(contents, elementId) + const target = pageTargetForElement(elementId) if (frameExecutionTarget(target, contents)) { throw new ToolError( `Dragging elements inside embedded frames is not supported. Use ${which}X/${which}Y viewport coordinates instead.` diff --git a/apps/desktop/src/main/browser-credentials/os-auth.ts b/apps/desktop/src/main/browser-credentials/os-auth.ts index 23d9cf6c530..2aea35c5c0b 100644 --- a/apps/desktop/src/main/browser-credentials/os-auth.ts +++ b/apps/desktop/src/main/browser-credentials/os-auth.ts @@ -147,7 +147,7 @@ async function promptForSecret(reason: string, action: string): Promise ? await showShellDialog(parent, options) : await showShellDialog(options) return response === 1 - } catch (error) { + } catch { // Fail closed: if the confirmation cannot be shown, nothing is revealed. logger.warn('Could not present the credential confirmation') return false diff --git a/apps/docs/components/icons.tsx b/apps/docs/components/icons.tsx index ef5cc9b3895..ff3d66b7c84 100644 --- a/apps/docs/components/icons.tsx +++ b/apps/docs/components/icons.tsx @@ -9613,7 +9613,7 @@ export function EnrowIcon(props: SVGProps) { ) } -// ---- Competitor brand logos (sourced via Context.dev brand-intelligence API, 2026-07-02) ---- +// Competitor brand logos (sourced via Context.dev brand-intelligence API, 2026-07-02) export function N8nIcon(props: SVGProps) { return ( diff --git a/apps/docs/components/ui/block-info-card.tsx b/apps/docs/components/ui/block-info-card.tsx index d20380c3d93..5bd1bc7e802 100644 --- a/apps/docs/components/ui/block-info-card.tsx +++ b/apps/docs/components/ui/block-info-card.tsx @@ -1,6 +1,7 @@ 'use client' import type * as React from 'react' +import { isLightTileColor } from '@sim/workflow-renderer/tile-icon-color' import { blockTypeToIconMap } from '@/components/ui/icon-mapping' interface BlockInfoCardProps { @@ -9,33 +10,6 @@ interface BlockInfoCardProps { icon?: React.ComponentType<{ className?: string }> } -/** - * Brightness above which a tile background is "clearly light" and a white - * foreground icon would wash out. Mirrors apps/sim's LIGHT_TILE_THRESHOLD - * (blocks/icon-color.ts) so monochrome `currentColor` icons (e.g. Daytona, - * Notion) stay legible on white/pale tiles instead of white-on-white. - */ -const LIGHT_TILE_THRESHOLD = 0.75 - -function isLightTileColor(color: string): boolean { - const hex = color.trim().replace('#', '').toLowerCase() - let r: number - let g: number - let b: number - if (/^[0-9a-f]{3}$/.test(hex)) { - r = Number.parseInt(hex[0] + hex[0], 16) - g = Number.parseInt(hex[1] + hex[1], 16) - b = Number.parseInt(hex[2] + hex[2], 16) - } else if (/^[0-9a-f]{6}$/.test(hex)) { - r = Number.parseInt(hex.slice(0, 2), 16) - g = Number.parseInt(hex.slice(2, 4), 16) - b = Number.parseInt(hex.slice(4, 6), 16) - } else { - return false - } - return (0.299 * r + 0.587 * g + 0.114 * b) / 255 > LIGHT_TILE_THRESHOLD -} - export function BlockInfoCard({ type, color, diff --git a/apps/docs/content/docs/api-reference/(generated)/workflows/meta.json b/apps/docs/content/docs/api-reference/(generated)/workflows/meta.json index 28453d1b10f..0416e441abe 100644 --- a/apps/docs/content/docs/api-reference/(generated)/workflows/meta.json +++ b/apps/docs/content/docs/api-reference/(generated)/workflows/meta.json @@ -14,6 +14,7 @@ "applyWorkflowVariables", "listWorkflowVersionsV2", "getWorkflowVersionV2", + "compareWorkflowVersionsV2", "updateWorkflowVersionV2", "activateWorkflowVersion", "revertWorkflowVersion", diff --git a/apps/docs/content/docs/cli/files.mdx b/apps/docs/content/docs/cli/files.mdx index b79f79045f5..0ec32c4991a 100644 --- a/apps/docs/content/docs/cli/files.mdx +++ b/apps/docs/content/docs/cli/files.mdx @@ -247,7 +247,7 @@ sim files versions list [options] -## Read the text content of one version of a file +## Read the text content of one version of a file as JSON or YAML ```bash sim files versions read [options] @@ -474,7 +474,7 @@ Also available as `sim files mv`. -## Read a file’s text content +## Read a file’s text content as JSON or YAML ```bash sim files read [options] diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index c24d737698e..27c734ae675 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -1015,7 +1015,7 @@ sim files versions list [options] ### sim files versions read -Read the text content of one version of a file +Read the text content of one version of a file as JSON or YAML ```bash sim files versions read [options] @@ -1258,7 +1258,7 @@ Also available as `sim files mv`. ### sim files read -Read a file’s text content +Read a file’s text content as JSON or YAML ```bash sim files read [options] @@ -6277,6 +6277,113 @@ sim workflows runs wait [options] +### sim workflows versions compare + +Compare Workflow Versions + +```bash +sim workflows versions compare [options] +``` + +**Arguments** + + + +| Argument | Required | Description | +| --- | --- | --- | +| `workflowId` | Yes | Unique workflow identifier. | + + + +**Options** + + + +| Option | Required | Description | +| --- | --- | --- | +| `--base ` | Yes | Deployment version to compare from. | +| `--target ` | Yes | Deployment version to compare to, in the same workflow. | + + + +### sim workflows versions get + +Get Workflow Version + +```bash +sim workflows versions get +``` + +**Arguments** + + + +| Argument | Required | Description | +| --- | --- | --- | +| `workflowId` | Yes | Unique workflow identifier. | +| `version` | Yes | Numeric deployment version. | + + + +### sim workflows versions list + +List Workflow Versions + +```bash +sim workflows versions list [options] +``` + +**Arguments** + + + +| Argument | Required | Description | +| --- | --- | --- | +| `workflowId` | Yes | Unique workflow identifier. | + + + +**Options** + + + +| Option | Required | Description | +| --- | --- | --- | +| `--limit ` | No | Maximum items to return (0 for everything). Defaults to `100`. | +| `--cursor ` | No | Continue from nextCursor returned by a previous result. | + + + +### sim workflows versions update + +Update Workflow Version + +```bash +sim workflows versions update [options] +``` + +**Arguments** + + + +| Argument | Required | Description | +| --- | --- | --- | +| `workflowId` | Yes | Unique workflow identifier. | +| `version` | Yes | Numeric deployment version. | + + + +**Options** + + + +| Option | Required | Description | +| --- | --- | --- | +| `--name ` | No | New label for the deployment version. | +| `--description ` | No | New release note for the deployment version, or null to clear it. (--description null sends the word, not JSON null). | + + + ### sim workflows create Create Workflow @@ -6293,7 +6400,7 @@ sim workflows create [options] | --- | --- | --- | | `--name ` | Yes | Workflow name. | | `--description ` | No | Optional workflow description. | -| `--folder ` | No | Folder path as shown in the app; the leading / is optional. | +| `--folder ` | No | Existing folder path (leading / optional); create it first with sim workflows mkdir <path>. | @@ -6750,84 +6857,6 @@ sim workflows state replace [options] -### sim workflows versions get - -Get Workflow Version - -```bash -sim workflows versions get -``` - -**Arguments** - - - -| Argument | Required | Description | -| --- | --- | --- | -| `workflowId` | Yes | Unique workflow identifier. | -| `version` | Yes | Numeric deployment version. | - - - -### sim workflows versions list - -List Workflow Versions - -```bash -sim workflows versions list [options] -``` - -**Arguments** - - - -| Argument | Required | Description | -| --- | --- | --- | -| `workflowId` | Yes | Unique workflow identifier. | - - - -**Options** - - - -| Option | Required | Description | -| --- | --- | --- | -| `--limit ` | No | Maximum items to return (0 for everything). Defaults to `100`. | -| `--cursor ` | No | Continue from nextCursor returned by a previous result. | - - - -### sim workflows versions update - -Update Workflow Version - -```bash -sim workflows versions update [options] -``` - -**Arguments** - - - -| Argument | Required | Description | -| --- | --- | --- | -| `workflowId` | Yes | Unique workflow identifier. | -| `version` | Yes | Numeric deployment version. | - - - -**Options** - - - -| Option | Required | Description | -| --- | --- | --- | -| `--name ` | No | New label for the deployment version. | -| `--description ` | No | New release note for the deployment version, or null to clear it. (--description null sends the word, not JSON null). | - - - ### sim workflows import Import Workflow diff --git a/apps/docs/content/docs/cli/workflows.mdx b/apps/docs/content/docs/cli/workflows.mdx index 9a98eb0f113..359f2be643a 100644 --- a/apps/docs/content/docs/cli/workflows.mdx +++ b/apps/docs/content/docs/cli/workflows.mdx @@ -239,6 +239,105 @@ sim workflows runs wait [options] +## Compare workflow versions + +```bash +sim workflows versions compare [options] +``` + +**Arguments** + + + +| Argument | Required | Description | +| --- | --- | --- | +| `workflowId` | Yes | Unique workflow identifier. | + + + +**Options** + + + +| Option | Required | Description | +| --- | --- | --- | +| `--base ` | Yes | Deployment version to compare from. | +| `--target ` | Yes | Deployment version to compare to, in the same workflow. | + + + +## Get workflow version + +```bash +sim workflows versions get +``` + +**Arguments** + + + +| Argument | Required | Description | +| --- | --- | --- | +| `workflowId` | Yes | Unique workflow identifier. | +| `version` | Yes | Numeric deployment version. | + + + +## List workflow versions + +```bash +sim workflows versions list [options] +``` + +**Arguments** + + + +| Argument | Required | Description | +| --- | --- | --- | +| `workflowId` | Yes | Unique workflow identifier. | + + + +**Options** + + + +| Option | Required | Description | +| --- | --- | --- | +| `--limit ` | No | Maximum items to return (0 for everything). Defaults to `100`. | +| `--cursor ` | No | Continue from nextCursor returned by a previous result. | + + + +## Update workflow version + +```bash +sim workflows versions update [options] +``` + +**Arguments** + + + +| Argument | Required | Description | +| --- | --- | --- | +| `workflowId` | Yes | Unique workflow identifier. | +| `version` | Yes | Numeric deployment version. | + + + +**Options** + + + +| Option | Required | Description | +| --- | --- | --- | +| `--name ` | No | New label for the deployment version. | +| `--description ` | No | New release note for the deployment version, or null to clear it. (--description null sends the word, not JSON null). | + + + ## Create workflow ```bash @@ -253,7 +352,7 @@ sim workflows create [options] | --- | --- | --- | | `--name ` | Yes | Workflow name. | | `--description ` | No | Optional workflow description. | -| `--folder ` | No | Folder path as shown in the app; the leading / is optional. | +| `--folder ` | No | Existing folder path (leading / optional); create it first with sim workflows mkdir <path>. | @@ -688,78 +787,6 @@ Replace Workflow State (OAuth login or personal API key required) -## Get workflow version - -```bash -sim workflows versions get -``` - -**Arguments** - - - -| Argument | Required | Description | -| --- | --- | --- | -| `workflowId` | Yes | Unique workflow identifier. | -| `version` | Yes | Numeric deployment version. | - - - -## List workflow versions - -```bash -sim workflows versions list [options] -``` - -**Arguments** - - - -| Argument | Required | Description | -| --- | --- | --- | -| `workflowId` | Yes | Unique workflow identifier. | - - - -**Options** - - - -| Option | Required | Description | -| --- | --- | --- | -| `--limit ` | No | Maximum items to return (0 for everything). Defaults to `100`. | -| `--cursor ` | No | Continue from nextCursor returned by a previous result. | - - - -## Update workflow version - -```bash -sim workflows versions update [options] -``` - -**Arguments** - - - -| Argument | Required | Description | -| --- | --- | --- | -| `workflowId` | Yes | Unique workflow identifier. | -| `version` | Yes | Numeric deployment version. | - - - -**Options** - - - -| Option | Required | Description | -| --- | --- | --- | -| `--name ` | No | New label for the deployment version. | -| `--description ` | No | New release note for the deployment version, or null to clear it. (--description null sends the word, not JSON null). | - - - ## Import workflow ```bash diff --git a/apps/docs/content/docs/integrations/greptile.mdx b/apps/docs/content/docs/integrations/greptile.mdx index 5df5d794748..43438c7019a 100644 --- a/apps/docs/content/docs/integrations/greptile.mdx +++ b/apps/docs/content/docs/integrations/greptile.mdx @@ -53,35 +53,6 @@ Query repositories in natural language and get answers with relevant code refere | ↳ `summary` | string | Summary of the code section | | ↳ `distance` | number | Similarity score \(lower = more relevant\) | -### Greptile Search - -Search repositories in natural language and get relevant code references without generating an answer. Useful for finding specific code locations. - -#### Input - -| Parameter | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `query` | string | Yes | Natural language search query to find relevant code. Example: "authentication middleware" or "database connection handling" | -| `repositories` | string | Yes | Comma-separated list of repositories. Format: "github:branch:owner/repo" or just "owner/repo" \(defaults to github:main\). Example: "facebook/react" or "github:main:facebook/react,github:main:facebook/relay" | -| `sessionId` | string | No | Session ID for conversation continuity. Use the same sessionId across multiple searches to maintain context. Example: "session-abc123" | -| `genius` | boolean | No | Enable genius mode for more thorough search \(slower but more accurate\) | -| `apiKey` | string | Yes | Greptile API key | -| `githubToken` | string | Yes | GitHub Personal Access Token with repo read access | - -#### Output - -| Parameter | Type | Description | -| --------- | ---- | ----------- | -| `sources` | array | Relevant code references matching the search query | -| ↳ `repository` | string | Repository name \(owner/repo\) | -| ↳ `remote` | string | Git remote \(github/gitlab\) | -| ↳ `branch` | string | Branch name | -| ↳ `filepath` | string | Path to the file | -| ↳ `linestart` | number | Starting line number | -| ↳ `lineend` | number | Ending line number | -| ↳ `summary` | string | Summary of the code section | -| ↳ `distance` | number | Similarity score \(lower = more relevant\) | - ### Greptile Index Repository Submit a repository to be indexed by Greptile. Indexing must complete before the repository can be queried. Small repos take 3-5 minutes, larger ones can take over an hour. diff --git a/apps/docs/content/docs/platform/self-hosting/integrations-oauth.mdx b/apps/docs/content/docs/platform/self-hosting/integrations-oauth.mdx index 392b0785f8d..99c522a3f49 100644 --- a/apps/docs/content/docs/platform/self-hosting/integrations-oauth.mdx +++ b/apps/docs/content/docs/platform/self-hosting/integrations-oauth.mdx @@ -207,7 +207,7 @@ The private key must include its PEM header, footer, and contents. Sim accepts a Keep the private key and client secret in the deployment's server configuration; organization admins select installations in Sim without entering these secrets. -The live Search application needs these five variables. If a worker also indexes ordinary GitHub knowledge bases, or Search has explicitly reverted to `SIM_SEARCH_LIVE=false`, configure the variables in that worker’s matching environment too. Updating the app’s secret store does not update a separately configured worker. Live Search does not schedule a GitHub indexing worker. +The live Search application needs these five variables. If a worker also indexes ordinary GitHub knowledge bases, configure the variables in that worker’s matching environment too. Updating the app’s secret store does not update a separately configured worker. Live Search does not schedule a GitHub indexing worker. The **Client ID** is different from the numeric **App ID**. Use credentials from **Developer settings → GitHub Apps**. `GITHUB_CLIENT_ID` and `GITHUB_CLIENT_SECRET` belong to the separate GitHub sign-in integration and remain unchanged. Search does not read `GITHUB_REPO_CLIENT_ID` or `GITHUB_REPO_CLIENT_SECRET`. diff --git a/apps/docs/content/docs/search/index.mdx b/apps/docs/content/docs/search/index.mdx index cb498ac484c..59c23f27f13 100644 --- a/apps/docs/content/docs/search/index.mdx +++ b/apps/docs/content/docs/search/index.mdx @@ -80,6 +80,6 @@ Conversations remain private to their author. Connecting an external account doe ## Legacy indexing and workspace knowledge bases -Live Search is the default. An operator can explicitly set `SIM_SEARCH_LIVE=false` to restore the legacy indexed Search backend. Its content sync, document processing, and stored permission maintenance are specific to that mode. These guides describe live Search. +Enterprise Search uses live provider queries. Search sources do not run content indexing or stored permission maintenance; ordinary knowledge-base connectors retain their indexing behavior. Ordinary workspace [knowledge bases](/knowledgebase) still ingest, chunk, and index documents for their own features. Their connector setup, processing status, and access settings remain separate. diff --git a/apps/docs/content/docs/search/mcp.mdx b/apps/docs/content/docs/search/mcp.mdx index 25fab917a3d..0e838e09fca 100644 --- a/apps/docs/content/docs/search/mcp.mdx +++ b/apps/docs/content/docs/search/mcp.mdx @@ -44,7 +44,7 @@ The connection applies to the organization whose URL you copied. Sim checks curr `chat` accepts questions up to 8,192 characters. Tool responses are limited to 1 MiB. API rate limits apply; follow any retry delay. Provider failures or incomplete retrieval are reported explicitly, and the tools do not fall back to an old index when a live provider is unavailable. -These are the live-backend schemas. If an operator explicitly selects legacy Search with `SIM_SEARCH_LIVE=false`, the server advertises its indexed search/read schemas instead. Refresh your MCP client's tool discovery after changing backends. +These are the Search MCP schemas. Refresh tool discovery in clients that previously connected to the retired indexed backend. ## Reconnect or revoke access diff --git a/apps/docs/openapi-v2-workflows.json b/apps/docs/openapi-v2-workflows.json index efd74c8ae62..eee81168e3c 100644 --- a/apps/docs/openapi-v2-workflows.json +++ b/apps/docs/openapi-v2-workflows.json @@ -1200,6 +1200,101 @@ } } }, + "/api/v2/workflows/{workflowId}/versions/compare": { + "get": { + "operationId": "compareWorkflowVersionsV2", + "summary": "Compare Workflow Versions", + "description": "Compare two deployment versions of the same workflow. Reports semantic changes, excluding canvas layout; credential-bearing values are withheld while their changes remain visible. Connections include stable block and port identifiers. The combined snapshots and the comparison result must each fit within 16 MiB.\n\nOAuth scope: `api:read`.", + "x-sim-operation": "workflows.versions.compare", + "x-oauth-scope": "api:read", + "tags": ["Workflows"], + "parameters": [ + { + "name": "workflowId", + "in": "path", + "required": true, + "description": "Unique workflow identifier.", + "schema": { + "type": "string", + "minLength": 1, + "description": "Unique workflow identifier.", + "examples": ["3b1f7c92-8d4e-4a6b-9c0d-5e2f8a714b36"] + } + }, + { + "name": "base", + "in": "query", + "required": true, + "description": "Deployment version to compare from.", + "schema": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 2147483647, + "description": "Deployment version to compare from." + } + }, + { + "name": "target", + "in": "query", + "required": true, + "description": "Deployment version to compare to, in the same workflow.", + "schema": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 2147483647, + "description": "Deployment version to compare to, in the same workflow." + } + } + ], + "responses": { + "200": { + "description": "Changes from the base deployment to the target deployment.", + "headers": { + "X-RateLimit-Limit": { + "$ref": "#/components/headers/X-RateLimit-Limit" + }, + "X-RateLimit-Remaining": { + "$ref": "#/components/headers/X-RateLimit-Remaining" + }, + "X-RateLimit-Reset": { + "$ref": "#/components/headers/X-RateLimit-Reset" + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowVersionComparisonResponse" + } + } + } + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "403": { + "$ref": "#/components/responses/Forbidden" + }, + "404": { + "$ref": "#/components/responses/NotFound" + }, + "413": { + "$ref": "#/components/responses/PayloadTooLarge" + }, + "429": { + "$ref": "#/components/responses/RateLimited" + }, + "500": { + "$ref": "#/components/responses/InternalError" + }, + "503": { + "$ref": "#/components/responses/ServiceUnavailable" + } + } + } + }, "/api/v2/workflows/{workflowId}/versions/{version}": { "get": { "operationId": "getWorkflowVersionV2", @@ -9171,6 +9266,620 @@ } ] }, + "WorkflowVersionComparison": { + "type": "object", + "properties": { + "workflowId": { + "type": "string", + "description": "Workflow compared." + }, + "base": { + "type": "integer", + "minimum": 1, + "maximum": 2147483647, + "description": "Base deployment version." + }, + "target": { + "type": "integer", + "minimum": 1, + "maximum": 2147483647, + "description": "Target deployment version." + }, + "diff": { + "type": "object", + "properties": { + "addedBlocks": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "Stable block identifier." + }, + "type": { + "type": "string", + "description": "Block type." + }, + "name": { + "description": "Block name.", + "type": "string" + } + }, + "required": ["id", "type"], + "additionalProperties": false + }, + "description": "Blocks present only in the target." + }, + "removedBlocks": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "Stable block identifier." + }, + "type": { + "type": "string", + "description": "Block type." + }, + "name": { + "description": "Block name.", + "type": "string" + } + }, + "required": ["id", "type"], + "additionalProperties": false + }, + "description": "Blocks present only in the base." + }, + "modifiedBlocks": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "Stable block identifier." + }, + "type": { + "type": "string", + "description": "Block type." + }, + "name": { + "description": "Block name.", + "type": "string" + }, + "changes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "field": { + "type": "string", + "description": "Changed field identifier." + }, + "oldValue": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "unset", + "description": "The field has no configured value." + } + }, + "required": ["kind"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "redacted", + "description": "The value is withheld by the credential policy." + } + }, + "required": ["kind"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "value", + "description": "The field value is included." + }, + "value": { + "description": "The complete user-authored field value." + } + }, + "required": ["kind", "value"], + "additionalProperties": false + } + ], + "description": "Base version value." + }, + "newValue": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "unset", + "description": "The field has no configured value." + } + }, + "required": ["kind"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "redacted", + "description": "The value is withheld by the credential policy." + } + }, + "required": ["kind"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "value", + "description": "The field value is included." + }, + "value": { + "description": "The complete user-authored field value." + } + }, + "required": ["kind", "value"], + "additionalProperties": false + } + ], + "description": "Target version value." + }, + "scope": { + "type": "string", + "enum": ["block", "subblock"], + "description": "Whether the field is a block setting or a subblock input." + } + }, + "required": ["field", "oldValue", "newValue", "scope"], + "additionalProperties": false + }, + "description": "Changed field values." + } + }, + "required": ["id", "type", "changes"], + "additionalProperties": false + }, + "description": "Blocks with changed fields." + }, + "edgeChanges": { + "type": "object", + "properties": { + "added": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991, + "description": "Number added." + }, + "removed": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991, + "description": "Number removed." + }, + "addedDetails": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string", + "description": "Stable source block identifier." + }, + "target": { + "type": "string", + "description": "Stable target block identifier." + }, + "sourceHandle": { + "description": "Source port identifier; omitted for the default port.", + "type": "string" + }, + "targetHandle": { + "description": "Target port identifier; omitted for the default port.", + "type": "string" + }, + "sourceName": { + "type": "string", + "description": "Source block name." + }, + "targetName": { + "type": "string", + "description": "Target block name." + } + }, + "required": ["source", "target", "sourceName", "targetName"], + "additionalProperties": false + }, + "description": "Added connections." + }, + "removedDetails": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string", + "description": "Stable source block identifier." + }, + "target": { + "type": "string", + "description": "Stable target block identifier." + }, + "sourceHandle": { + "description": "Source port identifier; omitted for the default port.", + "type": "string" + }, + "targetHandle": { + "description": "Target port identifier; omitted for the default port.", + "type": "string" + }, + "sourceName": { + "type": "string", + "description": "Source block name." + }, + "targetName": { + "type": "string", + "description": "Target block name." + } + }, + "required": ["source", "target", "sourceName", "targetName"], + "additionalProperties": false + }, + "description": "Removed connections." + } + }, + "required": ["added", "removed", "addedDetails", "removedDetails"], + "additionalProperties": false, + "description": "Connection changes." + }, + "loopChanges": { + "type": "object", + "properties": { + "added": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991, + "description": "Number added." + }, + "removed": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991, + "description": "Number removed." + }, + "modified": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991, + "description": "Number modified." + } + }, + "required": ["added", "removed", "modified"], + "additionalProperties": false, + "description": "Loop change counts." + }, + "parallelChanges": { + "type": "object", + "properties": { + "added": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991, + "description": "Number added." + }, + "removed": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991, + "description": "Number removed." + }, + "modified": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991, + "description": "Number modified." + } + }, + "required": ["added", "removed", "modified"], + "additionalProperties": false, + "description": "Parallel change counts." + }, + "containerChanges": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "Container block ID." + }, + "kind": { + "type": "string", + "enum": ["loop", "parallel"], + "description": "Container type." + }, + "name": { + "description": "Container name.", + "type": "string" + }, + "changes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "field": { + "type": "string", + "description": "Changed field identifier." + }, + "oldValue": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "unset", + "description": "The field has no configured value." + } + }, + "required": ["kind"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "redacted", + "description": "The value is withheld by the credential policy." + } + }, + "required": ["kind"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "value", + "description": "The field value is included." + }, + "value": { + "description": "The complete user-authored field value." + } + }, + "required": ["kind", "value"], + "additionalProperties": false + } + ], + "description": "Base version value." + }, + "newValue": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "unset", + "description": "The field has no configured value." + } + }, + "required": ["kind"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "redacted", + "description": "The value is withheld by the credential policy." + } + }, + "required": ["kind"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "value", + "description": "The field value is included." + }, + "value": { + "description": "The complete user-authored field value." + } + }, + "required": ["kind", "value"], + "additionalProperties": false + } + ], + "description": "Target version value." + } + }, + "required": ["field", "oldValue", "newValue"], + "additionalProperties": false + }, + "description": "Changed field values." + }, + "nodesAdded": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Added member block IDs." + }, + "nodesRemoved": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Removed member block IDs." + } + }, + "required": ["id", "kind", "changes", "nodesAdded", "nodesRemoved"], + "additionalProperties": false + }, + "description": "Container configuration and membership changes." + }, + "variableChanges": { + "type": "object", + "properties": { + "added": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991, + "description": "Number added." + }, + "removed": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991, + "description": "Number removed." + }, + "modified": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991, + "description": "Number modified." + }, + "addedNames": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Names of added variables." + }, + "removedNames": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Names of removed variables." + }, + "modifiedNames": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Names of modified variables." + } + }, + "required": [ + "added", + "removed", + "modified", + "addedNames", + "removedNames", + "modifiedNames" + ], + "additionalProperties": false, + "description": "Variable change counts and names." + }, + "hasChanges": { + "type": "boolean", + "description": "Whether any semantic change was found." + } + }, + "required": [ + "addedBlocks", + "removedBlocks", + "modifiedBlocks", + "edgeChanges", + "loopChanges", + "parallelChanges", + "containerChanges", + "variableChanges", + "hasChanges" + ], + "additionalProperties": false, + "description": "Changes from base to target." + } + }, + "required": ["workflowId", "base", "target", "diff"], + "additionalProperties": false, + "title": "Workflow version comparison", + "description": "Semantic changes from base to target within one workflow. Credential-bearing fields are redacted; changes to them are still reported. Canvas layout is excluded." + }, + "WorkflowVersionComparisonResponse": { + "type": "object", + "properties": { + "data": { + "description": "Response data.", + "$ref": "#/components/schemas/WorkflowVersionComparison" + } + }, + "required": ["data"], + "additionalProperties": false, + "title": "Workflow version comparison response", + "description": "Changes from base to target, with credential values withheld.", + "examples": [ + { + "data": { + "workflowId": "3b1f7c92-8d4e-4a6b-9c0d-5e2f8a714b36", + "base": 1, + "target": 2, + "diff": { + "addedBlocks": [], + "removedBlocks": [], + "modifiedBlocks": [], + "edgeChanges": { + "added": 0, + "removed": 0, + "addedDetails": [], + "removedDetails": [] + }, + "loopChanges": { + "added": 0, + "removed": 0, + "modified": 0 + }, + "parallelChanges": { + "added": 0, + "removed": 0, + "modified": 0 + }, + "containerChanges": [], + "variableChanges": { + "added": 0, + "removed": 0, + "modified": 0, + "addedNames": [], + "removedNames": [], + "modifiedNames": [] + }, + "hasChanges": false + } + } + } + ] + }, "DeployedWorkflowState": { "title": "Deployed workflow state", "description": "Workflow graph snapshot pinned by a deployment version.", diff --git a/apps/realtime/src/auth.ts b/apps/realtime/src/auth.ts index 40491a62af7..3fa013917d8 100644 --- a/apps/realtime/src/auth.ts +++ b/apps/realtime/src/auth.ts @@ -1,16 +1,6 @@ import { createVerifyAuth } from '@sim/auth/verify' import { env } from '@/env' -export const ANONYMOUS_USER_ID = '00000000-0000-0000-0000-000000000000' - -export const ANONYMOUS_USER = { - id: ANONYMOUS_USER_ID, - name: 'Anonymous', - email: 'anonymous@localhost', - emailVerified: true, - image: null, -} as const - export const auth = createVerifyAuth({ secret: env.BETTER_AUTH_SECRET, baseURL: env.BETTER_AUTH_URL, diff --git a/apps/realtime/src/database/operations.ts b/apps/realtime/src/database/operations.ts index cd4316448bf..7d6f444eb85 100644 --- a/apps/realtime/src/database/operations.ts +++ b/apps/realtime/src/database/operations.ts @@ -291,7 +291,6 @@ function isSubflowBlockType(blockType: string): blockType is SubflowType { export async function updateSubflowNodeList(dbOrTx: any, workflowId: string, parentId: string) { try { - // Get all child blocks of this parent const childBlocks = await dbOrTx .select({ id: workflowBlocks.id }) .from(workflowBlocks) @@ -304,7 +303,6 @@ export async function updateSubflowNodeList(dbOrTx: any, workflowId: string, par const childNodeIds = childBlocks.map((block: any) => block.id) - // Get current subflow config const subflowData = await dbOrTx .select({ config: workflowSubflows.config }) .from(workflowSubflows) @@ -437,7 +435,6 @@ export async function persistWorkflowOperation(workflowId: string, operation: an } }) - // Audit workflow-level lock/unlock operations if ( target === OPERATION_TARGETS.BLOCKS && op === BLOCKS_OPERATIONS.BATCH_TOGGLE_LOCKED && @@ -541,7 +538,6 @@ async function handleBlockOperationTx( throw new Error('Missing required fields for update name operation') } - // Check if block is protected (locked or inside locked parent) const blockToRename = await tx .select({ id: workflowBlocks.id, @@ -612,7 +608,6 @@ async function handleBlockOperationTx( throw new Error('Missing block ID for toggle enabled operation') } - // Get current enabled state const currentBlock = await tx .select({ enabled: workflowBlocks.enabled }) .from(workflowBlocks) @@ -654,7 +649,6 @@ async function handleBlockOperationTx( const isRemovingFromParent = !payload.parentId - // Get current data to update const [currentBlock] = await tx .select({ data: workflowBlocks.data }) .from(workflowBlocks) @@ -663,7 +657,6 @@ async function handleBlockOperationTx( const currentData = currentBlock?.data || {} - // Update data with parentId and extent const { parentId: _removedParentId, extent: _removedExtent, ...restData } = currentData const updatedData = isRemovingFromParent ? restData @@ -686,11 +679,9 @@ async function handleBlockOperationTx( throw new Error(`Block ${payload.id} not found in workflow ${workflowId}`) } - // If the block now has a parent, update the new parent's subflow node list if (payload.parentId) { await updateSubflowNodeList(tx, workflowId, payload.parentId) } - // If the block had a previous parent, update that parent's node list as well if (existing?.parentId && existing.parentId !== payload.parentId) { await updateSubflowNodeList(tx, workflowId, existing.parentId) } @@ -918,7 +909,6 @@ async function handleBlocksOperationTx( }) if (blocks && blocks.length > 0) { - // Fetch existing blocks to check for locked parents const existingBlocks = await tx .select({ id: workflowBlocks.id, locked: workflowBlocks.locked }) .from(workflowBlocks) @@ -931,7 +921,6 @@ async function handleBlocksOperationTx( .map((b: ExistingBlockRecord) => b.id) ) - // Filter out blocks being added to locked parents const allowedBlocks = (blocks as Array>).filter((block) => { const parentId = (block.data as Record | null)?.parentId as | string @@ -1050,7 +1039,6 @@ async function handleBlocksOperationTx( } } - // Update parent subflow node lists const parentIds = new Set() for (const block of allowedBlocks) { const parentId = (block.data as Record)?.parentId as string | undefined @@ -1166,7 +1154,6 @@ async function handleBlocksOperationTx( logger.info(`Batch removing ${ids.length} blocks from workflow ${workflowId}`) - // Fetch all blocks to check lock status and filter out protected blocks const allBlocks = await tx .select({ id: workflowBlocks.id, @@ -1182,7 +1169,6 @@ async function handleBlocksOperationTx( allBlocks.map((b: BlockRecord) => [b.id, b]) ) - // Filter out protected blocks from deletion request const deletableIds = ids.filter((id) => !isWorkflowBlockProtected(id, blocksById)) if (deletableIds.length === 0) { logger.info('All requested blocks are protected, skipping deletion') @@ -1195,7 +1181,6 @@ async function handleBlocksOperationTx( ) } - // Collect all block IDs including all descendants of subflows const allBlocksToDelete = new Set(deletableIds) for (const id of deletableIds) { @@ -1226,7 +1211,6 @@ async function handleBlocksOperationTx( // lifecycle is managed by deploy.ts, lifecycle.ts, and the /api/webhooks/[id] route. // Removing a trigger block from the draft canvas does not touch any webhook rows. - // Delete edges connected to any of the blocks await tx .delete(workflowEdges) .where( @@ -1239,7 +1223,6 @@ async function handleBlocksOperationTx( ) ) - // Delete subflow entries await tx .delete(workflowSubflows) .where( @@ -1249,14 +1232,12 @@ async function handleBlocksOperationTx( ) ) - // Delete all blocks await tx .delete(workflowBlocks) .where( and(eq(workflowBlocks.workflowId, workflowId), inArray(workflowBlocks.id, blockIdsArray)) ) - // Update parent subflow node lists using pre-collected parent IDs for (const parentId of parentIds) { await updateSubflowNodeList(tx, workflowId, parentId) } @@ -1277,7 +1258,6 @@ async function handleBlocksOperationTx( `Batch toggling enabled state for ${blockIds.length} blocks in workflow ${workflowId}` ) - // Get all blocks in workflow to find children and check locked state const allBlocks = await tx .select({ id: workflowBlocks.id, @@ -1295,14 +1275,12 @@ async function handleBlocksOperationTx( ) const blocksToToggle = new Set() - // Collect all blocks to toggle including descendants of containers for (const id of blockIds) { const block = blocksById[id] if (!block || isWorkflowBlockProtected(id, blocksById)) continue blocksToToggle.add(id) - // If it's a loop or parallel, also include all non-locked descendants if (block.type === 'loop' || block.type === 'parallel') { for (const descId of findDbDescendants(id, allBlocks)) { if (!isWorkflowBlockProtected(descId, blocksById)) { @@ -1319,7 +1297,6 @@ async function handleBlocksOperationTx( if (!firstBlock) break const targetEnabled = !firstBlock.enabled - // Update all affected blocks for (const blockId of blocksToToggle) { await tx .update(workflowBlocks) @@ -1342,7 +1319,6 @@ async function handleBlocksOperationTx( logger.info(`Batch toggling handles for ${blockIds.length} blocks in workflow ${workflowId}`) - // Fetch all blocks to check lock status and filter out protected blocks const allBlocks = await tx .select({ id: workflowBlocks.id, @@ -1358,7 +1334,6 @@ async function handleBlocksOperationTx( allBlocks.map((b: HandleBlockRecord) => [b.id, b]) ) - // Filter to only toggle handles on unprotected blocks const blocksToToggle = blockIds.filter( (id) => blocksById[id] && !isWorkflowBlockProtected(id, blocksById) ) @@ -1390,7 +1365,6 @@ async function handleBlocksOperationTx( logger.info(`Batch toggling locked for ${blockIds.length} blocks in workflow ${workflowId}`) - // Get all blocks in workflow to find children const allBlocks = await tx .select({ id: workflowBlocks.id, @@ -1407,14 +1381,12 @@ async function handleBlocksOperationTx( ) const blocksToToggle = new Set() - // Collect all blocks to toggle including descendants of containers for (const id of blockIds) { const block = blocksById[id] if (!block) continue blocksToToggle.add(id) - // If it's a loop or parallel, also include all descendants if (block.type === 'loop' || block.type === 'parallel') { for (const descId of findDbDescendants(id, allBlocks)) { blocksToToggle.add(descId) @@ -1429,7 +1401,6 @@ async function handleBlocksOperationTx( if (!firstBlock) break const targetLocked = !firstBlock.locked - // Update all affected blocks for (const blockId of blocksToToggle) { await tx .update(workflowBlocks) @@ -1452,7 +1423,6 @@ async function handleBlocksOperationTx( logger.info(`Batch updating parent for ${updates.length} blocks in workflow ${workflowId}`) - // Fetch all blocks to check lock status const allBlocks = await tx .select({ id: workflowBlocks.id, @@ -1471,7 +1441,6 @@ async function handleBlocksOperationTx( const { id, parentId, position } = update if (!id) continue - // Skip protected blocks (locked or inside locked container) if (isWorkflowBlockProtected(id, blocksById)) { logger.info(`Skipping block ${id} parent update - block is protected`) continue @@ -1496,7 +1465,6 @@ async function handleBlocksOperationTx( const isRemovingFromParent = !parentId - // Get current data and position const [currentBlock] = await tx .select({ data: workflowBlocks.data, @@ -1527,11 +1495,9 @@ async function handleBlocksOperationTx( }) .where(and(eq(workflowBlocks.id, id), eq(workflowBlocks.workflowId, workflowId))) - // If the block now has a parent, update the new parent's subflow node list if (parentId) { await updateSubflowNodeList(tx, workflowId, parentId) } - // If the block had a previous parent, update that parent's node list as well if (existingParentId && existingParentId !== parentId) { await updateSubflowNodeList(tx, workflowId, existingParentId) } @@ -1596,7 +1562,6 @@ async function handleEdgeOperationTx(tx: any, workflowId: string, operation: str throw new Error('Missing edge ID for remove operation') } - // Get the edge to check if target block is protected const [edgeToRemove] = await tx .select({ sourceBlockId: workflowEdges.sourceBlockId, @@ -1610,7 +1575,6 @@ async function handleEdgeOperationTx(tx: any, workflowId: string, operation: str throw new Error(`Edge ${payload.id} not found in workflow ${workflowId}`) } - // Check if target block is protected const connectedBlocks = await tx .select({ id: workflowBlocks.id, @@ -1630,7 +1594,6 @@ async function handleEdgeOperationTx(tx: any, workflowId: string, operation: str connectedBlocks.map((b: RemoveEdgeBlockRecord) => [b.id, b]) ) - // Collect parent IDs that need to be fetched const parentIds = new Set() for (const block of connectedBlocks) { const parentId = (block.data as Record | null)?.parentId as @@ -1641,7 +1604,6 @@ async function handleEdgeOperationTx(tx: any, workflowId: string, operation: str } } - // Fetch parent blocks if needed if (parentIds.size > 0) { const parentBlocks = await tx .select({ @@ -1696,7 +1658,6 @@ async function handleEdgesOperationTx( logger.info(`Batch removing ${ids.length} edges from workflow ${workflowId}`) - // Get edges to check connected blocks const edgesToRemove = await tx .select({ id: workflowEdges.id, @@ -1713,14 +1674,12 @@ async function handleEdgesOperationTx( type EdgeToRemove = (typeof edgesToRemove)[number] - // Get all connected block IDs const connectedBlockIds = new Set() edgesToRemove.forEach((e: EdgeToRemove) => { connectedBlockIds.add(e.sourceBlockId) connectedBlockIds.add(e.targetBlockId) }) - // Fetch blocks to check lock status const connectedBlocks = await tx .select({ id: workflowBlocks.id, @@ -1740,7 +1699,6 @@ async function handleEdgesOperationTx( connectedBlocks.map((b: EdgeBlockRecord) => [b.id, b]) ) - // Collect parent IDs that need to be fetched const parentIds = new Set() for (const block of connectedBlocks) { const parentId = (block.data as Record | null)?.parentId as @@ -1751,7 +1709,6 @@ async function handleEdgesOperationTx( } } - // Fetch parent blocks if needed if (parentIds.size > 0) { const parentBlocks = await tx .select({ @@ -1983,7 +1940,6 @@ async function handleSubflowOperationTx( break } - // Add other subflow operations as needed default: logger.warn(`Unknown subflow operation: ${operation}`) throw new Error(`Unsupported subflow operation: ${operation}`) @@ -2022,7 +1978,7 @@ function getWritableSubblockUpdateBlock( return block } -// Subblock operations - targeted value updates without replacing workflow state +/** Applies targeted subblock value updates without replacing the workflow state. */ async function handleSubblockOperationTx( tx: any, workflowId: string, @@ -2109,14 +2065,13 @@ async function handleSubblockOperationTx( } } -// Variable operations - updates workflow.variables JSON field +/** Applies variable operations to the `workflow.variables` JSON field. */ async function handleVariableOperationTx( tx: any, workflowId: string, operation: string, payload: any ) { - // Get current workflow variables const workflowData = await tx .select({ variables: workflow.variables }) .from(workflow) @@ -2135,7 +2090,6 @@ async function handleVariableOperationTx( throw new Error('Missing required fields for add variable operation') } - // Add the new variable const updatedVariables = { ...currentVariables, [payload.id]: { @@ -2164,7 +2118,6 @@ async function handleVariableOperationTx( throw new Error('Missing variable ID for remove operation') } - // Remove the variable const { [payload.variableId]: _, ...updatedVariables } = currentVariables await tx @@ -2185,7 +2138,7 @@ async function handleVariableOperationTx( } } -// Workflow operations - handles complete state replacement +/** Replaces the complete workflow state. */ async function handleWorkflowOperationTx( tx: any, workflowId: string, @@ -2209,10 +2162,8 @@ async function handleWorkflowOperationTx( await tx.delete(workflowBlocks).where(eq(workflowBlocks.workflowId, workflowId)) - // Delete all existing subflows await tx.delete(workflowSubflows).where(eq(workflowSubflows.workflowId, workflowId)) - // Insert all blocks from the new state if (blocks && Object.keys(blocks).length > 0) { const blockValues = Object.values(blocks).map((block: any) => ({ id: block.id, @@ -2237,7 +2188,6 @@ async function handleWorkflowOperationTx( await tx.insert(workflowBlocks).values(blockValues) } - // Insert all edges from the new state if (edges && edges.length > 0) { const canonicalEdges = (edges as Array>).map((edge) => canonicalizeEdgeAddCandidate({ @@ -2269,7 +2219,6 @@ async function handleWorkflowOperationTx( } } - // Insert all loops from the new state if (loops && Object.keys(loops).length > 0) { const loopValues = Object.entries(loops).map(([id, loop]: [string, any]) => ({ id, @@ -2281,7 +2230,6 @@ async function handleWorkflowOperationTx( await tx.insert(workflowSubflows).values(loopValues) } - // Insert all parallels from the new state if (parallels && Object.keys(parallels).length > 0) { const parallelValues = Object.entries(parallels).map(([id, parallel]: [string, any]) => ({ id, diff --git a/apps/realtime/src/middleware/auth.ts b/apps/realtime/src/middleware/auth.ts index 6ea22f02d91..77919e9a70d 100644 --- a/apps/realtime/src/middleware/auth.ts +++ b/apps/realtime/src/middleware/auth.ts @@ -1,7 +1,8 @@ +import { ANONYMOUS_USER, ANONYMOUS_USER_ID } from '@sim/auth/principal' import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import type { Socket } from 'socket.io' -import { ANONYMOUS_USER, ANONYMOUS_USER_ID, auth } from '@/auth' +import { auth } from '@/auth' import { isAuthDisabled } from '@/env' const logger = createLogger('SocketAuth') diff --git a/apps/sim/app/(interfaces)/resume/[workflowId]/[executionId]/resume-page-client.tsx b/apps/sim/app/(interfaces)/resume/[workflowId]/[executionId]/resume-page-client.tsx index bb09919dc63..5a88444c95a 100644 --- a/apps/sim/app/(interfaces)/resume/[workflowId]/[executionId]/resume-page-client.tsx +++ b/apps/sim/app/(interfaces)/resume/[workflowId]/[executionId]/resume-page-client.tsx @@ -208,9 +208,7 @@ export default function ResumeExecutionPage({ const queuePosition = selectedDetail?.pausePoint.queuePosition const resumeInputsRef = useRef>({}) const [resumeInput, setResumeInput] = useState('') - const [formValuesByContext, setFormValuesByContext] = useState< - Record> - >({}) + const formValuesByContextRef = useRef>>({}) const [formValues, setFormValues] = useState>({}) const [formErrors, setFormErrors] = useState>({}) const [loadingAction, setLoadingAction] = useState(false) @@ -308,17 +306,6 @@ export default function ResumeExecutionPage({ [formatValueForInputField] ) - const formatStructureValue = useCallback((value: any): string => { - if (value === null || value === undefined) return '—' - if (typeof value === 'string') return value - if (typeof value === 'number' || typeof value === 'boolean') return String(value) - try { - return JSON.stringify(value, null, 2) - } catch { - return String(value) - } - }, []) - const parseFormValue = useCallback( (field: NormalizedInputField, rawValue: string): { value: any; error?: string } => { const value = rawValue ?? '' @@ -358,11 +345,12 @@ export default function ResumeExecutionPage({ const handleFormFieldChange = useCallback( (fieldName: string, newValue: string) => { if (!selectedContextId) return - setFormValues((prev) => { - const updated = { ...prev, [fieldName]: newValue } - setFormValuesByContext((map) => ({ ...map, [selectedContextId]: updated })) - return updated - }) + const updated = { + ...formValuesByContextRef.current[selectedContextId], + [fieldName]: newValue, + } + formValuesByContextRef.current[selectedContextId] = updated + setFormValues(updated) setFormErrors((prev) => { if (!prev[fieldName]) return prev const { [fieldName]: _, ...rest } = prev @@ -516,12 +504,9 @@ export default function ResumeExecutionPage({ : undefined if (operation === 'human' && fetchedInputFields.length > 0) { const baseValues = buildInitialFormValues(fetchedInputFields, submission) - let mergedValues = baseValues - setFormValuesByContext((prev) => { - const existingValues = prev[detail.pausePoint.contextId] - if (existingValues) mergedValues = { ...baseValues, ...existingValues } - return { ...prev, [detail.pausePoint.contextId]: mergedValues } - }) + const existingValues = formValuesByContextRef.current[detail.pausePoint.contextId] + const mergedValues = existingValues ? { ...baseValues, ...existingValues } : baseValues + formValuesByContextRef.current[detail.pausePoint.contextId] = mergedValues setFormValues(mergedValues) setFormErrors({}) if (resumeInputsRef.current[detail.pausePoint.contextId] !== undefined) { diff --git a/apps/sim/app/api/auth/oauth2/callback/instagram/route.ts b/apps/sim/app/api/auth/oauth2/callback/instagram/route.ts index afca4590c1f..62603299bbc 100644 --- a/apps/sim/app/api/auth/oauth2/callback/instagram/route.ts +++ b/apps/sim/app/api/auth/oauth2/callback/instagram/route.ts @@ -1,5 +1,6 @@ import { db } from '@sim/db' import { account } from '@sim/db/schema' +import { EnvCapabilityConfigurationError } from '@sim/deployment-config/env-capabilities' import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' import { and, eq } from 'drizzle-orm' @@ -7,7 +8,6 @@ import { type NextRequest, NextResponse } from 'next/server' import { instagramCallbackContract } from '@/lib/api/contracts/oauth-connections' import { parseRequest } from '@/lib/api/server' import { getSession } from '@/lib/auth' -import { EnvCapabilityConfigurationError } from '@/lib/core/config/env-capabilities' import { requireConfiguredOAuthClient } from '@/lib/core/config/env-capabilities.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, diff --git a/apps/sim/app/api/auth/oauth2/callback/shopify/route.ts b/apps/sim/app/api/auth/oauth2/callback/shopify/route.ts index 7df3318106e..9eb23807984 100644 --- a/apps/sim/app/api/auth/oauth2/callback/shopify/route.ts +++ b/apps/sim/app/api/auth/oauth2/callback/shopify/route.ts @@ -1,3 +1,4 @@ +import { EnvCapabilityConfigurationError } from '@sim/deployment-config/env-capabilities' import { createLogger } from '@sim/logger' import { safeCompare } from '@sim/security/compare' import { hmacSha256Hex } from '@sim/security/hmac' @@ -7,7 +8,6 @@ import { shopifyShopDomainSchema, } from '@/lib/api/contracts/oauth-connections' import { getSession } from '@/lib/auth' -import { EnvCapabilityConfigurationError } from '@/lib/core/config/env-capabilities' import { requireConfiguredOAuthClient } from '@/lib/core/config/env-capabilities.server' import { getBaseUrl } from '@/lib/core/utils/urls' import { isSameOrigin } from '@/lib/core/utils/validation' diff --git a/apps/sim/app/api/auth/sso/register/route.test.ts b/apps/sim/app/api/auth/sso/register/route.test.ts index 668fbd217a7..7dc11402814 100644 --- a/apps/sim/app/api/auth/sso/register/route.test.ts +++ b/apps/sim/app/api/auth/sso/register/route.test.ts @@ -374,7 +374,7 @@ describe('POST /api/auth/sso/register', () => { it('does not SSRF-validate userInfoEndpoint when skipUserInfoEndpoint is requested', async () => { queueMembers([{ organizationId: 'org1', role: 'owner' }]) - mockValidateUrlWithDNS.mockImplementation(async (url: string, label: string) => { + mockValidateUrlWithDNS.mockImplementation(async (_url: string, label: string) => { if (label === 'OIDC userInfoEndpoint') { return { isValid: false, error: 'resolves to a private IP address' } } diff --git a/apps/sim/app/api/chat/[identifier]/otp/route.ts b/apps/sim/app/api/chat/[identifier]/otp/route.ts index 91c283e7e8a..6f3afae5d70 100644 --- a/apps/sim/app/api/chat/[identifier]/otp/route.ts +++ b/apps/sim/app/api/chat/[identifier]/otp/route.ts @@ -60,7 +60,7 @@ async function deliverOtp(requestId: string, deploymentId: string, title: string const otp = generateOTP() await storeOTP('chat', deploymentId, email, otp) - const emailHtml = await renderOTPEmail(otp, email, 'email-verification', title) + const emailHtml = await renderOTPEmail(otp, 'email-verification', title) const emailResult = await sendEmail({ to: email, subject: getOtpSubject(title), diff --git a/apps/sim/app/api/chat/utils.ts b/apps/sim/app/api/chat/utils.ts index 005aedfc91f..7775d3032a8 100644 --- a/apps/sim/app/api/chat/utils.ts +++ b/apps/sim/app/api/chat/utils.ts @@ -26,30 +26,6 @@ export async function setChatAuthCookie( }) } -/** - * Check if user has permission to create a chat for a specific workflow - */ -export async function checkWorkflowAccessForChatCreation( - workflowId: string, - userId: string -): Promise<{ hasAccess: boolean; workflow?: any }> { - const authorization = await authorizeWorkflowByWorkspacePermission({ - workflowId, - userId, - action: 'admin', - }) - - if (!authorization.workflow) { - return { hasAccess: false } - } - - if (authorization.allowed) { - return { hasAccess: true, workflow: authorization.workflow } - } - - return { hasAccess: false } -} - /** * Check if user has access to view/edit/delete a specific chat */ diff --git a/apps/sim/app/api/copilot/feedback/route.ts b/apps/sim/app/api/copilot/feedback/route.ts index 78e2edd0b2f..e973145c4f0 100644 --- a/apps/sim/app/api/copilot/feedback/route.ts +++ b/apps/sim/app/api/copilot/feedback/route.ts @@ -117,7 +117,7 @@ export const POST = withRouteHandler(async (req: NextRequest) => { * GET /api/copilot/feedback * Get feedback records for the authenticated user */ -export const GET = withRouteHandler(async (req: NextRequest) => { +export const GET = withRouteHandler(async () => { const tracker = createRequestTracker() try { diff --git a/apps/sim/app/api/cron/cleanup-file-versions/route.test.ts b/apps/sim/app/api/cron/cleanup-file-versions/route.test.ts new file mode 100644 index 00000000000..4b3707a81e0 --- /dev/null +++ b/apps/sim/app/api/cron/cleanup-file-versions/route.test.ts @@ -0,0 +1,73 @@ +import { createMockRequest } from '@sim/testing' +import { asyncJobsMock, asyncJobsMockFns } from '@sim/testing/mocks/async-jobs.mock' +import { authInternalMock, authInternalMockFns } from '@sim/testing/mocks/auth-internal.mock' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/auth/internal', () => authInternalMock) +vi.mock('@/lib/core/async-jobs', () => asyncJobsMock) + +import { GET } from '@/app/api/cron/cleanup-file-versions/route' + +const { mockVerifyCronAuth } = authInternalMockFns + +const mockEnqueue = asyncJobsMockFns.mockJobQueue.enqueue + +function request() { + return createMockRequest( + 'GET', + undefined, + {}, + 'http://localhost:3000/api/cron/cleanup-file-versions' + ) +} + +/** The job id a dispatch was keyed to, as the queue reports it back. */ +async function dispatchedJobId(): Promise { + const response = await GET(request()) + expect(response.status).toBe(200) + const body = (await response.json()) as { jobId: string } + return body.jobId +} + +describe('file version cleanup route', () => { + beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-10-01T03:30:00Z')) + mockVerifyCronAuth.mockReturnValue(null) + mockEnqueue.mockReset() + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('fails the cron invocation when the dispatch cannot be enqueued', async () => { + mockEnqueue.mockRejectedValueOnce(new Error('queue unavailable')) + + const response = await GET(request()) + + expect(response.status).toBe(500) + }) + + describe('with a queue that keys each job by the id it is given', () => { + beforeEach(() => { + mockEnqueue.mockImplementation( + async (_type: string, _payload: unknown, options: { jobId: string }) => options.jobId + ) + }) + + it('dispatches a retry on the same day as the same job', async () => { + const first = await dispatchedJobId() + vi.setSystemTime(new Date('2026-10-01T23:59:59.999Z')) + + expect(await dispatchedJobId()).toBe(first) + }) + + it('dispatches a new job on the next day', async () => { + const first = await dispatchedJobId() + vi.setSystemTime(new Date('2026-10-02T00:00:00.000Z')) + + expect(await dispatchedJobId()).not.toBe(first) + }) + }) +}) diff --git a/apps/sim/app/api/cron/cleanup-file-versions/route.ts b/apps/sim/app/api/cron/cleanup-file-versions/route.ts index a4b312347ce..13f7565b599 100644 --- a/apps/sim/app/api/cron/cleanup-file-versions/route.ts +++ b/apps/sim/app/api/cron/cleanup-file-versions/route.ts @@ -1,12 +1,14 @@ import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' import { verifyCronAuth } from '@/lib/auth/internal' -import { dispatchCleanupJobs } from '@/lib/billing/cleanup-dispatcher' +import { getJobQueue } from '@/lib/core/async-jobs' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { CLEANUP_DISPATCH_MAX_ATTEMPTS } from '@/background/cleanup-dispatch' export const dynamic = 'force-dynamic' const logger = createLogger('FileVersionCleanupAPI') +const FILE_VERSION_CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1000 /** GET /api/cron/cleanup-file-versions — dispatch retention for superseded workspace file versions. */ export const GET = withRouteHandler(async (request: NextRequest) => { @@ -14,11 +16,26 @@ export const GET = withRouteHandler(async (request: NextRequest) => { const authError = verifyCronAuth(request, 'file version cleanup') if (authError) return authError - const result = await dispatchCleanupJobs('cleanup-file-versions') + const queue = await getJobQueue() + const scheduleWindow = Math.floor(Date.now() / FILE_VERSION_CLEANUP_INTERVAL_MS) + const jobId = await queue.enqueue( + 'cleanup-dispatch', + { jobType: 'cleanup-file-versions' }, + { + maxAttempts: CLEANUP_DISPATCH_MAX_ATTEMPTS, + jobId: `cleanup-dispatch:cleanup-file-versions:${scheduleWindow}`, + name: 'File version cleanup dispatch', + concurrencyKey: 'cleanup-dispatch:cleanup-file-versions', + concurrencyLimit: 1, + runner: async () => { + const { dispatchCleanupJobs } = await import('@/lib/billing/cleanup-dispatcher') + return dispatchCleanupJobs('cleanup-file-versions') + }, + } + ) - logger.info('File version cleanup jobs dispatched', result) - - return NextResponse.json({ triggered: true, ...result }) + logger.info('File version cleanup dispatch enqueued', { jobId }) + return NextResponse.json({ triggered: true, jobId }) } catch (error) { logger.error('Failed to dispatch file version cleanup jobs:', { error }) return NextResponse.json({ error: 'Failed to dispatch file version cleanup' }, { status: 500 }) diff --git a/apps/sim/app/api/cron/cleanup-stale-executions/route.test.ts b/apps/sim/app/api/cron/cleanup-stale-executions/route.test.ts index 08476db1683..7d7c8bfb72a 100644 --- a/apps/sim/app/api/cron/cleanup-stale-executions/route.test.ts +++ b/apps/sim/app/api/cron/cleanup-stale-executions/route.test.ts @@ -1,62 +1,18 @@ -import { asyncJobs, tableJobs, workflowExecutionLogs } from '@sim/db/schema' -import { createLogger } from '@sim/logger' -import { createMockRequest, dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' +import { createMockRequest } from '@sim/testing' +import { asyncJobsMock, asyncJobsMockFns } from '@sim/testing/mocks/async-jobs.mock' import { authInternalMock, authInternalMockFns } from '@sim/testing/mocks/auth-internal.mock' -import { storageServiceMock, storageServiceMockFns } from '@sim/testing/mocks/storage-service.mock' -import { beforeEach, describe, expect, it, vi } from 'vitest' -import { JOB_RETENTION_HOURS } from '@/lib/core/async-jobs' -import { - SCHEDULE_CARRIER_IRRECOVERABLE_METADATA_KEY, - SCHEDULE_CARRIER_IRRECOVERABLE_RETENTION_HOURS, - SCHEDULE_CARRIER_RECONCILED_METADATA_KEY, -} from '@/lib/workflows/schedules/carrier-metadata' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('@/lib/auth/internal', () => authInternalMock) -vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock) +vi.mock('@/lib/core/async-jobs', () => asyncJobsMock) import { GET } from '@/app/api/cron/cleanup-stale-executions/route' -const mockVerifyCronAuth = authInternalMockFns.mockVerifyCronAuth -const mockDeleteFile = storageServiceMockFns.mockDeleteFile -mockDeleteFile.mockResolvedValue(undefined) +const { mockVerifyCronAuth } = authInternalMockFns -const cleanupLogger = - vi.mocked(createLogger).mock.results[ - vi.mocked(createLogger).mock.calls.findIndex(([name]) => name === 'CleanupStaleExecutions') - ].value +const mockEnqueue = asyncJobsMockFns.mockJobQueue.enqueue -interface MockCondition { - type?: string - conditions?: unknown[] - left?: unknown - right?: unknown - column?: unknown - values?: unknown - toSQL?: () => { sql: string; params: unknown[] } -} - -function flattenConditions(condition: unknown): MockCondition[] { - if (!condition || typeof condition !== 'object') return [] - const node = condition as MockCondition - return [node, ...(node.conditions?.flatMap((child) => flattenConditions(child)) ?? [])] -} - -function hasToSQL(value: unknown): value is { toSQL: () => { sql: string; params: unknown[] } } { - return typeof value === 'object' && value !== null && 'toSQL' in value -} - -/** - * Collects the leaves of a nested `sql` expression. The duration expression is - * built by a shared helper, so the values it binds sit one level below the - * fragment this route assembles rather than directly in its own params. - */ -function flattenSqlParams(expression: { sql: string; params: unknown[] }): unknown[] { - return expression.params.flatMap((param) => - hasToSQL(param) ? flattenSqlParams(param.toSQL()) : [param] - ) -} - -function createRequest() { +function request() { return createMockRequest( 'GET', undefined, @@ -65,419 +21,62 @@ function createRequest() { ) } -/** Recursively renders a mocked drizzle fragment, expanding nested fragments. */ -function renderSql(fragment: unknown): string { - if (fragment === null || fragment === undefined) return '' - if (typeof fragment !== 'object') return String(fragment) - const candidate = fragment as { rawSql?: string; strings?: string[]; values?: unknown[] } - if (typeof candidate.rawSql === 'string') return candidate.rawSql - if (!candidate.strings) return '' - return candidate.strings - .map((part, index) => - index < (candidate.values?.length ?? 0) - ? `${part}${renderSql(candidate.values?.[index])}` - : part - ) - .join('') +/** The job id a dispatch was keyed to, as the queue reports it back. */ +async function dispatchedJobId(): Promise { + const response = await GET(request()) + expect(response.status).toBe(200) + const body = (await response.json()) as { jobId: string } + return body.jobId } -/** Recursively collects the non-fragment bind values of a mocked fragment. */ -function collectSqlParams(fragment: unknown): unknown[] { - if (!fragment || typeof fragment !== 'object') return [] - const candidate = fragment as { values?: unknown[] } - if (!candidate.values) return [] - return candidate.values.flatMap((value) => { - const nested = collectSqlParams(value) - return nested.length > 0 ? nested : [value] - }) -} - -describe('stale execution cleanup deadline grace', () => { +describe('stale execution cleanup route', () => { beforeEach(() => { - resetDbChainMock() - cleanupLogger.info.mockReset() - mockVerifyCronAuth.mockReturnValue(null) - }) - - it('waits five minutes past a workflow execution deadline in both cleanup predicates', async () => { vi.useFakeTimers() - vi.setSystemTime(new Date('2026-08-03T12:10:00.000Z')) - queueTableRows(workflowExecutionLogs, [{ id: 'log-1' }]) - dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'log-1' }]) - - try { - const response = await GET(createRequest()) - - expect(response.status).toBe(200) - const expectedThreshold = new Date('2026-08-03T12:05:00.000Z') - const deadlineComparisons = dbChainMockFns.where.mock.calls - .flatMap(([condition]) => flattenConditions(condition)) - .filter( - (condition) => - condition.type === 'lt' && - condition.right instanceof Date && - condition.right.getTime() === expectedThreshold.getTime() - ) - - expect(deadlineComparisons).toHaveLength(2) - expect(deadlineComparisons.map(({ right }) => right)).toEqual([ - expectedThreshold, - expectedThreshold, - ]) - - const executionUpdateIndex = dbChainMockFns.update.mock.calls.findIndex( - ([table]) => table === workflowExecutionLogs - ) - const update = dbChainMockFns.set.mock.calls[executionUpdateIndex]?.[0] as { - endedAt: Date - totalDurationMs: { toSQL: () => { sql: string; params: unknown[] } } - executionData: { toSQL: () => { sql: string; params: unknown[] } } - } - const totalDurationLeaves = flattenSqlParams(update.totalDurationMs.toSQL()) - const renderedError = renderSql(update.executionData) - const errorLeaves = collectSqlParams(update.executionData) - - expect(renderedError).toContain('CASE') - expect(renderedError).toContain('IS NOT NULL') - expect(renderedError).toContain('ROUND') - expect(renderedError).toContain('EXTRACT(EPOCH') - expect(errorLeaves).toContain(workflowExecutionLogs.executionDeadlineAt) - expect(errorLeaves).toContain('Execution timed out') - expect(errorLeaves).toContain('Execution terminated: worker timeout or crash after ') - expect(errorLeaves).toContain(workflowExecutionLogs.startedAt) - expect(totalDurationLeaves).toContain(2_147_483_647) - expect(totalDurationLeaves).toContain(workflowExecutionLogs.startedAt) - expect(totalDurationLeaves).toContainEqual(new Date('2026-08-03T12:10:00.000Z')) - expect(update.endedAt).toEqual(new Date('2026-08-03T12:10:00.000Z')) - } finally { - vi.useRealTimers() - } - }) - - it('sweeps redacting logs on the generic window, never the execution deadline', async () => { - const response = await GET(createRequest()) - - expect(response.status).toBe(200) - const redactingPredicates = dbChainMockFns.where.mock.calls - .map(([condition]) => flattenConditions(condition)) - .filter((conditions) => - conditions.some( - (condition) => - condition.type === 'eq' && - condition.left === workflowExecutionLogs.status && - condition.right === 'redacting' - ) - ) - - expect(redactingPredicates.length).toBeGreaterThan(0) - for (const conditions of redactingPredicates) { - expect( - conditions.some((condition) => condition.left === workflowExecutionLogs.executionDeadlineAt) - ).toBe(false) - expect( - conditions.some( - (condition) => - condition.type === 'lt' && condition.left === workflowExecutionLogs.startedAt - ) - ).toBe(true) - } + vi.setSystemTime(new Date('2026-10-01T17:31:00Z')) + mockVerifyCronAuth.mockReturnValue(null) + mockEnqueue.mockReset() }) - it('leaves pending and processing schedule jobs to schedule recovery', async () => { - const response = await GET(createRequest()) - - expect(response.status).toBe(200) - const activeAsyncPredicates = dbChainMockFns.where.mock.calls - .map(([condition]) => flattenConditions(condition)) - .filter((conditions) => - conditions.some( - (condition) => - condition.type === 'ne' && - condition.left === asyncJobs.type && - condition.right === 'schedule-execution' - ) - ) - - expect( - activeAsyncPredicates.some((conditions) => - conditions.some( - (condition) => - condition.type === 'eq' && - condition.left === asyncJobs.status && - condition.right === 'processing' - ) - ) - ).toBe(true) - expect( - activeAsyncPredicates.some((conditions) => - conditions.some( - (condition) => - condition.type === 'eq' && - condition.left === asyncJobs.status && - condition.right === 'pending' - ) - ) - ).toBe(true) + afterEach(() => { + vi.useRealTimers() }) - it('retains terminal schedule carriers until reconciliation is recorded', async () => { - const response = await GET(createRequest()) + it('answers with the dispatched job once the cleanup is enqueued', async () => { + mockEnqueue.mockResolvedValueOnce('job-stale-1') - expect(response.status).toBe(200) - const retentionConditions = dbChainMockFns.where.mock.calls.flatMap(([condition]) => - flattenConditions(condition) - ) - const reconciliationMarker = retentionConditions.find((condition) => - renderSql(condition).includes(SCHEDULE_CARRIER_RECONCILED_METADATA_KEY) - ) - - expect(collectSqlParams(reconciliationMarker)).toContain(asyncJobs.metadata) - expect( - retentionConditions.some( - (condition) => - condition.type === 'ne' && - condition.left === asyncJobs.type && - condition.right === 'schedule-execution' - ) - ).toBe(true) - }) - - it('spells carrier metadata keys as SQL literals so the partial index matches', async () => { - const response = await GET(createRequest()) + const response = await GET(request()) expect(response.status).toBe(200) - const reconciliationMarker = dbChainMockFns.where.mock.calls - .flatMap(([condition]) => flattenConditions(condition)) - .find((condition) => renderSql(condition).includes(SCHEDULE_CARRIER_RECONCILED_METADATA_KEY)) - - expect(renderSql(reconciliationMarker)).toContain( - `'${SCHEDULE_CARRIER_RECONCILED_METADATA_KEY}'` - ) - expect(collectSqlParams(reconciliationMarker)).not.toContain( - SCHEDULE_CARRIER_RECONCILED_METADATA_KEY - ) + await expect(response.json()).resolves.toEqual({ triggered: true, jobId: 'job-stale-1' }) }) - it('deletes irrecoverable schedule carrier tombstones once their longer window lapses', async () => { - const response = await GET(createRequest()) - - expect(response.status).toBe(200) - const retentionConditions = dbChainMockFns.where.mock.calls.flatMap(([condition]) => - flattenConditions(condition) - ) - const irrecoverableExclusion = retentionConditions.find((condition) => - renderSql(condition).includes(SCHEDULE_CARRIER_IRRECOVERABLE_METADATA_KEY) - ) + it('fails the cron invocation when the job cannot be enqueued', async () => { + mockEnqueue.mockRejectedValueOnce(new Error('queue unavailable')) - expect(renderSql(irrecoverableExclusion)).toContain("<> 'true'") - expect(collectSqlParams(irrecoverableExclusion)).toContain(asyncJobs.metadata) + const response = await GET(request()) - const tombstoneWindow = retentionConditions.filter( - (condition) => - condition.type === 'lt' && - condition.left === asyncJobs.completedAt && - condition.right instanceof Date - ) - const oldest = Math.min(...tombstoneWindow.map(({ right }) => (right as Date).getTime())) - const newest = Math.max(...tombstoneWindow.map(({ right }) => (right as Date).getTime())) - expect(newest - oldest).toBe( - (SCHEDULE_CARRIER_IRRECOVERABLE_RETENTION_HOURS - JOB_RETENTION_HOURS) * 60 * 60 * 1000 - ) + expect(response.status).toBe(500) }) - it('keeps table-job heartbeat cleanup independent from workflow timeout policy', async () => { - vi.useFakeTimers() - vi.setSystemTime(new Date('2026-08-03T12:00:00.000Z')) - queueTableRows(tableJobs, [{ id: 'table-job-1' }]) - dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'table-job-1' }]) - - try { - const response = await GET(createRequest()) - - expect(response.status).toBe(200) - const expectedThreshold = new Date('2026-08-03T10:25:00.000Z') - const tableJobComparisons = dbChainMockFns.where.mock.calls - .flatMap(([condition]) => flattenConditions(condition)) - .filter( - (condition) => - condition.type === 'lt' && - condition.left === tableJobs.updatedAt && - condition.right instanceof Date && - condition.right.getTime() === expectedThreshold.getTime() - ) - - expect(tableJobComparisons).toHaveLength(2) - expect(tableJobComparisons.map(({ right }) => right)).toEqual([ - expectedThreshold, - expectedThreshold, - ]) - - const tableJobUpdateIndex = dbChainMockFns.update.mock.calls.findIndex( - ([table]) => table === tableJobs + describe('with a queue that keys each job by the id it is given', () => { + beforeEach(() => { + mockEnqueue.mockImplementation( + async (_type: string, _payload: unknown, options: { jobId: string }) => options.jobId ) - const update = dbChainMockFns.set.mock.calls[tableJobUpdateIndex]?.[0] as { - error: string - } - expect(update.error).toBe( - 'Job terminated: no progress for more than 95 minutes (worker timeout or crash)' - ) - } finally { - vi.useRealTimers() - } - }) - - it('claims every cleanup page without overlapping concurrent workers', async () => { - const response = await GET(createRequest()) - - expect(response.status).toBe(200) - // Nine batched arms: the connector sync-log retention pass is the newest. - expect(dbChainMockFns.transaction).toHaveBeenCalledTimes(9) - expect(dbChainMockFns.for).toHaveBeenCalledTimes(9) - for (const [strength, options] of dbChainMockFns.for.mock.calls) { - expect(strength).toBe('update') - expect(options).toEqual({ skipLocked: true }) - } - }) - - it('caps every bulk mutation and returns only scalar export cleanup fields', async () => { - const stateBatch = Array.from({ length: 1000 }, (_, index) => ({ id: `state-${index}` })) - const retentionBatch = Array.from({ length: 2000 }, (_, index) => ({ - id: `retention-${index}`, - })) - const exportBatch = Array.from({ length: 100 }, (_, index) => ({ - type: 'export', - resultKey: `workspace/workspace-1/exports/table-1/job-${index}/export.csv`, - })) - const exportCandidates = Array.from({ length: 100 }, (_, index) => ({ - id: `export-${index}`, - })) - - for (let batch = 0; batch < 10; batch++) { - const workflowBatch = Array.from({ length: 100 }, (_, index) => ({ - id: `workflow-state-${batch}-${index}`, - })) - queueTableRows(workflowExecutionLogs, workflowBatch) - dbChainMockFns.returning.mockResolvedValueOnce(workflowBatch) - } - for (let batch = 0; batch < 10; batch++) { - queueTableRows(asyncJobs, stateBatch) - dbChainMockFns.returning.mockResolvedValueOnce(stateBatch) - } - for (let batch = 0; batch < 10; batch++) { - queueTableRows(tableJobs, stateBatch) - dbChainMockFns.returning.mockResolvedValueOnce(stateBatch) - } - for (let batch = 0; batch < 10; batch++) { - queueTableRows(tableJobs, exportCandidates) - dbChainMockFns.returning.mockResolvedValueOnce(exportBatch) - } - for (let batch = 0; batch < 10; batch++) { - queueTableRows(asyncJobs, stateBatch) - dbChainMockFns.returning.mockResolvedValueOnce(stateBatch) - } - for (let batch = 0; batch < 10; batch++) { - queueTableRows(asyncJobs, retentionBatch) - dbChainMockFns.returning.mockResolvedValueOnce(retentionBatch) - } - dbChainMockFns.returning.mockResolvedValueOnce([]) - - const response = await GET(createRequest()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - executions: { - found: 1000, - cleaned: 1000, - failed: 0, - }, - asyncJobs: { - staleProcessingMarkedFailed: 10_000, - stalePendingMarkedFailed: 10_000, - oldDeleted: 20_000, - }, - tableJobs: { - staleMarkedFailed: 10_000, - }, }) - expect(mockDeleteFile).toHaveBeenCalledTimes(1000) - - const limits = dbChainMockFns.limit.mock.calls.map(([limit]) => limit) - expect(limits.filter((limit) => limit === 100)).toHaveLength(20) - expect(limits.filter((limit) => limit === 1000)).toHaveLength(30) - expect(limits.filter((limit) => limit === 2000)).toHaveLength(12) - - const workflowUpdates = dbChainMockFns.update.mock.calls.filter( - ([table]) => table === workflowExecutionLogs - ) - expect(workflowUpdates).toHaveLength(10) - const returningShapes = dbChainMockFns.returning.mock.calls - .map(([shape]) => shape) - .filter((shape): shape is Record => Boolean(shape)) - expect(returningShapes.some((shape) => 'payload' in shape)).toBe(false) - expect(returningShapes.some((shape) => 'type' in shape && 'resultKey' in shape)).toBe(true) + it('dispatches a retry inside the same thirty-minute window as the same job', async () => { + const first = await dispatchedJobId() + vi.setSystemTime(new Date('2026-10-01T17:59:59.999Z')) - const claimedIds = dbChainMockFns.where.mock.calls - .flatMap(([condition]) => flattenConditions(condition)) - .filter((condition) => condition.type === 'inArray') - .map((condition) => condition.values) - expect(claimedIds.length).toBeGreaterThan(0) - expect(claimedIds.every((ids) => Array.isArray(ids))).toBe(true) - }) - - it('preserves committed workflow cleanup counts when a later batch fails', async () => { - const firstBatch = Array.from({ length: 100 }, (_, index) => ({ - id: `execution-${index}`, - })) - const failedBatch = Array.from({ length: 37 }, (_, index) => ({ - id: `failed-execution-${index}`, - })) - queueTableRows(workflowExecutionLogs, firstBatch) - queueTableRows(workflowExecutionLogs, failedBatch) - queueTableRows(asyncJobs, [{ id: 'async-job-1' }]) - dbChainMockFns.returning - .mockResolvedValueOnce(firstBatch) - .mockRejectedValueOnce(new Error('database unavailable')) - .mockResolvedValueOnce([{ id: 'async-job-1' }]) - - const response = await GET(createRequest()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - executions: { - found: 137, - cleaned: 100, - failed: 37, - }, + expect(await dispatchedJobId()).toBe(first) }) - expect( - dbChainMockFns.update.mock.calls.filter(([table]) => table === workflowExecutionLogs) - ).toHaveLength(2) - expect(dbChainMockFns.update.mock.calls.some(([table]) => table === asyncJobs)).toBe(true) - }) - it('continues draining when an atomic race updates fewer rows than were selected', async () => { - const firstCandidates = Array.from({ length: 100 }, (_, index) => ({ - id: `execution-${index}`, - })) - const firstUpdated = firstCandidates.slice(0, 99) - const secondBatch = [{ id: 'execution-100' }] - queueTableRows(workflowExecutionLogs, firstCandidates) - queueTableRows(workflowExecutionLogs, secondBatch) - dbChainMockFns.returning.mockResolvedValueOnce(firstUpdated).mockResolvedValueOnce(secondBatch) + it('dispatches a new job once the next thirty-minute window begins', async () => { + const first = await dispatchedJobId() + vi.setSystemTime(new Date('2026-10-01T18:00:00.000Z')) - const response = await GET(createRequest()) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - executions: { - found: 101, - cleaned: 100, - failed: 0, - }, + expect(await dispatchedJobId()).not.toBe(first) }) - expect( - dbChainMockFns.update.mock.calls.filter(([table]) => table === workflowExecutionLogs) - ).toHaveLength(2) }) }) diff --git a/apps/sim/app/api/cron/cleanup-stale-executions/route.ts b/apps/sim/app/api/cron/cleanup-stale-executions/route.ts index 0eba10da8cc..cf514015843 100644 --- a/apps/sim/app/api/cron/cleanup-stale-executions/route.ts +++ b/apps/sim/app/api/cron/cleanup-stale-executions/route.ts @@ -1,794 +1,46 @@ -import { db } from '@sim/db' -import { - asyncJobs, - knowledgeConnectorSyncLog, - tableJobs, - workflowDeploymentOperation, - workflowExecutionLogs, -} from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { and, eq, exists, gt, inArray, isNull, lt, ne, or, sql } from 'drizzle-orm' -import { alias } from 'drizzle-orm/pg-core' import { type NextRequest, NextResponse } from 'next/server' import { verifyCronAuth } from '@/lib/auth/internal' -import { - JOB_PENDING_RETENTION_HOURS, - JOB_RETENTION_HOURS, - JOB_STATUS, - MAX_JOB_DURATION_SECONDS, - MIN_JOB_DURATION_SECONDS, - TERMINAL_JOB_STATUSES, -} from '@/lib/core/async-jobs' -import { - getExecutionReservationTtlMs, - getTimeoutErrorMessage, - RESERVATION_TTL_BUFFER_MS, -} from '@/lib/core/execution-limits' +import { getJobQueue } from '@/lib/core/async-jobs' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { DbTransaction } from '@/lib/db/types' -import { elapsedDurationMsSql } from '@/lib/logs/execution/duration' -import { - STALE_SWEEPABLE_EXECUTION_STATUSES, - type StaleSweepableExecutionStatus, -} from '@/lib/logs/types' -import { sweepOrphanedRuns } from '@/lib/mothership/async-runs/orphaned-runs' -import { cancelStaleDispatches } from '@/lib/table/dispatcher' -import { deleteFile } from '@/lib/uploads/core/storage-service' -import { - carrierNotIrrecoverableSql, - carrierReconciledSql, - SCHEDULE_CARRIER_IRRECOVERABLE_RETENTION_HOURS, -} from '@/lib/workflows/schedules/carrier-metadata' -import { SCHEDULE_EXECUTION_QUEUE_NAME } from '@/lib/workflows/schedules/execution-limits' -const logger = createLogger('CleanupStaleExecutions') +export const dynamic = 'force-dynamic' -const STALE_THRESHOLD_MS = getExecutionReservationTtlMs() -const STALE_THRESHOLD_MINUTES = Math.ceil(STALE_THRESHOLD_MS / 60000) -const GENERIC_STALE_PROCESSING_ERROR = `Job terminated: stuck in processing for more than ${STALE_THRESHOLD_MINUTES} minutes` -const EXECUTION_DEADLINE_ERROR = getTimeoutErrorMessage(undefined) -/** - * Table jobs run as detached workers with progress heartbeats, independently of workflow timeout - * policy. Preserve their historical 90-minute task window plus five-minute cleanup grace. - */ -const TABLE_JOB_STALE_THRESHOLD_MINUTES = 95 -/** Terminal table-jobs older than this are pruned; only the latest job per table is ever read. */ -const TABLE_JOB_RETENTION_HOURS = 24 -/** - * A table run dispatch whose holder has not made progress for this long is - * treated as dead. Same shape and window as the table-job threshold above: the - * 90-minute Trigger.dev task ceiling (`maxDuration` in `trigger.config.ts`) plus - * five minutes of cleanup grace, measured from the dispatcher's own per-window - * heartbeat rather than from when the run was requested. - */ -const TABLE_DISPATCH_STALE_THRESHOLD_MINUTES = 95 -/** Per-run ceiling on reaped dispatches, so one tick cannot fan out unbounded SSE. */ -const TABLE_DISPATCH_MAX_PER_RUN = 200 -/** - * Terminal deployment operations older than this are pruned. Every reader of - * this table is latest-generation-only, and idempotency keys only need to - * survive a client retry window, so 30 days is generous. - */ -const DEPLOYMENT_OPERATION_RETENTION_DAYS = 30 -/** - * Terminal connector sync logs older than this are pruned. Nothing pruned them - * before, so the table grew by one row per sync run forever — a connector on a - * fifteen-minute interval writes about 35,000 rows a year on its own. That cost - * lands on `loadPreviousListingObservation`, which reads the newest `completed` - * row per connector through an index covering `connector_id` alone, so every - * retained row makes the sort behind the deletion-safety corroboration slower. - */ -const CONNECTOR_SYNC_LOG_RETENTION_DAYS = 30 -const CONNECTOR_SYNC_LOG_PRUNE_BATCH_SIZE = 2000 -const CONNECTOR_SYNC_LOG_MAX_ROWS_PER_RUN = 20_000 -const DEPLOYMENT_OPERATION_PRUNE_BATCH_SIZE = 2000 -const DEPLOYMENT_OPERATION_PRUNE_MAX_BATCHES = 10 -const WORKFLOW_EXECUTION_MUTATION_BATCH_SIZE = 100 -const WORKFLOW_EXECUTION_MAX_ROWS_PER_RUN = 1000 -const STATE_MUTATION_BATCH_SIZE = 1000 -const STATE_MUTATION_MAX_ROWS_PER_RUN = 10_000 -const RETENTION_DELETE_BATCH_SIZE = 2000 -const RETENTION_DELETE_MAX_ROWS_PER_RUN = 20_000 -const TABLE_JOB_PRUNE_BATCH_SIZE = 100 -const TABLE_JOB_PRUNE_MAX_ROWS_PER_RUN = 1000 - -interface BatchedMutationResult { - affected: number - reachedLimit: boolean -} - -interface RunBatchedMutationOptions { - batchSize: number - maxRowsPerRun: number - claim: (tx: DbTransaction, limit: number) => Promise - mutation: (tx: DbTransaction, candidateIds: string[]) => Promise - onBatch?: (rows: TRow[]) => Promise -} - -/** - * Runs a mutation in bounded, atomic pages. Each page claims explicit rows with - * `FOR UPDATE SKIP LOCKED` and mutates those same rows before committing, so - * concurrent cleanup workers cannot overlap and `LIMIT` is evaluated exactly once. - */ -async function runBatchedMutation({ - batchSize, - maxRowsPerRun, - claim, - mutation, - onBatch, -}: RunBatchedMutationOptions): Promise { - let affected = 0 - - while (affected < maxRowsPerRun) { - const limit = Math.min(batchSize, maxRowsPerRun - affected) - const { candidates, rows } = await db.transaction(async (tx) => { - const candidates = await claim(tx, limit) - if (candidates.length > limit) { - throw new Error(`Cleanup claimed ${candidates.length} rows for a ${limit}-row batch`) - } - if (candidates.length === 0) return { candidates, rows: [] as TRow[] } - - const rows = await mutation( - tx, - candidates.map(({ id }) => id) - ) - if (rows.length !== candidates.length) { - throw new Error( - `Cleanup mutation returned ${rows.length} rows for ${candidates.length} claimed rows` - ) - } - return { candidates, rows } - }) - if (candidates.length === 0) break - - affected += rows.length - if (onBatch) await onBatch(rows) - if (candidates.length < limit) break - } - - return { affected, reachedLimit: affected >= maxRowsPerRun } -} +const logger = createLogger('CleanupStaleExecutionsApi') +const STALE_EXECUTION_CLEANUP_INTERVAL_MS = 30 * 60 * 1000 export const GET = withRouteHandler(async (request: NextRequest) => { try { const authError = verifyCronAuth(request, 'Stale execution cleanup') - if (authError) { - return authError - } - - logger.info('Starting stale execution cleanup job') - - const now = new Date() - const staleDeadlineThreshold = new Date(now.getTime() - RESERVATION_TTL_BUFFER_MS) - const staleThreshold = new Date(now.getTime() - STALE_THRESHOLD_MINUTES * 60 * 1000) - const stalePendingThreshold = new Date( - now.getTime() - JOB_PENDING_RETENTION_HOURS * 60 * 60 * 1000 - ) - const staleTableJobThreshold = new Date( - now.getTime() - TABLE_JOB_STALE_THRESHOLD_MINUTES * 60 * 1000 - ) - const staleDispatchThreshold = new Date( - now.getTime() - TABLE_DISPATCH_STALE_THRESHOLD_MINUTES * 60 * 1000 - ) - - let staleExecutionsFound = 0 - let cleaned = 0 - let failed = 0 - let currentWorkflowBatchSize = 0 - - try { - /** - * `running` is swept on its execution deadline plus the cleanup grace, or - * on the generic stale window when it has no deadline. - * - * `redacting` gets the generic window only. That status is set *after* the - * run finished, while a live worker masks the payload, so the execution - * deadline is already in the past by the time redaction starts — the - * deadline rule would fail a masking pass that is merely slow, and - * schedule recovery would read that as a failed occurrence even though - * the worker is about to persist `completed`. A redaction that genuinely - * crashed still clears within the generic window. - */ - const staleExecutionTimePredicate = (status: StaleSweepableExecutionStatus) => - status === 'redacting' - ? lt(workflowExecutionLogs.startedAt, staleThreshold) - : or( - lt(workflowExecutionLogs.executionDeadlineAt, staleDeadlineThreshold), - and( - isNull(workflowExecutionLogs.executionDeadlineAt), - lt(workflowExecutionLogs.startedAt, staleThreshold) - ) - ) - const cleanupTimestamp = sql.param(now, workflowExecutionLogs.startedAt) - const staleDurationMinutes = sql`ROUND( - EXTRACT(EPOCH FROM (${cleanupTimestamp} - ${workflowExecutionLogs.startedAt})) / 60 - )::integer` - const totalDurationMs = elapsedDurationMsSql(now) - const staleDurationError = sql`${'Execution terminated: worker timeout or crash after '}::text - || ${staleDurationMinutes}::text - || ' minutes'` - /** - * A `redacting` row is never swept by the deadline rule, so the deadline - * message cannot apply to it. - */ - const staleExecutionError = (status: StaleSweepableExecutionStatus) => - status === 'redacting' - ? staleDurationError - : sql`CASE - WHEN ${workflowExecutionLogs.executionDeadlineAt} IS NOT NULL - THEN ${EXECUTION_DEADLINE_ERROR}::text - ELSE ${staleDurationError} - END` - /** - * Swept one status at a time so each pass stays on its own partial index; - * `status IN (...)` would match neither. The row budget is shared across - * both passes so the per-run cap keeps meaning what its name says. - */ - let workflowRowsConsidered = 0 - for (const executionStatus of STALE_SWEEPABLE_EXECUTION_STATUSES) { - const staleExecutionPredicate = and( - eq(workflowExecutionLogs.status, executionStatus), - staleExecutionTimePredicate(executionStatus) - ) - while (workflowRowsConsidered < WORKFLOW_EXECUTION_MAX_ROWS_PER_RUN) { - const limit = Math.min( - WORKFLOW_EXECUTION_MUTATION_BATCH_SIZE, - WORKFLOW_EXECUTION_MAX_ROWS_PER_RUN - workflowRowsConsidered + if (authError) return authError + + const queue = await getJobQueue() + const scheduleWindow = Math.floor(Date.now() / STALE_EXECUTION_CLEANUP_INTERVAL_MS) + const jobId = await queue.enqueue( + 'cleanup-stale-executions', + {}, + { + maxAttempts: 1, + jobId: `cleanup-stale-executions:${scheduleWindow}`, + name: 'Stale execution cleanup', + concurrencyKey: 'cleanup:stale-executions', + concurrencyLimit: 1, + runner: async () => { + const { runCleanupStaleExecutions } = await import( + '@/background/cleanup-stale-executions' ) - currentWorkflowBatchSize = 0 - const { candidates, updatedExecutions } = await db.transaction(async (tx) => { - const candidates = await tx - .select({ id: workflowExecutionLogs.id }) - .from(workflowExecutionLogs) - .where(staleExecutionPredicate) - .limit(limit) - .for('update', { skipLocked: true }) - currentWorkflowBatchSize = candidates.length - if (candidates.length === 0) return { candidates, updatedExecutions: [] } - - const updatedExecutions = await tx - .update(workflowExecutionLogs) - .set({ - status: 'failed', - endedAt: now, - executionDeadlineAt: null, - totalDurationMs, - executionData: sql`jsonb_set( - COALESCE(execution_data, '{}'::jsonb), - ARRAY['error'], - to_jsonb(${staleExecutionError(executionStatus)}) - )`, - }) - .where( - and( - staleExecutionPredicate, - inArray( - workflowExecutionLogs.id, - candidates.map(({ id }) => id) - ) - ) - ) - .returning({ id: workflowExecutionLogs.id }) - - return { candidates, updatedExecutions } - }) - currentWorkflowBatchSize = 0 - staleExecutionsFound += candidates.length - if (candidates.length === 0) break - - cleaned += updatedExecutions.length - workflowRowsConsidered += candidates.length - if (candidates.length < limit) break - } - } - - if (workflowRowsConsidered >= WORKFLOW_EXECUTION_MAX_ROWS_PER_RUN) { - logger.info('Deferred remaining stale workflow executions after reaching the per-run cap', { - maxRowsPerRun: WORKFLOW_EXECUTION_MAX_ROWS_PER_RUN, - }) - } - } catch (error) { - logger.error('Failed to clean up stale workflow executions:', { - error: toError(error).message, - }) - staleExecutionsFound += currentWorkflowBatchSize - failed += currentWorkflowBatchSize - } - - logger.info(`Stale execution cleanup completed. Cleaned: ${cleaned}, Failed: ${failed}`) - - // Clean up stale async jobs (stuck in processing) - let asyncJobsMarkedFailed = 0 - - try { - const hasPositiveMaxDuration = sql`CASE - WHEN jsonb_typeof(${asyncJobs.metadata}->'maxDurationSeconds') = 'number' - THEN (${asyncJobs.metadata}->>'maxDurationSeconds')::numeric >= ${MIN_JOB_DURATION_SECONDS} - AND (${asyncJobs.metadata}->>'maxDurationSeconds')::numeric <= ${MAX_JOB_DURATION_SECONDS} - AND trunc((${asyncJobs.metadata}->>'maxDurationSeconds')::numeric) - = (${asyncJobs.metadata}->>'maxDurationSeconds')::numeric - ELSE FALSE - END` - // A bare `Date` in a raw template reaches the driver unserialized; `sql.param` - // binds it through the column encoder, as `lt(column, date)` does elsewhere here. - const staleProcessingCutoff = sql.param(now, asyncJobs.startedAt) - const staleProcessingFallbackCutoff = sql.param(staleThreshold, asyncJobs.startedAt) - const staleProcessingDurationPredicate = sql`CASE - WHEN ${hasPositiveMaxDuration} - THEN ${asyncJobs.startedAt} + ((${asyncJobs.metadata}->>'maxDurationSeconds')::double precision * interval '1 second') < ${staleProcessingCutoff} - ELSE ${asyncJobs.startedAt} < ${staleProcessingFallbackCutoff} - END` - const staleProcessingPredicate = and( - eq(asyncJobs.status, JOB_STATUS.PROCESSING), - ne(asyncJobs.type, SCHEDULE_EXECUTION_QUEUE_NAME), - staleProcessingDurationPredicate - ) - const staleProcessingResult = await runBatchedMutation({ - batchSize: STATE_MUTATION_BATCH_SIZE, - maxRowsPerRun: STATE_MUTATION_MAX_ROWS_PER_RUN, - claim: (tx, limit) => - tx - .select({ id: asyncJobs.id }) - .from(asyncJobs) - .where(staleProcessingPredicate) - .limit(limit) - .for('update', { skipLocked: true }), - mutation: (tx, candidateIds) => - tx - .update(asyncJobs) - .set({ - status: JOB_STATUS.FAILED, - completedAt: new Date(), - error: sql`CASE - WHEN ${hasPositiveMaxDuration} - THEN 'Job terminated: stuck in processing for more than ' - || (${asyncJobs.metadata}->>'maxDurationSeconds') - || ' seconds (worker cleanup deadline)' - ELSE ${GENERIC_STALE_PROCESSING_ERROR} - END`, - updatedAt: new Date(), - }) - .where(and(staleProcessingPredicate, inArray(asyncJobs.id, candidateIds))) - .returning({ id: asyncJobs.id }), - }) - - asyncJobsMarkedFailed = staleProcessingResult.affected - if (asyncJobsMarkedFailed > 0) { - logger.info(`Marked ${asyncJobsMarkedFailed} stale async jobs as failed`) - } - if (staleProcessingResult.reachedLimit) { - logger.info('Deferred remaining stale async jobs after reaching the per-run cap', { - maxRowsPerRun: STATE_MUTATION_MAX_ROWS_PER_RUN, - }) - } - } catch (error) { - logger.error('Failed to clean up stale async jobs:', { - error: toError(error).message, - }) - } - - // Mark stale table jobs (import, export, or delete) as failed. Jobs run detached on the web container - // and are lost if the pod is killed mid-run. `updated_at` is bumped by progress updates, so a - // `running` job with no recent update has stalled (not merely slow). Committed work is left in - // place (no rollback); the user retries. Also prune long-settled terminal jobs so the table - // doesn't grow unbounded (the latest job per table is what list/detail reads surface). - let staleTableJobsMarkedFailed = 0 - try { - const staleTableJobPredicate = and( - eq(tableJobs.status, 'running'), - lt(tableJobs.updatedAt, staleTableJobThreshold) - ) - const staleTableJobResult = await runBatchedMutation({ - batchSize: STATE_MUTATION_BATCH_SIZE, - maxRowsPerRun: STATE_MUTATION_MAX_ROWS_PER_RUN, - claim: (tx, limit) => - tx - .select({ id: tableJobs.id }) - .from(tableJobs) - .where(staleTableJobPredicate) - .limit(limit) - .for('update', { skipLocked: true }), - mutation: (tx, candidateIds) => - tx - .update(tableJobs) - .set({ - status: 'failed', - error: `Job terminated: no progress for more than ${TABLE_JOB_STALE_THRESHOLD_MINUTES} minutes (worker timeout or crash)`, - completedAt: now, - updatedAt: now, - }) - .where(and(staleTableJobPredicate, inArray(tableJobs.id, candidateIds))) - .returning({ id: tableJobs.id }), - }) - - staleTableJobsMarkedFailed = staleTableJobResult.affected - if (staleTableJobsMarkedFailed > 0) { - logger.info(`Marked ${staleTableJobsMarkedFailed} stale table jobs as failed`) - } - - const terminalRetention = new Date(Date.now() - TABLE_JOB_RETENTION_HOURS * 60 * 60 * 1000) - const terminalTableJobPredicate = and( - inArray(tableJobs.status, ['ready', 'failed', 'canceled']), - lt(tableJobs.updatedAt, terminalRetention) - ) - const terminalTableJobResult = await runBatchedMutation({ - batchSize: TABLE_JOB_PRUNE_BATCH_SIZE, - maxRowsPerRun: TABLE_JOB_PRUNE_MAX_ROWS_PER_RUN, - claim: (tx, limit) => - tx - .select({ id: tableJobs.id }) - .from(tableJobs) - .where(terminalTableJobPredicate) - .limit(limit) - .for('update', { skipLocked: true }), - mutation: (tx, candidateIds) => - tx - .delete(tableJobs) - .where(and(terminalTableJobPredicate, inArray(tableJobs.id, candidateIds))) - .returning({ - type: tableJobs.type, - resultKey: sql`${tableJobs.payload}->>'resultKey'`, - }), - onBatch: async (jobs) => { - /** - * Pruned export jobs carry the generated file's storage key. The scalar - * key is returned instead of the full JSON payload, and cleanup stays - * sequential so storage concurrency is bounded at one request. - */ - for (const { type, resultKey } of jobs) { - if (type !== 'export' || !resultKey) continue - await deleteFile({ key: resultKey, context: 'workspace' }).catch((err) => { - logger.warn('Failed to delete pruned export file', { - resultKey, - error: toError(err).message, - }) - }) - } + return runCleanupStaleExecutions() }, - }) - if (terminalTableJobResult.reachedLimit) { - logger.info('Deferred remaining terminal table jobs after reaching the per-run cap', { - maxRowsPerRun: TABLE_JOB_PRUNE_MAX_ROWS_PER_RUN, - }) - } - } catch (error) { - logger.error('Failed to clean up stale table jobs:', { - error: toError(error).message, - }) - } - - // Clean up stale pending jobs (never started, e.g., due to server crash before startJob()) - let stalePendingJobsMarkedFailed = 0 - - try { - const stalePendingPredicate = and( - eq(asyncJobs.status, JOB_STATUS.PENDING), - ne(asyncJobs.type, SCHEDULE_EXECUTION_QUEUE_NAME), - lt(asyncJobs.createdAt, stalePendingThreshold) - ) - const stalePendingResult = await runBatchedMutation({ - batchSize: STATE_MUTATION_BATCH_SIZE, - maxRowsPerRun: STATE_MUTATION_MAX_ROWS_PER_RUN, - claim: (tx, limit) => - tx - .select({ id: asyncJobs.id }) - .from(asyncJobs) - .where(stalePendingPredicate) - .limit(limit) - .for('update', { skipLocked: true }), - mutation: (tx, candidateIds) => - tx - .update(asyncJobs) - .set({ - status: JOB_STATUS.FAILED, - completedAt: new Date(), - error: `Job terminated: stuck in pending state for more than ${JOB_PENDING_RETENTION_HOURS} hours (never started)`, - updatedAt: new Date(), - }) - .where(and(stalePendingPredicate, inArray(asyncJobs.id, candidateIds))) - .returning({ id: asyncJobs.id }), - }) - - stalePendingJobsMarkedFailed = stalePendingResult.affected - if (stalePendingJobsMarkedFailed > 0) { - logger.info(`Marked ${stalePendingJobsMarkedFailed} stale pending jobs as failed`) } - if (stalePendingResult.reachedLimit) { - logger.info('Deferred remaining stale pending jobs after reaching the per-run cap', { - maxRowsPerRun: STATE_MUTATION_MAX_ROWS_PER_RUN, - }) - } - } catch (error) { - logger.error('Failed to clean up stale pending jobs:', { - error: toError(error).message, - }) - } - - const retentionNow = Date.now() - const retentionThreshold = new Date(retentionNow - JOB_RETENTION_HOURS * 60 * 60 * 1000) - const irrecoverableCarrierRetentionThreshold = new Date( - retentionNow - SCHEDULE_CARRIER_IRRECOVERABLE_RETENTION_HOURS * 60 * 60 * 1000 ) - let asyncJobsDeleted = 0 - try { - const retainedJobPredicate = and( - inArray(asyncJobs.status, TERMINAL_JOB_STATUSES), - or( - ne(asyncJobs.type, SCHEDULE_EXECUTION_QUEUE_NAME), - /** - * Schedule recovery owns a carrier until it stamps the reconciled - * marker, so retention waits for it rather than deleting an - * occurrence that has not been accounted for yet. - */ - and( - carrierReconciledSql(asyncJobs.metadata), - or( - carrierNotIrrecoverableSql(asyncJobs.metadata), - lt(asyncJobs.completedAt, irrecoverableCarrierRetentionThreshold) - ) - ) - ), - lt(asyncJobs.completedAt, retentionThreshold) - ) - const retainedJobResult = await runBatchedMutation({ - batchSize: RETENTION_DELETE_BATCH_SIZE, - maxRowsPerRun: RETENTION_DELETE_MAX_ROWS_PER_RUN, - claim: (tx, limit) => - tx - .select({ id: asyncJobs.id }) - .from(asyncJobs) - .where(retainedJobPredicate) - .limit(limit) - .for('update', { skipLocked: true }), - mutation: (tx, candidateIds) => - tx - .delete(asyncJobs) - .where(and(retainedJobPredicate, inArray(asyncJobs.id, candidateIds))) - .returning({ id: asyncJobs.id }), - }) - - asyncJobsDeleted = retainedJobResult.affected - if (asyncJobsDeleted > 0) { - logger.info( - `Deleted ${asyncJobsDeleted} old async jobs (retention: ${JOB_RETENTION_HOURS}h)` - ) - } - if (retainedJobResult.reachedLimit) { - logger.info('Deferred remaining retained async jobs after reaching the per-run cap', { - maxRowsPerRun: RETENTION_DELETE_MAX_ROWS_PER_RUN, - }) - } - } catch (error) { - logger.error('Failed to delete old async jobs:', { - error: toError(error).message, - }) - } - - /** - * Prune terminal connector sync logs past retention. - * - * HARD INVARIANT: the newest row per connector must survive, and so must the - * newest `completed` row. `loadPreviousListingObservation` reconstructs the - * previous listing from the latest `completed` log, and that reconstruction - * decides whether a suspect listing is corroborated — i.e. whether - * reconciliation may delete documents. Pruning the last `completed` row - * would silently change deletion behaviour, so both `exists` guards below - * are load-bearing rather than defensive. - * - * `started` rows are never eligible: they are either in flight or waiting on - * the scheduler's own sweep to close them. - */ - let connectorSyncLogsPruned = 0 - try { - const syncLogRetention = new Date( - Date.now() - CONNECTOR_SYNC_LOG_RETENTION_DAYS * 24 * 60 * 60 * 1000 - ) - const newerSyncLog = alias(knowledgeConnectorSyncLog, 'newer_sync_log') - const newerCompletedSyncLog = alias(knowledgeConnectorSyncLog, 'newer_completed_sync_log') - const syncLogPredicate = and( - inArray(knowledgeConnectorSyncLog.status, ['completed', 'failed']), - lt(knowledgeConnectorSyncLog.startedAt, syncLogRetention), - exists( - db - .select({ id: newerSyncLog.id }) - .from(newerSyncLog) - .where( - and( - eq(newerSyncLog.connectorId, knowledgeConnectorSyncLog.connectorId), - gt(newerSyncLog.startedAt, knowledgeConnectorSyncLog.startedAt) - ) - ) - ), - or( - ne(knowledgeConnectorSyncLog.status, 'completed'), - exists( - db - .select({ id: newerCompletedSyncLog.id }) - .from(newerCompletedSyncLog) - .where( - and( - eq(newerCompletedSyncLog.connectorId, knowledgeConnectorSyncLog.connectorId), - eq(newerCompletedSyncLog.status, 'completed'), - gt(newerCompletedSyncLog.startedAt, knowledgeConnectorSyncLog.startedAt) - ) - ) - ) - ) - ) - const syncLogResult = await runBatchedMutation({ - batchSize: CONNECTOR_SYNC_LOG_PRUNE_BATCH_SIZE, - maxRowsPerRun: CONNECTOR_SYNC_LOG_MAX_ROWS_PER_RUN, - claim: (tx, limit) => - tx - .select({ id: knowledgeConnectorSyncLog.id }) - .from(knowledgeConnectorSyncLog) - .where(syncLogPredicate) - .limit(limit) - .for('update', { skipLocked: true }), - mutation: (tx, candidateIds) => - tx - .delete(knowledgeConnectorSyncLog) - .where(inArray(knowledgeConnectorSyncLog.id, candidateIds)) - .returning({ id: knowledgeConnectorSyncLog.id }), - }) - connectorSyncLogsPruned = syncLogResult.affected - if (connectorSyncLogsPruned > 0) { - logger.info( - `Pruned ${connectorSyncLogsPruned} old connector sync logs (retention: ${CONNECTOR_SYNC_LOG_RETENTION_DAYS}d)` - ) - } - if (syncLogResult.reachedLimit) { - logger.info('Deferred remaining connector sync logs after reaching the per-run cap', { - maxRowsPerRun: CONNECTOR_SYNC_LOG_MAX_ROWS_PER_RUN, - }) - } - } catch (error) { - logger.error('Failed to prune old connector sync logs:', { - error: toError(error).message, - }) - } - - /** - * Prune terminal deployment operations past retention. HARD INVARIANT: - * the newest-generation row per workflow must always survive — the next - * deploy computes `generation = MAX(generation) + 1`, and the webhook - * registration store fences rows with lt/gt comparisons against stored - * generations, so generation reuse after a full wipe would permanently - * wedge that workflow's deployments. The `exists(newer)` predicate - * guarantees the max-generation row is never eligible; the status filter - * keeps in-flight rows (an outbox worker may still hold their fence). - */ - let deploymentOperationsPruned = 0 - try { - const deploymentOpRetention = new Date( - Date.now() - DEPLOYMENT_OPERATION_RETENTION_DAYS * 24 * 60 * 60 * 1000 - ) - const newerOperation = alias(workflowDeploymentOperation, 'newer_operation') - const deploymentOpPredicate = and( - inArray(workflowDeploymentOperation.status, ['active', 'failed', 'superseded']), - lt(workflowDeploymentOperation.completedAt, deploymentOpRetention), - exists( - db - .select({ id: newerOperation.id }) - .from(newerOperation) - .where( - and( - eq(newerOperation.workflowId, workflowDeploymentOperation.workflowId), - gt(newerOperation.generation, workflowDeploymentOperation.generation) - ) - ) - ) - ) - const deploymentOpResult = await runBatchedMutation({ - batchSize: DEPLOYMENT_OPERATION_PRUNE_BATCH_SIZE, - maxRowsPerRun: - DEPLOYMENT_OPERATION_PRUNE_BATCH_SIZE * DEPLOYMENT_OPERATION_PRUNE_MAX_BATCHES, - claim: (tx, limit) => - tx - .select({ id: workflowDeploymentOperation.id }) - .from(workflowDeploymentOperation) - .where(deploymentOpPredicate) - .limit(limit) - .for('update', { skipLocked: true }), - mutation: (tx, candidateIds) => - tx - .delete(workflowDeploymentOperation) - .where(inArray(workflowDeploymentOperation.id, candidateIds)) - .returning({ id: workflowDeploymentOperation.id }), - }) - deploymentOperationsPruned = deploymentOpResult.affected - if (deploymentOperationsPruned > 0) { - logger.info( - `Pruned ${deploymentOperationsPruned} old deployment operations (retention: ${DEPLOYMENT_OPERATION_RETENTION_DAYS}d)` - ) - } - if (deploymentOpResult.reachedLimit) { - logger.info('Deferred remaining deployment operations after reaching the per-run cap', { - maxRowsPerRun: - DEPLOYMENT_OPERATION_PRUNE_BATCH_SIZE * DEPLOYMENT_OPERATION_PRUNE_MAX_BATCHES, - }) - } - } catch (error) { - logger.error('Failed to prune old deployment operations:', { - error: toError(error).message, - }) - } - - /** - * Cancel table run dispatches abandoned by a dead dispatcher. Nothing else - * reclaims them — every other terminal transition is user- or flow-initiated - * — so a dispatcher killed mid-loop left the row `dispatching` forever and - * the client's "X running" overlay with it. Ages from the dispatcher's - * per-window heartbeat, so a slow-but-live dispatch is spared. - */ - let staleDispatchesCancelled = 0 - try { - staleDispatchesCancelled = ( - await cancelStaleDispatches(staleDispatchThreshold, TABLE_DISPATCH_MAX_PER_RUN) - ).length - if (staleDispatchesCancelled > 0) { - logger.warn(`Cancelled ${staleDispatchesCancelled} abandoned table run dispatches`, { - thresholdMinutes: TABLE_DISPATCH_STALE_THRESHOLD_MINUTES, - }) - } - } catch (error) { - logger.error('Failed to cancel abandoned table run dispatches:', { - error: toError(error).message, - }) - } - - /** - * Settle Chat runs no controller will finish: their process died, their - * controller was superseded without a successor, or Stop found none. Without - * this they stay unfinished forever and keep their chat marked as busy. - */ - let orphanedRunsSettled = 0 - try { - orphanedRunsSettled = (await sweepOrphanedRuns()).settledRunIds.length - } catch (error) { - logger.error('Failed to settle orphaned Chat runs:', { - error: toError(error).message, - }) - } - - return NextResponse.json({ - success: true, - executions: { - found: staleExecutionsFound, - cleaned, - failed, - thresholdMinutes: STALE_THRESHOLD_MINUTES, - }, - asyncJobs: { - staleProcessingMarkedFailed: asyncJobsMarkedFailed, - stalePendingMarkedFailed: stalePendingJobsMarkedFailed, - oldDeleted: asyncJobsDeleted, - staleThresholdMinutes: STALE_THRESHOLD_MINUTES, - retentionHours: JOB_RETENTION_HOURS, - }, - tableJobs: { - staleMarkedFailed: staleTableJobsMarkedFailed, - }, - connectorSyncLogs: { - pruned: connectorSyncLogsPruned, - retentionDays: CONNECTOR_SYNC_LOG_RETENTION_DAYS, - }, - tableRunDispatches: { - staleCancelled: staleDispatchesCancelled, - thresholdMinutes: TABLE_DISPATCH_STALE_THRESHOLD_MINUTES, - }, - deploymentOperations: { - pruned: deploymentOperationsPruned, - retentionDays: DEPLOYMENT_OPERATION_RETENTION_DAYS, - }, - chatRuns: { - orphanedSettled: orphanedRunsSettled, - }, - }) + logger.info('Stale execution cleanup dispatched', { jobId }) + return NextResponse.json({ triggered: true, jobId }) } catch (error) { - logger.error('Error in stale execution cleanup job:', error) - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) + logger.error('Failed to dispatch stale execution cleanup', { error }) + return NextResponse.json( + { error: 'Failed to dispatch stale execution cleanup' }, + { status: 500 } + ) } }) diff --git a/apps/sim/app/api/environment/route.ts b/apps/sim/app/api/environment/route.ts index 10034a9e495..dbef90ae0e7 100644 --- a/apps/sim/app/api/environment/route.ts +++ b/apps/sim/app/api/environment/route.ts @@ -115,7 +115,7 @@ export const POST = withRouteHandler(async (req: NextRequest) => { } }) -export const GET = withRouteHandler(async (request: Request) => { +export const GET = withRouteHandler(async () => { const requestId = generateRequestId() try { diff --git a/apps/sim/app/api/files/authorization.test.ts b/apps/sim/app/api/files/authorization.test.ts index 26b588f6d90..9d601a564f3 100644 --- a/apps/sim/app/api/files/authorization.test.ts +++ b/apps/sim/app/api/files/authorization.test.ts @@ -142,7 +142,7 @@ describe('verifyKBFileWriteAccess (binding-only delete authorization)', () => { describe('public-context access (profile-pictures / og-images / workspace-logos)', () => { function write(cloudKey: string, context: 'profile-pictures' | 'og-images' | 'workspace-logos') { - return verifyFileAccess(cloudKey, USER_ID, undefined, context, false, { requireWrite: true }) + return verifyFileAccess(cloudKey, USER_ID, undefined, context, { requireWrite: true }) } it('allows organization logo reads and denies generic deletes even for the uploader', async () => { @@ -152,10 +152,10 @@ describe('public-context access (profile-pictures / og-images / workspace-logos) true ) await expect( - verifyFileAccess(key, USER_ID, undefined, 'organization-logos', false, { requireWrite: true }) + verifyFileAccess(key, USER_ID, undefined, 'organization-logos', { requireWrite: true }) ).resolves.toBe(false) await expect( - verifyFileAccess(key, USER_ID, undefined, 'general', false, { requireWrite: true }) + verifyFileAccess(key, USER_ID, undefined, 'general', { requireWrite: true }) ).resolves.toBe(false) expect(mockGetFileMetadata).not.toHaveBeenCalled() expect(mockGetUserEntityPermissions).not.toHaveBeenCalled() @@ -203,7 +203,7 @@ describe('workspace-scoped access (workspace files and mothership attachments)', }) function read(cloudKey: string, context: 'workspace' | 'mothership') { - return verifyFileAccess(cloudKey, USER_ID, undefined, context, false) + return verifyFileAccess(cloudKey, USER_ID, undefined, context) } interface BoundRow { @@ -315,7 +315,7 @@ describe('organization connector cache access', () => { 'denies the uploader a raw download even with a forged %s context', async (context) => { await expect( - verifyFileAccess(CLOUD_KEY, USER_ID, undefined, context, false, { knowledgeAccess: 'user' }) + verifyFileAccess(CLOUD_KEY, USER_ID, undefined, context, { knowledgeAccess: 'user' }) ).resolves.toBe(false) expect(mockGetFileMetadata).not.toHaveBeenCalled() expect(mockGetUserEntityPermissions).not.toHaveBeenCalled() @@ -325,7 +325,7 @@ describe('organization connector cache access', () => { it('denies system reads after the cache loses its active document reference', async () => { dbChainMockFns.limit.mockResolvedValue([]) await expect( - verifyFileAccess(CLOUD_KEY, USER_ID, undefined, 'knowledge-base', false, { + verifyFileAccess(CLOUD_KEY, USER_ID, undefined, 'knowledge-base', { knowledgeAccess: SYSTEM_ACCESS_SCOPE, }) ).resolves.toBe(false) @@ -338,7 +338,7 @@ describe('organization connector cache access', () => { deletedAt: null, }) await expect( - verifyFileAccess(CLOUD_KEY, USER_ID, undefined, 'knowledge-base', false, { + verifyFileAccess(CLOUD_KEY, USER_ID, undefined, 'knowledge-base', { knowledgeAccess: SYSTEM_ACCESS_SCOPE, }) ).resolves.toBe(false) @@ -347,7 +347,7 @@ describe('organization connector cache access', () => { it('does not let a raw download endpoint delete organization caches', async () => { await expect( - verifyFileAccess(CLOUD_KEY, USER_ID, undefined, 'general', false, { + verifyFileAccess(CLOUD_KEY, USER_ID, undefined, 'general', { requireWrite: true, knowledgeAccess: SYSTEM_ACCESS_SCOPE, }) @@ -378,7 +378,7 @@ describe('KB file live source authorization', () => { queueTableRows(schemaMock.knowledgeConnector, [{ connectorId: 'confluence-source' }]) queueTableRows(schemaMock.document, allowed ? [{ id: 'doc-1' }] : []) await expect( - verifyFileAccess(CLOUD_KEY, USER_ID, undefined, 'knowledge-base', false, { + verifyFileAccess(CLOUD_KEY, USER_ID, undefined, 'knowledge-base', { knowledgeAccess: access, }) ).resolves.toBe(allowed) @@ -415,7 +415,7 @@ describe('KB file live source authorization', () => { mockGetUserEntityPermissions.mockResolvedValue(null) const get = vi.fn() await expect( - verifyFileAccess(CLOUD_KEY, USER_ID, undefined, 'knowledge-base', false, { + verifyFileAccess(CLOUD_KEY, USER_ID, undefined, 'knowledge-base', { knowledgeAccess: { get, getForConnectors: vi.fn(), getForDocuments: vi.fn() }, }) ).resolves.toBe(false) diff --git a/apps/sim/app/api/files/authorization.ts b/apps/sim/app/api/files/authorization.ts index f96381bbddd..ab28084e50c 100644 --- a/apps/sim/app/api/files/authorization.ts +++ b/apps/sim/app/api/files/authorization.ts @@ -19,7 +19,6 @@ import { getFileMetadataByKey } from '@/lib/uploads/server/metadata' import { isWorkspaceScopedContext } from '@/lib/uploads/shared/types' import { inferContextFromKey } from '@/lib/uploads/utils/file-utils' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' -import { isUuid } from '@/executor/constants' const logger = createLogger('FileAuthorization') @@ -31,12 +30,6 @@ export class FileAccessDeniedError extends Error { } } -interface AuthorizationResult { - granted: boolean - reason: string - workspaceId?: string -} - type WorkspacePermission = 'read' | 'write' | 'admin' /** @@ -113,34 +106,12 @@ async function lookupWorkspaceFileByKey( } } -/** - * Extract workspace ID from workspace file key pattern - * Pattern: {workspaceId}/{timestamp}-{random}-{filename} - */ -function extractWorkspaceIdFromKey(key: string): string | null { - const inferredContext = inferContextFromKey(key) - if (inferredContext !== 'workspace') { - return null - } - - // Use the proper parsing utility from workspace context module - const parts = key.split('/') - const workspaceId = parts[0] - - if (workspaceId && isUuid(workspaceId)) { - return workspaceId - } - - return null -} - /** * Verify file access based on file path patterns and metadata * @param cloudKey The file key/path (e.g., "workspace_id/workflow_id/execution_id/filename" or "kb/filename") * @param userId The authenticated user ID * @param customConfig Optional custom storage configuration * @param context Optional explicit storage context - * @param isLocal Optional flag indicating if this is local storage * @returns Promise True if user has access, false otherwise */ export async function verifyFileAccess( @@ -148,7 +119,6 @@ export async function verifyFileAccess( userId: string, customConfig?: StorageConfig, context?: StorageContext | 'general', - isLocal?: boolean, options?: { requireWrite?: boolean; knowledgeAccess?: KnowledgeFileAccess } ): Promise { /** Organization images require the Principal-aware Assistant application resolver. */ @@ -161,10 +131,10 @@ export async function verifyFileAccess( if (keyContext === 'knowledge-base') { return requireWrite ? verifyKBFileWriteAccess(cloudKey, userId) - : verifyKBFileAccess(cloudKey, userId, customConfig, options?.knowledgeAccess) + : verifyKBFileAccess(cloudKey, userId, options?.knowledgeAccess) } if (context === 'general') { - return await verifyRegularFileAccess(cloudKey, userId, customConfig, isLocal, requireWrite) + return await verifyRegularFileAccess(cloudKey, userId, customConfig, requireWrite) } // Infer context from key if not explicitly provided @@ -185,12 +155,12 @@ export async function verifyFileAccess( // 1. Workspace / mothership files: Check database first (most reliable for both local and cloud) if (isWorkspaceScopedContext(inferredContext)) { - return await verifyWorkspaceFileAccess(cloudKey, userId, customConfig, isLocal, requireWrite) + return await verifyWorkspaceFileAccess(cloudKey, userId, customConfig, requireWrite) } // 2. Execution files: workspace_id/workflow_id/execution_id/filename if (inferredContext === 'execution') { - return await verifyExecutionFileAccess(cloudKey, userId, customConfig, requireWrite) + return await verifyExecutionFileAccess(cloudKey, userId, requireWrite) } // 3. Copilot files: Check database first, then metadata, then path pattern (legacy) @@ -202,7 +172,7 @@ export async function verifyFileAccess( if (inferredContext === 'knowledge-base') { return requireWrite ? verifyKBFileWriteAccess(cloudKey, userId) - : verifyKBFileAccess(cloudKey, userId, customConfig, options?.knowledgeAccess) + : verifyKBFileAccess(cloudKey, userId, options?.knowledgeAccess) } // 5. Chat files: chat/filename @@ -212,7 +182,7 @@ export async function verifyFileAccess( // 6. Regular uploads: UUID-filename or timestamp-filename // Check metadata for userId/workspaceId, or database for workspace files - return await verifyRegularFileAccess(cloudKey, userId, customConfig, isLocal, requireWrite) + return await verifyRegularFileAccess(cloudKey, userId, customConfig, requireWrite) } catch (error) { logger.error('Error verifying file access:', { cloudKey, userId, error }) // Deny access on error to be safe @@ -240,7 +210,6 @@ async function verifyWorkspaceFileAccess( cloudKey: string, userId: string, customConfig?: StorageConfig, - isLocal?: boolean, requireWrite = false ): Promise { try { @@ -397,7 +366,6 @@ async function verifyPublicAssetWriteAccess( async function verifyExecutionFileAccess( cloudKey: string, userId: string, - customConfig?: StorageConfig, requireWrite = false ): Promise { const parts = cloudKey.split('/') @@ -580,7 +548,6 @@ async function resolveKnowledgeFileAccess( async function verifyKBFileAccess( cloudKey: string, userId: string, - customConfig?: StorageConfig, knowledgeAccess?: KnowledgeFileAccess ): Promise { try { @@ -731,7 +698,6 @@ async function verifyRegularFileAccess( cloudKey: string, userId: string, customConfig?: StorageConfig, - isLocal?: boolean, requireWrite = false ): Promise { try { diff --git a/apps/sim/app/api/files/delete/route.test.ts b/apps/sim/app/api/files/delete/route.test.ts index b7596b12b51..6c331f9cd06 100644 --- a/apps/sim/app/api/files/delete/route.test.ts +++ b/apps/sim/app/api/files/delete/route.test.ts @@ -54,7 +54,6 @@ describe('File Delete API Route', () => { filesAuthorizationMockFns.mockVerifyFileAccess.mockResolvedValue(true) storageServiceMockFns.mockDeleteFile.mockResolvedValue(undefined) storageServiceMockFns.mockHasCloudStorage.mockReturnValue(true) - uploadsMockFns.mockGetStorageProvider.mockReturnValue('s3') uploadsMockFns.mockIsUsingCloudStorage.mockReturnValue(true) }) diff --git a/apps/sim/app/api/files/delete/route.ts b/apps/sim/app/api/files/delete/route.ts index 34903aa22e1..f50db010c50 100644 --- a/apps/sim/app/api/files/delete/route.ts +++ b/apps/sim/app/api/files/delete/route.ts @@ -6,7 +6,7 @@ import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import type { StorageContext } from '@/lib/uploads/config' -import { deleteFile, hasCloudStorage } from '@/lib/uploads/core/storage-service' +import { deleteFile } from '@/lib/uploads/core/storage-service' import { deleteFileMetadata } from '@/lib/uploads/server/metadata' import { extractStorageKey, inferContextFromKey } from '@/lib/uploads/utils/file-utils' import { verifyFileAccess, verifyKBFileWriteAccess } from '@/app/api/files/authorization' @@ -73,7 +73,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const hasAccess = storageContext === 'knowledge-base' ? await verifyKBFileWriteAccess(key, userId) - : await verifyFileAccess(key, userId, undefined, storageContext, !hasCloudStorage(), { + : await verifyFileAccess(key, userId, undefined, storageContext, { requireWrite: true, }) diff --git a/apps/sim/app/api/files/export/[id]/route.ts b/apps/sim/app/api/files/export/[id]/route.ts index 37737a8ba84..9ec91812e13 100644 --- a/apps/sim/app/api/files/export/[id]/route.ts +++ b/apps/sim/app/api/files/export/[id]/route.ts @@ -60,7 +60,7 @@ export const GET = withRouteHandler( } const knowledgeAccess = authResult.authType === AuthType.SESSION ? 'user' : undefined - const hasAccess = await verifyFileAccess(record.key, userId, undefined, undefined, undefined, { + const hasAccess = await verifyFileAccess(record.key, userId, undefined, undefined, { knowledgeAccess, }) if (!hasAccess) { @@ -147,7 +147,7 @@ export const GET = withRouteHandler( const imgRecord = await getFileMetadataById(storedFileId(imageId)) if (!imgRecord) return null if ( - !(await verifyFileAccess(imgRecord.key, userId, undefined, undefined, undefined, { + !(await verifyFileAccess(imgRecord.key, userId, undefined, undefined, { knowledgeAccess, })) ) { diff --git a/apps/sim/app/api/files/public/[token]/otp/route.ts b/apps/sim/app/api/files/public/[token]/otp/route.ts index 26fa53f8f8b..57a707c189b 100644 --- a/apps/sim/app/api/files/public/[token]/otp/route.ts +++ b/apps/sim/app/api/files/public/[token]/otp/route.ts @@ -73,7 +73,7 @@ async function deliverOtp(requestId: string, shareId: string, email: string): Pr const otp = generateOTP() await storeOTP('file', shareId, email, otp) - const emailHtml = await renderOTPEmail(otp, email, 'email-verification', SHARE_EMAIL_LABEL) + const emailHtml = await renderOTPEmail(otp, 'email-verification', SHARE_EMAIL_LABEL) const emailResult = await sendEmail({ to: email, subject: getOtpSubject(SHARE_EMAIL_LABEL), diff --git a/apps/sim/app/api/files/serve/[...path]/route.ts b/apps/sim/app/api/files/serve/[...path]/route.ts index 666135163fc..eeff30cd6d7 100644 --- a/apps/sim/app/api/files/serve/[...path]/route.ts +++ b/apps/sim/app/api/files/serve/[...path]/route.ts @@ -473,7 +473,6 @@ async function handleLocalFile( userId, undefined, // customConfig context, - true, // isLocal { knowledgeAccess } ) @@ -547,8 +546,7 @@ async function handleCloudProxy( cloudKey, userId, undefined, // customConfig - context, // context - false, // isLocal + context, { knowledgeAccess } ) diff --git a/apps/sim/app/api/files/utils.ts b/apps/sim/app/api/files/utils.ts index 0bd29f54288..bbd20fe6b52 100644 --- a/apps/sim/app/api/files/utils.ts +++ b/apps/sim/app/api/files/utils.ts @@ -14,11 +14,6 @@ export interface ApiSuccessResponse { [key: string]: any } -interface ApiErrorResponse { - error: string - message?: string -} - export interface FileResponse { buffer: Buffer contentType: string @@ -40,7 +35,7 @@ export class InvalidRequestError extends Error { } } -export const contentTypeMap: Record = { +const contentTypeMap: Record = { txt: 'text/plain', csv: 'text/csv', json: 'application/json', diff --git a/apps/sim/app/api/files/view/[id]/route.ts b/apps/sim/app/api/files/view/[id]/route.ts index 47ab06d164f..26a38f073d9 100644 --- a/apps/sim/app/api/files/view/[id]/route.ts +++ b/apps/sim/app/api/files/view/[id]/route.ts @@ -38,7 +38,6 @@ export const GET = withRouteHandler( authResult.userId, undefined, record.context as StorageContext | 'general', - undefined, { knowledgeAccess: authResult.authType === AuthType.SESSION ? 'user' : undefined } ) if (!hasAccess) { diff --git a/apps/sim/app/api/folders/reorder/route.test.ts b/apps/sim/app/api/folders/reorder/route.test.ts index b9a1a759e76..18acecc4de6 100644 --- a/apps/sim/app/api/folders/reorder/route.test.ts +++ b/apps/sim/app/api/folders/reorder/route.test.ts @@ -4,18 +4,6 @@ import { authMockFns, createMockRequest, permissionsMock, permissionsMockFns } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockLogger } = vi.hoisted(() => ({ - mockLogger: { - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - debug: vi.fn(), - trace: vi.fn(), - fatal: vi.fn(), - child: vi.fn(), - }, -})) - const mockGetUserEntityPermissions = permissionsMockFns.mockGetUserEntityPermissions vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock) diff --git a/apps/sim/app/api/knowledge/connectors/directory-sync/route.test.ts b/apps/sim/app/api/knowledge/connectors/directory-sync/route.test.ts index f9c6c7c53ae..d3fded53e48 100644 --- a/apps/sim/app/api/knowledge/connectors/directory-sync/route.test.ts +++ b/apps/sim/app/api/knowledge/connectors/directory-sync/route.test.ts @@ -4,7 +4,6 @@ import { hasMockCondition, resetEnvFlagsMock, schemaMock, - setEnvFlags, } from '@sim/testing' import { authInternalMock, authInternalMockFns } from '@sim/testing/mocks/auth-internal.mock' import { dbChainMockFns } from '@sim/testing/mocks/database.mock' @@ -108,24 +107,6 @@ describe('connector directory sync scheduler', () => { await expect(run()).resolves.toMatchObject({ dispatched: 1, failed: 1 }) }) - it.each([true, false])( - 'excludes Search directories from scheduled pages only when live Search is %s', - async (liveSearch) => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: liveSearch }) - mockConnectorRows.mockResolvedValue([]) - await run() - expect( - hasMockCondition( - mockWhere.mock.calls[0][0], - (node) => - node.type === 'eq' && - node.left === schemaMock.knowledgeBase.isSearchIndex && - node.right === false - ) - ).toBe(liveSearch) - } - ) - it('does not enqueue a connector another scheduler claimed or paused', async () => { mockConnectorRows.mockResolvedValue([connector()]) mockClaim.mockResolvedValueOnce([]) diff --git a/apps/sim/app/api/knowledge/search/route.ts b/apps/sim/app/api/knowledge/search/route.ts index 1186b6da0f0..610507ea53e 100644 --- a/apps/sim/app/api/knowledge/search/route.ts +++ b/apps/sim/app/api/knowledge/search/route.ts @@ -6,68 +6,9 @@ import { } from '@/lib/api/server/routes' import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { DEFAULT_RERANKER_MODEL } from '@/lib/knowledge/reranker-models' -import { sourceAuthor } from '@/lib/knowledge/search/author' -import { searchScopedKnowledge } from '@/lib/sim-search/indexed' -import { isIndexedOrgSearchEnabled } from '@/lib/sim-search/indexed/gate' import { searchLiveKnowledge } from '@/lib/sim-search/live/application' -const DIRECT_SEARCH_VECTOR_BUDGET_MS = 3000 - -const indexedSearchRoute = defineInternalJsonRoute({ - contract: searchWorkspaceKnowledgeContract, - auth: internalSessionAuth, - operation: knowledgeOperations.search, - rateLimit: internalRateLimits.none({ - reason: - 'A person typing queries; the embedding call is metered against the canonical search owner', - }), - errorPolicy: internalKnowledgeErrorPolicies.search, - mapInput: ({ body }, { request }) => ({ - workspaceId: body.workspaceId, - organizationId: body.organizationId, - filters: body.filters, - query: body.query, - topK: body.topK, - allowPartialResults: true, - vectorBudgetMs: DIRECT_SEARCH_VECTOR_BUDGET_MS, - /** - * A person's search is reranked by a cross-encoder whenever the workspace or the platform - * holds a key for one; the use case checks that before spending a call, and reranking stays - * best-effort, so a provider outage leaves the fused order in place. - */ - rerankerEnabled: true, - rerankerModel: DEFAULT_RERANKER_MODEL, - surface: 'dashboard' as const, - signal: request.signal, - }), - useCase: searchScopedKnowledge, - present: ({ results, knowledgeBases, retrieval }, { input }) => { - const knowledgeBaseNames = new Map(knowledgeBases.map((kb) => [kb.id, kb.name])) - return { - success: true as const, - data: { - query: input.query ?? '', - retrieval, - results: results.map((result) => ({ - documentId: result.documentId, - knowledgeBaseId: result.knowledgeBaseId, - knowledgeBaseName: knowledgeBaseNames.get(result.knowledgeBaseId) ?? '', - documentName: result.documentName, - sourceUrl: result.sourceUrl, - connectorType: result.connectorType, - sourceModifiedAt: result.sourceModifiedAt?.toISOString() ?? null, - author: sourceAuthor(result.metadata), - content: result.content, - chunkIndex: result.chunkIndex, - similarity: result.similarity, - })), - }, - } - }, -}) - -const liveSearchRoute = defineInternalJsonRoute({ +export const POST = defineInternalJsonRoute({ contract: searchWorkspaceKnowledgeContract, auth: internalSessionAuth, operation: knowledgeOperations.search, @@ -80,6 +21,3 @@ const liveSearchRoute = defineInternalJsonRoute({ useCase: searchLiveKnowledge, present: (data) => ({ success: true as const, data }), }) - -/** Indexed organization search is dormant unless its gate is on; Live Search serves otherwise. */ -export const POST = isIndexedOrgSearchEnabled() ? indexedSearchRoute : liveSearchRoute diff --git a/apps/sim/app/api/knowledge/sim-search/connect/route.ts b/apps/sim/app/api/knowledge/sim-search/connect/route.ts deleted file mode 100644 index 521b6b570ef..00000000000 --- a/apps/sim/app/api/knowledge/sim-search/connect/route.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { connectSimSearchConnectorContract } from '@/lib/api/contracts/knowledge' -import { - defineInternalJsonRoute, - internalRateLimits, - internalSessionAuth, -} from '@/lib/api/server/routes' -import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' -import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { connectSimSearchConnector } from '@/lib/knowledge/application/sim-search' - -export const POST = defineInternalJsonRoute({ - contract: connectSimSearchConnectorContract, - auth: internalSessionAuth, - operation: knowledgeOperations.simSearchConnect, - rateLimit: internalRateLimits.none({ reason: 'One click per source; mints a single-use link' }), - errorPolicy: internalKnowledgeErrorPolicies.connectAccount, - mapInput: ({ body }) => body, - useCase: connectSimSearchConnector, - present: (result) => ({ success: true as const, data: result }), -}) diff --git a/apps/sim/app/api/knowledge/sim-search/integrations/overview/route.ts b/apps/sim/app/api/knowledge/sim-search/integrations/overview/route.ts deleted file mode 100644 index 325f40ea7c8..00000000000 --- a/apps/sim/app/api/knowledge/sim-search/integrations/overview/route.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { readOrganizationSearchOverviewContract } from '@/lib/api/contracts/knowledge/connectors' -import { - defineInternalJsonRoute, - internalRateLimits, - internalSessionAuth, -} from '@/lib/api/server/routes' -import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' -import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { readOrganizationSearchOverview } from '@/lib/knowledge/application/organization-search-overview' - -export const GET = defineInternalJsonRoute({ - contract: readOrganizationSearchOverviewContract, - auth: internalSessionAuth, - operation: knowledgeOperations.readOrganizationSearchOverview, - rateLimit: internalRateLimits.none({ - reason: 'Bounded provider operational aggregates for organization integration settings', - }), - errorPolicy: internalKnowledgeErrorPolicies.connectors, - mapInput: ({ query }) => query, - useCase: readOrganizationSearchOverview, - present: (overview) => ({ success: true as const, data: overview }), - staticResponseHeaders: { 'Cache-Control': 'private, no-store' }, -}) diff --git a/apps/sim/app/api/knowledge/sim-search/personal-source-setup/route.ts b/apps/sim/app/api/knowledge/sim-search/personal-source-setup/route.ts deleted file mode 100644 index 8fff7a8ff6e..00000000000 --- a/apps/sim/app/api/knowledge/sim-search/personal-source-setup/route.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { - listPersonalSourceSetupAccountsContract, - personalSourceSetupContract, -} from '@/lib/api/contracts/knowledge/personal-source-setup' -import { - defineInternalJsonRoute, - extendInternalErrorPolicy, - internalErrorResponse, - internalRateLimits, - internalSessionAuth, -} from '@/lib/api/server/routes' -import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' -import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { - listPersonalSourceSetupAccounts, - personalSourceSetup, -} from '@/lib/knowledge/application/personal-source-setup' -import { - SelectorConnectionUnavailableError, - SelectorContextUnavailableError, - SelectorOptionsUnavailableError, -} from '@/lib/selectors/server/errors' -import { IntegrationNotAllowedError } from '@/ee/access-control/utils/permission-check' - -const errorPolicy = extendInternalErrorPolicy( - internalKnowledgeErrorPolicies.connectAccount, - (error) => { - if (error instanceof SelectorConnectionUnavailableError) - return internalErrorResponse(error.status, { - error: 'Reconnect your account to choose projects or spaces', - }) - if (error instanceof SelectorContextUnavailableError) - return internalErrorResponse(400, { - error: 'Enter your Atlassian site to choose projects or spaces', - }) - if (error instanceof SelectorOptionsUnavailableError) - return internalErrorResponse(error.status, { - error: - 'Could not load projects or spaces. Check the site and account access, then try again.', - }) - if (error instanceof IntegrationNotAllowedError) - return internalErrorResponse(403, { error: error.message }) - return null - } -) - -export const GET = defineInternalJsonRoute({ - contract: listPersonalSourceSetupAccountsContract, - auth: internalSessionAuth, - operation: knowledgeOperations.listPersonalSourceSetupAccounts, - rateLimit: internalRateLimits.user({ bucketName: 'knowledge.search.personal-setup.accounts' }), - errorPolicy, - mapInput: ({ query }) => query, - useCase: listPersonalSourceSetupAccounts, - present: (data) => ({ success: true as const, data }), - staticResponseHeaders: { 'Cache-Control': 'private, no-store' }, -}) - -export const POST = defineInternalJsonRoute({ - contract: personalSourceSetupContract, - auth: internalSessionAuth, - operation: knowledgeOperations.personalSourceSetup, - rateLimit: internalRateLimits.user({ bucketName: 'knowledge.search.personal-setup' }), - errorPolicy, - parseOptions: { maxBodyBytes: 384 * 1024 }, - mapInput: ({ body }) => body, - useCase: personalSourceSetup, - present: (data) => ({ success: true as const, data }), - staticResponseHeaders: { 'Cache-Control': 'private, no-store' }, -}) diff --git a/apps/sim/app/api/knowledge/sim-search/sources/overview/route.ts b/apps/sim/app/api/knowledge/sim-search/sources/overview/route.ts deleted file mode 100644 index f6bdf9abb5e..00000000000 --- a/apps/sim/app/api/knowledge/sim-search/sources/overview/route.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { readSearchSourceOverviewContract } from '@/lib/api/contracts/knowledge/connectors' -import { - defineInternalJsonRoute, - internalRateLimits, - internalSessionAuth, -} from '@/lib/api/server/routes' -import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' -import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { readSearchSourceOverview } from '@/lib/knowledge/application/search-source-overview' - -export const GET = defineInternalJsonRoute({ - contract: readSearchSourceOverviewContract, - auth: internalSessionAuth, - operation: knowledgeOperations.readSearchSourceOverview, - rateLimit: internalRateLimits.none({ - reason: 'Bounded provider existence probes for source setup and indexing progress', - }), - errorPolicy: internalKnowledgeErrorPolicies.connectors, - mapInput: ({ query }) => query, - useCase: readSearchSourceOverview, - present: (overview) => ({ success: true as const, data: overview }), - staticResponseHeaders: { 'Cache-Control': 'private, no-store' }, -}) diff --git a/apps/sim/app/api/knowledge/sim-search/sources/progress/route.ts b/apps/sim/app/api/knowledge/sim-search/sources/progress/route.ts deleted file mode 100644 index a6a793a28af..00000000000 --- a/apps/sim/app/api/knowledge/sim-search/sources/progress/route.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { readSearchSourceProgressContract } from '@/lib/api/contracts/knowledge/connectors' -import { - defineInternalJsonRoute, - internalRateLimits, - internalSessionAuth, -} from '@/lib/api/server/routes' -import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' -import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { readSearchSourceProgress } from '@/lib/knowledge/application/search-source-progress' - -export const POST = defineInternalJsonRoute({ - contract: readSearchSourceProgressContract, - auth: internalSessionAuth, - operation: knowledgeOperations.readSearchSourceProgress, - rateLimit: internalRateLimits.none({ - reason: 'Bounded viewer-authorized indexing progress polling', - }), - errorPolicy: internalKnowledgeErrorPolicies.connectors, - mapInput: ({ body }) => body, - useCase: readSearchSourceProgress, - present: ({ sources }) => ({ success: true as const, data: sources }), - staticResponseHeaders: { 'Cache-Control': 'private, no-store' }, -}) diff --git a/apps/sim/app/api/knowledge/sim-search/sources/route.test.ts b/apps/sim/app/api/knowledge/sim-search/sources/route.test.ts index 4b2f0c54243..d255e78bcf6 100644 --- a/apps/sim/app/api/knowledge/sim-search/sources/route.test.ts +++ b/apps/sim/app/api/knowledge/sim-search/sources/route.test.ts @@ -5,32 +5,16 @@ import type { SearchSourceSummary } from '@/lib/api/contracts/knowledge/connecto const mocks = vi.hoisted(() => ({ execute: vi.fn(), - overview: vi.fn(), - adminOverview: vi.fn(), -})) -vi.mock('@/lib/knowledge/application/organization-search-overview', () => ({ - readOrganizationSearchOverview: { - operation: { id: 'knowledge.search.integrations.overview' }, - execute: mocks.adminOverview, - }, })) vi.mock('@/lib/knowledge/application/search-sources', () => ({ listSearchSources: { operation: { id: 'knowledge.search.sources.list' }, execute: mocks.execute }, })) -vi.mock('@/lib/knowledge/application/search-source-overview', () => ({ - readSearchSourceOverview: { - operation: { id: 'knowledge.search.sources.overview' }, - execute: mocks.overview, - }, -})) vi.mock('@/lib/knowledge/application/search', () => knowledgeSearchUseCaseMock) vi.mock('@/lib/knowledge/application/upload-sessions', () => ({ KnowledgeDocumentUnsupportedMediaTypeError: class extends Error {}, })) import { NoWorkspaceAccessError } from '@/lib/core/application/workspace-authorization' -import { OrchestrationError } from '@/lib/core/orchestration/types' -import { GET as getAdminOverview } from '@/app/api/knowledge/sim-search/integrations/overview/route' import { GET } from '@/app/api/knowledge/sim-search/sources/route' const WORKSPACE_ID = '7d28e5e2-fb03-4118-9c52-4ab77ccff369' @@ -42,15 +26,7 @@ const source = { accessMode: 'admin', availability: 'available', enabled: true, - isSyncing: false, - lastSyncAt: null, - hasSyncError: false, - hasViewerDocuments: false, - viewerFailedDocumentCount: 0, - viewerEmailVerified: true, - viewerAccounts: [], - connectionRequired: false, - viewerMembership: null, + isGitHubInstallation: false, } satisfies SearchSourceSummary beforeEach(() => { @@ -112,21 +88,3 @@ describe('Search pagination boundary', () => { expect(mocks.execute).not.toHaveBeenCalled() }) }) - -describe('organization administration overview boundary', () => { - it('preserves a role refusal without exposing health data', async () => { - mocks.adminOverview.mockRejectedValue( - new OrchestrationError('forbidden', 'Organization administrator access is required') - ) - const response = await getAdminOverview( - createMockRequest( - 'GET', - undefined, - {}, - `http://localhost/api/knowledge/sim-search/integrations/overview?organizationId=${WORKSPACE_ID}` - ) - ) - expect(response.status).toBe(403) - expect(await response.json()).not.toHaveProperty('data') - }) -}) diff --git a/apps/sim/app/api/knowledge/sim-search/stats/route.ts b/apps/sim/app/api/knowledge/sim-search/stats/route.ts deleted file mode 100644 index e7ae7eb9777..00000000000 --- a/apps/sim/app/api/knowledge/sim-search/stats/route.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { readOrganizationSearchStatsContract } from '@/lib/api/contracts/knowledge/search-stats' -import { - defineInternalJsonRoute, - internalRateLimits, - internalSessionAuth, -} from '@/lib/api/server/routes' -import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' -import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { readOrganizationSearchStats } from '@/lib/knowledge/application/organization-search-stats' - -export const GET = defineInternalJsonRoute({ - contract: readOrganizationSearchStatsContract, - auth: internalSessionAuth, - operation: knowledgeOperations.readOrganizationSearchStats, - rateLimit: internalRateLimits.user({ - bucketName: 'organization-search-stats', - config: { maxTokens: 30, refillRate: 30, refillIntervalMs: 60_000 }, - }), - errorPolicy: internalKnowledgeErrorPolicies.connectors, - mapInput: ({ query }) => query, - useCase: readOrganizationSearchStats, - present: (data) => ({ success: true as const, data }), - staticResponseHeaders: { 'Cache-Control': 'private, no-store' }, -}) diff --git a/apps/sim/app/api/mcp/oauth/callback/route.test.ts b/apps/sim/app/api/mcp/oauth/callback/route.test.ts index d9b38f12508..28fdcff930f 100644 --- a/apps/sim/app/api/mcp/oauth/callback/route.test.ts +++ b/apps/sim/app/api/mcp/oauth/callback/route.test.ts @@ -1,3 +1,5 @@ +import { runInNewContext } from 'node:vm' +import { BroadcastChannel } from 'node:worker_threads' import { authMockFns, dbChainMockFns, @@ -5,10 +7,14 @@ import { mcpOauthMockFns, resetDbChainMock, } from '@sim/testing' +import { flushMicrotasks } from '@sim/testing/helpers/async' +import { emcnMock } from '@sim/testing/mocks/emcn.mock' +import { getMockLogger } from '@sim/testing/mocks/logger.mock' import { mcpServiceMock, mcpServiceMockFns } from '@sim/testing/mocks/mcp-service.mock' import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' import { CredentialGroupOAuthStateVersionError } from '@/lib/credential-groups/oauth-attempt-version' +import { connectCredentialGroupInPopup } from '@/lib/credential-groups/oauth-popup' const { mockAuthenticateEnrollment, @@ -24,6 +30,7 @@ const { vi.mock('@/lib/mcp/oauth', () => mcpOauthMock) vi.mock('@/lib/mcp/service', () => mcpServiceMock) +vi.mock('@sim/emcn', () => emcnMock) vi.mock('@/lib/credential-groups/application/enrollment-auth', () => ({ credentialGroupOAuthAttemptPrincipal: mockAuthenticateEnrollment, })) @@ -88,30 +95,41 @@ describe('MCP OAuth callback route', () => { mockEnforceCallbackRateLimit.mockResolvedValue(null) }) - it.each([undefined, 'denied'])( - 'finishes a direct connection without the invitation form: %s', - async (error) => { - const completionId = '00000000-0000-4000-8000-000000000002' - mockConsumeManagedAttempt.mockResolvedValueOnce({ - state: 'mcp_cg_direct', - organizationId: 'organization-1', - invitationToken: 'invitation-token', - mcpServerId: 'server-1', - completionId, - returnTo: 'integrations', - }) - const response = await GET( - new NextRequest( - `http://localhost:3000/api/mcp/oauth/callback?state=mcp_cg_direct&${error ? 'error=denied' : 'code=code-1'}` - ) + it.each([ + [undefined, null], + ['access_denied', 'denied'], + ['invalid_request', 'failed'], + ['invalid_scope', 'failed'], + ['server_error', 'provider_unavailable'], + ['temporarily_unavailable', 'provider_unavailable'], + ['unexpected-secret-value', 'failed'], + ])('finishes a direct connection without the invitation form: %s', async (error, expected) => { + const completionId = '00000000-0000-4000-8000-000000000002' + mockConsumeManagedAttempt.mockResolvedValueOnce({ + state: 'mcp_cg_direct', + organizationId: 'organization-1', + invitationToken: 'invitation-token', + mcpServerId: 'server-1', + completionId, + returnTo: 'integrations', + }) + const response = await GET( + new NextRequest( + `http://localhost:3000/api/mcp/oauth/callback?state=mcp_cg_direct&${error ? `error=${error}&error_description=private-provider-detail` : 'code=code-1'}` ) - const destination = new URL(response.headers.get('location')!, 'http://localhost:3000') - expect(destination.pathname).toBe('/credential-groups/complete') - expect(destination.searchParams.get('completionId')).toBe(completionId) - expect(destination.searchParams.get('organizationId')).toBe('organization-1') - expect(destination.searchParams.get('oauth')).toBe(error ?? null) + ) + const destination = new URL(response.headers.get('location')!, 'http://localhost:3000') + expect(destination.pathname).toBe('/credential-groups/complete') + expect(destination.searchParams.get('completionId')).toBe(completionId) + expect(destination.searchParams.get('organizationId')).toBe('organization-1') + expect(destination.searchParams.get('oauth')).toBe(expected) + if (error) { + const warnings = JSON.stringify(getMockLogger('McpOauthCallbackAPI').warn.mock.calls) + expect(warnings).toContain(error === 'unexpected-secret-value' ? 'unknown' : error) + expect(warnings).not.toContain('unexpected-secret-value') + expect(warnings).not.toContain('private-provider-detail') } - ) + }) it('performs the token exchange through the SSRF-guarded mcpAuthGuarded wrapper', async () => { const request = new NextRequest( @@ -185,4 +203,78 @@ describe('MCP OAuth callback route', () => { expect(mockAuthenticateEnrollment).not.toHaveBeenCalled() expect(mockCompleteManagedMcpOAuth).not.toHaveBeenCalled() }) + + it.each(['missing', 'version'])( + 'delivers a %s managed-state failure to only the initiating popup', + async (kind) => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + vi.stubGlobal('BroadcastChannel', BroadcastChannel) + vi.stubGlobal('window', { + open: () => ({ closed: false, opener: null, location: { replace() {} }, close() {} }), + }) + const controller = new AbortController() + const completionId = '00000000-0000-4000-8000-000000000010' + const secondId = '00000000-0000-4000-8000-000000000011' + const state = 'mcp_cg_first' + let first: string | undefined + let second: string | undefined + const start = (nonce: string) => async () => ({ + invitationLink: 'http://localhost:3000/credential-groups/enroll/fixture', + authorizationUrl: `https://oauth.example.com/authorize?state=${nonce}`, + }) + const outcomes = [ + connectCredentialGroupInPopup(completionId, start(state), controller.signal).then( + () => { + first = 'connected' + }, + (error: Error) => { + first = error.message + } + ), + connectCredentialGroupInPopup(secondId, start('mcp_cg_second'), controller.signal).then( + () => { + second = 'connected' + }, + (error: Error) => { + second = error.message + } + ), + ] + const observer = new BroadcastChannel('mcp-oauth') + const success = new BroadcastChannel(`sim:credential-group-oauth:${secondId}`) + try { + await flushMicrotasks() + if (kind === 'missing') mockConsumeManagedAttempt.mockResolvedValueOnce(null) + else + mockConsumeManagedAttempt.mockRejectedValueOnce( + new CredentialGroupOAuthStateVersionError() + ) + const response = await GET( + new NextRequest(`http://localhost:3000/api/mcp/oauth/callback?state=${state}&code=unused`) + ) + const body = await response.text() + const script = body.match(/