From 71501745ff077721e56a9ad501a40994a23748a3 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 5 Nov 2026 13:51:55 -0500 Subject: [PATCH 1/8] feat(mongodb): add the MongoDB destination package A MongoDB 5.0+ destination with the same CLI, container image and Destination semantics as the SQL packages: workos_* collections in the application's database, _live views over them, the single-writer lease, the fenced checkpoint, server-side reconcile and the quarantine store. Every write is one update pipeline with upsert, so the staleness gate runs in the operation that writes, and payloads go in as $literal so a string like "$100 plan" is never read as a field path. The lease and every cursor for an environment share one document, which makes setStateIfHeld a single conditional update with no transaction: a standalone mongod, a replica set and Atlas all behave the same. Cursors are stored as canonical JSON because document equality in MongoDB is field-order sensitive and the engine's is not. Catalog columns become indexes on _raw.; there are no generated fields and so no column drift. The suite runs on an embedded mongod (mongodb-memory-server), whose postinstall download is turned off in pnpm-workspace.yaml so install and the Docker build stay network-light; the binary downloads on first test run and CI caches it. The Docker workflow gains the mongodb matrix entry and release-please gains the package. --- .claude/worktrees/docs-neon-guide | 1 + .github/workflows/ci.yml | 35 + .github/workflows/docker.yml | 1 + .release-please-manifest.json | 3 +- README.md | 13 + packages/sync-engine/README.md | 1 + packages/sync-mongodb/LICENSE.txt | 21 + packages/sync-mongodb/README.md | 259 ++++ packages/sync-mongodb/package.json | 55 + packages/sync-mongodb/src/cli.ts | 490 ++++++++ packages/sync-mongodb/src/db/client.ts | 63 + packages/sync-mongodb/src/db/errors.ts | 65 + packages/sync-mongodb/src/db/lease.ts | 206 ++++ packages/sync-mongodb/src/db/reconcile.ts | 167 +++ packages/sync-mongodb/src/db/rejected.ts | 106 ++ packages/sync-mongodb/src/db/timestamps.ts | 75 ++ packages/sync-mongodb/src/db/upsert.ts | 402 ++++++ packages/sync-mongodb/src/destination.ts | 308 +++++ packages/sync-mongodb/src/health.ts | 39 + packages/sync-mongodb/src/index.ts | 48 + packages/sync-mongodb/src/otel.ts | 75 ++ packages/sync-mongodb/src/schema/migrate.ts | 197 +++ packages/sync-mongodb/src/schema/names.ts | 93 ++ packages/sync-mongodb/src/schema/views.ts | 78 ++ packages/sync-mongodb/test/cli.test.ts | 372 ++++++ .../sync-mongodb/test/destination.test.ts | 106 ++ packages/sync-mongodb/test/engine.test.ts | 1083 +++++++++++++++++ packages/sync-mongodb/test/errors.test.ts | 79 ++ packages/sync-mongodb/test/global-setup.ts | 26 + packages/sync-mongodb/test/health-cli.test.ts | 38 + packages/sync-mongodb/test/health.test.ts | 194 +++ packages/sync-mongodb/test/helpers/ca.ts | 17 + packages/sync-mongodb/test/helpers/faulty.ts | 62 + packages/sync-mongodb/test/helpers/mongo.ts | 52 + packages/sync-mongodb/test/lease.test.ts | 181 +++ packages/sync-mongodb/test/migrate.test.ts | 242 ++++ packages/sync-mongodb/test/otel.test.ts | 219 ++++ packages/sync-mongodb/test/reconcile.test.ts | 183 +++ packages/sync-mongodb/test/rejected.test.ts | 229 ++++ packages/sync-mongodb/test/setup.ts | 5 + packages/sync-mongodb/test/timestamps.test.ts | 74 ++ packages/sync-mongodb/test/upsert.test.ts | 412 +++++++ packages/sync-mongodb/tsconfig.build.json | 11 + packages/sync-mongodb/tsconfig.json | 22 + packages/sync-mongodb/vitest.config.ts | 16 + pnpm-lock.yaml | 493 +++++++- pnpm-workspace.yaml | 4 + release-please-config.json | 4 + 48 files changed, 6921 insertions(+), 4 deletions(-) create mode 160000 .claude/worktrees/docs-neon-guide create mode 100644 packages/sync-mongodb/LICENSE.txt create mode 100644 packages/sync-mongodb/README.md create mode 100644 packages/sync-mongodb/package.json create mode 100644 packages/sync-mongodb/src/cli.ts create mode 100644 packages/sync-mongodb/src/db/client.ts create mode 100644 packages/sync-mongodb/src/db/errors.ts create mode 100644 packages/sync-mongodb/src/db/lease.ts create mode 100644 packages/sync-mongodb/src/db/reconcile.ts create mode 100644 packages/sync-mongodb/src/db/rejected.ts create mode 100644 packages/sync-mongodb/src/db/timestamps.ts create mode 100644 packages/sync-mongodb/src/db/upsert.ts create mode 100644 packages/sync-mongodb/src/destination.ts create mode 100644 packages/sync-mongodb/src/health.ts create mode 100644 packages/sync-mongodb/src/index.ts create mode 100644 packages/sync-mongodb/src/otel.ts create mode 100644 packages/sync-mongodb/src/schema/migrate.ts create mode 100644 packages/sync-mongodb/src/schema/names.ts create mode 100644 packages/sync-mongodb/src/schema/views.ts create mode 100644 packages/sync-mongodb/test/cli.test.ts create mode 100644 packages/sync-mongodb/test/destination.test.ts create mode 100644 packages/sync-mongodb/test/engine.test.ts create mode 100644 packages/sync-mongodb/test/errors.test.ts create mode 100644 packages/sync-mongodb/test/global-setup.ts create mode 100644 packages/sync-mongodb/test/health-cli.test.ts create mode 100644 packages/sync-mongodb/test/health.test.ts create mode 100644 packages/sync-mongodb/test/helpers/ca.ts create mode 100644 packages/sync-mongodb/test/helpers/faulty.ts create mode 100644 packages/sync-mongodb/test/helpers/mongo.ts create mode 100644 packages/sync-mongodb/test/lease.test.ts create mode 100644 packages/sync-mongodb/test/migrate.test.ts create mode 100644 packages/sync-mongodb/test/otel.test.ts create mode 100644 packages/sync-mongodb/test/reconcile.test.ts create mode 100644 packages/sync-mongodb/test/rejected.test.ts create mode 100644 packages/sync-mongodb/test/setup.ts create mode 100644 packages/sync-mongodb/test/timestamps.test.ts create mode 100644 packages/sync-mongodb/test/upsert.test.ts create mode 100644 packages/sync-mongodb/tsconfig.build.json create mode 100644 packages/sync-mongodb/tsconfig.json create mode 100644 packages/sync-mongodb/vitest.config.ts diff --git a/.claude/worktrees/docs-neon-guide b/.claude/worktrees/docs-neon-guide new file mode 160000 index 0000000..0c969a7 --- /dev/null +++ b/.claude/worktrees/docs-neon-guide @@ -0,0 +1 @@ +Subproject commit 0c969a7736624f52af179b3ed1cd9d7cc43403eb diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4834588..d112088 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,3 +108,38 @@ jobs: # packages rather than the sources the unit tests import. - name: End-to-end example run: pnpm --filter workos-sync-postgres-example start + + mongodb: + name: MongoDB Node ${{ matrix.node }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + node: [22, 24] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: ${{ matrix.node }} + + # The suite runs on an embedded mongod (mongodb-memory-server), which + # downloads the binary on first use; the cache keeps that off the + # critical path. The version is pinned in the package's package.json. + - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ~/.cache/mongodb-binaries + key: mongodb-binaries-${{ runner.os }}-${{ hashFiles('packages/sync-mongodb/package.json') }} + + - name: Install + run: pnpm install --frozen-lockfile + + - name: Test + run: pnpm --filter @workos/sync-mongodb test + + - name: Build + run: pnpm --filter "@workos/sync-mongodb..." build diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d034365..14d09b7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -22,6 +22,7 @@ jobs: # One entry per destination package: packages/sync-. destination: - postgres + - mongodb runner: [ubuntu-latest, ubuntu-24.04-arm] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.release-please-manifest.json b/.release-please-manifest.json index ab0272a..9e1e382 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,4 +1,5 @@ { "packages/sync-engine": "0.1.0", - "packages/sync-postgres": "0.1.0" + "packages/sync-postgres": "0.1.0", + "packages/sync-mongodb": "0.0.0" } diff --git a/README.md b/README.md index a42afae..5640782 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,19 @@ SELECT o.name AS organization, u.email, m.role_slug The `_live` views are generated next to the tables and hide deleted rows; the base tables keep them as tombstones for when you want to ask who left. +## Destinations + +One engine, one package per database. Each ships a CLI with the same +commands and a container image under the same name: + +| Destination | Package | Mirror | +| ----------- | ------------------------------------------------- | ----------------------------------------------------------------------- | +| Postgres | [`@workos/sync-postgres`](packages/sync-postgres) | A `workos` schema of tables with generated columns and `_live` views | +| MongoDB | [`@workos/sync-mongodb`](packages/sync-mongodb) | `workos_*` collections in your database, with indexes and `_live` views | + +The rest of this page shows Postgres; every command and option below has a +MongoDB counterpart, and each package's README is the reference for it. + ## Try it in 30 seconds No credentials, no Docker, no network. Runs the whole lifecycle against a diff --git a/packages/sync-engine/README.md b/packages/sync-engine/README.md index b678799..6b06505 100644 --- a/packages/sync-engine/README.md +++ b/packages/sync-engine/README.md @@ -11,6 +11,7 @@ package and re-exports it, so it is the only thing to install: | Destination | Package | | ----------- | ------------------------------------------------------------------------------ | | Postgres | [`@workos/sync-postgres`](https://www.npmjs.com/package/@workos/sync-postgres) | +| MongoDB | [`@workos/sync-mongodb`](https://www.npmjs.com/package/@workos/sync-mongodb) | Install this package directly only to write a new destination. diff --git a/packages/sync-mongodb/LICENSE.txt b/packages/sync-mongodb/LICENSE.txt new file mode 100644 index 0000000..c0cf745 --- /dev/null +++ b/packages/sync-mongodb/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 WorkOS + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/sync-mongodb/README.md b/packages/sync-mongodb/README.md new file mode 100644 index 0000000..0c9e0e1 --- /dev/null +++ b/packages/sync-mongodb/README.md @@ -0,0 +1,259 @@ +# @workos/sync-mongodb + +Mirror your WorkOS data into your own MongoDB, and query it next to your +application's collections. + +Backfills what already exists from the REST API, tails the Events API from a +durable cursor, and reconciles the two so drift is found rather than assumed. +The result is a set of `workos_*` collections in your database that you can +`$lookup` against your own: + +```js +db.workos_organization_memberships_live.aggregate([ + { $match: { _env: "production", "_raw.organization_id": "org_01H..." } }, + { + $lookup: { + from: "workos_users_live", + localField: "_raw.user_id", + foreignField: "_raw.id", + as: "user", + }, + }, + { $unwind: "$user" }, + { $project: { email: "$user._raw.email", role: "$_raw.role.slug" } }, +]); +``` + +The `_live` views hide deleted documents; the base collections keep them as +tombstones for when you want to ask who left. + +## Install + +```bash +npm install @workos/sync-mongodb +``` + +This package ships the MongoDB destination, the `workos-sync-mongodb` +command, and re-exports the engine from +[`@workos/sync-engine`](https://www.npmjs.com/package/@workos/sync-engine), +so it is the only package you need for MongoDB. It needs MongoDB 5.0 or +later: update pipelines with `$$NOW`, `$expr` in update filters, and `.` and +`$` inside the field names of stored payloads (directory users' SCIM +attributes carry both) all arrived in 5.0. Atlas, a replica set and a +standalone `mongod` are all fine; nothing here needs a transaction. + +## Command line + +```bash +export DATABASE_URL=mongodb://localhost:27017/app +export WORKOS_API_KEY=sk_live_... + +npx workos-sync-mongodb init # create the collections, indexes and views +npx workos-sync-mongodb backfill # load everything that already exists +npx workos-sync-mongodb listen # tail the Events API, forever +``` + +| Command | What it does | +| --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `init [--repair]` | Create the collections, indexes, `_live` views and bookkeeping. Idempotent; safe on every deploy. Fails if a generated `_live` view has a pipeline that is not the generated one; `--repair` redefines it. Reports indexes the catalog no longer calls for, and never drops them. | +| `backfill [--resume] [--max-pages ]` | Load current state from the REST list endpoints. `--resume` continues each stream from its checkpoint and skips streams already complete. `--max-pages` stops after `n` pages across all streams, for a host with a time limit; run again with `--resume` until every stream reports `complete`. | +| `listen [--health-port ]` | Poll the Events API continuously as the sole writer; waits if another listener holds the lease. Finishes the current batch on `SIGINT` or `SIGTERM`. With a port, serves `GET /healthz` and `/metrics` on it. | +| `tick` | One poll, then exit. For a cron job instead of a long-running process. Exits 3 if a listener holds the lease. | +| `reconcile [--repair] [--resume] [--max-pages ]` | Diff ids against the API. Exits 2 on drift; `--repair` tombstones missed deletes and inserts missed creates. `--max-pages` stops starting streams once `n` pages have been fetched, for a host with a time limit; a stream that starts runs to its last page, since a diff needs the whole listing. Run again with `--resume` until every stream reports `complete`. | +| `refresh [--resume] [--max-pages ]` | Re-list top-level streams from the REST API, replacing each document; `--max-pages` and `--resume` slice it like `backfill`, with which it shares checkpoints. Re-run `backfill` for per-parent streams. | +| `status` | Cursor, time since the last successful poll, and per-stream document counts, tombstones and lag. | +| `redact --user ` | Erase one subject from the mirror, the event archive and the quarantine. | +| `rejected [--replay] [--id ]` | Show events the tail quarantined; exits 2 if there are any. `--replay` re-applies them (one, with `--id`) once the cause is fixed. | +| `streams` | List what this build can mirror, with group, mode, and whether it is on by default. | + +Flags every command takes: + +| Flag | Overrides | What it does | +| ---------------------- | ---------------------- | --------------------------------------------------------------------------------------- | +| `--database-url ` | `DATABASE_URL` | MongoDB connection string. | +| `--database ` | `WORKOS_SYNC_DATABASE` | Database the mirror lives in. Defaults to the one the connection string names. | +| `--prefix ` | `WORKOS_SYNC_PREFIX` | Collection prefix for the mirror, the views and the bookkeeping. Defaults to `workos_`. | +| `--streams ` | | Restrict to specific streams. `events` is always included. | +| `--help` | | Print usage. | + +Documents are labelled `_env: 'production'` by default; set +`WORKOS_ENVIRONMENT_NAME` if the key is for something else. The label is +written to every document and must never change once you have data. To mirror +several environments into one database, run one process per key with a +different label each; cursors are kept per label, so they never collide. + +The command reads its settings from the environment; where a flag exists for +the same setting, the flag wins. + +| Variable | Default | What it does | +| ------------------------------------ | ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `DATABASE_URL` | required | MongoDB connection string (`mongodb://` or `mongodb+srv://`). | +| `WORKOS_API_KEY` | required | API key for the environment to mirror. One key, one environment, one process; dedicate it to the sync. | +| `WORKOS_ENVIRONMENT_NAME` | `production` | Label written to `_env` on every document. Must not change once you have data. Run one process per environment to mirror several. | +| `WORKOS_API_URL` | `https://api.workos.com` | API base URL. | +| `WORKOS_SYNC_DATABASE` | from the connection string | Database the mirror lives in; `workos` when the connection string names none. | +| `WORKOS_SYNC_PREFIX` | `workos_` | Collection prefix. Letters, digits and underscores. | +| `WORKOS_SYNC_ID` | `default` | Identifier when several syncs share one database. Each keeps its own cursors, lease and quarantine. | +| `WORKOS_SYNC_HYDRATE` | `off` | Re-fetch objects when their events arrive: `off`, `on_create`, `always`, or per stream as `users=always,organizations=off`. | +| `WORKOS_SYNC_POLL_MS` | `5000` | Idle wait between polls, in ms, when the last poll returned nothing. | +| `WORKOS_SYNC_LAG_MS` | `5000` | Hold back events younger than this, in ms, before advancing the cursor. | +| `WORKOS_SYNC_MAX_EVENTS_PER_TICK` | one page (100) | Events to apply per poll, fetched across as many pages as needed. | +| `WORKOS_SYNC_HEALTH_PORT` | off (`8080` in the container image) | `listen` only: serve `GET /healthz` and `GET /metrics` on this port, from in-memory state, with no database query. | +| `WORKOS_SYNC_HEALTH_HOST` | `0.0.0.0` | Interface the health server binds. | +| `WORKOS_SYNC_HEALTH_MAX_POLL_AGE_MS` | `60000` | `/healthz` returns 503 once any environment's last successful poll is older than this. | +| `WORKOS_SYNC_OTEL` | off | `1` starts the OpenTelemetry Node SDK for the command, configured by the standard `OTEL_*` variables, and flushes it on exit. Every command, `tick` included. | +| `WORKOS_SYNC_LEASE_TTL_MS` | `60000` | How long the tail lease outlives a crashed listener, in ms; at least 1. The most lag a crash can add before a replacement takes over. | +| `WORKOS_SYNC_MONGODB_TLS_CA` | system trust store | CA certificate the database's certificate must chain to, as PEM or base64 of it. TLS is then required and verified against it. | +| `LOG_LEVEL` | `info` | `debug`, `info`, `warn`, `error` or `silent`. Logs are JSON lines on stdout. | + +`listen` with a health port answers `GET /healthz` with `200` while the tail +is polling (and while it is still starting) and `503` once a poll is overdue +or the engine has stopped, and `GET /metrics` in Prometheus text format. Both +are served from memory, with no database query, so a probe keeps working when +the database does not. See +[Operations](https://github.com/workos/sync-engine/blob/main/docs/operations.md) +for the metric names and a Kubernetes probe snippet. + +## Embed it + +Most deployments embed the engine in a service that already has a +`MongoClient`: + +```ts +import { MongoClient } from "mongodb"; +import { SyncEngine, MongoDestination } from "@workos/sync-mongodb"; + +const client = new MongoClient(process.env.DATABASE_URL!); +const engine = new SyncEngine({ + destination: new MongoDestination({ client, database: "app" }), + environments: [{ name: "production", apiKey: process.env.WORKOS_API_KEY! }], +}); + +await engine.backfill(); +await engine.listen(); +``` + +A pre-built `client` is never closed by the destination. Pass `url` instead +and the destination opens and closes a client of its own. + +| Option | Default | What it does | +| ------------- | ------------------------- | ----------------------------------------------------------------------------------------------------------- | +| `url` | | Connection string. Either this or `client`. | +| `client` | | A pre-built `MongoClient`, e.g. the application's own. | +| `sslCa` | | PEM CA certificate(s) the server must chain to; TLS is then required. Only with `url`. | +| `database` | from `url`, else `workos` | Database the mirror lives in. | +| `prefix` | `workos_` | Collection prefix. | +| `syncId` | `default` | Identifier when several syncs share one database. | +| `batchSize` | `500` | Documents per bulk write. | +| `repairViews` | `false` | On `init()`, redefine a generated `_live` view whose pipeline is not the generated one, instead of failing. | + +The engine options (`streams`, `extraColumns`, `hydrate`, `pollIntervalMs`, +`lagWindowMs`, `maxEventsPerTick`, `pageSize`, `logger`) are in the +[`@workos/sync-engine` README](https://github.com/workos/sync-engine/blob/main/packages/sync-engine/README.md). + +## Querying the mirror + +Every stream becomes a collection named after it with the prefix in front, +and every document in it has the same shape: + +```js +{ + _id: ObjectId("..."), + _env: "production", // the environment label + _raw: { id: "user_01H...", email: "ada@example.com", ... }, // the object, verbatim + _updated_at: ISODate("..."), // source time; the staleness gate compares this + _synced_at: ISODate("..."), // write time, by the server's clock; lag is the difference + _deleted_at: null // tombstone; absent on append-only streams +} +``` + +`_raw` is the WorkOS object as the API returned it, timestamps included as +ISO-8601 strings. There are no projected fields to keep in step with it: +query `_raw.` directly, and sort ISO-8601 strings lexically, which is +chronological order. + +```js +// Users of one organization, with their role. +db.workos_organization_memberships_live.find( + { _env: "production", "_raw.organization_id": "org_01H..." }, + { "_raw.user_id": 1, "_raw.role.slug": 1 }, +); + +// Failed sign-ins in the last hour. authentication.* events have no REST +// endpoint; this collection is the only record of them. +db.workos_auth_attempts.countDocuments({ + _env: "production", + "_raw.status": "failed", + "_raw.created_at": { $gt: new Date(Date.now() - 3600_000).toISOString() }, +}); + +// Users whose domain verification lapsed, joined to your own accounts. +db.workos_organization_domains_live.aggregate([ + { $match: { _env: "production", "_raw.state": "failed" } }, + { + $lookup: { + from: "accounts", + localField: "_raw.organization_id", + foreignField: "workos_org_id", + as: "account", + }, + }, +]); +``` + +What the Postgres destination calls a column is an index here. Every stream's +natural key is a unique index on `_env` plus the key's `_raw.`, every +column the catalog marks `index: true` is a compound index on `_env` and +`_raw.`, and `extraColumns` add indexes the same way: + +```ts +new SyncEngine({ + destination, + environments, + extraColumns: { + users: [ + { + name: "region", + type: "text", + path: ["metadata", "region"], + index: true, + }, + ], + }, +}); +``` + +The `type` is accepted for parity with the other destinations and ignored. An +index the catalog no longer calls for is reported by `init` as an orphan and +left in place. + +Each mirror collection has a `_live` view next to it — +`workos_users_live` — that is the collection without its tombstones, so the +common query needs no `_deleted_at` filter. Append-only streams (`events`, +`auth_attempts`) have no tombstones and no view. + +There are three bookkeeping collections. `workos__sync` holds the +events-tail lease and every cursor for an environment in one document per +`(sync_id, env)`, which is what lets the tail's checkpoint be a single +conditional update — the holder, the lease's expiry by the server's clock and +the cursor it expects are all in the filter — without a multi-document +transaction, so a standalone `mongod` is as safe a target as a replica set. +`workos__rejected` is the quarantine for events the tail could not apply, and +`workos__migrations` records which catalogs have been applied so `init` is one +query when nothing changed. + +## Testing + +`pnpm test` runs against an embedded `mongod` from mongodb-memory-server, +downloaded on first run and cached. Set `TEST_MONGODB_URL` to run the same +suite against a server of your own instead: + +```sh +TEST_MONGODB_URL=mongodb://127.0.0.1:27017 pnpm --filter @workos/sync-mongodb test +``` + +Each test creates a database of its own and drops it on completion. + +## License + +MIT diff --git a/packages/sync-mongodb/package.json b/packages/sync-mongodb/package.json new file mode 100644 index 0000000..f6b461a --- /dev/null +++ b/packages/sync-mongodb/package.json @@ -0,0 +1,55 @@ +{ + "name": "@workos/sync-mongodb", + "version": "0.0.0", + "description": "Mirror your WorkOS data into MongoDB. Backfill, tail the Events API, and reconcile drift.", + "type": "module", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/workos/sync-engine.git", + "directory": "packages/sync-mongodb" + }, + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "bin": { + "workos-sync-mongodb": "./dist/cli.js" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "files": [ + "dist" + ], + "scripts": { + "build": "tsc -p tsconfig.build.json", + "test": "vitest run", + "test:watch": "vitest", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@workos/sync-engine": "workspace:*", + "mongodb": "^6.17.0" + }, + "optionalDependencies": { + "@opentelemetry/sdk-node": "^0.300.0" + }, + "devDependencies": { + "@opentelemetry/api": "^1.10.0", + "@opentelemetry/sdk-metrics": "^3.0.0", + "@types/node": "^26.6.5", + "mongodb-memory-server": "^10.1.4", + "typescript": "^7.0.2", + "vitest": "^5.0.5" + }, + "engines": { + "node": ">=22" + }, + "config": { + "mongodbMemoryServer": { + "version": "8.0.4" + } + } +} diff --git a/packages/sync-mongodb/src/cli.ts b/packages/sync-mongodb/src/cli.ts new file mode 100644 index 0000000..ea1f455 --- /dev/null +++ b/packages/sync-mongodb/src/cli.ts @@ -0,0 +1,490 @@ +#!/usr/bin/env node +/** + * workos-sync-mongodb — command line interface. + * + * workos-sync-mongodb init [--repair] create the collections, indexes and views; --repair + * redefines generated views whose pipeline changed + * workos-sync-mongodb backfill [--resume] [--max-pages ] + * page the list endpoints, optionally n pages at a time + * workos-sync-mongodb listen tail the Events API + * workos-sync-mongodb tick one poll, then exit (for cron) + * workos-sync-mongodb reconcile [--repair] detect (and optionally fix) drift + * workos-sync-mongodb refresh re-list to refresh partial event fields + * workos-sync-mongodb status cursor age and per-stream freshness + * workos-sync-mongodb redact --user erase one subject + * workos-sync-mongodb rejected [--replay] quarantined events; replay them once fixed + * workos-sync-mongodb streams list what this build can mirror + */ +import { + SyncEngine, + LeaseHeldError, + STREAMS, + DEFAULT_HYDRATE, + defaultStreams, + createLogger, + startHealthServer, + type EnvironmentConfig, + type HydrateMode, + type Logger, + type LogLevel, + type SyncEngineConfig, +} from '@workos/sync-engine' +import { X509Certificate } from 'node:crypto' +import type { MongoClient } from 'mongodb' +import { MongoDestination } from './destination.js' +import { healthOptionsFromEnv } from './health.js' +import { startOtel } from './otel.js' +import { DEFAULT_PREFIX } from './schema/names.js' + +interface Args { + command: string + flags: Record +} + +function parseArgs(argv: string[]): Args { + const [command = 'help', ...rest] = argv + const flags: Record = {} + for (let i = 0; i < rest.length; i++) { + const arg = rest[i]! + if (!arg.startsWith('--')) continue + const key = arg.slice(2) + const next = rest[i + 1] + if (next && !next.startsWith('--')) { + flags[key] = next + i++ + } else { + flags[key] = true + } + } + return { command, flags } +} + +/** + * One key, one environment, one process. `WORKOS_ENVIRONMENT_NAME` is the label + * written to every document as `_env`. It is yours to choose because no API + * call reports which environment a key belongs to, and it must stay stable + * across key rotation. To mirror several environments into one database, run + * one process per key with a different label each; cursors are kept per label. + */ +function environmentsFromEnv(): EnvironmentConfig[] { + const apiKey = process.env['WORKOS_API_KEY'] + if (!apiKey) throw new Error('set WORKOS_API_KEY') + return [ + { + name: process.env['WORKOS_ENVIRONMENT_NAME'] ?? 'production', + apiKey, + apiUrl: process.env['WORKOS_API_URL'], + }, + ] +} + +const HYDRATE_MODES: HydrateMode[] = ['off', 'on_create', 'always'] + +/** + * `WORKOS_SYNC_HYDRATE` is either one mode for every stream (`off`, + * `on_create`, `always`) or a per-stream list (`users=always,organizations=off`). + * Unset means the engine default: off. + */ +export function parseHydrate(value: string | undefined): SyncEngineConfig['hydrate'] { + if (!value || !value.trim()) return undefined + const v = value.trim() + if ((HYDRATE_MODES as string[]).includes(v)) return v as HydrateMode + const out: Record = { ...DEFAULT_HYDRATE } + for (const pair of v.split(',')) { + const idx = pair.indexOf('=') + const stream = pair.slice(0, idx).trim() + const mode = pair.slice(idx + 1).trim() + if (idx < 1 || !(HYDRATE_MODES as string[]).includes(mode)) { + throw new Error( + `invalid WORKOS_SYNC_HYDRATE entry "${pair}": expected off | on_create | always, or stream=mode,...`, + ) + } + out[stream] = mode as HydrateMode + } + return out +} + +/** + * `--repair` on `init` means redefining generated `_live` views; on + * `reconcile` it means repairing rows. Sharing the flag name is deliberate — + * it is "fix what you found" in both — but each command reads only its own + * meaning. + */ +export function repairRequested(command: string, flags: Record): boolean { + return command === 'init' && Boolean(flags['repair']) +} + +const PEM_CERTIFICATE = /-----BEGIN CERTIFICATE-----[\s\S]*?-----END CERTIFICATE-----/g + +/** True when `text` holds at least one PEM certificate and every one of them parses. */ +function isCertificateBundle(text: string): boolean { + const blocks = text.match(PEM_CERTIFICATE) + if (!blocks) return false + try { + for (const block of blocks) new X509Certificate(block) + return true + } catch { + return false + } +} + +/** + * `WORKOS_SYNC_MONGODB_TLS_CA` is the CA certificate the database's + * certificate must chain to: PEM, or the PEM encoded as base64 for a secret + * store that cannot hold a multi-line value. A PEM whose newlines arrived as + * the two characters `\n` is straightened out, since that is how some hosts + * hand back a pasted certificate. Unset or empty means the driver's own + * defaults. Every certificate in the value is parsed, so a truncated or + * damaged one is refused here rather than handed to the driver, where it + * would surface as a handshake failure against the server and read as the + * server's fault rather than the configuration's. + */ +export function parseSslCa(value: string | undefined): string | undefined { + if (!value || !value.trim()) return undefined + const v = value.trim().replace(/\\n/g, '\n') + if (isCertificateBundle(v)) return v + const decoded = Buffer.from(v, 'base64').toString('utf8') + if (isCertificateBundle(decoded)) return decoded + throw new Error( + 'invalid WORKOS_SYNC_MONGODB_TLS_CA: expected one or more PEM certificates, or them encoded as base64', + ) +} + +function databaseFromFlags(flags: Record): string | undefined { + const v = (flags['database'] as string | undefined) ?? process.env['WORKOS_SYNC_DATABASE'] + return v && v.trim() ? v.trim() : undefined +} + +function prefixFromFlags(flags: Record): string { + return (flags['prefix'] as string) ?? process.env['WORKOS_SYNC_PREFIX'] ?? DEFAULT_PREFIX +} + +function streamsFromFlags(flags: Record): string[] | undefined { + return typeof flags['streams'] === 'string' ? flags['streams'].split(',') : undefined +} + +/** + * Built apart from the rest of the config because `init` reads the report + * back off it after the engine has run; the engine's own `init()` returns + * nothing. `client` is a test seam: a pre-built client in place of a + * connection string, so the CLI can be exercised end to end. + */ +function destinationFromEnv( + flags: Record, + repair: boolean, + client?: MongoClient, +): MongoDestination { + const url = (flags['database-url'] as string) ?? process.env['DATABASE_URL'] + if (!client && !url) throw new Error('set DATABASE_URL (or pass --database-url)') + return new MongoDestination({ + url, + sslCa: parseSslCa(process.env['WORKOS_SYNC_MONGODB_TLS_CA']), + client, + database: databaseFromFlags(flags), + prefix: prefixFromFlags(flags), + syncId: process.env['WORKOS_SYNC_ID'] ?? 'default', + repairViews: repair, + }) +} + +/** + * `WORKOS_SYNC_LEASE_TTL_MS` is a number of milliseconds, at least 1. Unset + * or empty means the engine default. Anything else is refused here rather + * than further down: an empty value would otherwise become zero, and a lease + * that expires on arrival excludes nobody. + */ +export function parseLeaseTtl(value: string | undefined): number | undefined { + if (!value || !value.trim()) return undefined + const n = Number(value) + if (!Number.isFinite(n) || Math.floor(n) < 1) { + throw new Error(`invalid WORKOS_SYNC_LEASE_TTL_MS "${value}": expected a number of milliseconds of at least 1`) + } + return n +} + +/** + * `--max-pages ` is a positive integer. Absent means unbounded. A bare + * `--max-pages` with no value, zero, or a fraction is refused here, before the + * engine connects: a budget of nothing would stop before the first page and + * look like a backfill that found nothing. + */ +export function parseMaxPages(value: string | boolean | undefined): number | undefined { + if (value === undefined || value === false) return undefined + const n = typeof value === 'string' ? Number(value.trim()) : NaN + if (typeof value !== 'string' || !value.trim() || !Number.isInteger(n) || n < 1) { + throw new Error(`invalid --max-pages "${String(value)}": expected a positive integer`) + } + return n +} + +function configFromEnv( + flags: Record, + destination: MongoDestination, +): SyncEngineConfig & { logger: Logger } { + const streams = streamsFromFlags(flags) + return { + destination, + environments: environmentsFromEnv(), + streams, + hydrate: parseHydrate(process.env['WORKOS_SYNC_HYDRATE']), + pollIntervalMs: Number(process.env['WORKOS_SYNC_POLL_MS'] ?? 5000), + lagWindowMs: Number(process.env['WORKOS_SYNC_LAG_MS'] ?? 5000), + maxEventsPerTick: process.env['WORKOS_SYNC_MAX_EVENTS_PER_TICK'] + ? Number(process.env['WORKOS_SYNC_MAX_EVENTS_PER_TICK']) + : undefined, + leaseTtlMs: parseLeaseTtl(process.env['WORKOS_SYNC_LEASE_TTL_MS']), + logger: createLogger((process.env['LOG_LEVEL'] as LogLevel) ?? 'info'), + } +} + +const HELP = `workos-sync-mongodb — mirror WorkOS data into MongoDB + +Usage + workos-sync-mongodb [flags] + +Commands + init [--repair] Create the collections, indexes, _live views and bookkeeping. + Fails if a generated _live view has a pipeline that is not the + generated one; --repair redefines it + backfill [--resume] [--max-pages ] + Load current state from the REST list endpoints. + --resume continues each stream from its checkpoint and + skips streams already complete; --max-pages stops after + n pages across all streams, for hosts with a time limit + (run again with --resume until every stream is complete) + listen [--health-port ] + Tail the Events API and apply changes continuously + (waits while another listener holds the lease); + with a port, serve GET /healthz and /metrics on it + tick One poll of the Events API, then exit (for a cron job); + exits 3 if a listener holds the lease + reconcile [--repair] [--resume] [--max-pages ] + Compare the mirror against the API; --repair fixes drift. + --max-pages stops starting streams once n pages have been + fetched (a stream that starts runs to its last page, since + a diff needs the whole listing); run again with --resume + until every stream reports complete + refresh [--resume] [--max-pages ] + Re-list top-level streams, replacing each document; + --max-pages and --resume slice it like backfill, with + which it shares checkpoints + status Cursor position, poll age, and per-stream freshness + redact --user Delete one subject from the mirror and event archive + rejected Show events quarantined by the tail; exits 2 if there are any + --replay Re-apply them all (or one with --id ) once the cause is fixed + streams Show the streams this build can mirror + +Flags + --database-url MongoDB connection string (or DATABASE_URL) + --database Database the mirror lives in (or WORKOS_SYNC_DATABASE; + default: the one the connection string names, else workos) + --prefix Collection prefix (or WORKOS_SYNC_PREFIX; default: workos_) + --streams Restrict to specific streams (events is always included) + +Environment + DATABASE_URL MongoDB connection string (mongodb:// or mongodb+srv://) + WORKOS_SYNC_MONGODB_TLS_CA CA certificate the database's certificate must chain to, as PEM + or base64 of it; TLS is then required and verified against it + (default: the system trust store) + WORKOS_API_KEY API key for the environment to mirror + WORKOS_ENVIRONMENT_NAME Label written to _env on every document (default: production) + WORKOS_API_URL API base URL (default: https://api.workos.com) + WORKOS_SYNC_DATABASE Database the mirror lives in (default: from the connection string) + WORKOS_SYNC_PREFIX Collection prefix (default: workos_) + WORKOS_SYNC_ID Sync identifier when several syncs share a database (default: default) + WORKOS_SYNC_HYDRATE off | on_create | always, or users=always,organizations=off + (default: off) + WORKOS_SYNC_POLL_MS Idle wait between polls (default: 5000) + WORKOS_SYNC_LAG_MS Hold back events younger than this (default: 5000) + WORKOS_SYNC_MAX_EVENTS_PER_TICK Events to apply per poll (default: one page) + WORKOS_SYNC_LEASE_TTL_MS How long the tail lease outlives a crashed listener, at least 1 (default: 60000) + WORKOS_SYNC_HEALTH_PORT listen only: serve /healthz and /metrics on this port (default: off) + WORKOS_SYNC_HEALTH_HOST Interface the health server binds (default: 0.0.0.0) + WORKOS_SYNC_HEALTH_MAX_POLL_AGE_MS + /healthz returns 503 once any environment's last successful + poll is older than this (default: 60000) + WORKOS_SYNC_OTEL 1 starts the OpenTelemetry Node SDK for the command, configured by + the OTEL_* variables, and flushes it on exit (default: off) + LOG_LEVEL debug | info | warn | error | silent +` + +export interface MainDeps { + /** A pre-built client in place of `DATABASE_URL`, so tests can run the CLI against a test server. */ + client?: MongoClient + /** A whole engine in place of one built from the environment, for tests of commands that need faults injected. */ + engine?: SyncEngine +} + +export async function main(argv = process.argv.slice(2), deps: MainDeps = {}): Promise { + const { command, flags } = parseArgs(argv) + + if (command === 'help' || flags['help']) { + process.stdout.write(HELP) + return 0 + } + + if (command === 'streams') { + const defaults = new Set(defaultStreams()) + const rows = STREAMS.map((s) => ({ + stream: s.name, + group: s.group, + mode: s.mode, + default: defaults.has(s.name) ? 'yes' : 'no (opt-in)', + backfill: s.list ? 'yes' : s.name === 'organization_domains' ? 'from organizations' : 'no', + events: (s.events?.upsert?.length ?? 0) + (s.events?.delete?.length ?? 0), + })) + process.stdout.write(JSON.stringify(rows, null, 2) + '\n') + return 0 + } + + // The SDK, when asked for, goes up before the engine is built, so that the + // instruments the engine makes at construction are real ones, and comes + // down after the command returns, which is what exports a one-shot + // command's metrics at all. + const otel = await startOtel() + try { + return await runCommand(command, flags, deps) + } finally { + await otel?.shutdown() + } +} + +async function runCommand(command: string, flags: Record, deps: MainDeps): Promise { + // With an injected engine there is no destination of ours to read the + // report from, and nothing to build from the environment. + const destination = deps.engine ? undefined : destinationFromEnv(flags, repairRequested(command, flags), deps.client) + const config = destination ? configFromEnv(flags, destination) : undefined + const engine = deps.engine ?? new SyncEngine(config!) + const logger = config?.logger ?? createLogger((process.env['LOG_LEVEL'] as LogLevel) ?? 'info') + try { + switch (command) { + case 'init': { + await engine.init() + const orphans = destination?.lastMigration?.orphans ?? [] + const views = destination?.lastMigration?.views ?? [] + // Orphans are advisory: an index the catalog no longer calls for is + // still serving whatever query it was made for. + if (orphans.length > 0 || views.length > 0) { + process.stdout.write(JSON.stringify({ views, orphans }, null, 2) + '\n') + } + process.stdout.write('schema ready\n') + return 0 + } + case 'backfill': { + const results = await engine.backfill({ + resume: Boolean(flags['resume']), + maxPages: parseMaxPages(flags['max-pages']), + }) + process.stdout.write(JSON.stringify(results, null, 2) + '\n') + // A budget that stops short is the outcome asked for, not an error: + // the JSON says which streams are complete, and the next run resumes. + return 0 + } + case 'listen': { + // Up before the engine connects, so a probe during a slow schema init + // sees "starting" rather than a refused connection. + const healthOptions = healthOptionsFromEnv(flags) + const health = healthOptions ? await startHealthServer(engine, healthOptions) : null + if (healthOptions && health) { + logger.info('health server listening', { + host: healthOptions.host, + port: health.port, + maxPollAgeMs: healthOptions.maxPollAgeMs, + }) + } + const shutdown = () => { + engine.stop() + } + process.on('SIGINT', shutdown) + process.on('SIGTERM', shutdown) + try { + await engine.listen() + } finally { + await health?.close() + } + return 0 + } + case 'tick': { + try { + const r = await engine.tick() + process.stdout.write(JSON.stringify(r, null, 2) + '\n') + return 0 + } catch (err) { + // Distinct from a failure (1) and from drift (2): the sync is fine, + // something else is tailing it. Usually a `listen` left running. + if (err instanceof LeaseHeldError) { + process.stderr.write(`${err.message}; nothing to do\n`) + return 3 + } + throw err + } + } + case 'reconcile': { + const r = await engine.reconcile({ + repair: Boolean(flags['repair']), + resume: Boolean(flags['resume']), + maxPages: parseMaxPages(flags['max-pages']), + }) + process.stdout.write(JSON.stringify(r, null, 2) + '\n') + // A non-zero exit makes this usable as a scheduled check. + const drift = Object.values(r.diffs) + .flat() + .some((d) => d.localOnly.length > 0 || d.remoteOnly.length > 0) + return drift && !flags['repair'] ? 2 : 0 + } + case 'refresh': { + const r = await engine.refresh({ + resume: Boolean(flags['resume']), + maxPages: parseMaxPages(flags['max-pages']), + }) + process.stdout.write(JSON.stringify(r, null, 2) + '\n') + return 0 + } + case 'status': { + const r = await engine.status() + process.stdout.write(JSON.stringify(r, null, 2) + '\n') + return 0 + } + case 'redact': { + const user = flags['user'] + if (typeof user !== 'string') throw new Error('redact requires --user ') + const r = await engine.redact(user) + process.stdout.write(JSON.stringify(r, null, 2) + '\n') + return 0 + } + case 'rejected': { + const id = flags['id'] + if (id !== undefined && typeof id !== 'string') throw new Error('--id requires an event id') + const only = typeof id === 'string' ? { ids: [id] } : {} + if (flags['replay']) { + const r = await engine.replayRejected(only) + process.stdout.write(JSON.stringify(r, null, 2) + '\n') + // Anything still rejected after a replay is still a problem. + return r.stillRejected.length > 0 ? 2 : 0 + } + const r = await engine.rejected(only) + process.stdout.write(JSON.stringify(r, null, 2) + '\n') + // Like `reconcile`: a non-zero exit makes this a scheduled check. + const any = Object.values(r).some((rows) => rows.length > 0) + return any ? 2 : 0 + } + default: + process.stderr.write(`unknown command: ${command}\n\n${HELP}`) + return 1 + } + } finally { + await engine.close() + } +} + +// Run when invoked directly, not when imported by a test. +const invokedDirectly = process.argv[1] && (process.argv[1].endsWith('cli.js') || process.argv[1].endsWith('cli.ts')) +if (invokedDirectly) { + main().then( + (code) => process.exit(code), + (err) => { + process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`) + process.exit(1) + }, + ) +} diff --git a/packages/sync-mongodb/src/db/client.ts b/packages/sync-mongodb/src/db/client.ts new file mode 100644 index 0000000..90e7c45 --- /dev/null +++ b/packages/sync-mongodb/src/db/client.ts @@ -0,0 +1,63 @@ +/** + * The connection. + * + * One `MongoClient` is a pool, so the destination holds one for its life and + * every method draws on it. The application may hand in its own instead + * (`fromClient`), in which case the destination never closes it. + */ +import { MongoClient, type MongoClientOptions } from 'mongodb' +import type { Logger } from '@workos/sync-engine' + +/** + * How the engine's connections introduce themselves to the server, so they + * can be told apart from the application's own in `currentOp`, the server + * log and a provider's connection dashboard. Set only when the connection + * string names no `appName` of its own: the operator's choice always wins. + * Kept a constant rather than carrying a version so that filters written + * against it do not break on release. + */ +export const APPLICATION_NAME = 'workos-sync-engine' + +export interface ConnectOptions { + /** + * PEM-encoded CA certificate(s) the server's certificate must chain to. + * TLS is then required, whatever the connection string says short of + * `tls=false`, which is refused as a contradiction. + */ + sslCa?: string + logger?: Logger +} + +/** The database the connection string names, if any. */ +export function databaseFromUrl(url: string): string | undefined { + const m = /^mongodb(?:\+srv)?:\/\/(?:[^@/?]*@)?[^/?]*(?:\/([^?]*))?/i.exec(url) + const db = m?.[1] + return db ? decodeURIComponent(db) : undefined +} + +/** What `new MongoClient(url, ...)` is given. */ +export function clientOptions(url: string, options: ConnectOptions = {}): MongoClientOptions { + const out: MongoClientOptions = {} + if (!/[?&]appName=/i.test(url)) out.appName = APPLICATION_NAME + if (options.sslCa !== undefined) { + if (/[?&](?:tls|ssl)=false(?:&|$)/i.test(url)) { + throw new Error( + 'a CA certificate was given but the connection string turns TLS off (tls=false or ssl=false): drop one or the other', + ) + } + out.tls = true + out.ca = options.sslCa + } + return out +} + +export async function connect(url: string, options: ConnectOptions = {}): Promise { + const client = new MongoClient(url, clientOptions(url, options)) + await client.connect() + return client +} + +/** A pre-built client, e.g. the application's own. Not closed by the destination. */ +export function fromClient(client: MongoClient): MongoClient { + return client +} diff --git a/packages/sync-mongodb/src/db/errors.ts b/packages/sync-mongodb/src/db/errors.ts new file mode 100644 index 0000000..de15521 --- /dev/null +++ b/packages/sync-mongodb/src/db/errors.ts @@ -0,0 +1,65 @@ +/** + * Classifying MongoDB errors for the quarantine. + * + * The engine only ever sets an event aside for an error that will not go away + * on retry. Getting this wrong in either direction is expensive: treat a + * dropped connection as deterministic and a flaky network quarantines good + * events; treat a bad payload as transient and the tail stalls forever, which + * is the failure the quarantine exists to prevent. + * + * The driver says which kind an error was, three ways: the class + * (`MongoNetworkError` for a socket that died, `MongoServerSelectionError` + * for a cluster with nothing to talk to), the error labels the server + * attaches to a retryable failure, and the numeric server code. + */ +import { MongoError, MongoNetworkError, MongoServerSelectionError } from 'mongodb' + +/** + * Server codes that describe the server's state, not the document's: + * + * 6 HostUnreachable 7 HostNotFound + * 24 LockTimeout 46 LockBusy + * 50 MaxTimeMSExpired 64 WriteConcernTimeout + * 89 NetworkTimeout 91 ShutdownInProgress + * 112 WriteConflict 134 ReadConcernMajorityNotAvailableYet + * 189 PrimarySteppedDown 262 ExceededTimeLimit + * 10107 NotWritablePrimary 11600 InterruptedAtShutdown + * 11602 InterruptedDueToReplStateChange + * 13435 NotPrimaryNoSecondaryOk 13436 NotPrimaryOrSecondary + * + * Not in the list, on purpose: 11000 (duplicate key), 2 (BadValue), 121 + * (document failed validation), 10334 (BSON object too large), 17280 (key too + * large to index). Those are verdicts on the document and will be returned + * again, identically, however many times it is retried. + */ +const TRANSIENT_SERVER_CODES = new Set([ + 6, 7, 24, 46, 50, 64, 89, 91, 112, 134, 189, 262, 10107, 11600, 11602, 13435, 13436, +]) + +/** Labels the server attaches when it is telling the client to try again. */ +const TRANSIENT_LABELS = ['RetryableWriteError', 'TransientTransactionError'] + +/** Node socket errors, as they surface on an error's `code` or `cause`. */ +const TRANSIENT_SOCKET_CODES = new Set(['ECONNRESET', 'ECONNREFUSED', 'ETIMEDOUT', 'EPIPE']) + +/** + * Whether `err` is a MongoDB or socket error that a retry could fix. Anything + * without a recognisable class, label or code is reported as deterministic, + * which is the conservative answer: the engine's two-sightings rule covers + * the gap. + */ +export function isTransientMongoError(err: unknown): boolean { + if (typeof err !== 'object' || err === null) return false + if (err instanceof MongoNetworkError || err instanceof MongoServerSelectionError) return true + if (err instanceof MongoError && TRANSIENT_LABELS.some((label) => err.hasErrorLabel(label))) return true + const code = (err as { code?: unknown }).code + if (typeof code === 'number') return TRANSIENT_SERVER_CODES.has(code) + if (typeof code === 'string') return TRANSIENT_SOCKET_CODES.has(code) + // A driver error wrapping the socket error that caused it. + const cause = (err as { cause?: unknown }).cause + if (typeof cause === 'object' && cause !== null) { + const causeCode = (cause as { code?: unknown }).code + return typeof causeCode === 'string' && TRANSIENT_SOCKET_CODES.has(causeCode) + } + return false +} diff --git a/packages/sync-mongodb/src/db/lease.ts b/packages/sync-mongodb/src/db/lease.ts new file mode 100644 index 0000000..efd82d9 --- /dev/null +++ b/packages/sync-mongodb/src/db/lease.ts @@ -0,0 +1,206 @@ +/** + * The single-writer lease and the cursors, as one document. + * + * In Postgres the lease is one row and each cursor another, and the tail's + * checkpoint checks the two in a transaction. MongoDB only promises atomicity + * for one document unless the deployment is a replica set, and a standalone + * `mongod` — a developer's laptop, an embedded test server — is not. So the + * lease and every cursor for an environment share a document in `_sync`, + * keyed by `(sync_id, env)`: + * + * { sync_id, env, + * lease: { holder, acquired_at, expires_at }, + * state: { __tail__: { json, updated_at }, users: { json, updated_at }, ... } } + * + * That makes `setStateIfHeld` one conditional `updateOne` whose filter names + * the holder, the unexpired lease and the expected cursor all at once: either + * every condition holds and the write lands, or nothing happens. There is no + * window between the check and the write, on any deployment. + * + * Taking the lease is one `findOneAndUpdate` with an update pipeline whose + * condition only lets the lease change hands when it is ours, free or + * expired. Expiry is judged by `$$NOW`, the server's clock, never the + * client's, so two hosts with skewed clocks still agree on who holds it. + * + * Cursor state is stored as canonical JSON — keys sorted, recursively — + * because MongoDB compares embedded documents field by field *in order*, and + * the engine compares cursors by value: `{a, b}` and `{b, a}` are the same + * cursor, as they are to Postgres's `jsonb`. + */ +import type { Lease, LeaseResult } from '@workos/sync-engine' +import type { Collection, Document } from 'mongodb' + +export interface LeaseFields { + holder: string + acquired_at: Date + expires_at: Date +} + +export interface StateField { + /** Canonical JSON; see `canonicalJson`. */ + json: string + updated_at: Date +} + +export interface SyncDoc extends Document { + sync_id: string + env: string + lease?: LeaseFields | null + state?: Record +} + +/** `value` as JSON with every object's keys sorted, so equal values serialise equally. */ +export function canonicalJson(value: unknown): string { + return JSON.stringify(sortKeys(value)) +} + +function sortKeys(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortKeys) + if (value !== null && typeof value === 'object') { + const out: Record = {} + for (const key of Object.keys(value as Record).sort()) { + const v = (value as Record)[key] + if (v !== undefined) out[key] = sortKeys(v) + } + return out + } + return value +} + +/** The field a stream's cursor lives under. Stream names are catalog-controlled, but a path separator in one would silently address the wrong thing. */ +function stateField(stream: string): string { + if (stream === '' || stream.includes('.') || stream.startsWith('$')) { + throw new Error(`invalid state key ${JSON.stringify(stream)}: must not be empty, contain "." or start with "$"`) + } + return `state.${stream}` +} + +const DUPLICATE_KEY = 11000 + +export async function acquireLease( + coll: Collection, + syncId: string, + env: string, + holder: string, + ttlMs: number, +): Promise { + const ttl = Math.max(0, Math.floor(ttlMs)) + // Free when there is no lease, when it is ours already, or when the holder + // let it expire. Anything else leaves the lease exactly as it was. + const free = { + $or: [ + { $in: [{ $type: '$lease' }, ['missing', 'null']] }, + { $eq: ['$lease.holder', holder] }, + { $lte: ['$lease.expires_at', '$$NOW'] }, + ], + } + const update: Document[] = [ + { + $set: { + sync_id: syncId, + env, + lease: { + $cond: [ + free, + { + holder, + // A renewal keeps the original takeover time; a takeover starts a new one. + acquired_at: { $cond: [{ $eq: ['$lease.holder', holder] }, '$lease.acquired_at', '$$NOW'] }, + expires_at: { $add: ['$$NOW', ttl] }, + }, + '$lease', + ], + }, + }, + }, + ] + const attempt = () => + coll.findOneAndUpdate({ sync_id: syncId, env }, update, { upsert: true, returnDocument: 'after' }) + let doc: SyncDoc | null + try { + doc = await attempt() + } catch (err) { + // Two processes asking at once while no document exists: both try to + // insert, the unique index lets one through, and the other sees the + // document on its second try and is judged against it. + if ((err as { code?: unknown }).code !== DUPLICATE_KEY) throw err + doc = await attempt() + } + const lease = doc?.lease + if (!lease) return { acquired: false, holder: null, expiresAt: null } + return { acquired: lease.holder === holder, holder: lease.holder, expiresAt: lease.expires_at.toISOString() } +} + +export async function releaseLease( + coll: Collection, + syncId: string, + env: string, + holder: string, +): Promise { + // Removing the lease rather than expiring it keeps `status` honest: a + // released lease reads as free, not as a stale field with a past date. + await coll.updateOne({ sync_id: syncId, env, 'lease.holder': holder }, { $unset: { lease: '' } }) +} + +export async function getLease(coll: Collection, syncId: string, env: string): Promise { + const doc = await coll.findOne( + { sync_id: syncId, env, lease: { $ne: null }, $expr: { $gt: ['$lease.expires_at', '$$NOW'] } }, + { projection: { lease: 1 } }, + ) + const lease = doc?.lease + return lease ? { holder: lease.holder, expiresAt: lease.expires_at.toISOString() } : null +} + +export async function getState( + coll: Collection, + syncId: string, + env: string, + stream: string, +): Promise { + const field = stateField(stream) + const doc = await coll.findOne({ sync_id: syncId, env }, { projection: { [field]: 1 } }) + const stored = doc?.state?.[stream] + return stored ? (JSON.parse(stored.json) as T) : null +} + +export async function setState( + coll: Collection, + syncId: string, + env: string, + stream: string, + state: unknown, +): Promise { + await coll.updateOne( + { sync_id: syncId, env }, + { $set: { [stateField(stream)]: { json: canonicalJson(state), updated_at: new Date() } } }, + { upsert: true }, + ) +} + +/** + * One document, one filter, one write: the holder, the lease's expiry by + * the server's clock, and the cursor as it was expected to be are all in + * the filter, so the write either matches and lands or matches nothing. + */ +export async function setStateIfHeld( + coll: Collection, + syncId: string, + env: string, + stream: string, + state: unknown, + holder: string, + expected: unknown, +): Promise { + const field = stateField(stream) + const r = await coll.updateOne( + { + sync_id: syncId, + env, + 'lease.holder': holder, + $expr: { $gt: ['$lease.expires_at', '$$NOW'] }, + [`${field}.json`]: expected == null ? { $exists: false } : canonicalJson(expected), + }, + { $set: { [field]: { json: canonicalJson(state), updated_at: new Date() } } }, + ) + return r.matchedCount === 1 +} diff --git a/packages/sync-mongodb/src/db/reconcile.ts b/packages/sync-mongodb/src/db/reconcile.ts new file mode 100644 index 0000000..d48497f --- /dev/null +++ b/packages/sync-mongodb/src/db/reconcile.ts @@ -0,0 +1,167 @@ +/** + * The reconcile set difference, computed in MongoDB. + * + * The engine's `diffStream` once held every live local id in a `Set` and every + * remote payload in a `Map`, so the memory a reconcile needed grew with the + * size of the stream rather than the size of the drift: a directory with a + * million users turned a weekly health check into an out-of-memory. Fivetran + * and Airbyte stage the remote side into a table and diff in the database, + * and so does this. + * + * The live keys are copied into one staging collection by the server + * (`$out`) before the first page is requested, the remote pages are written + * into another as they arrive, then three aggregations produce the answer: a + * `$lookup` for `matched`, an anti-join from the local side for `localOnly`, + * an anti-join from the remote side for `remoteOnly`. Node holds one page of + * remote rows at a time, plus the drift. + */ +import { encodeKey, type IdDiff, type TableDef } from '@workos/sync-engine' +import { randomBytes } from 'node:crypto' +import { MongoBulkWriteError, type Collection, type Db, type Document } from 'mongodb' +import { fieldPath, keyPath, type Names } from '../schema/names.js' +import { chunk, keyValues, type MirrorDoc, type WriteOptions } from './upsert.js' + +export interface MongoDiff extends IdDiff { + /** Remote rows with no value at the key path. Counted rather than thrown. */ + skipped: number +} + +interface StagedKey extends Document { + /** The whole key as one string; see `SEPARATOR`. */ + k: string +} + +interface StagedRemote extends StagedKey { + raw: Record +} + +/** + * Composite keys are joined with a control character that no id or slug + * contains, so that `$lookup` compares one string against one string. The + * encoding the engine sees is still `encodeKey`'s: the join is undone before + * a key leaves this module. + */ +const SEPARATOR = '\u0001' + +const DUPLICATE_KEY = 11000 +const NAMESPACE_NOT_FOUND = 26 +const DEFAULT_BATCH_SIZE = 500 + +/** The key of a live row as one string, built by the server. */ +function keyExpression(stream: TableDef): Document { + const parts = stream.key.map((k) => ({ $toString: `$${fieldPath(keyPath(stream, k))}` })) + if (parts.length === 1) return parts[0]! + const concat: (Document | string)[] = [] + for (const [i, part] of parts.entries()) { + if (i > 0) concat.push(SEPARATOR) + concat.push(part) + } + return { $concat: concat } +} + +const decode = (stream: TableDef, k: string) => encodeKey(stream.key.length === 1 ? [k] : k.split(SEPARATOR)) + +export async function diffIdsInMongo( + db: Db, + names: Names, + opts: WriteOptions, + stream: TableDef, + env: string, + remote: AsyncIterable[]>, +): Promise { + const token = randomBytes(6).toString('hex') + const localName = names.staging('local', token) + const remoteName = names.staging('remote', token) + const local = db.collection(localName) + const staged = db.collection(remoteName) + const mirror = db.collection(names.mirror(stream)) + + try { + // Append-only collections have no tombstone field; every row is live. + const live: Document = stream.mode === 'append' ? { _env: env } : { _env: env, _deleted_at: null } + + // The live keys are frozen before the first page is requested, not read + // once the pages are in. Listing a large stream takes minutes under the + // API's rate limits and the tail keeps writing throughout: a row it + // inserts after the listing has passed that object's position would be + // live in the mirror and absent from the pages, and `--repair` would + // tombstone an object that exists upstream. Deletion is terminal, so no + // later event would undo that. Captured here, such a row is simply not a + // candidate, which is how the in-memory diff behaved when it read + // `liveIds` first. + await mirror + .aggregate( + [ + { $match: live }, + { $project: { _id: 0, k: keyExpression(stream) } }, + // A row with no complete key cannot be reported, so it cannot be a candidate. + { $match: { k: { $ne: null } } }, + { $out: localName }, + ], + { allowDiskUse: true }, + ) + .toArray() + await local.createIndex({ k: 1 }, { unique: true }) + await staged.createIndex({ k: 1 }, { unique: true }) + + let skipped = 0 + for await (const page of remote) { + const keyed: StagedRemote[] = [] + for (const data of page) { + const id = keyValues(stream, data) + if (!id) { + skipped++ + continue + } + keyed.push({ k: id.join(SEPARATOR), raw: data }) + } + for (const batch of chunk(keyed, opts.batchSize ?? DEFAULT_BATCH_SIZE)) { + try { + // An id the API returns twice (a page boundary moving under a + // concurrent write) keeps its first copy. Nothing is lost that the + // next reconcile would not find again. + await staged.insertMany(batch, { ordered: false }) + } catch (err) { + if (!(err instanceof MongoBulkWriteError)) throw err + const errors = Array.isArray(err.writeErrors) ? err.writeErrors : [err.writeErrors] + if (!errors.every((e) => e.code === DUPLICATE_KEY)) throw err + } + } + } + + const lookup = (from: string) => ({ $lookup: { from, localField: 'k', foreignField: 'k', as: 'm' } }) + const matched = await local + .aggregate<{ n: number }>([lookup(remoteName), { $match: { 'm.0': { $exists: true } } }, { $count: 'n' }]) + .toArray() + const localOnly = await local + .aggregate( + [lookup(remoteName), { $match: { m: { $size: 0 } } }, { $sort: { k: 1 } }, { $project: { _id: 0, k: 1 } }], + { allowDiskUse: true }, + ) + .toArray() + const remoteOnly = await staged + .aggregate( + [lookup(localName), { $match: { m: { $size: 0 } } }, { $sort: { k: 1 } }, { $project: { _id: 0, raw: 1 } }], + { allowDiskUse: true }, + ) + .toArray() + + return { + matched: matched[0]?.n ?? 0, + localOnly: localOnly.map((r) => decode(stream, r.k)), + remoteOnly: remoteOnly.map((r) => r.raw), + skipped, + } + } finally { + await dropQuietly(local) + await dropQuietly(staged) + } +} + +async function dropQuietly(coll: Collection): Promise { + try { + await coll.drop() + } catch (err) { + if ((err as { code?: unknown }).code !== NAMESPACE_NOT_FOUND) throw err + } +} diff --git a/packages/sync-mongodb/src/db/rejected.ts b/packages/sync-mongodb/src/db/rejected.ts new file mode 100644 index 0000000..47ac608 --- /dev/null +++ b/packages/sync-mongodb/src/db/rejected.ts @@ -0,0 +1,106 @@ +/** + * The quarantine store: `_rejected`, one document per event the tail set aside. + * + * Keyed by `(sync_id, env, event_id)` like the rest of the bookkeeping, so + * several syncs sharing a database never see each other's rejects. The whole + * event is stored verbatim in `_raw`, which is what makes a replay possible + * and is also why `redact` has to reach in here as well as into `events`. + */ +import type { RejectedEvent, WorkOSEvent } from '@workos/sync-engine' +import type { Collection, Document } from 'mongodb' +import { parseInstantOrNull } from './timestamps.js' + +export interface RejectedDoc extends Document { + sync_id: string + env: string + event_id: string + event_type: string + object_id: string | null + /** The event's own `created_at`, as a date when it is one; null otherwise. */ + created_at: Date | null + rejected_at: Date + /** The error from the most recent attempt to apply it. */ + error: string + _raw: WorkOSEvent +} + +export interface RejectedOptions { + syncId: string +} + +export async function rejectEvents( + coll: Collection, + opts: RejectedOptions, + env: string, + rows: { event: WorkOSEvent; error: string }[], +): Promise { + if (rows.length === 0) return + await coll.bulkWrite( + rows.map(({ event, error }) => { + const objectId = (event.data as Record | undefined)?.['id'] + return { + updateOne: { + filter: { sync_id: opts.syncId, env, event_id: event.id }, + // A replay that fails again lands here a second time: keep the + // latest error and timestamp so `rejected` reports what is wrong + // now, not what was. `created_at` is parsed leniently rather than + // strictly: an unparseable timestamp is one of the ways an event + // ends up here, and the quarantine write must not fail on the data + // it is quarantining. Unparseable input stores null. + update: { + $set: { + event_type: event.event, + object_id: typeof objectId === 'string' ? objectId : null, + created_at: parseInstantOrNull(event.created_at), + error, + _raw: event, + }, + $currentDate: { rejected_at: true }, + $setOnInsert: { sync_id: opts.syncId, env, event_id: event.id }, + }, + upsert: true, + }, + } + }), + { ordered: true }, + ) +} + +export async function listRejected( + coll: Collection, + opts: RejectedOptions, + env: string, + ids?: string[], +): Promise { + if (ids && ids.length === 0) return [] + const docs = await coll + .find({ sync_id: opts.syncId, env, ...(ids ? { event_id: { $in: ids } } : {}) }) + .sort({ event_id: 1 }) + .toArray() + return docs.map((doc) => ({ + eventId: doc.event_id, + eventType: doc.event_type, + objectId: doc.object_id, + // The event's own timestamp, as it arrived; the date field is only for queries. + createdAt: doc._raw.created_at, + rejectedAt: doc.rejected_at.toISOString(), + error: doc.error, + raw: doc._raw, + })) +} + +/** How many events are quarantined, without reading their payloads. */ +export function countRejected(coll: Collection, opts: RejectedOptions, env: string): Promise { + return coll.countDocuments({ sync_id: opts.syncId, env }) +} + +export async function clearRejected( + coll: Collection, + opts: RejectedOptions, + env: string, + ids: string[], +): Promise { + if (ids.length === 0) return 0 + const r = await coll.deleteMany({ sync_id: opts.syncId, env, event_id: { $in: ids } }) + return r.deletedCount +} diff --git a/packages/sync-mongodb/src/db/timestamps.ts b/packages/sync-mongodb/src/db/timestamps.ts new file mode 100644 index 0000000..b62ecf7 --- /dev/null +++ b/packages/sync-mongodb/src/db/timestamps.ts @@ -0,0 +1,75 @@ +/** + * ISO-8601 parsing, strict. + * + * `Row.updatedAt` and an event's `created_at` arrive as strings and are + * stored as BSON dates, so the staleness gate compares instants rather than + * text. `new Date(iso)` is not strict enough for that: V8 rolls `2026-02-30` + * over into March instead of rejecting it, so a timestamp the Postgres + * destination refuses — and quarantines the event for — would be silently + * mis-stored here. This parser mirrors `parse_iso8601` in + * `@workos/sync-postgres`: the same shape, the same range checks, and a + * verdict rather than a guess for anything else. + */ + +const ISO = /^(\d{4})-(\d{2})-(\d{2})[Tt ](\d{2}):(\d{2}):(\d{2})(?:\.(\d+))?\s*(?:([Zz])|([+-])(\d{2}):?(\d{2}))?$/ + +// Day counts of a non-leap year, with February corrected for the year in hand. +const daysInMonth = (year: number, month: number) => + new Date(Date.UTC(2001, month, 0)).getUTCDate() + (month === 2 && isLeap(year) ? 1 : 0) +const isLeap = (year: number) => (year % 4 === 0 && year % 100 !== 0) || year % 400 === 0 + +export type ParseFailure = 'shape' | 'range' + +/** The instant `iso` names, or why it names none. */ +export function parseIso8601(iso: string): { date: Date } | { failure: ParseFailure } { + const m = ISO.exec(iso) + if (!m) return { failure: 'shape' } + const [, y, mo, d, h, mi, s, frac, zulu, sign, oh, om] = m + const year = Number(y) + const month = Number(mo) + const day = Number(d) + const hour = Number(h) + const minute = Number(mi) + const second = Number(s) + if (year < 1 || month < 1 || month > 12) return { failure: 'range' } + if (day < 1 || day > daysInMonth(year, month)) return { failure: 'range' } + if (hour > 23 || minute > 59 || second > 60) return { failure: 'range' } + // Millisecond precision is what BSON dates carry; further digits are dropped. + const ms = frac ? Number(frac.slice(0, 3).padEnd(3, '0')) : 0 + // Built in two steps so that years below 100 are not read as 19xx. + const date = new Date(Date.UTC(2000, month - 1, day, hour, minute, second, ms)) + date.setUTCFullYear(year) + if (!zulu && sign) { + const offset = (Number(oh) * 60 + Number(om)) * 60_000 + date.setTime(date.getTime() - (sign === '-' ? -offset : offset)) + } + return { date } +} + +/** Deterministic: the same input fails the same way every time, which is what lets the engine quarantine on it. */ +export class InvalidTimestampError extends Error { + readonly code = 'INVALID_TIMESTAMP' + + constructor(iso: string, failure: ParseFailure) { + super( + `invalid timestamp ${JSON.stringify(iso)}: ${ + failure === 'range' ? 'date/time field value out of range' : 'not an ISO-8601 instant' + }`, + ) + this.name = 'InvalidTimestampError' + } +} + +/** `iso` as a date, or an error the engine can quarantine on. */ +export function parseInstant(iso: string): Date { + const r = parseIso8601(iso) + if ('date' in r) return r.date + throw new InvalidTimestampError(iso, r.failure) +} + +/** `iso` as a date, or null: for fields that must never fail a write. */ +export function parseInstantOrNull(iso: unknown): Date | null { + if (typeof iso !== 'string') return null + const r = parseIso8601(iso) + return 'date' in r ? r.date : null +} diff --git a/packages/sync-mongodb/src/db/upsert.ts b/packages/sync-mongodb/src/db/upsert.ts new file mode 100644 index 0000000..a0b66f4 --- /dev/null +++ b/packages/sync-mongodb/src/db/upsert.ts @@ -0,0 +1,402 @@ +/** + * Write path. + * + * Three rules, each of which exists because of a documented failure in a + * comparable engine: + * + * 1. **Staleness gate, `>=` not `>`.** Writes are rejected when the incoming + * row is older than the row already stored. It is `>=` because event + * timestamps are not infinitely precise: with a strict `<` comparison two + * events sharing a timestamp cause the second to be dropped forever, which + * is exactly how a comparable billing sync engine kept subscriptions + * "trialing" after they converted. Ties resolve last-writer-wins, and the + * backstop is a re-list rather than the comparison — `refresh` for + * top-level streams, `backfill` for those listed per parent — because + * `reconcile` compares ids only and cannot see a wrong payload. + * + * 2. **Merge, don't replace, when writing from an event.** Most WorkOS event + * payloads are the full REST object, but deprecated fields are left out + * of them, and any stream whose events are narrower than its REST object + * would have those fields dropped by a whole-document replace. Event + * writes shallow-merge into `_raw` (`$mergeObjects`); REST-sourced writes + * (backfill, reconcile, hydrate) replace it. + * + * 3. **Deletion is terminal.** Tombstones are not staleness-gated, and + * `_deleted_at` is never cleared. A late `updated` event arriving after a + * delete refreshes the payload but must not resurrect the row. WorkOS IDs + * are ULIDs and are never reused, so this is safe. Memberships are the + * exception, in both directions: a re-add (`relinkRows`) clears the + * tombstone and a removal (`unlinkRows`) sets it, each only when it is at + * least as new as the row. Events can be replayed out of order from + * quarantine, and an older link or unlink must not undo a newer one. + * + * Every write is an update pipeline run by the server with `upsert: true`, + * so the gate is applied against the stored timestamp in the same operation + * that writes — there is no window between a read and a write for another + * writer to slip through. Payloads go into the pipeline as `$literal`: an + * expression context reads a string like `"$100 plan"` as a field path, and + * `$literal` is what stops it. + */ +import { emptyStats, type Row, type TableDef, type WriteMode, type WriteStats } from '@workos/sync-engine' +import type { AnyBulkWriteOperation, Collection, Document, Filter } from 'mongodb' +import { fieldPath, keyPath } from '../schema/names.js' +import { parseInstant } from './timestamps.js' + +/** One mirrored row. */ +export interface MirrorDoc extends Document { + _env: string + /** The payload as it arrived, the only thing the engine ever writes about an object. */ + _raw: Record + /** Source-derived. The staleness gate compares this. */ + _updated_at: Date + /** Destination write time, by the server's clock. Lag is `_synced_at - _updated_at`. */ + _synced_at: Date + /** Tombstone. Absent on append-only streams. */ + _deleted_at?: Date | null +} + +export function readPath(data: Record, path: string | string[]): unknown { + const parts = Array.isArray(path) ? path : [path] + let cur: unknown = data + for (const p of parts) { + if (cur == null || typeof cur !== 'object') return undefined + cur = (cur as Record)[p] + } + return cur +} + +/** Key values for a row, in `stream.key` order. Undefined if any part is missing. */ +export function keyValues(stream: TableDef, data: Record): string[] | undefined { + const out: string[] = [] + for (const k of stream.key) { + const v = readPath(data, keyPath(stream, k)) + if (v == null || v === '') return undefined + out.push(String(v)) + } + return out +} + +/** + * Collapse rows that share a primary key, keeping the newest. It serves the + * delete, relink and unlink paths, whose rows are whole; upserts use + * `wavesByKey`. + */ +export function dedupeByKey(stream: TableDef, rows: Row[]): { rows: Row[]; dropped: number } { + const byKey = new Map() + let dropped = 0 + for (const row of rows) { + const key = keyValues(stream, row.data) + if (!key) { + dropped++ + continue + } + const k = key.join('\0') + const existing = byKey.get(k) + if (!existing || row.updatedAt >= existing.updatedAt) byKey.set(k, row) + } + return { rows: [...byKey.values()], dropped } +} + +/** + * Split a batch into waves in which no key appears twice, keeping order: the + * n-th row for a key goes in wave n. Written one after another, the waves + * are exactly the rows written one at a time — each row meets the staleness + * gate against what the row before it left stored, and a merge overlays it + * on that — while a batch with no repeated key, which is nearly every batch, + * is still a single bulk write. + * + * An ordered bulk write already applies its operations one after another, + * so the waves are not what makes this correct; they are what keeps the + * write's shape the same as the Postgres destination's, where they are. + * + * Keeping the newest row, as `dedupeByKey` does, is wrong for a merge write: + * a hydrated create (the whole REST object) followed in the same poll by an + * update event whose payload is narrower would lose every field the event + * omits until the next refresh, which is what hydration exists to prevent. + * Folding the payloads here instead would be wrong the other way: a row + * older than the stored one is refused on its own, but folded under a newer + * row's timestamp its fields would overwrite the stored values. Only the + * database knows the stored timestamp, so the database applies the gate. + */ +export function wavesByKey(stream: TableDef, rows: Row[]): { waves: Row[][]; dropped: number } { + const waves: Row[][] = [] + const seen = new Map() + let dropped = 0 + for (const row of rows) { + const key = keyValues(stream, row.data) + if (!key) { + dropped++ + continue + } + const k = key.join('\0') + const n = seen.get(k) ?? 0 + seen.set(k, n + 1) + const wave = waves[n] ?? (waves[n] = []) + wave.push(row) + } + return { waves, dropped } +} + +export const chunk = (items: T[], size: number): T[][] => { + const out: T[][] = [] + for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size)) + return out +} + +export interface WriteOptions { + /** Rows per bulk write. Defaults to 500. */ + batchSize?: number +} + +const DEFAULT_BATCH_SIZE = 500 + +/** The filter that names one row: its environment and every key column at its catalog path. */ +export function keyFilter(stream: TableDef, env: string, data: Record): Filter { + const filter: Record = { _env: env } + for (const k of stream.key) { + const path = keyPath(stream, k) + filter[fieldPath(path)] = readPath(data, path) + } + return filter as Filter +} + +type Op = AnyBulkWriteOperation + +/** + * The gate, as an expression. True when the document is being inserted: + * a missing `_updated_at` sorts below any date in BSON order. + */ +const gate = (at: Date) => ({ $gte: [at, '$_updated_at'] }) +/** Shallow merge, matching Postgres's `jsonb || jsonb`: a field in `data` replaces the stored one whole. */ +const merged = (data: Record) => ({ $mergeObjects: ['$_raw', { $literal: data }] }) +/** `_synced_at` moves to the server's clock when `when` holds, and is set on insert either way. */ +const synced = (when: Document) => ({ $cond: [when, '$$NOW', { $ifNull: ['$_synced_at', '$$NOW'] }] }) +/** Whether this operation is inserting rather than updating. */ +const inserting = { $eq: [{ $type: '$_updated_at' }, 'missing'] } + +function upsertOp(stream: TableDef, env: string, row: Row, mode: WriteMode): Op { + const at = parseInstant(row.updatedAt) + const filter = keyFilter(stream, env, row.data) + + if (stream.mode === 'append') { + // Immutable log. Re-delivery is a no-op, which is the whole idempotency story. + return { + updateOne: { + filter, + update: [ + { + $set: { + _env: env, + _raw: { $cond: [inserting, { $literal: row.data }, '$_raw'] }, + _updated_at: { $ifNull: ['$_updated_at', at] }, + _synced_at: { $ifNull: ['$_synced_at', '$$NOW'] }, + }, + }, + ], + upsert: true, + }, + } + } + + const newRaw = mode === 'merge' ? merged(row.data) : { $literal: row.data } + const passes = gate(at) + // An equal timestamp with an identical payload is the same observation + // again; rewriting it would move `_synced_at` for nothing and report an + // update that changed nothing. (`$ne` compares documents field by field in + // order, so a payload that differs only in field order counts as changed.) + const changed = { $and: [passes, { $or: [{ $gt: [at, '$_updated_at'] }, { $ne: [newRaw, '$_raw'] }] }] } + return { + updateOne: { + filter, + update: [ + { + $set: { + _env: env, + _raw: { $cond: [passes, newRaw, '$_raw'] }, + _updated_at: { $cond: [passes, at, '$_updated_at'] }, + _synced_at: synced(changed), + }, + }, + ], + upsert: true, + }, + } +} + +export async function upsertRows( + coll: Collection, + opts: WriteOptions, + stream: TableDef, + env: string, + rows: Row[], + mode: WriteMode, +): Promise { + const stats = emptyStats() + if (rows.length === 0) return stats + const { waves, dropped } = wavesByKey(stream, rows) + // Rows we could not key (a payload with no id and no slug) are counted as + // skipped rather than silently vanishing. + stats.skipped += dropped + for (const wave of waves) { + for (const batch of chunk(wave, opts.batchSize ?? DEFAULT_BATCH_SIZE)) { + const ops = batch.map((row) => upsertOp(stream, env, row, mode)) + const r = await coll.bulkWrite(ops, { ordered: true }) + // A gate that refuses modifies nothing, so the counts fall out of the + // result: inserted, changed, and the rest. + stats.created += r.upsertedCount + stats.updated += r.modifiedCount + stats.skipped += batch.length - r.upsertedCount - r.modifiedCount + } + } + return stats +} + +export async function deleteRows( + coll: Collection, + opts: WriteOptions, + stream: TableDef, + env: string, + rows: Row[], +): Promise { + const stats = emptyStats() + if (rows.length === 0) return stats + const { rows: deduped, dropped } = dedupeByKey(stream, rows) + stats.skipped += dropped + + for (const batch of chunk(deduped, opts.batchSize ?? DEFAULT_BATCH_SIZE)) { + const ops: Op[] = batch.map((row) => { + const at = parseInstant(row.updatedAt) + // `$max`, not `$ifNull`, for `_deleted_at`: a membership removed, + // re-added and removed again must carry the *latest* removal, because + // that is what `relinkRows` compares a re-add against. `$max` ignores a + // missing value, so a live row is tombstoned at the incoming time. + return { + updateOne: { + filter: keyFilter(stream, env, row.data), + update: [ + { + $set: { + _env: env, + _raw: merged(row.data), + _updated_at: { $ifNull: ['$_updated_at', at] }, + _deleted_at: { $max: ['$_deleted_at', at] }, + _synced_at: '$$NOW', + }, + }, + ], + upsert: true, + }, + } + }) + const r = await coll.bulkWrite(ops, { ordered: true }) + stats.deleted += r.upsertedCount + r.matchedCount + } + return stats +} + +/** Restore a tombstoned membership row (a re-add after a removal). */ +export async function relinkRows( + coll: Collection, + opts: WriteOptions, + stream: TableDef, + env: string, + rows: Row[], +): Promise { + const stats = emptyStats() + if (rows.length === 0) return stats + const { rows: deduped, dropped } = dedupeByKey(stream, rows) + stats.skipped += dropped + + for (const batch of chunk(deduped, opts.batchSize ?? DEFAULT_BATCH_SIZE)) { + const ops: Op[] = batch.map((row) => { + const at = parseInstant(row.updatedAt) + // A membership is a fact that can legitimately come back: added, removed, + // added again. Unlike object deletes, a re-link clears the tombstone — + // but only a re-link at least as new as the removal. `_updated_at` alone + // is not enough: a tombstone written by `deleteRows` (a reconcile repair) + // leaves it untouched, so a link replayed from quarantine after that + // removal would pass the staleness gate and resurrect the membership. + const passes = { + $and: [ + gate(at), + { $or: [{ $eq: [{ $ifNull: ['$_deleted_at', null] }, null] }, { $gte: [at, '$_deleted_at'] }] }, + ], + } + return { + updateOne: { + filter: keyFilter(stream, env, row.data), + update: [ + { + $set: { + _env: env, + _raw: { $cond: [passes, merged(row.data), '$_raw'] }, + _updated_at: { $cond: [passes, at, '$_updated_at'] }, + _deleted_at: { $cond: [passes, null, '$_deleted_at'] }, + _synced_at: synced(passes), + }, + }, + ], + upsert: true, + }, + } + }) + const r = await coll.bulkWrite(ops, { ordered: true }) + stats.created += r.upsertedCount + stats.updated += r.modifiedCount + stats.skipped += batch.length - r.upsertedCount - r.modifiedCount + } + return stats +} + +/** + * Tombstone membership rows (a removal). The counterpart of `relinkRows`, + * and staleness-gated like it, which is what sets it apart from `deleteRows`: + * a membership can come back, so a removal is a fact with a time, not a + * terminal state, and one replayed from quarantine after a newer re-add has + * landed must not hide a membership that is active now. + */ +export async function unlinkRows( + coll: Collection, + opts: WriteOptions, + stream: TableDef, + env: string, + rows: Row[], +): Promise { + const stats = emptyStats() + if (rows.length === 0) return stats + const { rows: deduped, dropped } = dedupeByKey(stream, rows) + stats.skipped += dropped + + for (const batch of chunk(deduped, opts.batchSize ?? DEFAULT_BATCH_SIZE)) { + const ops: Op[] = batch.map((row) => { + const at = parseInstant(row.updatedAt) + // `_updated_at` moves to the removal time, so the next re-add is gated + // against the removal by the same comparison every membership write + // uses. `_deleted_at` keeps the latest removal for the same reason + // `deleteRows` does. + const passes = gate(at) + return { + updateOne: { + filter: keyFilter(stream, env, row.data), + update: [ + { + $set: { + _env: env, + _raw: { $cond: [passes, merged(row.data), '$_raw'] }, + _updated_at: { $cond: [passes, at, '$_updated_at'] }, + _deleted_at: { $cond: [passes, { $max: ['$_deleted_at', at] }, '$_deleted_at'] }, + _synced_at: synced(passes), + }, + }, + ], + upsert: true, + }, + } + }) + const r = await coll.bulkWrite(ops, { ordered: true }) + const deleted = r.upsertedCount + r.modifiedCount + stats.deleted += deleted + stats.skipped += batch.length - deleted + } + return stats +} diff --git a/packages/sync-mongodb/src/destination.ts b/packages/sync-mongodb/src/destination.ts new file mode 100644 index 0000000..ac3d9e2 --- /dev/null +++ b/packages/sync-mongodb/src/destination.ts @@ -0,0 +1,308 @@ +/** + * The MongoDB destination: everything `@workos/sync-engine` needs to know + * about a database, implemented with the official driver. + */ +import type { + Destination, + FreshnessRow, + IdDiff, + Lease, + LeaseResult, + Logger, + RejectedEvent, + Row, + TableDef, + WorkOSEvent, + WriteMode, + WriteStats, +} from '@workos/sync-engine' +import { encodeKey } from '@workos/sync-engine' +import type { Collection, Db, Document, MongoClient } from 'mongodb' +import { connect, databaseFromUrl } from './db/client.js' +import { isTransientMongoError } from './db/errors.js' +import { acquireLease, getLease, getState, releaseLease, setState, setStateIfHeld, type SyncDoc } from './db/lease.js' +import { diffIdsInMongo } from './db/reconcile.js' +import { clearRejected, countRejected, listRejected, rejectEvents, type RejectedDoc } from './db/rejected.js' +import { + deleteRows, + readPath, + relinkRows, + unlinkRows, + upsertRows, + type MirrorDoc, + type WriteOptions, +} from './db/upsert.js' +import { migrate, type MigrateResult } from './schema/migrate.js' +import { fieldPath, keyPath, Names } from './schema/names.js' + +export interface MongoDestinationOptions { + /** Connection string (`mongodb://` or `mongodb+srv://`). Alternatively pass a pre-built `client`. */ + url?: string + /** A pre-built client, e.g. the application's own. Not closed by `close()`. */ + client?: MongoClient + /** + * PEM-encoded CA certificate(s) the server's certificate must chain to, + * for a database whose certificate a public CA did not issue. TLS is then + * required, whatever the connection string carries short of `tls=false`, + * which is refused as a contradiction. Only with `url`: a pre-built + * `client` brings its own TLS settings. `WORKOS_SYNC_MONGODB_TLS_CA` on the CLI. + */ + sslCa?: string + /** + * Database the mirror lives in. Defaults to the one the connection string + * names, or the pre-built client's default database; `workos` when + * neither names one. The mirror shares the application's database on + * purpose: `$lookup` cannot cross databases. + */ + database?: string + /** Collection prefix, applied to the mirror, the views and the bookkeeping. Defaults to `workos_`. */ + prefix?: string + /** Identifier when several syncs share one database. Defaults to `default`. */ + syncId?: string + /** Rows per bulk write. Defaults to 500. */ + batchSize?: number + /** + * On `init()`, redefine a generated `_live` view whose pipeline is not the + * generated one. Off, `init()` fails and names the view instead. A view + * over some other collection, or a plain collection under the name, is + * never touched either way. `init --repair` on the CLI. + */ + repairViews?: boolean +} + +export class MongoDestination implements Destination { + readonly names: Names + readonly prefix: string + private client: MongoClient | undefined + private ownsClient = false + private db: Db | undefined + private readonly database: string | undefined + private readonly syncId: string + private readonly batchSize: number | undefined + /** + * What the most recent `init()` did, including any orphan indexes it found. + * The engine's `init()` returns nothing, so this is how a caller — the + * CLI's `init` command — gets the report back. + */ + lastMigration: MigrateResult | undefined + + constructor(private options: MongoDestinationOptions) { + if (!options.client && !options.url) throw new Error('url or client is required') + // With both `client` and `url`, `client` is what gets used and `init()` + // never connects, so a CA given alongside would be silently dropped. + if (options.sslCa !== undefined && (options.client || !options.url)) { + throw new Error('sslCa applies to url; a pre-built client carries its own TLS settings') + } + this.names = new Names(options.prefix) + this.prefix = this.names.prefix + this.syncId = options.syncId ?? 'default' + this.batchSize = options.batchSize + this.client = options.client + this.database = options.database ?? (options.url ? databaseFromUrl(options.url) : undefined) + } + + /** The database the mirror lives in. Known once `init()` has run, or sooner when configured. */ + get databaseName(): string | undefined { + return this.db?.databaseName ?? this.database + } + + private get store(): Db { + if (!this.db) throw new Error('destination not initialised: call init() first') + return this.db + } + + private get writeOpts(): WriteOptions { + return { batchSize: this.batchSize } + } + + private get bookkeepingOpts() { + return { syncId: this.syncId } + } + + private mirror(stream: TableDef): Collection { + return this.store.collection(this.names.mirror(stream)) + } + + private get sync(): Collection { + return this.store.collection(this.names.sync) + } + + private get quarantine(): Collection { + return this.store.collection(this.names.rejected) + } + + async init(streams: TableDef[], logger?: Logger): Promise { + if (!this.client) { + this.client = await connect(this.options.url!, { sslCa: this.options.sslCa, logger }) + this.ownsClient = true + } + this.db ??= this.client.db(this.database ?? (this.options.client ? this.client.db().databaseName : 'workos')) + this.lastMigration = await migrate(this.db, streams, this.names, logger, { + repairViews: this.options.repairViews, + }) + } + + async close(): Promise { + if (this.ownsClient && this.client) await this.client.close() + } + + upsert(stream: TableDef, env: string, rows: Row[], mode: WriteMode): Promise { + return upsertRows(this.mirror(stream), this.writeOpts, stream, env, rows, mode) + } + + delete(stream: TableDef, env: string, rows: Row[]): Promise { + return deleteRows(this.mirror(stream), this.writeOpts, stream, env, rows) + } + + relink(stream: TableDef, env: string, rows: Row[]): Promise { + return relinkRows(this.mirror(stream), this.writeOpts, stream, env, rows) + } + + unlink(stream: TableDef, env: string, rows: Row[]): Promise { + return unlinkRows(this.mirror(stream), this.writeOpts, stream, env, rows) + } + + async liveIds(stream: TableDef, env: string): Promise { + const paths = stream.key.map((k) => keyPath(stream, k)) + const fields = paths.map((p) => fieldPath(p)) + const filter: Document = stream.mode === 'append' ? { _env: env } : { _env: env, _deleted_at: null } + const docs = await this.mirror(stream) + .find(filter, { + projection: { _id: 0, ...Object.fromEntries(fields.map((f) => [f, 1])) }, + sort: Object.fromEntries(fields.map((f) => [f, 1])), + }) + .toArray() + return docs.map((doc) => encodeKey(paths.map((p) => String(readPath(doc._raw, p))))) + } + + async rawRows(stream: TableDef, env: string): Promise[]> { + const docs = await this.mirror(stream) + .find({ _env: env }, { projection: { _id: 0, _raw: 1 } }) + .toArray() + return docs.map((doc) => doc._raw) + } + + /** Staged into collections of their own and diffed by the server; see `db/reconcile.ts`. */ + diffIds(stream: TableDef, env: string, remote: AsyncIterable[]>): Promise { + return diffIdsInMongo(this.store, this.names, this.writeOpts, stream, env, remote) + } + + async freshness(stream: TableDef, env: string, lagWindowHours: number): Promise { + const windowMs = lagWindowHours * 3_600_000 + const recent = { $gt: ['$_updated_at', { $subtract: ['$$NOW', windowMs] }] } + const [row] = await this.mirror(stream) + .aggregate<{ + rows: number + deleted: number + lastSynced: Date | null + staleness: number | null + lag: number | null + lagN: number + }>([ + { $match: { _env: env } }, + { + $group: { + _id: null, + rows: { $sum: 1 }, + deleted: { $sum: { $cond: [{ $ne: [{ $ifNull: ['$_deleted_at', null] }, null] }, 1, 0] } }, + lastSynced: { $max: '$_synced_at' }, + lastUpdated: { $max: '$_updated_at' }, + // `$avg` ignores nulls, so only rows inside the window count. + lag: { $avg: { $cond: [recent, { $subtract: ['$_synced_at', '$_updated_at'] }, null] } }, + lagN: { $sum: { $cond: [recent, 1, 0] } }, + }, + }, + { + $project: { + _id: 0, + rows: 1, + deleted: 1, + lastSynced: 1, + lag: 1, + lagN: 1, + staleness: { $subtract: ['$$NOW', '$lastUpdated'] }, + }, + }, + ]) + .toArray() + const lagN = row?.lagN ?? 0 + return { + stream: stream.name, + rows: row?.rows ?? 0, + deleted: row?.deleted ?? 0, + lastSyncedAt: row?.lastSynced ? row.lastSynced.toISOString() : null, + stalenessSeconds: row?.staleness != null ? row.staleness / 1000 : null, + lagSeconds: lagN > 0 && row?.lag != null ? row.lag / 1000 : null, + lagSampleSize: lagN, + } + } + + async redact(streams: TableDef[], env: string, userId: string): Promise<{ table: string; rows: number }[]> { + const out: { table: string; rows: number }[] = [] + for (const stream of streams) { + const col = stream.columns.find((c) => c.name === 'user_id' || (c.name === 'id' && stream.name === 'users')) + if (!col) continue + const r = await this.mirror(stream).deleteMany({ _env: env, [fieldPath(col.path)]: userId }) + if (r.deletedCount > 0) out.push({ table: stream.name, rows: r.deletedCount }) + } + // Archived events embed the payload, so they must be scrubbed too. + const ev = await this.store + .collection(this.names.mirror('events')) + .deleteMany({ _env: env, $or: [{ '_raw.data.id': userId }, { '_raw.data.user_id': userId }] }) + if (ev.deletedCount > 0) out.push({ table: 'events', rows: ev.deletedCount }) + // Quarantined events embed the payload too, and they may be the only copy + // of it: an event rejected before its archive write succeeded is nowhere + // in `events`. + const rj = await this.quarantine.deleteMany({ + sync_id: this.syncId, + env, + $or: [{ object_id: userId }, { '_raw.data.user_id': userId }], + }) + if (rj.deletedCount > 0) out.push({ table: '_rejected', rows: rj.deletedCount }) + return out + } + + reject(env: string, rows: { event: WorkOSEvent; error: string }[]): Promise { + return rejectEvents(this.quarantine, this.bookkeepingOpts, env, rows) + } + + rejected(env: string, opts: { ids?: string[] } = {}): Promise { + return listRejected(this.quarantine, this.bookkeepingOpts, env, opts.ids) + } + + rejectedCount(env: string): Promise { + return countRejected(this.quarantine, this.bookkeepingOpts, env) + } + + clearRejected(env: string, ids: string[]): Promise { + return clearRejected(this.quarantine, this.bookkeepingOpts, env, ids) + } + + isTransientError(err: unknown): boolean { + return isTransientMongoError(err) + } + + getState(env: string, stream: string): Promise { + return getState(this.sync, this.syncId, env, stream) + } + + setState(env: string, stream: string, state: unknown): Promise { + return setState(this.sync, this.syncId, env, stream, state) + } + + setStateIfHeld(env: string, stream: string, state: unknown, holder: string, expected: unknown): Promise { + return setStateIfHeld(this.sync, this.syncId, env, stream, state, holder, expected) + } + + acquireLease(env: string, holder: string, ttlMs: number): Promise { + return acquireLease(this.sync, this.syncId, env, holder, ttlMs) + } + + releaseLease(env: string, holder: string): Promise { + return releaseLease(this.sync, this.syncId, env, holder) + } + + getLease(env: string): Promise { + return getLease(this.sync, this.syncId, env) + } +} diff --git a/packages/sync-mongodb/src/health.ts b/packages/sync-mongodb/src/health.ts new file mode 100644 index 0000000..4e91bca --- /dev/null +++ b/packages/sync-mongodb/src/health.ts @@ -0,0 +1,39 @@ +/** + * Health server settings for the `listen` command. + * + * Off unless a port is given, because the CLI also runs as one-shot commands + * and a cron `tick`, where a listening socket would be a surprise. The Docker + * image sets the port for the same reason it sets `NODE_ENV`: a container is + * the deployment where something external wants to ask whether the process is + * alive. Everything the server needs is read here, in one place, so `cli.ts` + * only has to ask for a config and hand it to `startHealthServer`. + */ +import { DEFAULT_MAX_POLL_AGE_MS, type HealthServerOptions } from '@workos/sync-engine' + +type Flags = Record + +/** + * `--health-port` or `WORKOS_SYNC_HEALTH_PORT` turns the server on; + * `WORKOS_SYNC_HEALTH_HOST` and `WORKOS_SYNC_HEALTH_MAX_POLL_AGE_MS` tune it. + * Returns null when it should stay off. + */ +export function healthOptionsFromEnv(flags: Flags, env: NodeJS.ProcessEnv = process.env): HealthServerOptions | null { + const raw = typeof flags['health-port'] === 'string' ? flags['health-port'] : env['WORKOS_SYNC_HEALTH_PORT'] + if (raw === undefined || raw.trim() === '') return null + const port = Number(raw) + if (!Number.isInteger(port) || port < 0 || port > 65535) { + throw new Error(`invalid health port "${raw}": expected an integer between 0 and 65535`) + } + const maxRaw = env['WORKOS_SYNC_HEALTH_MAX_POLL_AGE_MS'] + const maxPollAgeMs = maxRaw === undefined || maxRaw.trim() === '' ? DEFAULT_MAX_POLL_AGE_MS : Number(maxRaw) + if (!Number.isFinite(maxPollAgeMs) || maxPollAgeMs <= 0) { + throw new Error( + `invalid WORKOS_SYNC_HEALTH_MAX_POLL_AGE_MS "${maxRaw}": expected a positive number of milliseconds`, + ) + } + return { + port, + host: env['WORKOS_SYNC_HEALTH_HOST']?.trim() || '0.0.0.0', + maxPollAgeMs, + } +} diff --git a/packages/sync-mongodb/src/index.ts b/packages/sync-mongodb/src/index.ts new file mode 100644 index 0000000..34f130d --- /dev/null +++ b/packages/sync-mongodb/src/index.ts @@ -0,0 +1,48 @@ +/** + * @workos/sync-mongodb + * + * Mirror WorkOS data into your own MongoDB so you can query it next to your + * application's collections, and `$lookup` across them. + * + * The engine itself lives in `@workos/sync-engine` and is re-exported here, so + * this one package is all an application needs to import. This package adds + * the MongoDB destination, the collections, indexes and write path behind + * it, and the CLI. + * + * The mirror is a **read model**. It is eventually consistent, typically within + * seconds. Read from it for lookups, lists, dashboards, analytics and agents; + * keep writes and authoritative post-signup checks on the WorkOS API. + */ +export * from '@workos/sync-engine' + +export { MongoDestination } from './destination.js' +export type { MongoDestinationOptions } from './destination.js' + +export { migrate, fingerprint, indexSpecs, indexName, ensureBookkeeping, DDL_VERSION } from './schema/migrate.js' +export type { MigrateOptions, MigrateResult, OrphanIndex } from './schema/migrate.js' +export { ensureLiveView, LIVE_PIPELINE } from './schema/views.js' +export type { ApplyViewOptions, ViewOutcome } from './schema/views.js' +export { Names, DEFAULT_PREFIX, LIVE_VIEW_SUFFIX, validatePrefix, fieldPath, keyPath } from './schema/names.js' + +export { + upsertRows, + deleteRows, + relinkRows, + unlinkRows, + dedupeByKey, + wavesByKey, + keyValues, + keyFilter, + readPath, +} from './db/upsert.js' +export type { MirrorDoc, WriteOptions } from './db/upsert.js' +export { diffIdsInMongo } from './db/reconcile.js' +export type { MongoDiff } from './db/reconcile.js' +export { rejectEvents, listRejected, countRejected, clearRejected } from './db/rejected.js' +export type { RejectedDoc } from './db/rejected.js' +export { acquireLease, releaseLease, getLease, getState, setState, setStateIfHeld, canonicalJson } from './db/lease.js' +export type { SyncDoc } from './db/lease.js' +export { isTransientMongoError } from './db/errors.js' +export { parseIso8601, parseInstant, parseInstantOrNull, InvalidTimestampError } from './db/timestamps.js' +export { connect, fromClient, clientOptions, databaseFromUrl, APPLICATION_NAME } from './db/client.js' +export type { ConnectOptions } from './db/client.js' diff --git a/packages/sync-mongodb/src/otel.ts b/packages/sync-mongodb/src/otel.ts new file mode 100644 index 0000000..fb0ab10 --- /dev/null +++ b/packages/sync-mongodb/src/otel.ts @@ -0,0 +1,75 @@ +/** + * OpenTelemetry for the command line. + * + * The engine depends on the OpenTelemetry API only, and the API does nothing + * until a process installs an SDK. The CLI is a process of its own, so it is + * where one can live: `WORKOS_SYNC_OTEL=1` starts the Node SDK, configured by + * the standard `OTEL_*` variables (exporter, endpoint, headers, service + * name), before the engine is built, and shuts it down once the command + * returns. The shutdown is the part that matters for `tick` and the sliced + * commands: they exit seconds after they start, long before the SDK's + * periodic export would have fired, and shutting down flushes. + * + * The SDK is an optional dependency of this package, so a host that embeds + * the library and runs its own SDK can leave it out; the container image has + * it. A missing module is reported as such rather than as a stack trace. + */ +export interface OtelHandle { + /** Flush and stop the SDK. Call it once the command has returned. */ + shutdown(): Promise +} + +/** The part of `@opentelemetry/sdk-node` this module uses, so nothing here types against the package. */ +export interface SdkModule { + NodeSDK: new (configuration?: { serviceName?: string }) => { start(): void; shutdown(): Promise } +} + +/** The `service.name` reported, unless `OTEL_SERVICE_NAME` says otherwise. */ +export const DEFAULT_SERVICE_NAME = 'workos-sync-mongodb' + +const ON = new Set(['1', 'true', 'yes', 'on']) +const OFF = new Set(['0', 'false', 'no', 'off']) + +/** `WORKOS_SYNC_OTEL` is on, off, or unset, which is off. Anything else is refused up front. */ +export function otelEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + const raw = env['WORKOS_SYNC_OTEL'] + if (raw === undefined || raw.trim() === '') return false + const v = raw.trim().toLowerCase() + if (ON.has(v)) return true + if (OFF.has(v)) return false + throw new Error(`invalid WORKOS_SYNC_OTEL "${raw}": expected 1 or 0`) +} + +/** + * Start the SDK when `WORKOS_SYNC_OTEL` asks for it; null when it does not. + * The SDK reads its own configuration from the process environment; `env` + * only decides whether to start it and what to call the service. `load` is + * a test seam standing in for the import. + */ +export async function startOtel( + env: NodeJS.ProcessEnv = process.env, + load: () => Promise = () => import('@opentelemetry/sdk-node'), +): Promise { + if (!otelEnabled(env)) return null + let sdkNode: SdkModule + try { + sdkNode = await load() + } catch (err) { + if (isModuleNotFound(err)) { + throw new Error( + 'WORKOS_SYNC_OTEL is set but @opentelemetry/sdk-node is not installed: ' + + 'add it to the host (npm install @opentelemetry/sdk-node), or unset WORKOS_SYNC_OTEL', + ) + } + throw err + } + const named = Boolean(env['OTEL_SERVICE_NAME']?.trim()) + const sdk = new sdkNode.NodeSDK(named ? {} : { serviceName: DEFAULT_SERVICE_NAME }) + sdk.start() + return { shutdown: () => sdk.shutdown() } +} + +function isModuleNotFound(err: unknown): boolean { + const code = (err as { code?: unknown } | null)?.code + return code === 'ERR_MODULE_NOT_FOUND' || code === 'MODULE_NOT_FOUND' +} diff --git a/packages/sync-mongodb/src/schema/migrate.ts b/packages/sync-mongodb/src/schema/migrate.ts new file mode 100644 index 0000000..8fc503d --- /dev/null +++ b/packages/sync-mongodb/src/schema/migrate.ts @@ -0,0 +1,197 @@ +/** + * Schema setup. + * + * MongoDB has no columns to declare, so "schema" here is the collections, + * their indexes and the `_live` views, all derived from the stream catalog + * rather than hand-written per field. A comparable sync engine accumulated 42 + * hand-written migration files, one per upstream schema change, each needing + * a release; deriving the indexes instead means promoting a field to an + * index is a catalog edit. + * + * Every step is idempotent — `createCollection` tolerates an existing one, + * `createIndexes` is a no-op for an index that already exists with the same + * specification, a view is compared before it is touched — so two processes + * running `init` at once need no lock. A fingerprint of the catalog gates the + * whole pass, so an unchanged catalog costs one query on every start. + * + * There are no typed columns to drift, so there is no repair pass. What can + * linger is an index for a column the catalog no longer indexes; it is + * reported as an orphan and never dropped, since it may be doing work for + * somebody's query. + */ +import { createHash } from 'node:crypto' +import type { Logger, TableDef } from '@workos/sync-engine' +import type { Db, Document, IndexDescription } from 'mongodb' +import { fieldPath, keyPath, type Names } from './names.js' +import { ensureLiveView, type ApplyViewOptions } from './views.js' + +/** + * Bumped whenever what is generated for an unchanged catalog changes: a new + * kind of index, a reworked view. The fingerprint otherwise hashes only the + * catalog, so without this a database whose streams did not change would + * never see the new work on upgrade. + */ +export const DDL_VERSION = 1 + +export function fingerprint(streams: TableDef[], prefix: string): string { + const canonical = JSON.stringify({ + ddl: DDL_VERSION, + prefix, + streams: streams + .map((s) => ({ + n: s.name, + k: s.key, + m: s.mode, + c: s.columns.map((c) => [c.name, c.type, c.path, c.index ?? false]), + })) + .sort((a, b) => a.n.localeCompare(b.n)), + }) + return createHash('sha256').update(canonical).digest('hex').slice(0, 16) +} + +/** + * The indexes a stream's collection carries: the unique key, one per + * `index: true` column, and the two the engine's own queries scan. Every + * index key leads with `_env`, since every query does. + */ +export function indexSpecs(stream: TableDef): IndexDescription[] { + const specs: IndexDescription[] = [] + const key: Record = { _env: 1 } + for (const k of stream.key) key[fieldPath(keyPath(stream, k))] = 1 + specs.push({ key, unique: true }) + const seen = new Set([indexName(key)]) + for (const c of stream.columns) { + if (!c.index) continue + const spec = { _env: 1 as const, [fieldPath(c.path)]: 1 as const } + if (seen.has(indexName(spec))) continue + seen.add(indexName(spec)) + specs.push({ key: spec }) + } + // Freshness and reconciliation scan these. + specs.push({ key: { _synced_at: 1 } }) + if (stream.mode !== 'append') { + specs.push({ key: { _deleted_at: 1 }, partialFilterExpression: { _deleted_at: { $type: 'date' } } }) + } + return specs +} + +/** The name MongoDB gives an index left unnamed: fields and directions joined by `_`. */ +export function indexName(key: Document): string { + return Object.entries(key) + .map(([field, direction]) => `${field}_${String(direction)}`) + .join('_') +} + +export interface OrphanIndex { + collection: string + index: string + key: Document +} + +export interface MigrateOptions extends ApplyViewOptions {} + +export interface MigrateResult { + /** Work was done. `false` means the fingerprint was already recorded. */ + applied: boolean + fingerprint: string + collections: number + indexes: number + /** `_live` views created or redefined by this run. */ + views: string[] + /** + * Indexes on the engine's collections that the catalog does not call for: + * a column no longer indexed, or an `extraColumns` path that changed. + * Reported, never dropped. + */ + orphans: OrphanIndex[] +} + +interface MigrationDoc extends Document { + _id: string + applied_at: Date + detail: Document +} + +const NAMESPACE_EXISTS = 48 +const DUPLICATE_KEY = 11000 + +async function ensureCollection(db: Db, name: string): Promise { + try { + await db.createCollection(name) + return true + } catch (err) { + if ((err as { code?: unknown }).code !== NAMESPACE_EXISTS) throw err + return false + } +} + +/** + * Engine bookkeeping: the lease-and-cursors document, the quarantine, and the + * applied fingerprints. Ensured on every `migrate` before the fingerprint + * check, so an index added here reaches existing deployments without a new + * fingerprint. + */ +export async function ensureBookkeeping(db: Db, names: Names): Promise { + await db.collection(names.sync).createIndexes([{ key: { sync_id: 1, env: 1 }, unique: true }]) + await db.collection(names.rejected).createIndexes([{ key: { sync_id: 1, env: 1, event_id: 1 }, unique: true }]) +} + +export async function migrate( + db: Db, + streams: TableDef[], + names: Names, + logger?: Logger, + options: MigrateOptions = {}, +): Promise { + const fp = fingerprint(streams, names.prefix) + await ensureBookkeeping(db, names) + + const migrations = db.collection(names.migrations) + if (await migrations.findOne({ _id: fp }, { projection: { _id: 1 } })) { + logger?.debug('schema up to date', { fingerprint: fp }) + return { applied: false, fingerprint: fp, collections: 0, indexes: 0, views: [], orphans: [] } + } + + let collections = 0 + let indexes = 0 + const views: string[] = [] + const orphans: OrphanIndex[] = [] + for (const stream of streams) { + const name = names.mirror(stream) + if (await ensureCollection(db, name)) collections++ + const coll = db.collection(name) + const specs = indexSpecs(stream) + await coll.createIndexes(specs) + indexes += specs.length + + const expected = new Set(['_id_', ...specs.map((s) => s.name ?? indexName(s.key))]) + for (const existing of await coll.listIndexes().toArray()) { + if (expected.has(existing['name'] as string)) continue + const orphan = { collection: name, index: existing['name'] as string, key: existing['key'] as Document } + orphans.push(orphan) + // Still serving whatever query it was made for, so it is harmless to + // leave; reported so a renamed or removed column does not linger unnoticed. + logger?.warn('orphan index', orphan) + } + + // Append-only streams are immutable logs: no tombstones, nothing to hide. + if (stream.mode !== 'append') { + const outcome = await ensureLiveView(db, names, stream, logger, options) + if (outcome !== 'kept') views.push(names.liveView(stream)) + } + } + + try { + await migrations.insertOne({ + _id: fp, + applied_at: new Date(), + detail: { streams: streams.map((s) => s.name), collections, indexes }, + }) + } catch (err) { + // A concurrent init recorded the same fingerprint first; the work was the same. + if ((err as { code?: unknown }).code !== DUPLICATE_KEY) throw err + } + + logger?.info('schema applied', { fingerprint: fp, collections, indexes, views: views.length }) + return { applied: true, fingerprint: fp, collections, indexes, views, orphans } +} diff --git a/packages/sync-mongodb/src/schema/names.ts b/packages/sync-mongodb/src/schema/names.ts new file mode 100644 index 0000000..8945834 --- /dev/null +++ b/packages/sync-mongodb/src/schema/names.ts @@ -0,0 +1,93 @@ +/** + * Collection naming. + * + * The mirror lives in the application's own database, under a prefix, rather + * than in a database of its own the way the Postgres mirror lives in a + * schema: `$lookup` cannot cross databases, and joining the mirror with the + * application's collections is the reason it exists. The prefix is what keeps + * `workos_users` from colliding with an application's `users`. + * + * Bookkeeping collections carry a second underscore (`workos__sync`), which + * marks them as the engine's own the way `_sync_state` does in Postgres. + */ +import type { ColumnDef, TableDef } from '@workos/sync-engine' + +export const DEFAULT_PREFIX = 'workos_' +export const LIVE_VIEW_SUFFIX = '_live' + +/** + * Letters, digits and underscores, not starting with a digit. No `$`, which + * MongoDB reserves; no `.`, which would make a collection name look like a + * namespace; nothing that could become `system.*`. + */ +const PREFIX = /^[A-Za-z_][A-Za-z0-9_]*$/ + +export function validatePrefix(prefix: string): string { + if (!PREFIX.test(prefix)) { + throw new Error( + `invalid collection prefix "${prefix}": expected letters, digits and underscores, not starting with a digit`, + ) + } + return prefix +} + +/** Every collection the destination touches, derived from one prefix. */ +export class Names { + readonly prefix: string + + constructor(prefix: string = DEFAULT_PREFIX) { + this.prefix = validatePrefix(prefix) + } + + /** The mirror collection for a stream: `workos_users`. */ + mirror(stream: TableDef | string): string { + return `${this.prefix}${typeof stream === 'string' ? stream : stream.name}` + } + + /** The view over a mirror collection without its tombstones: `workos_users_live`. */ + liveView(stream: TableDef | string): string { + return `${this.mirror(stream)}${LIVE_VIEW_SUFFIX}` + } + + /** The lease and the cursors, one document per `(sync_id, env)`. */ + get sync(): string { + return `${this.prefix}_sync` + } + + /** The quarantine. */ + get rejected(): string { + return `${this.prefix}_rejected` + } + + /** Applied catalog fingerprints. */ + get migrations(): string { + return `${this.prefix}_migrations` + } + + /** A reconcile staging collection, unique to one run. */ + staging(side: 'local' | 'remote', token: string): string { + return `${this.prefix}_reconcile_${side}_${token}` + } +} + +/** + * The dotted path to a payload field inside `_raw`: `_raw.role.slug`. A + * segment with a `.` in it or a leading `$` cannot be addressed this way, so + * it is refused here rather than silently indexing the wrong thing. + */ +export function fieldPath(path: ColumnDef['path']): string { + const parts = Array.isArray(path) ? path : [path] + for (const p of parts) { + if (p === '' || p.includes('.') || p.startsWith('$')) { + throw new Error( + `cannot address payload path ${JSON.stringify(parts)}: a segment is empty, contains "." or starts with "$"`, + ) + } + } + return `_raw.${parts.join('.')}` +} + +/** The catalog path of a key column, or the column name when it has no column. */ +export function keyPath(stream: TableDef, key: string): ColumnDef['path'] { + return stream.columns.find((c) => c.name === key)?.path ?? key +} diff --git a/packages/sync-mongodb/src/schema/views.ts b/packages/sync-mongodb/src/schema/views.ts new file mode 100644 index 0000000..269bf73 --- /dev/null +++ b/packages/sync-mongodb/src/schema/views.ts @@ -0,0 +1,78 @@ +/** + * Live views. + * + * Every query against a mirror collection has to remember + * `{ _deleted_at: null }`, and forgetting it is the most common query + * mistake: a seat count that includes users who left, a `$lookup` that + * resurrects a revoked membership. Fivetran and dbt users solve this with a + * view over each soft-deleted table, and MongoDB has views too. Ours are + * generated from the catalog, in the same pass as the collections, so they + * can never drift from them. + * + * Only the view layer is new. The engine keeps writing, redacting, + * reconciling and reporting freshness against the base collections, and + * the base collections remain the place to tell "deleted" from "never seen". + */ +import type { Logger, TableDef } from '@workos/sync-engine' +import type { Db, Document } from 'mongodb' +import type { Names } from './names.js' + +/** `{ _deleted_at: null }` matches a missing field as well as an explicit null. */ +export const LIVE_PIPELINE: Document[] = [{ $match: { _deleted_at: null } }] + +export interface ApplyViewOptions { + /** + * Redefine a `_live` view over the right collection whose pipeline is not + * the generated one. Off, `init()` fails and says which view it would have + * redefined. A view over some other collection, or a plain collection under + * the name, is never touched either way. + */ + repairViews?: boolean +} + +const NAMESPACE_EXISTS = 48 + +export type ViewOutcome = 'created' | 'kept' | 'rebuilt' + +/** + * Make sure `_live` exists over the stream's mirror collection with + * the generated pipeline. Idempotent; a concurrent `init` racing to create + * the same view is tolerated. + */ +export async function ensureLiveView( + db: Db, + names: Names, + stream: TableDef, + logger?: Logger, + options: ApplyViewOptions = {}, +): Promise { + const source = names.mirror(stream) + const view = names.liveView(stream) + const existing = (await db.listCollections({ name: view }, { nameOnly: false }).toArray())[0] + if (!existing) { + try { + await db.createCollection(view, { viewOn: source, pipeline: LIVE_PIPELINE }) + return 'created' + } catch (err) { + if ((err as { code?: unknown }).code !== NAMESPACE_EXISTS) throw err + // Another init got there first; judge what it made. + return ensureLiveView(db, names, stream, logger, options) + } + } + if (existing.type !== 'view' || existing.options?.['viewOn'] !== source) { + throw new Error( + `${db.databaseName}.${view} exists and is not the generated view over ${source}: ` + + `rename or drop it, or choose another prefix`, + ) + } + if (JSON.stringify(existing.options?.['pipeline']) === JSON.stringify(LIVE_PIPELINE)) return 'kept' + if (!options.repairViews) { + throw new Error( + `${db.databaseName}.${view} is a view over ${source} with a pipeline that is not the generated one; ` + + `run init --repair to redefine it, or drop it`, + ) + } + await db.command({ collMod: view, viewOn: source, pipeline: LIVE_PIPELINE }) + logger?.warn('live view redefined to the generated pipeline', { view, source }) + return 'rebuilt' +} diff --git a/packages/sync-mongodb/test/cli.test.ts b/packages/sync-mongodb/test/cli.test.ts new file mode 100644 index 0000000..58ac456 --- /dev/null +++ b/packages/sync-mongodb/test/cli.test.ts @@ -0,0 +1,372 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { SyncEngine, WorkOSClient, silentLogger } from '@workos/sync-engine' +import { main, parseHydrate, parseLeaseTtl, parseMaxPages, parseSslCa, repairRequested } from '../src/cli.js' +import { createTestDb, type TestDb } from './helpers/mongo.js' +import { FaultyDestination } from './helpers/faulty.js' +import { MongoDestination } from '../src/destination.js' +import { TEST_CA_PEM } from './helpers/ca.js' +import { FakeWorkOS, resetIds } from '../../sync-engine/test/helpers/fake-workos.js' + +describe('WORKOS_SYNC_HYDRATE', () => { + it('leaves the engine default in place when unset', () => { + expect(parseHydrate(undefined)).toBeUndefined() + expect(parseHydrate('')).toBeUndefined() + }) + + it('accepts one mode for every stream', () => { + expect(parseHydrate('off')).toBe('off') + expect(parseHydrate('always')).toBe('always') + }) + + it('accepts per-stream overrides on top of the default', () => { + expect(parseHydrate('users=always')).toEqual({ users: 'always' }) + expect(parseHydrate('users=off, organizations=off')).toEqual({ users: 'off', organizations: 'off' }) + }) + + it('rejects anything else loudly', () => { + expect(() => parseHydrate('sometimes')).toThrow(/invalid WORKOS_SYNC_HYDRATE/) + expect(() => parseHydrate('users=maybe')).toThrow(/invalid WORKOS_SYNC_HYDRATE/) + }) +}) + +describe('backfill --max-pages', () => { + it('means unbounded when absent', () => { + expect(parseMaxPages(undefined)).toBeUndefined() + }) + + it('accepts a positive integer', () => { + expect(parseMaxPages('25')).toBe(25) + expect(parseMaxPages(' 1 ')).toBe(1) + }) + + it('refuses a bare flag, zero, a fraction, and anything that is not a number', () => { + for (const bad of [true, '', '0', '-3', '1.5', 'ten']) { + expect(() => parseMaxPages(bad)).toThrow(/invalid --max-pages/) + } + }) +}) + +describe('init --repair', () => { + it('asks for repair on init and nothing else', () => { + expect(repairRequested('init', { repair: true })).toBe(true) + expect(repairRequested('init', {})).toBe(false) + // `reconcile --repair` repairs rows, not views. + expect(repairRequested('reconcile', { repair: true })).toBe(false) + }) + + it('is in the help text, which names this destination', async () => { + const out: string[] = [] + const spy = vi.spyOn(process.stdout, 'write').mockImplementation(((chunk: string | Uint8Array) => { + out.push(String(chunk)) + return true + }) as typeof process.stdout.write) + try { + expect(await main(['help'])).toBe(0) + expect(await main(['init', '--help'])).toBe(0) + } finally { + spy.mockRestore() + } + const help = out.join('') + expect(help).toContain('init [--repair]') + expect(help).toContain('workos-sync-mongodb') + expect(help).toContain('--prefix') + expect(help).toContain('WORKOS_SYNC_MONGODB_TLS_CA') + expect(help).not.toContain('--print-sql') + expect(help).not.toContain('Postgres') + }) +}) + +describe('WORKOS_SYNC_LEASE_TTL_MS', () => { + it('leaves the engine default in place when unset or empty', () => { + expect(parseLeaseTtl(undefined)).toBeUndefined() + expect(parseLeaseTtl('')).toBeUndefined() + expect(parseLeaseTtl(' ')).toBeUndefined() + }) + + it('accepts a number of milliseconds', () => { + expect(parseLeaseTtl('250')).toBe(250) + }) + + it('refuses a value that would expire on arrival', () => { + for (const bad of ['0', '-1', '0.5', 'abc', 'Infinity']) { + expect(() => parseLeaseTtl(bad)).toThrow(/invalid WORKOS_SYNC_LEASE_TTL_MS/) + } + }) +}) + +describe('WORKOS_SYNC_MONGODB_TLS_CA', () => { + const pem = TEST_CA_PEM + + it('leaves the driver defaults in place when unset or empty', () => { + expect(parseSslCa(undefined)).toBeUndefined() + expect(parseSslCa('')).toBeUndefined() + expect(parseSslCa(' ')).toBeUndefined() + }) + + it('accepts PEM as is, as a bundle, with escaped newlines, or as base64', () => { + expect(parseSslCa(pem)).toBe(pem.trim()) + expect(parseSslCa(pem + pem)).toBe((pem + pem).trim()) + expect(parseSslCa(pem.trim().replace(/\n/g, '\\n'))).toBe(pem.trim()) + const b64 = Buffer.from(pem).toString('base64') + expect(parseSslCa(b64)).toBe(pem) + expect(parseSslCa(b64.replace(/(.{20})/g, '$1\n'))).toBe(pem) + }) + + it('refuses anything that is not a certificate, damaged ones included', () => { + const lines = pem.trim().split('\n') + const header = lines[0]! + const footer = lines[lines.length - 1]! + const truncated = [header, ...lines.slice(1, 4), footer].join('\n') + for (const bad of [ + '/etc/ssl/ca.crt', + 'not a cert', + Buffer.from('nope').toString('base64'), + header, + truncated, + Buffer.from(truncated).toString('base64'), + ]) { + expect(() => parseSslCa(bad)).toThrow(/invalid WORKOS_SYNC_MONGODB_TLS_CA/) + } + }) +}) + +/** Every command below is run through `main` against a real server. */ +describe('commands', () => { + let t: TestDb + let fake: FakeWorkOS + let out: string[] + const env = { ...process.env } + + beforeEach(async () => { + t = await createTestDb() + resetIds() + fake = new FakeWorkOS() + out = [] + vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => { + out.push(String(chunk)) + return true + }) + process.env['WORKOS_API_KEY'] = 'sk_test_1' + delete process.env['DATABASE_URL'] + }) + afterEach(async () => { + vi.restoreAllMocks() + process.env = { ...env } + await t.close() + }) + + const lastJson = () => JSON.parse(out[out.length - 1]!) as Record + const engineOver = (destination: MongoDestination, streams: string[]) => + new SyncEngine( + { + destination, + environments: [{ name: 'prod', apiKey: 'sk_test_1' }], + streams, + hydrate: 'off', + logger: silentLogger, + lagWindowMs: 0, + }, + { + clientFor: () => + new WorkOSClient({ + apiKey: 'sk_test_1', + apiUrl: 'http://fake.workos.local', + fetchImpl: fake.fetchImpl, + pageSize: 100, + }), + }, + ) + + it('init creates the schema from the environment and the flags, and is quiet when nothing changed', async () => { + expect( + await main(['init', '--database', t.name, '--streams', 'users', '--prefix', 'mirror_'], { client: t.client }), + ).toBe(0) + expect(out.join('')).toContain('schema ready') + const names = (await t.db.listCollections().toArray()) + .map((c) => c.name) + .filter((n) => !n.startsWith('system.')) + .sort() + expect(names).toEqual([ + 'mirror__migrations', + 'mirror__rejected', + 'mirror__sync', + 'mirror_events', + 'mirror_users', + 'mirror_users_live', + ]) + // The views it made are reported the first time, not the second. + expect(out.some((o) => o.includes('mirror_users_live'))).toBe(true) + out.length = 0 + expect( + await main(['init', '--database', t.name, '--streams', 'users', '--prefix', 'mirror_'], { client: t.client }), + ).toBe(0) + expect(out.join('')).toBe('schema ready\n') + // The environment works where the flags do. + process.env['WORKOS_SYNC_DATABASE'] = t.name + process.env['WORKOS_SYNC_PREFIX'] = 'other_' + expect(await main(['init', '--streams', 'users'], { client: t.client })).toBe(0) + expect(await t.db.listCollections({ name: 'other_users' }).toArray()).toHaveLength(1) + }) + + it('init --repair redefines a generated view whose pipeline changed, and plain init refuses to', async () => { + expect(await main(['init', '--database', t.name, '--streams', 'users'], { client: t.client })).toBe(0) + await t.db.command({ + collMod: 'workos_users_live', + viewOn: 'workos_users', + pipeline: [{ $match: { _env: 'prod' } }], + }) + // The fingerprint is recorded, so a plain init does not look; a catalog change makes it look. + await expect( + main(['init', '--database', t.name, '--streams', 'users,organizations'], { client: t.client }), + ).rejects.toThrow(/init --repair/) + expect( + await main(['init', '--repair', '--database', t.name, '--streams', 'users,organizations'], { client: t.client }), + ).toBe(0) + const info = (await t.db.listCollections({ name: 'workos_users_live' }, { nameOnly: false }).toArray())[0]! + expect(info.options?.['pipeline']).toEqual([{ $match: { _deleted_at: null } }]) + }) + + it('needs a connection string, a key, and a known stream', async () => { + await expect(main(['init'])).rejects.toThrow(/DATABASE_URL/) + delete process.env['WORKOS_API_KEY'] + await expect(main(['init', '--database', t.name], { client: t.client })).rejects.toThrow(/WORKOS_API_KEY/) + process.env['WORKOS_API_KEY'] = 'sk_test_1' + await expect(main(['init', '--database', t.name, '--streams', 'customers'], { client: t.client })).rejects.toThrow( + /unknown stream: customers/, + ) + await expect(main(['init', '--database', t.name, '--prefix', 'bad prefix'], { client: t.client })).rejects.toThrow( + /invalid collection prefix/, + ) + }) + + it('streams lists the catalog without touching a database; an unknown command exits 1', async () => { + expect(await main(['streams'])).toBe(0) + const rows = JSON.parse(out.join('')) as { stream: string; default: string }[] + expect(rows.find((r) => r.stream === 'users')).toMatchObject({ default: 'yes' }) + const err: string[] = [] + vi.spyOn(process.stderr, 'write').mockImplementation((chunk) => { + err.push(String(chunk)) + return true + }) + expect(await main(['frobnicate', '--database', t.name], { client: t.client })).toBe(1) + expect(err.join('')).toContain('unknown command: frobnicate') + }) + + it('backfill, status and redact report what they did', async () => { + const user = fake.createUser({ email: 'ada@example.com' }) + const engine = engineOver(new MongoDestination({ client: t.client, database: t.name }), ['users']) + expect(await main(['backfill'], { engine })).toBe(0) + expect( + (lastJson() as unknown as { stream: string; complete: boolean }[]).find((r) => r.stream === 'users'), + ).toMatchObject({ complete: true }) + expect(await t.db.collection('workos_users').countDocuments()).toBe(1) + + expect(await main(['status'], { engine })).toBe(0) + const status = lastJson() as Record< + string, + { started: boolean; rejected: number; streams: { stream: string; rows: number }[] } + > + expect(status['prod']!.started).toBe(true) + expect(status['prod']!.rejected).toBe(0) + expect(status['prod']!.streams.find((s) => s.stream === 'users')).toMatchObject({ rows: 1 }) + + await expect(main(['redact'], { engine })).rejects.toThrow(/--user/) + expect(await main(['redact', '--user', user['id'] as string], { engine })).toBe(0) + expect(lastJson()).toEqual([{ table: 'users', rows: 1 }]) + expect(await t.db.collection('workos_users').countDocuments()).toBe(0) + }) + + it('tick exits 3 while a listener holds the lease', async () => { + fake.createUser() + const other = new MongoDestination({ client: t.client, database: t.name }) + await other.init([]) + await other.acquireLease('prod', 'listener', 60_000) + const err: string[] = [] + vi.spyOn(process.stderr, 'write').mockImplementation((chunk) => { + err.push(String(chunk)) + return true + }) + const engine = engineOver(new MongoDestination({ client: t.client, database: t.name }), ['users']) + expect(await main(['tick'], { engine })).toBe(3) + expect(err.join('')).toContain('listener') + await other.releaseLease('prod', 'listener') + expect( + await main(['tick'], { + engine: engineOver(new MongoDestination({ client: t.client, database: t.name }), ['users']), + }), + ).toBe(0) + expect(lastJson()).toMatchObject({ processed: 1 }) + }) + + it('rejected exits 2 while quarantined events exist, and 0 once they replay', async () => { + fake.createUser({ email: 'good@example.com' }) + const bad = fake.createUser({ email: 'bad@example.com' }) + const destination = new FaultyDestination({ client: t.client, database: t.name }) + destination.poison = bad['id'] as string + const engine = engineOver(destination, ['users']) + + // Nothing quarantined yet: a clean exit, so a scheduled check stays quiet. + expect(await main(['rejected'], { engine })).toBe(0) + + // Two polls: first sighting, then quarantine. + await engine.tick() + await engine.tick() + expect(await main(['rejected'], { engine })).toBe(2) + const listed = lastJson() as { prod: { eventId: string; eventType: string; error: string }[] } + expect(listed.prod).toHaveLength(1) + expect(listed.prod[0]!.eventType).toBe('user.created') + expect(listed.prod[0]!.error).toMatch(/Document failed validation/) + + // Still broken: a replay reports it and keeps the non-zero exit. + expect(await main(['rejected', '--replay'], { engine })).toBe(2) + expect((lastJson() as { stillRejected: unknown[] }).stillRejected).toHaveLength(1) + + // Fixed: replaying one by id clears it, and the check goes quiet again. + destination.poison = null + expect(await main(['rejected', '--replay', '--id', listed.prod[0]!.eventId], { engine })).toBe(0) + expect(lastJson()).toEqual({ replayed: 1, stillRejected: [] }) + expect(await main(['rejected'], { engine })).toBe(0) + + const users = await t.db.collection('workos_users').find().sort({ '_raw.email': 1 }).toArray() + expect(users.map((u) => u['_raw']['email'])).toEqual(['bad@example.com', 'good@example.com']) + }) + + it('reconcile exits 2 on drift, 0 after a repair, and slices with --max-pages', async () => { + for (let i = 0; i < 250; i++) fake.createUser() + const gone = fake.createUser({ email: 'gone@example.com' }) + const engine = engineOver(new MongoDestination({ client: t.client, database: t.name }), ['users', 'invitations']) + await engine.backfill() + fake.users.delete(gone['id'] as string) + + const diffs = () => + ( + lastJson() as { + diffs: Record + } + ).diffs['prod']! + // A slice that stops short is the outcome asked for, not an error, but drift is. + expect(await main(['reconcile', '--max-pages', '2'], { engine })).toBe(2) + expect(diffs().find((d) => d.stream === 'users')).toMatchObject({ + pages: 3, + complete: true, + localOnly: [gone['id']], + }) + expect(diffs().find((d) => d.stream === 'invitations')).toMatchObject({ pages: 0, complete: false }) + + expect(await main(['reconcile', '--repair'], { engine })).toBe(0) + expect(await main(['reconcile'], { engine })).toBe(0) + expect(await t.db.collection('workos_users').countDocuments({ _deleted_at: { $type: 'date' } })).toBe(1) + }) + + it('refresh slices with --max-pages and converges with --resume', async () => { + for (let i = 0; i < 250; i++) fake.createUser() + const engine = engineOver(new MongoDestination({ client: t.client, database: t.name }), ['users']) + await engine.backfill() + + const results = () => lastJson() as unknown as { stream: string; pages: number; complete: boolean }[] + expect(await main(['refresh', '--max-pages', '2'], { engine })).toBe(0) + expect(results().find((r) => r.stream === 'users')).toMatchObject({ pages: 2, complete: false }) + expect(await main(['refresh', '--resume'], { engine })).toBe(0) + expect(results().find((r) => r.stream === 'users')).toMatchObject({ pages: 1, complete: true }) + }) +}) diff --git a/packages/sync-mongodb/test/destination.test.ts b/packages/sync-mongodb/test/destination.test.ts new file mode 100644 index 0000000..3727d9c --- /dev/null +++ b/packages/sync-mongodb/test/destination.test.ts @@ -0,0 +1,106 @@ +import { describe, it, expect } from 'vitest' +import { MongoClient } from 'mongodb' +import { MongoDestination } from '../src/destination.js' +import { clientOptions, databaseFromUrl, APPLICATION_NAME } from '../src/db/client.js' +import { Names, fieldPath, validatePrefix } from '../src/schema/names.js' +import { TEST_CA_PEM } from './helpers/ca.js' + +/** Enough of a client for the constructor, which never connects. */ +const stub = new MongoClient('mongodb://db.example.internal/app') + +describe('MongoDestination options', () => { + it('needs a connection string or a client', () => { + expect(() => new MongoDestination({})).toThrow(/url or client is required/) + }) + + it('takes a CA alongside a connection string', () => { + // Constructing does not connect, so no server is needed to accept the option. + expect( + () => new MongoDestination({ url: 'mongodb://db.example.internal/mirror', sslCa: TEST_CA_PEM }), + ).not.toThrow() + }) + + it('refuses a CA with a pre-built client, which carries its own TLS settings', () => { + expect(() => new MongoDestination({ client: stub, sslCa: TEST_CA_PEM })).toThrow(/sslCa applies to url/) + // With both given, the client is what gets used and `init()` never + // connects, so accepting the CA here would mean silently dropping it. + expect( + () => new MongoDestination({ client: stub, url: 'mongodb://db.example.internal/mirror', sslCa: TEST_CA_PEM }), + ).toThrow(/sslCa applies to url/) + }) + + it('reads the database off the connection string unless told otherwise', () => { + expect(new MongoDestination({ url: 'mongodb://db.example.internal/app?retryWrites=true' }).databaseName).toBe('app') + expect(new MongoDestination({ url: 'mongodb://db.example.internal/app', database: 'mirror' }).databaseName).toBe( + 'mirror', + ) + // Nothing named: decided at init, where the client's default applies. + expect(new MongoDestination({ url: 'mongodb://db.example.internal' }).databaseName).toBeUndefined() + }) + + it('validates the prefix up front', () => { + expect(() => new MongoDestination({ url: 'mongodb://x/app', prefix: 'work os' })).toThrow( + /invalid collection prefix/, + ) + expect(() => new MongoDestination({ url: 'mongodb://x/app', prefix: '$w' })).toThrow(/invalid collection prefix/) + expect(() => new MongoDestination({ url: 'mongodb://x/app', prefix: '1w' })).toThrow(/invalid collection prefix/) + expect(new MongoDestination({ url: 'mongodb://x/app', prefix: 'mirror_' }).prefix).toBe('mirror_') + expect(new MongoDestination({ url: 'mongodb://x/app' }).prefix).toBe('workos_') + }) +}) + +describe('databaseFromUrl', () => { + it('reads the path of plain, SRV, multi-host and credentialed strings', () => { + expect(databaseFromUrl('mongodb://localhost:27017/app')).toBe('app') + expect(databaseFromUrl('mongodb+srv://cluster0.example.net/app?retryWrites=true&w=majority')).toBe('app') + expect(databaseFromUrl('mongodb://a:27017,b:27017,c:27017/app?replicaSet=rs0')).toBe('app') + expect(databaseFromUrl('mongodb://user:p%40ss@localhost/my%20app')).toBe('my app') + }) + + it('says when there is none', () => { + expect(databaseFromUrl('mongodb://localhost:27017')).toBeUndefined() + expect(databaseFromUrl('mongodb://localhost:27017/')).toBeUndefined() + expect(databaseFromUrl('mongodb://localhost:27017/?appName=x')).toBeUndefined() + }) +}) + +describe('clientOptions', () => { + it('introduces the connection as the engine unless the string names an appName', () => { + expect(clientOptions('mongodb://localhost/app')).toEqual({ appName: APPLICATION_NAME }) + expect(clientOptions('mongodb://localhost/app?appName=mine')).toEqual({}) + expect(clientOptions('mongodb://localhost/app?w=majority&appname=mine')).toEqual({}) + }) + + it('requires TLS when a CA is given, and refuses a string that turns TLS off', () => { + expect(clientOptions('mongodb://localhost/app', { sslCa: TEST_CA_PEM })).toMatchObject({ + tls: true, + ca: TEST_CA_PEM, + }) + expect(() => clientOptions('mongodb://localhost/app?tls=false', { sslCa: TEST_CA_PEM })).toThrow(/turns TLS off/) + expect(() => clientOptions('mongodb://localhost/app?ssl=false&w=1', { sslCa: TEST_CA_PEM })).toThrow( + /turns TLS off/, + ) + expect(() => clientOptions('mongodb://localhost/app?tls=true', { sslCa: TEST_CA_PEM })).not.toThrow() + }) +}) + +describe('Names', () => { + it('derives every collection from the prefix', () => { + const n = new Names('workos_') + expect(n.mirror('users')).toBe('workos_users') + expect(n.liveView('users')).toBe('workos_users_live') + expect(n.sync).toBe('workos__sync') + expect(n.rejected).toBe('workos__rejected') + expect(n.migrations).toBe('workos__migrations') + expect(n.staging('local', 'abc')).toBe('workos__reconcile_local_abc') + expect(validatePrefix('ok_1')).toBe('ok_1') + }) + + it('addresses payload fields under _raw and refuses what dotted paths cannot say', () => { + expect(fieldPath('id')).toBe('_raw.id') + expect(fieldPath(['role', 'slug'])).toBe('_raw.role.slug') + expect(() => fieldPath(['a.b'])).toThrow(/cannot address/) + expect(() => fieldPath(['$a'])).toThrow(/cannot address/) + expect(() => fieldPath([''])).toThrow(/cannot address/) + }) +}) diff --git a/packages/sync-mongodb/test/engine.test.ts b/packages/sync-mongodb/test/engine.test.ts new file mode 100644 index 0000000..fe1f90f --- /dev/null +++ b/packages/sync-mongodb/test/engine.test.ts @@ -0,0 +1,1083 @@ +/** + * Whole-engine tests: backfill, tail, drift and repair, driven against the + * fake WorkOS API and a real MongoDB. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { + SyncEngine, + WorkOSClient, + LeaseHeldError, + StateStore, + silentLogger, + type Destination, +} from '@workos/sync-engine' +import { MongoNetworkError } from 'mongodb' +import { createTestDb, type TestDb } from './helpers/mongo.js' +import { FakeWorkOS, resetIds, id } from '../../sync-engine/test/helpers/fake-workos.js' +import { MongoDestination } from '../src/destination.js' +import type { MirrorDoc } from '../src/db/upsert.js' +import type { SyncDoc } from '../src/db/lease.js' +import { FaultyDestination, captureLogger } from './helpers/faulty.js' + +const TIER1 = [ + 'users', + 'organizations', + 'organization_memberships', + 'organization_domains', + 'invitations', + 'events', + 'auth_attempts', +] + +function makeEngine( + t: TestDb, + fake: FakeWorkOS, + streams = TIER1, + extra = {}, + /** Runs before every API call, so a test can make something happen mid-poll. */ + beforeFetch?: () => Promise, +) { + const fetchImpl: typeof fetch = beforeFetch + ? async (input, init) => { + await beforeFetch() + return fake.fetchImpl(input, init) + } + : fake.fetchImpl + return new SyncEngine( + { + destination: new MongoDestination({ client: t.client, database: t.name }), + environments: [{ name: 'prod', apiKey: 'sk_test_1' }], + streams, + logger: silentLogger, + lagWindowMs: 0, + ...extra, + }, + { + clientFor: () => + new WorkOSClient({ + apiKey: 'sk_test_1', + apiUrl: 'http://fake.workos.local', + fetchImpl, + pageSize: 100, + }), + }, + ) +} + +const coll = (t: TestDb, stream: string) => t.db.collection(`workos_${stream}`) +const count = (t: TestDb, stream: string, filter = {}) => coll(t, stream).countDocuments(filter) +const raws = async (t: TestDb, stream: string, filter = {}, sort: Record = { '_raw.id': 1 }) => + (await coll(t, stream).find(filter).sort(sort).toArray()).map((d) => d._raw) +const emails = async (t: TestDb) => (await raws(t, 'users', {}, { '_raw.email': 1 })).map((r) => r['email'] as string) + +describe('sync engine', () => { + let t: TestDb + let fake: FakeWorkOS + + beforeEach(async () => { + t = await createTestDb() + resetIds() + fake = new FakeWorkOS() + }) + afterEach(async () => { + await t.close() + }) + + it('backfills existing state, including the field events omit', async () => { + const org = fake.createOrganization({ name: 'Acme' }) + const user = fake.createUser({ email: 'ada@acme.com' }) + fake.createMembership(user['id'] as string, org['id'] as string, 'admin') + + const engine = makeEngine(t, fake) + await engine.backfill() + + expect(await count(t, 'users')).toBe(1) + expect(await count(t, 'organizations')).toBe(1) + expect(await count(t, 'organization_memberships')).toBe(1) + + // Every listing the backfill made says which phase it was. + const listings = fake.requestLog + .map((path, i) => [path, fake.userAgents[i]!] as const) + .filter(([path]) => !path.startsWith('/events')) + expect(listings.length).toBeGreaterThan(0) + expect(listings.every(([, ua]) => ua.endsWith('(phase=backfill)'))).toBe(true) + + const [u] = await raws(t, 'users') + expect(u!['email']).toBe('ada@acme.com') + expect(u!['external_id']).toMatch(/^ext_/) + expect(u!['metadata']).toEqual({ plan: 'pro' }) + + const [o] = await raws(t, 'organizations') + // Deprecated fields are left out of events; only the REST object carries it. + expect(o!['allow_profiles_outside_organization']).toBe(false) + + const [m] = await raws(t, 'organization_memberships') + expect((m!['role'] as { slug: string }).slug).toBe('admin') + // Every document carries the bookkeeping fields and the environment label. + const doc = (await coll(t, 'users').findOne({}))! + expect(doc._env).toBe('prod') + expect(doc._updated_at).toBeInstanceOf(Date) + expect(doc._synced_at).toBeInstanceOf(Date) + }) + + it('captures the stream position before backfilling so nothing is missed', async () => { + fake.createUser() + const engine = makeEngine(t, fake) + await engine.backfill() + + const before = await engine.status() + expect(before['prod']!.started).toBe(true) + expect(before['prod']!.rangeStart).not.toBeNull() + expect(before['prod']!.cursor).toBeNull() + + await engine.tick() + const after = await engine.status() + expect(after['prod']!.cursor).not.toBeNull() + expect(after['prod']!.rangeStart).toBeNull() + }) + + it('stops at the page budget and resumes from the checkpoint', async () => { + for (let i = 0; i < 250; i++) fake.createUser() + const engine = makeEngine(t, fake, ['users', 'invitations']) + + const first = await engine.backfill({ maxPages: 2 }) + expect(first.find((r) => r.stream === 'users')).toMatchObject({ pages: 2, complete: false }) + expect(await count(t, 'users')).toBe(200) + expect(first.find((r) => r.stream === 'invitations')).toMatchObject({ pages: 0, complete: false }) + + const before = fake.requestLog.length + const second = await engine.backfill({ resume: true, maxPages: 10 }) + expect(second.find((r) => r.stream === 'users')).toMatchObject({ pages: 1, complete: true }) + expect(second.find((r) => r.stream === 'invitations')).toMatchObject({ pages: 1, complete: true }) + expect(await count(t, 'users')).toBe(250) + const users = fake.requestLog.slice(before).filter((r) => r.startsWith('/user_management/users')) + expect(users).toHaveLength(1) + expect(users[0]).toContain('after=') + }) + + /** The real engine over a destination with one write fault injected; see helpers/faulty.ts. */ + const faultyEngine = (destination: FaultyDestination, streams: string[], logger = silentLogger) => + new SyncEngine( + { + destination, + environments: [{ name: 'prod', apiKey: 'sk_test_1' }], + streams, + hydrate: 'off', + logger, + lagWindowMs: 0, + }, + { + clientFor: () => + new WorkOSClient({ + apiKey: 'sk_test_1', + apiUrl: 'http://fake.workos.local', + fetchImpl: fake.fetchImpl, + pageSize: 100, + }), + }, + ) + + it('holds a child stream back while its parent is incomplete, so a resume fetches the missing children', async () => { + const orgA = fake.createOrganization() + const orgB = fake.createOrganization() + fake.createMembership(fake.createUser()['id'] as string, orgA['id'] as string) + fake.createMembership(fake.createUser()['id'] as string, orgB['id'] as string) + const destination = new FaultyDestination({ client: t.client, database: t.name }) + destination.poison = orgB['id'] as string + const { logger, lines } = captureLogger() + const engine = faultyEngine(destination, ['users', 'organizations', 'organization_memberships'], logger) + + const first = await engine.backfill() + expect(first.find((r) => r.stream === 'organizations')).toMatchObject({ complete: false }) + const held = first.find((r) => r.stream === 'organization_memberships')! + expect(held).toMatchObject({ pages: 0, complete: false }) + expect(held.skippedReason).toContain('organizations') + expect(lines.map((l) => l.msg)).not.toContain('backfill paused: page budget spent; resume to continue') + + destination.poison = null + const second = await engine.backfill({ resume: true }) + expect(second.find((r) => r.stream === 'organizations')).toMatchObject({ complete: true }) + expect(second.find((r) => r.stream === 'organization_memberships')).toMatchObject({ complete: true }) + expect(await count(t, 'organization_memberships')).toBe(2) + }) + + it('applies live events after backfill', async () => { + const user = fake.createUser({ email: 'before@example.com' }) + const engine = makeEngine(t, fake) + await engine.backfill() + + fake.updateUser(user['id'] as string, { email: 'after@example.com' }) + const org = fake.createOrganization({ name: 'New Co' }) + await engine.tick() + + expect(await emails(t)).toEqual(['after@example.com']) + expect(await count(t, 'organizations')).toBe(1) + expect((await raws(t, 'organizations'))[0]!['name']).toBe('New Co') + expect(org['id']).toBeTruthy() + }) + + it('does not drop backfilled fields when an event arrives', async () => { + // Events leave out deprecated fields, allow_profiles_outside_organization + // here. The event write has to merge into the backfilled document, not replace it. + const org = fake.createOrganization({ name: 'Acme' }) + const engine = makeEngine(t, fake) + await engine.backfill() + await engine.tick() + + fake.updateOrganization(org['id'] as string, { name: 'Acme Corp' }) + await engine.tick() + + const [row] = await raws(t, 'organizations') + expect(row!['name']).toBe('Acme Corp') + expect(row!['allow_profiles_outside_organization']).toBe(false) + }) + + it('is idempotent when the same events are delivered twice', async () => { + fake = new FakeWorkOS({ duplicateEvents: true }) + fake.createUser() + fake.createOrganization() + + const engine = makeEngine(t, fake) + await engine.tick() + await engine.tick() + + expect(await count(t, 'users')).toBe(1) + expect(await count(t, 'organizations')).toBe(1) + }) + + it('converges regardless of the order events are applied in', async () => { + const user = fake.createUser({ email: 'v1@example.com' }) + fake.updateUser(user['id'] as string, { email: 'v2@example.com' }) + fake.updateUser(user['id'] as string, { email: 'v3@example.com' }) + + // Serve the batch reversed, as out-of-order delivery would. + const shuffled = new FakeWorkOS({ shuffleEvents: true }) + shuffled.users = fake.users + shuffled.events = fake.events + + const engine = makeEngine(t, shuffled) + await engine.tick() + + // The staleness gate means the newest state wins whatever the arrival order. + expect(await emails(t)).toEqual(['v3@example.com']) + }) + + it('resolves events that share a timestamp as last-writer-wins', async () => { + const user = fake.createUser({ email: 'v1@example.com' }) + const uid = user['id'] as string + const frozen = new Date(Date.now() - 30_000).toISOString() + fake.users.set(uid, { ...fake.users.get(uid)!, email: 'tie-a@example.com', updated_at: frozen }) + fake.events.push({ + id: `event_tie_a`, + event: 'user.updated', + data: fake.userEventPayload({ ...fake.users.get(uid)!, email: 'tie-a@example.com', updated_at: frozen }), + created_at: frozen, + }) + fake.events.push({ + id: `event_tie_b`, + event: 'user.updated', + data: fake.userEventPayload({ ...fake.users.get(uid)!, email: 'tie-b@example.com', updated_at: frozen }), + created_at: frozen, + }) + + const engine = makeEngine(t, fake) + await engine.tick() + + expect(await emails(t)).toEqual(['tie-b@example.com']) + }) + + it('tombstones a deleted user: the live view drops it while the collection keeps it', async () => { + const gone = fake.createUser({ email: 'gone@example.com' }) + fake.createUser({ email: 'stays@example.com' }) + const engine = makeEngine(t, fake) + await engine.backfill() + await engine.tick() + expect(await count(t, 'users')).toBe(2) + expect(await count(t, 'users_live')).toBe(2) + + fake.deleteUser(gone['id'] as string) + await engine.tick() + + // No `_deleted_at: null` needed: that is the point of the view. + expect(await count(t, 'users')).toBe(2) + expect(await count(t, 'users', { _deleted_at: null })).toBe(1) + expect(await count(t, 'users_live')).toBe(1) + expect((await raws(t, 'users_live'))[0]!['email']).toBe('stays@example.com') + }) + + it('archives subscribed events that have no mirror collection, instead of throwing', async () => { + fake.createUser() + fake.emitUnknown() + + const engine = makeEngine(t, fake, TIER1, { + eventTypes: ['user.created', 'user.updated', 'user.deleted', 'quantum_widget.entangled'], + }) + await engine.tick() + + const names = (await raws(t, 'events')).map((r) => r['event']) + expect(names).toContain('user.created') + expect(names).toContain('quantum_widget.entangled') + // It was archived only — no mirror document was invented for it. + expect(await count(t, 'users')).toBe(1) + }) + + it('subscribes only to the event types its selected streams declare', async () => { + const usersOnly = makeEngine(t, fake, ['users', 'events']) + expect(usersOnly.eventTypes).toEqual(['user.created', 'user.deleted', 'user.updated']) + + const withAuth = makeEngine(t, fake, TIER1) + expect(withAuth.eventTypes).toContain('authentication.password_failed') + expect(withAuth.eventTypes).toContain('organization_membership.created') + expect(withAuth.eventTypes).not.toContain('dsync.user.created') + }) + + it('records sign-in attempts that exist nowhere else in the API', async () => { + const user = fake.createUser() + fake.emitAuthAttempt(user['id'] as string, true) + fake.emitAuthAttempt(user['id'] as string, false) + fake.emitAuthAttempt(user['id'] as string, false) + + const engine = makeEngine(t, fake) + await engine.tick() + + const byStatus = await coll(t, 'auth_attempts') + .aggregate<{ _id: string; n: number }>([ + { $group: { _id: '$_raw.status', n: { $sum: 1 } } }, + { $sort: { _id: 1 } }, + ]) + .toArray() + expect(byStatus).toEqual([ + { _id: 'failed', n: 2 }, + { _id: 'succeeded', n: 1 }, + ]) + const failed = (await coll(t, 'auth_attempts').findOne({ '_raw.status': 'failed' }))! + expect((failed._raw['error'] as { code: string }).code).toBe('invalid_credentials') + }) + + it('resumes from the stored cursor rather than replaying', async () => { + fake.createUser() + const engine = makeEngine(t, fake) + const first = await engine.tick() + expect(first.processed).toBe(1) + + const second = await engine.tick() + expect(second.processed).toBe(0) + + fake.createUser() + const third = await engine.tick() + expect(third.processed).toBe(1) + }) + + it('keeps two environments apart in one database', async () => { + const staging = new FakeWorkOS() + staging.createUser({ email: 'staging@example.com' }) + fake.createUser({ email: 'prod@example.com' }) + + const engine = new SyncEngine( + { + destination: new MongoDestination({ client: t.client, database: t.name }), + environments: [ + { name: 'prod', apiKey: 'sk_live' }, + { name: 'staging', apiKey: 'sk_test' }, + ], + streams: TIER1, + logger: silentLogger, + lagWindowMs: 0, + }, + { + clientFor: (env) => + new WorkOSClient({ + apiKey: env.apiKey, + apiUrl: 'http://fake.workos.local', + fetchImpl: env.name === 'prod' ? fake.fetchImpl : staging.fetchImpl, + }), + }, + ) + await engine.backfill() + + const docs = await coll(t, 'users').find().sort({ _env: 1 }).toArray() + expect(docs.map((d) => [d._env, d._raw['email']])).toEqual([ + ['prod', 'prod@example.com'], + ['staging', 'staging@example.com'], + ]) + + // Cursors are per environment and are not comparable across them. + const status = await engine.status() + expect(Object.keys(status)).toEqual(['prod', 'staging']) + }) + + it('always includes the events archive, whatever streams are selected', async () => { + fake.createUser() + const engine = makeEngine(t, fake, ['users']) + expect(engine.streams.map((s) => s.name)).toEqual(['users', 'events']) + + const r = await engine.tick() + expect(r.processed).toBe(1) + expect(await count(t, 'events')).toBe(1) + }) + + it('rejects a stream name it does not know', () => { + expect(() => makeEngine(t, fake, ['users', 'customers'])).toThrow(/unknown stream: customers/) + }) + + it('turns an extra column into an index on _raw without touching the catalog', async () => { + fake.createUser({ metadata: { plan: 'pro', region: 'eu' } }) + const engine = makeEngine(t, fake, TIER1, { + extraColumns: { + users: [{ name: 'region', type: 'text', path: ['metadata', 'region'], index: true }], + }, + }) + await engine.backfill() + + const indexes = await coll(t, 'users').listIndexes().toArray() + expect(indexes.map((i) => i.name)).toContain('_env_1__raw.metadata.region_1') + // There is no projected field to keep in step: the value is read where it lives. + expect(await count(t, 'users', { _env: 'prod', '_raw.metadata.region': 'eu' })).toBe(1) + // Every index leads with _env, so a query that names the environment is served by it. + const plan = await coll(t, 'users').find({ _env: 'prod', '_raw.metadata.region': 'eu' }).explain('queryPlanner') + expect(JSON.stringify(plan['queryPlanner']['winningPlan'])).toContain('_env_1__raw.metadata.region_1') + }) + + it('keeps the hydrated fields when the create and a later update share a tick', async () => { + const engine = makeEngine(t, fake, TIER1, { hydrate: { organizations: 'on_create' } }) + await engine.backfill() + + const org = fake.createOrganization({ name: 'Both' }) + fake.updateOrganization(org['id'] as string, { name: 'Both Corp' }) + expect((await engine.tick()).processed).toBe(2) + + const [row] = await raws(t, 'organizations', { '_raw.id': org['id'] }) + expect(row).toMatchObject({ name: 'Both Corp', allow_profiles_outside_organization: false }) + }) + + it('hydrates on create so a new organization has its full payload immediately', async () => { + const engine = makeEngine(t, fake, TIER1, { hydrate: { organizations: 'on_create' } }) + await engine.backfill() + + fake.createOrganization({ name: 'Fresh' }) + await engine.tick() + + const [row] = await raws(t, 'organizations', { '_raw.name': 'Fresh' }) + expect(row!['allow_profiles_outside_organization']).toBe(false) + expect(fake.userAgents.at(-1)).toMatch(/\(phase=hydrate\)$/) + + // Without hydration the field the event omits stays absent until a refresh. + const plain = makeEngine(t, fake) + fake.createOrganization({ name: 'Raw' }) + await plain.tick() + const [raw] = await raws(t, 'organizations', { '_raw.name': 'Raw' }) + expect(raw!['allow_profiles_outside_organization']).toBeUndefined() + }) + + it('looks up mirrored identity data next to application collections', async () => { + // The point of the whole exercise. + const org = fake.createOrganization({ name: 'Acme' }) + const u1 = fake.createUser({ email: 'active@acme.com' }) + const u2 = fake.createUser({ email: 'dormant@acme.com' }) + fake.createMembership(u1['id'] as string, org['id'] as string) + fake.createMembership(u2['id'] as string, org['id'] as string) + + const engine = makeEngine(t, fake) + await engine.backfill() + await engine.tick() + fake.emitAuthAttempt(u1['id'] as string, true) + await engine.tick() + + await t.db.collection('app_events').insertOne({ user_id: u1['id'], occurred_at: new Date() }) + + const rows = await t.db + .collection('workos_organization_memberships_live') + .aggregate<{ email: string; sign_ins: number; app_events: number }>([ + { $match: { _env: 'prod', '_raw.organization_id': org['id'] } }, + { + $lookup: { + from: 'workos_users_live', + let: { uid: '$_raw.user_id', env: '$_env' }, + pipeline: [{ $match: { $expr: { $and: [{ $eq: ['$_raw.id', '$$uid'] }, { $eq: ['$_env', '$$env'] }] } } }], + as: 'user', + }, + }, + { $unwind: '$user' }, + { + $lookup: { + from: 'workos_auth_attempts', + localField: '_raw.user_id', + foreignField: '_raw.user_id', + as: 'attempts', + }, + }, + { $lookup: { from: 'app_events', localField: '_raw.user_id', foreignField: 'user_id', as: 'app' } }, + { + $project: { + _id: 0, + email: '$user._raw.email', + sign_ins: { $size: '$attempts' }, + app_events: { $size: '$app' }, + }, + }, + { $sort: { email: 1 } }, + ]) + .toArray() + expect(rows).toEqual([ + { email: 'active@acme.com', sign_ins: 1, app_events: 1 }, + { email: 'dormant@acme.com', sign_ins: 0, app_events: 0 }, + ]) + }) +}) + +describe('reconciliation', () => { + let t: TestDb + let fake: FakeWorkOS + + beforeEach(async () => { + t = await createTestDb() + resetIds() + fake = new FakeWorkOS() + }) + afterEach(async () => { + await t.close() + }) + + it('reports a clean mirror as matching with no drift', async () => { + fake.createUser() + fake.createOrganization() + const engine = makeEngine(t, fake) + await engine.backfill() + + const { diffs } = await engine.reconcile() + const users = diffs['prod']!.find((d) => d.stream === 'users')! + expect(users.localOnly).toEqual([]) + expect(users.remoteOnly).toEqual([]) + expect(users.matched).toBe(1) + // Nothing staged is left behind. + expect(await t.db.listCollections({ name: /_reconcile_/ }).toArray()).toEqual([]) + }) + + it('detects a missed delete, which nothing else can', async () => { + const user = fake.createUser() + const engine = makeEngine(t, fake) + await engine.backfill() + + fake.users.delete(user['id'] as string) + + const { diffs } = await engine.reconcile() + const users = diffs['prod']!.find((d) => d.stream === 'users')! + expect(users.localOnly).toEqual([user['id']]) + expect(users.remoteOnly).toEqual([]) + }) + + it('detects a missed create', async () => { + fake.createUser() + const engine = makeEngine(t, fake) + await engine.backfill() + + const missed = fake.createUser({ email: 'missed@example.com' }) + fake.events = [] + + const { diffs } = await engine.reconcile() + const users = diffs['prod']!.find((d) => d.stream === 'users')! + expect(users.remoteOnly).toEqual([missed['id']]) + }) + + it('repairs drift in both directions', async () => { + const stale = fake.createUser({ email: 'gone@example.com' }) + const engine = makeEngine(t, fake) + await engine.backfill() + + fake.users.delete(stale['id'] as string) + const added = fake.createUser({ email: 'added@example.com' }) + fake.events = [] + + await engine.reconcile({ repair: true }) + + expect((await raws(t, 'users', { _deleted_at: null })).map((r) => r['email'])).toEqual(['added@example.com']) + expect((await raws(t, 'users', { _deleted_at: { $type: 'date' } }))[0]!['email']).toBe('gone@example.com') + expect(added['id']).toBeTruthy() + + // A second pass finds nothing left to fix. + const { diffs } = await engine.reconcile() + const users = diffs['prod']!.find((d) => d.stream === 'users')! + expect(users.localOnly).toEqual([]) + expect(users.remoteOnly).toEqual([]) + }) + + it('says plainly when a stream cannot be reconciled', async () => { + const engine = makeEngine(t, fake) + await engine.backfill() + const { diffs } = await engine.reconcile() + + const events = diffs['prod']!.find((d) => d.stream === 'events')! + expect(events.skippedReason).toMatch(/append-only/) + const domains = diffs['prod']!.find((d) => d.stream === 'organization_domains')! + expect(domains.skippedReason).toMatch(/no list endpoint/) + }) + + it('reports freshness and lag per stream', async () => { + fake.createUser() + const engine = makeEngine(t, fake) + await engine.backfill() + + const status = await engine.status() + const users = status['prod']!.streams.find((s) => s.stream === 'users')! + expect(users.rows).toBe(1) + expect(users.lastSyncedAt).toBeTruthy() + expect(users.lagSeconds).not.toBeNull() + expect(status['prod']!.rejected).toBe(0) + expect(status['prod']!.lease).toBeNull() + }) + + it('refreshes fields that event payloads omit', async () => { + const org = fake.createOrganization() + const engine = makeEngine(t, fake) + await engine.tick() + + let [row] = await raws(t, 'organizations') + expect(row!['allow_profiles_outside_organization']).toBeUndefined() + + await engine.refresh() + ;[row] = await raws(t, 'organizations') + expect(row!['allow_profiles_outside_organization']).toBe(false) + expect(org['id']).toBeTruthy() + }) +}) + +describe('single-writer lease', () => { + let t: TestDb + let fake: FakeWorkOS + + beforeEach(async () => { + t = await createTestDb() + resetIds() + fake = new FakeWorkOS() + }) + afterEach(async () => { + await t.close() + }) + + const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) + const until = async (ok: () => Promise, timeoutMs = 5_000) => { + const deadline = Date.now() + timeoutMs + while (!(await ok())) { + if (Date.now() > deadline) throw new Error('condition not met in time') + await sleep(10) + } + } + const expireLease = () => + t.db + .collection('workos__sync') + .updateOne({ sync_id: 'default', env: 'prod' }, { $set: { 'lease.expires_at': new Date(Date.now() - 1000) } }) + + it('tick() refuses to run while another process holds the lease', async () => { + fake.createUser() + const other = new MongoDestination({ client: t.client, database: t.name }) + await other.init([]) + expect((await other.acquireLease('prod', 'someone-else', 60_000)).acquired).toBe(true) + + const engine = makeEngine(t, fake) + await expect(engine.tick()).rejects.toThrow(LeaseHeldError) + expect(await count(t, 'users')).toBe(0) + + await other.releaseLease('prod', 'someone-else') + expect((await engine.tick()).processed).toBe(1) + // A tick gives the lease back when it is done. + expect(await other.getLease('prod')).toBeNull() + }) + + it('listen() waits for the lease, takes over when it is released, and gives it back on stop', async () => { + fake.createUser() + const other = new MongoDestination({ client: t.client, database: t.name }) + await other.init([]) + await other.acquireLease('prod', 'someone-else', 60_000) + + const engine = makeEngine(t, fake, TIER1, { pollIntervalMs: 10 }) + const listening = engine.listen() + await sleep(100) + expect(await count(t, 'users')).toBe(0) + expect(await other.getLease('prod')).toMatchObject({ holder: 'someone-else' }) + + await other.releaseLease('prod', 'someone-else') + await until(async () => (await count(t, 'users')) === 1) + const lease = await other.getLease('prod') + expect(lease).not.toBeNull() + expect(lease!.holder).not.toBe('someone-else') + + engine.stop() + await listening + expect(await other.getLease('prod')).toBeNull() + }) + + it('does not checkpoint after losing the lease mid-poll', async () => { + fake.createUser() + const other = new MongoDestination({ client: t.client, database: t.name }) + await other.init([]) + let stalled = false + // The poll stalls past the TTL while fetching and a replacement takes + // over in the meantime. The database clock is moved rather than the test. + const engine = makeEngine(t, fake, TIER1, {}, async () => { + if (stalled) return + stalled = true + await expireLease() + await other.acquireLease('prod', 'replacement', 60_000) + }) + await engine.init() + const r = await engine.tick() + + // The batch it had already fetched is applied, which is idempotent, but + // the cursor is left where the replacement will look for it. + expect(await count(t, 'users')).toBe(1) + expect(r.processed).toBe(0) + const status = await engine.status() + expect(status['prod']!.cursor).toBeNull() + expect(status['prod']!.lastPolledAt).toBeNull() + expect(await other.getLease('prod')).toMatchObject({ holder: 'replacement' }) + }) + + it('keeps a slow standalone tick covered so it can still checkpoint', async () => { + fake.createUser() + const other = new MongoDestination({ client: t.client, database: t.name }) + await other.init([]) + let slowed = false + const engine = makeEngine(t, fake, TIER1, { leaseTtlMs: 150 }, async () => { + if (slowed) return + slowed = true + await sleep(400) + }) + await engine.init() + const r = await engine.tick() + + expect(r.processed).toBe(1) + expect((await engine.status())['prod']!.cursor).not.toBeNull() + expect(await other.getLease('prod')).toBeNull() + }) + + it('does not overwrite a position that moved under it, even while it holds the lease', async () => { + fake.createUser() + const other = new MongoDestination({ client: t.client, database: t.name }) + await other.init([]) + let moved = false + const engine = makeEngine(t, fake, TIER1, {}, async () => { + if (moved) return + moved = true + await new StateStore(other as Destination).setTail('prod', { cursor: 'event_from_elsewhere' }) + }) + await engine.init() + const r = await engine.tick() + + expect(r.processed).toBe(0) + expect((await engine.status())['prod']!.cursor).toBe('event_from_elsewhere') + expect(await other.getLease('prod')).toBeNull() + }) +}) + +describe('redaction', () => { + let t: TestDb + let fake: FakeWorkOS + + beforeEach(async () => { + t = await createTestDb() + resetIds() + fake = new FakeWorkOS() + }) + afterEach(async () => { + await t.close() + }) + + it('erases a subject from the mirror and the event archive', async () => { + const user = fake.createUser({ email: 'forget@example.com' }) + fake.emitAuthAttempt(user['id'] as string, true) + const engine = makeEngine(t, fake) + await engine.backfill() + await engine.tick() + + const removed = await engine.redact(user['id'] as string) + expect(removed.length).toBeGreaterThan(0) + expect(await count(t, 'users')).toBe(0) + expect(await count(t, 'auth_attempts')).toBe(0) + // The archived payloads carry the same personal data and must go too. + const leftovers = await coll(t, 'events').find().toArray() + expect(leftovers.some((d) => JSON.stringify(d).includes(user['id'] as string))).toBe(false) + }) +}) + +describe('poison events', () => { + let t: TestDb + let fake: FakeWorkOS + + beforeEach(async () => { + t = await createTestDb() + resetIds() + fake = new FakeWorkOS() + }) + afterEach(async () => { + await t.close() + }) + + const rejectedCount = () => t.db.collection('workos__rejected').countDocuments() + const eventFor = (objectId: string) => fake.events.find((e) => e.data['id'] === objectId)! + + /** The engine over a FaultyDestination, with a capturing logger and no client retries. */ + function makeFaulty(extra: Record = {}, fetchImpl = fake.fetchImpl) { + const destination = new FaultyDestination({ client: t.client, database: t.name }) + const { logger, lines } = captureLogger() + const engine = new SyncEngine( + { + destination, + environments: [{ name: 'prod', apiKey: 'sk_test_1' }], + streams: TIER1, + hydrate: 'off', + logger, + lagWindowMs: 0, + ...extra, + }, + { + clientFor: () => + new WorkOSClient({ + apiKey: 'sk_test_1', + apiUrl: 'http://fake.workos.local', + fetchImpl, + maxRetries: 0, + }), + }, + ) + return { engine, destination, lines } + } + + it('quarantines one event after two sightings and keeps the rest of the environment moving', async () => { + fake.createUser({ email: 'good1@example.com' }) + const bad = fake.createUser({ email: 'bad@example.com' }) + fake.createUser({ email: 'good2@example.com' }) + const badEvent = eventFor(bad['id'] as string) + const { engine, destination, lines } = makeFaulty() + destination.poison = bad['id'] as string + + // Poll 1: first sighting. The tick fails, the cursor stays put, and the + // suspect is only remembered in memory. + const first = await engine.tick() + expect(first).toMatchObject({ processed: 0, rejected: 0 }) + let status = await engine.status() + expect(status['prod']!.cursor).toBeNull() + expect(status['prod']!.rejected).toBe(0) + expect(await rejectedCount()).toBe(0) + const warned = lines.find((l) => l.level === 'warn' && l.fields['eventId'] === badEvent.id) + expect(warned?.msg).toMatch(/quarantined if it fails again/) + expect(warned?.fields['error']).toMatch(/Document failed validation/) + + // Poll 2: the same batch, the same failure. The good events are written, + // the bad one is set aside, and the cursor moves past all three. + const second = await engine.tick() + expect(second).toMatchObject({ processed: 3, rejected: 1 }) + status = await engine.status() + expect(status['prod']!.cursor).toBe(fake.events[fake.events.length - 1]!.id) + expect(status['prod']!.rejected).toBe(1) + expect(await emails(t)).toEqual(['good1@example.com', 'good2@example.com']) + + const rejected = (await engine.rejected())['prod']! + expect(rejected).toHaveLength(1) + expect(rejected[0]).toMatchObject({ + eventId: badEvent.id, + eventType: 'user.created', + objectId: bad['id'], + createdAt: badEvent.created_at, + raw: badEvent, + }) + expect(rejected[0]!.error).toMatch(/Document failed validation/) + expect(rejected[0]!.rejectedAt).toMatch(/^\d{4}-\d{2}-\d{2}T/) + const logged = lines.find((l) => l.level === 'error' && l.msg.includes('quarantined')) + expect(logged?.fields).toMatchObject({ eventId: badEvent.id, eventType: 'user.created', objectId: bad['id'] }) + + // The tail is not stalled: the next poll is a no-op rather than a retry. + expect((await engine.tick()).processed).toBe(0) + // The archive still has every event, including the one set aside. + expect(await count(t, 'events')).toBe(3) + }) + + it('replays a quarantined event through the normal path once the cause is fixed', async () => { + fake.createUser({ email: 'good@example.com' }) + const bad = fake.createUser({ email: 'bad@example.com' }) + const { engine, destination } = makeFaulty() + destination.poison = bad['id'] as string + await engine.tick() + await engine.tick() + expect((await engine.status())['prod']!.rejected).toBe(1) + + // Still broken: the document stays, with its error refreshed. + destination.fault = () => Object.assign(new Error('still broken'), { code: 121 }) + const again = await engine.replayRejected() + expect(again.replayed).toBe(0) + expect(again.stillRejected).toEqual([{ eventId: eventFor(bad['id'] as string).id, error: 'Error: still broken' }]) + expect((await engine.rejected())['prod']![0]!.error).toBe('Error: still broken') + + // Fixed: the event is applied and forgotten. + destination.poison = null + const fixed = await engine.replayRejected() + expect(fixed).toEqual({ replayed: 1, stillRejected: [] }) + expect(await rejectedCount()).toBe(0) + expect((await engine.status())['prod']!.rejected).toBe(0) + expect(await emails(t)).toEqual(['bad@example.com', 'good@example.com']) + }) + + it('never quarantines on an error the destination classifies as transient', async () => { + const u1 = fake.createUser() + fake.createUser() + const { engine, destination, lines } = makeFaulty() + destination.poison = u1['id'] as string + + // A dead socket, a write conflict, then a socket error: all the database + // having a bad moment, and the tick must retry, however often it happens. + destination.fault = () => new MongoNetworkError('connection closed') + await engine.tick() + await engine.tick() + destination.fault = () => Object.assign(new Error('WriteConflict'), { code: 112 }) + await engine.tick() + destination.fault = () => Object.assign(new Error('read ECONNRESET'), { code: 'ECONNRESET' }) + await engine.tick() + + expect(await rejectedCount()).toBe(0) + expect((await engine.status())['prod']!.cursor).toBeNull() + expect(lines.filter((l) => l.msg === 'tick failed')).toHaveLength(4) + expect(lines.some((l) => l.level === 'warn')).toBe(false) + // It failed fast each time, without ever isolating event by event. + expect(destination.faults).toBe(4) + + // Once the database recovers the batch applies in full. + destination.poison = null + expect((await engine.tick()).processed).toBe(2) + expect(await count(t, 'users')).toBe(2) + }) + + it('treats a whole batch failing as an outage, not a batch of poison events', async () => { + fake.createUser() + fake.createUser() + fake.createUser() + const { engine, destination, lines } = makeFaulty() + // An unclassified error, so only the all-fail rule stands in the way. + destination.fault = () => new Error('ns not found: workos_users') + destination.failAll = true + + await engine.tick() + await engine.tick() + await engine.tick() + + expect(await rejectedCount()).toBe(0) + expect((await engine.status())['prod']!.cursor).toBeNull() + expect(lines.filter((l) => l.msg === 'tick failed')).toHaveLength(3) + expect(lines.some((l) => l.level === 'warn')).toBe(false) + }) + + it('forgets a suspect that succeeds on its second poll', async () => { + const flaky = fake.createUser({ email: 'flaky@example.com' }) + fake.createUser({ email: 'steady@example.com' }) + const { engine, destination } = makeFaulty() + // An unclassified error that happens once: exactly what two sightings is for. + destination.fault = () => new Error('connection closed unexpectedly') + destination.poison = flaky['id'] as string + await engine.tick() + destination.poison = null + + const second = await engine.tick() + expect(second).toMatchObject({ processed: 2, rejected: 0 }) + expect(await rejectedCount()).toBe(0) + expect(await emails(t)).toEqual(['flaky@example.com', 'steady@example.com']) + }) + + it('does not resurrect a membership removed after a quarantined link', async () => { + // A membership re-add is the one write that clears a tombstone, which + // makes it the one place an out-of-order replay could undo a removal that + // happened after the replayed event. + const at = (ms: number) => new Date(Date.now() - 50_000 + ms).toISOString() + const pair = { group_id: 'group_01', organization_membership_id: 'om_01' } + const push = (event: string, created_at: string) => { + const e = { id: id('event_'), event, data: pair, created_at } + fake.events.push(e) + return e + } + push('group.member_added', at(0)) + push('group.member_removed', at(1_000)) + const readd = push('group.member_added', at(2_000)) + push('group.member_removed', at(3_000)) + const { engine, destination } = makeFaulty({ streams: ['group_memberships', 'events'] }) + const live = () => count(t, 'group_memberships', { _deleted_at: null }) + + // The archive write for the re-add fails, so it alone is set aside; the + // removal that followed it is applied, and the membership ends up removed. + destination.poison = readd.id + await engine.tick() + expect(await engine.tick()).toMatchObject({ processed: 4, rejected: 1 }) + expect(await count(t, 'group_memberships')).toBe(1) + expect(await live()).toBe(0) + + // Replayed now, the re-add is older than the removal it would undo. It is + // a no-op, like any stale event, and is forgotten like any replayed one. + destination.poison = null + expect(await engine.replayRejected()).toEqual({ replayed: 1, stillRejected: [] }) + expect(await live()).toBe(0) + expect(await rejectedCount()).toBe(0) + }) + + it('does not hide a membership re-added after a quarantined removal', async () => { + const at = (ms: number) => new Date(Date.now() - 50_000 + ms).toISOString() + const pair = { group_id: 'group_01', organization_membership_id: 'om_01' } + const push = (event: string, created_at: string) => { + const e = { id: id('event_'), event, data: pair, created_at } + fake.events.push(e) + return e + } + push('group.member_added', at(0)) + const removal = push('group.member_removed', at(1_000)) + push('group.member_added', at(2_000)) + const { engine, destination } = makeFaulty({ streams: ['group_memberships', 'events'] }) + const live = () => count(t, 'group_memberships', { _deleted_at: null }) + + destination.poison = removal.id + await engine.tick() + expect(await engine.tick()).toMatchObject({ processed: 3, rejected: 1 }) + expect(await live()).toBe(1) + + destination.poison = null + expect(await engine.replayRejected()).toEqual({ replayed: 1, stillRejected: [] }) + expect(await live()).toBe(1) + expect(await rejectedCount()).toBe(0) + + // A removal that really is newer than the re-add still lands. + push('group.member_removed', at(3_000)) + expect(await engine.tick()).toMatchObject({ processed: 1, rejected: 0 }) + expect(await live()).toBe(0) + }) + + it('quarantines an event whose own timestamp is not an instant, with no fault injected', async () => { + // A `created_at` that has the ISO-8601 shape but is not a real instant + // fails the archive write's parse. The quarantine document stores the + // same value, and must not fail on it: that would leave the tail stuck on + // the very event the quarantine exists to set aside. + const bad = fake.createUser({ email: 'bad@example.com' }) + const badEvent = eventFor(bad['id'] as string) + badEvent.created_at = '2026-02-30T12:00:00Z' + fake.createUser({ email: 'good@example.com' }) + const { engine } = makeFaulty() + + expect((await engine.tick()).processed).toBe(0) + expect(await engine.tick()).toMatchObject({ processed: 2, rejected: 1 }) + const rejected = (await engine.rejected())['prod']! + expect(rejected).toHaveLength(1) + expect(rejected[0]).toMatchObject({ eventId: badEvent.id, createdAt: '2026-02-30T12:00:00Z' }) + expect(rejected[0]!.error).toMatch(/out of range/) + const stored = await t.db.collection('workos__rejected').findOne({ event_id: badEvent.id }) + expect(stored!['created_at']).toBeNull() + expect(await emails(t)).toEqual(['good@example.com']) + expect((await engine.status())['prod']!.cursor).not.toBeNull() + }) + + it('is scrubbed by redact like the event archive', async () => { + fake.createUser({ email: 'keep@example.com' }) + const subject = fake.createUser({ email: 'forget@example.com' }) + const { engine, destination } = makeFaulty() + destination.poison = subject['id'] as string + await engine.tick() + await engine.tick() + expect(await rejectedCount()).toBe(1) + + const removed = await engine.redact(subject['id'] as string) + expect(removed).toContainEqual({ table: '_rejected', rows: 1 }) + expect(await rejectedCount()).toBe(0) + }) +}) diff --git a/packages/sync-mongodb/test/errors.test.ts b/packages/sync-mongodb/test/errors.test.ts new file mode 100644 index 0000000..f82ff58 --- /dev/null +++ b/packages/sync-mongodb/test/errors.test.ts @@ -0,0 +1,79 @@ +/** + * The transient-error classifier that decides whether a failed write can ever + * lead to a quarantine. A wrong answer in either direction is costly: a + * connection drop called deterministic quarantines good events, and a bad + * document called transient stalls the tail forever. + */ +import { describe, it, expect } from 'vitest' +import { MongoNetworkError, MongoNetworkTimeoutError, MongoServerError, MongoServerSelectionError } from 'mongodb' +import { isTransientMongoError } from '../src/db/errors.js' + +const serverError = (code: number, errorLabels: string[] = []) => + new MongoServerError({ errmsg: `code ${code}`, code, errorLabels }) +const socketError = (code: string) => Object.assign(new Error(code), { code, syscall: 'read' }) + +describe('isTransientMongoError', () => { + it('calls a dead socket or an unreachable cluster transient', () => { + expect(isTransientMongoError(new MongoNetworkError('connection closed'))).toBe(true) + expect(isTransientMongoError(new MongoNetworkTimeoutError('timed out'))).toBe(true) + expect(isTransientMongoError(Object.create(MongoServerSelectionError.prototype) as Error)).toBe(true) + }) + + it('calls server-state codes transient: elections, shutdowns, lock contention, write conflicts, time limits', () => { + const codes = [ + 6, // HostUnreachable + 7, // HostNotFound + 24, // LockTimeout + 46, // LockBusy + 50, // MaxTimeMSExpired + 89, // NetworkTimeout + 91, // ShutdownInProgress + 112, // WriteConflict + 189, // PrimarySteppedDown + 262, // ExceededTimeLimit + 10107, // NotWritablePrimary + 11600, // InterruptedAtShutdown + 11602, // InterruptedDueToReplStateChange + 13435, // NotPrimaryNoSecondaryOk + 13436, // NotPrimaryOrSecondary + ] + for (const code of codes) expect(isTransientMongoError(serverError(code)), String(code)).toBe(true) + }) + + it('trusts the labels the server attaches to a retryable failure, whatever the code', () => { + expect(isTransientMongoError(serverError(2, ['RetryableWriteError']))).toBe(true) + expect(isTransientMongoError(serverError(2, ['TransientTransactionError']))).toBe(true) + expect(isTransientMongoError(serverError(2, ['SomethingElse']))).toBe(false) + }) + + it('calls verdicts on the document deterministic: duplicates, bad values, validation, size', () => { + const codes = [ + 11000, // DuplicateKey + 2, // BadValue + 121, // DocumentValidationFailure + 10334, // BSONObjectTooLarge + 17280, // KeyTooLong + 26, // NamespaceNotFound + 48, // NamespaceExists + ] + for (const code of codes) expect(isTransientMongoError(serverError(code)), String(code)).toBe(false) + }) + + it('recognises the socket errors node surfaces unwrapped, and as a cause', () => { + for (const code of ['ECONNRESET', 'ECONNREFUSED', 'ETIMEDOUT', 'EPIPE']) { + expect(isTransientMongoError(socketError(code)), code).toBe(true) + expect(isTransientMongoError(new Error('wrapped', { cause: socketError(code) })), code).toBe(true) + } + expect(isTransientMongoError(socketError('ENOENT'))).toBe(false) + }) + + it('presumes anything it cannot classify is deterministic', () => { + // The two-sightings rule is what guards this gap; the classifier itself + // must not guess in the direction that skips data. + expect(isTransientMongoError(new Error('connection closed unexpectedly'))).toBe(false) + expect(isTransientMongoError(Object.assign(new Error('x'), { code: 'INVALID_TIMESTAMP' }))).toBe(false) + expect(isTransientMongoError('ECONNRESET')).toBe(false) + expect(isTransientMongoError(null)).toBe(false) + expect(isTransientMongoError(undefined)).toBe(false) + }) +}) diff --git a/packages/sync-mongodb/test/global-setup.ts b/packages/sync-mongodb/test/global-setup.ts new file mode 100644 index 0000000..2e31474 --- /dev/null +++ b/packages/sync-mongodb/test/global-setup.ts @@ -0,0 +1,26 @@ +/** + * One MongoDB for the whole run. + * + * `TEST_MONGODB_URL` points the suite at a server of your own (a container, + * Atlas); without it, an embedded `mongod` from mongodb-memory-server is + * started here, once, and every test file connects to it. Each test still + * owns its state: `createTestDb` hands out a database of its own and drops + * it afterwards. The binary is downloaded on first use and cached, since its + * postinstall is skipped at `pnpm install`. + */ +import { MongoMemoryServer } from 'mongodb-memory-server' + +export default async function setup(): Promise<(() => Promise) | undefined> { + if (process.env['TEST_MONGODB_URL']) return undefined + const server = await MongoMemoryServer.create() + process.env['TEST_MONGODB_URL'] = server.getUri() + return async () => { + // Nothing in the data directory is worth a graceful shutdown, and after a + // full run has created and dropped a few thousand collections mongod + // takes longer to stop than the ten seconds mongodb-memory-server allows + // before it kills it anyway, with a warning. Kill it outright; `stop` + // then only has the directory to clean up. + server.instanceInfo?.instance.mongodProcess?.kill('SIGKILL') + await server.stop() + } +} diff --git a/packages/sync-mongodb/test/health-cli.test.ts b/packages/sync-mongodb/test/health-cli.test.ts new file mode 100644 index 0000000..c2df3ef --- /dev/null +++ b/packages/sync-mongodb/test/health-cli.test.ts @@ -0,0 +1,38 @@ +import { describe, it, expect } from 'vitest' +import { healthOptionsFromEnv } from '../src/health.js' + +describe('WORKOS_SYNC_HEALTH_*', () => { + it('stays off unless a port is given', () => { + expect(healthOptionsFromEnv({}, {})).toBeNull() + expect(healthOptionsFromEnv({}, { WORKOS_SYNC_HEALTH_PORT: '' })).toBeNull() + }) + + it('reads the port from the environment with the documented defaults', () => { + expect(healthOptionsFromEnv({}, { WORKOS_SYNC_HEALTH_PORT: '8080' })).toEqual({ + port: 8080, + host: '0.0.0.0', + maxPollAgeMs: 60_000, + }) + }) + + it('lets the flag win over the environment, and honours host and window overrides', () => { + expect( + healthOptionsFromEnv( + { 'health-port': '9000' }, + { + WORKOS_SYNC_HEALTH_PORT: '8080', + WORKOS_SYNC_HEALTH_HOST: '127.0.0.1', + WORKOS_SYNC_HEALTH_MAX_POLL_AGE_MS: '300000', + }, + ), + ).toEqual({ port: 9000, host: '127.0.0.1', maxPollAgeMs: 300_000 }) + }) + + it('rejects values that would otherwise fail later or silently', () => { + expect(() => healthOptionsFromEnv({}, { WORKOS_SYNC_HEALTH_PORT: 'eighty' })).toThrow(/invalid health port/) + expect(() => healthOptionsFromEnv({}, { WORKOS_SYNC_HEALTH_PORT: '70000' })).toThrow(/invalid health port/) + expect(() => + healthOptionsFromEnv({}, { WORKOS_SYNC_HEALTH_PORT: '8080', WORKOS_SYNC_HEALTH_MAX_POLL_AGE_MS: '0' }), + ).toThrow(/WORKOS_SYNC_HEALTH_MAX_POLL_AGE_MS/) + }) +}) diff --git a/packages/sync-mongodb/test/health.test.ts b/packages/sync-mongodb/test/health.test.ts new file mode 100644 index 0000000..e5d9ae8 --- /dev/null +++ b/packages/sync-mongodb/test/health.test.ts @@ -0,0 +1,194 @@ +/** + * `engine.health()` driven by real ticks: the fake WorkOS API on one side, a + * real MongoDB on the other, and the in-memory health state in between. The + * HTTP surface itself is unit-tested in sync-engine/test/health.test.ts; here + * it is only wired to a live engine to prove the two agree. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { SyncEngine, WorkOSClient, silentLogger, startHealthServer, type HealthServer } from '@workos/sync-engine' +import { createTestDb, type TestDb } from './helpers/mongo.js' +import { FakeWorkOS, resetIds } from '../../sync-engine/test/helpers/fake-workos.js' +import { MongoDestination } from '../src/destination.js' + +const STREAMS = ['users', 'organizations', 'events'] + +function makeEngine(t: TestDb, fetchImpl: typeof fetch, apiKey = 'sk_test_1') { + return new SyncEngine( + { + destination: new MongoDestination({ client: t.client, database: t.name }), + environments: [{ name: 'prod', apiKey }], + streams: STREAMS, + logger: silentLogger, + lagWindowMs: 0, + }, + { + clientFor: () => + new WorkOSClient({ apiKey, apiUrl: 'http://fake.workos.local', fetchImpl, maxRetries: 0, pageSize: 100 }), + }, + ) +} + +describe('engine health', () => { + let t: TestDb + let fake: FakeWorkOS + let server: HealthServer | undefined + + beforeEach(async () => { + t = await createTestDb() + resetIds() + fake = new FakeWorkOS() + }) + afterEach(async () => { + await server?.close() + server = undefined + await t.close() + }) + + it('is `starting` with nothing recorded before the first tick', () => { + const h = makeEngine(t, fake.fetchImpl).health() + expect(h.state).toBe('starting') + expect(h.fatal).toBeNull() + expect(Date.parse(h.startedAt)).not.toBeNaN() + expect(h.environments['prod']).toMatchObject({ + lastPolledAt: null, + lastEventAt: null, + cursor: null, + lastError: null, + counters: { ticks: 0, events: 0, errors: 0 }, + }) + }) + + it('advances counters and the cursor as ticks process events', async () => { + const u = fake.createUser() + fake.createOrganization() + const engine = makeEngine(t, fake.fetchImpl) + await engine.backfill() + + await engine.tick() + let h = engine.health() + expect(h.state).toBe('listening') + let e = h.environments['prod']! + expect(e.counters.ticks).toBe(1) + expect(e.counters.events).toBe(2) + // Backfill already wrote both rows, so the tail's replay updates them. + expect(e.counters.created + e.counters.updated + e.counters.skipped).toBeGreaterThan(0) + expect(e.cursor).toMatch(/^event_/) + expect(e.lastEventAt).not.toBeNull() + expect(e.lastPolledAt).not.toBeNull() + expect(e.lastError).toBeNull() + + // A quiet poll still counts as a poll, and still moves lastPolledAt. + const before = e.lastPolledAt! + await new Promise((r) => setTimeout(r, 5)) + await engine.tick() + e = engine.health().environments['prod']! + expect(e.counters.ticks).toBe(2) + expect(e.counters.events).toBe(2) + expect(e.lastPolledAt! >= before).toBe(true) + + fake.updateUser(u['id'] as string, { first_name: 'Grace' }) + await engine.tick() + h = engine.health() + e = h.environments['prod']! + expect(e.counters.events).toBe(3) + expect(e.counters.updated).toBeGreaterThanOrEqual(1) + }) + + it('records a retryable failure as lastError and keeps going', async () => { + fake.createUser() + const engine = makeEngine(t, fake.fetchImpl) + await engine.backfill() + await engine.tick() + + // Make the events endpoint fail with a 500. With no retry budget the + // client surfaces it on the first attempt, and the engine treats it as + // transient: logged, counted, and polled again next tick. + let broken = true + const flaky = ((input: string | URL | Request, init?: RequestInit) => { + const url = typeof input === 'string' ? input : input.toString() + if (broken && url.includes('/events')) { + return Promise.resolve(new Response('{"message":"boom"}', { status: 500 })) + } + return fake.fetchImpl(input, init) + }) as typeof fetch + const failing = makeEngine(t, flaky) + await failing.tick() + let e = failing.health().environments['prod']! + expect(e.counters.errors).toBe(1) + expect(e.counters.ticks).toBe(1) + expect(e.lastError?.message).toMatch(/500/) + expect(e.lastPolledAt).toBeNull() + expect(failing.health().fatal).toBeNull() + + broken = false + await failing.tick() + e = failing.health().environments['prod']! + expect(e.counters.ticks).toBe(2) + expect(e.lastPolledAt).not.toBeNull() + // The record of the failure stays; the counters say it is not ongoing. + expect(e.lastError?.message).toMatch(/500/) + }) + + it('records a rejected credential as fatal and stopped', async () => { + const strict = new FakeWorkOS({ apiKey: 'sk_test_1' }) + strict.createUser() + const engine = makeEngine(t, strict.fetchImpl, 'sk_test_WRONG') + await expect(engine.tick()).rejects.toThrow(/401/) + const h = engine.health() + expect(h.state).toBe('stopped') + expect(h.fatal).toMatch(/401/) + }) + + it('marks the engine stopped once listen() returns', async () => { + const engine = makeEngine(t, fake.fetchImpl) + // Schema setup first: `stop()` only takes once the listen loop has begun, + // and creating collections, indexes and views takes longer than the + // moment this test waits before stopping. + await engine.init() + const listening = engine.listen() + // Stop after the first tick has had a chance to run. + await new Promise((r) => setTimeout(r, 50)) + engine.stop() + await listening + const h = engine.health() + expect(h.state).toBe('stopped') + expect(h.environments['prod']!.counters.ticks).toBeGreaterThanOrEqual(1) + }) + + it('serves the same state over HTTP, and the status code follows it', async () => { + fake.createUser() + const engine = makeEngine(t, fake.fetchImpl) + let clock = Date.now() + server = await startHealthServer(engine, { port: 0, host: '127.0.0.1', now: () => clock }) + const base = `http://127.0.0.1:${server.port}` + + let res = await fetch(`${base}/healthz`) + expect(res.status).toBe(200) + expect(((await res.json()) as { state: string }).state).toBe('starting') + + await engine.backfill() + await engine.tick() + clock = Date.now() + res = await fetch(`${base}/healthz`) + expect(res.status).toBe(200) + const body = (await res.json()) as { + state: string + healthy: boolean + environments: Record + } + expect(body.state).toBe('listening') + expect(body.healthy).toBe(true) + expect(body.environments['prod']!.counters.events).toBe(1) + expect(body.environments['prod']!.pollAgeSeconds).toBeLessThan(5) + + const metrics = await (await fetch(`${base}/metrics`)).text() + expect(metrics).toMatch(/^workos_sync_poll_age_seconds\{env="prod"\} \d/m) + expect(metrics).toContain('workos_sync_events_total{env="prod"} 1\n') + expect(metrics).toContain('workos_sync_up 1\n') + + // Age the last poll past the window without waiting for it. + clock += 61_000 + res = await fetch(`${base}/healthz`) + expect(res.status).toBe(503) + }) +}) diff --git a/packages/sync-mongodb/test/helpers/ca.ts b/packages/sync-mongodb/test/helpers/ca.ts new file mode 100644 index 0000000..0c48621 --- /dev/null +++ b/packages/sync-mongodb/test/helpers/ca.ts @@ -0,0 +1,17 @@ +/** + * A self-signed certificate for tests that need a value `X509Certificate` + * accepts. Generated once with `openssl req -x509 -newkey ec ...` and its + * private key discarded on the spot, so it can sign nothing and is not a + * secret; it names no real host and is trusted by nothing. + */ +export const TEST_CA_PEM = `-----BEGIN CERTIFICATE----- +MIIBdTCCARygAwIBAgIUYA/j8vmXLoOxBg17MVsxr0WptZgwCgYIKoZIzj0EAwIw +IDEeMBwGA1UEAwwVc3luYy1wb3N0Z3JlcyB0ZXN0IENBMCAXDTI2MTAyOTE2NDE1 +OVoYDzIxMjYxMDA1MTY0MTU5WjAgMR4wHAYDVQQDDBVzeW5jLXBvc3RncmVzIHRl +c3QgQ0EwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAR/tIk0cVsFVh49bOYbZ2UB +blKkXKfCfgj3+6A5B7t+dz+phLfY0PJe/RfKCPnbvQI4ZDO8jkLctAjItiiNjbBm +ozIwMDAdBgNVHQ4EFgQUdosu7nttrBB5We4MUhtGRm7Q/gEwDwYDVR0TAQH/BAUw +AwEB/zAKBggqhkjOPQQDAgNHADBEAiBWaRIW/Ef4zN9nmK01dsPVor+p06ihoxO8 +G/oBUsR6zAIgOJSTKCW/ymLt0xjT42M7xm2AoJximsh0cy3Wbk+mohU= +-----END CERTIFICATE----- +` diff --git a/packages/sync-mongodb/test/helpers/faulty.ts b/packages/sync-mongodb/test/helpers/faulty.ts new file mode 100644 index 0000000..1140562 --- /dev/null +++ b/packages/sync-mongodb/test/helpers/faulty.ts @@ -0,0 +1,62 @@ +/** + * Test doubles for the poison-event quarantine. + * + * `FaultyDestination` is the real MongoDB destination with one write fault + * injected: it stands in for a document the server refuses on one payload, + * or a router bug on one shape, without having to construct the offending + * data for every scenario. The error it throws is configurable so the same + * double can imitate a deterministic failure, a dropped connection, or a + * socket error. + */ +import type { Logger, Row, TableDef, WriteMode, WriteStats } from '@workos/sync-engine' +import { MongoDestination } from '../../src/destination.js' + +export class FaultyDestination extends MongoDestination { + /** Object id whose mirror write fails. `null` means no fault. */ + poison: string | null = null + /** Fail every write, as an outage would. */ + failAll = false + /** + * Fail only multi-row writes: the batch attempt fails, every event succeeds + * alone. That is what an interaction between rows, or a transient error the + * classifier does not know, looks like to the quarantine. + */ + failBatches = false + /** What the fault throws; defaults to the shape of a server-side validation error. */ + fault: () => Error = () => + Object.assign(new Error('Document failed validation: "maybe" is not a boolean'), { code: 121 }) + /** How many times the fault fired, so a test can prove isolation happened. */ + faults = 0 + + override async upsert(stream: TableDef, env: string, rows: Row[], mode: WriteMode): Promise { + const hit = + this.failAll || + (this.failBatches && rows.length > 1) || + (this.poison !== null && rows.some((r) => r.data['id'] === this.poison)) + if (hit) { + this.faults++ + throw this.fault() + } + return super.upsert(stream, env, rows, mode) + } +} + +export interface LogLine { + level: 'debug' | 'info' | 'warn' | 'error' + msg: string + fields: Record +} + +/** A logger that keeps every line, so a test can assert on what was said. */ +export function captureLogger(): { logger: Logger; lines: LogLine[] } { + const lines: LogLine[] = [] + const at = + (level: LogLine['level']) => + (msg: string, fields: Record = {}) => { + lines.push({ level, msg, fields }) + } + return { + lines, + logger: { debug: at('debug'), info: at('info'), warn: at('warn'), error: at('error') }, + } +} diff --git a/packages/sync-mongodb/test/helpers/mongo.ts b/packages/sync-mongodb/test/helpers/mongo.ts new file mode 100644 index 0000000..52e8df8 --- /dev/null +++ b/packages/sync-mongodb/test/helpers/mongo.ts @@ -0,0 +1,52 @@ +/** + * A database per test on the server the global setup provides, so the tests + * exercise real update pipelines, `$lookup`, views and unique indexes — not a + * mock's approximation — and never see each other's state. + * + * One client serves every test in a file: a client is a pool with its own + * monitors, and opening forty of them in one worker is what keeps that + * worker from exiting promptly once the file is done. `test/setup.ts` closes + * it after the file's last test. + */ +import { randomBytes } from 'node:crypto' +import { MongoClient, type Db } from 'mongodb' + +export interface TestDb { + client: MongoClient + db: Db + /** The database name, for `new MongoDestination({ client, database })`. */ + name: string + /** Drop the database. The client is shared and stays open. */ + close(): Promise +} + +let shared: Promise | undefined + +async function sharedClient(): Promise { + const url = process.env['TEST_MONGODB_URL'] + if (!url) throw new Error('TEST_MONGODB_URL is not set; the global setup should have started a server') + shared ??= new MongoClient(url).connect() + return shared +} + +export async function createTestDb(): Promise { + const client = await sharedClient() + const name = `workos_test_${randomBytes(6).toString('hex')}` + const db = client.db(name) + return { + client, + db, + name, + async close() { + await db.dropDatabase() + }, + } +} + +/** Close the file's client. Registered by `test/setup.ts` as an `afterAll`. */ +export async function closeSharedClient(): Promise { + const client = await shared?.catch(() => undefined) + shared = undefined + // Forced: nothing is in flight once the file's last test has returned. + await client?.close(true) +} diff --git a/packages/sync-mongodb/test/lease.test.ts b/packages/sync-mongodb/test/lease.test.ts new file mode 100644 index 0000000..228b5cf --- /dev/null +++ b/packages/sync-mongodb/test/lease.test.ts @@ -0,0 +1,181 @@ +/** + * Lease and checkpoint semantics against a real server. The lease and the + * cursors share one document, so every check here is one atomic update. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { streamByName } from '@workos/sync-engine' +import { createTestDb, type TestDb } from './helpers/mongo.js' +import { MongoDestination } from '../src/destination.js' +import { canonicalJson, type SyncDoc } from '../src/db/lease.js' + +describe('events tail lease', () => { + let t: TestDb + let dest: MongoDestination + + beforeEach(async () => { + t = await createTestDb() + dest = new MongoDestination({ client: t.client, database: t.name }) + await dest.init([streamByName('users')!]) + }) + afterEach(async () => { + await t.close() + }) + + const sync = () => t.db.collection('workos__sync') + const expire = () => + sync().updateOne({ sync_id: 'default', env: 'prod' }, { $set: { 'lease.expires_at': new Date(Date.now() - 1000) } }) + + it('is taken by the first process to ask', async () => { + const r = await dest.acquireLease('prod', 'a', 60_000) + expect(r.acquired).toBe(true) + expect(r.holder).toBe('a') + expect(Date.parse(r.expiresAt!)).toBeGreaterThan(Date.now() + 30_000) + expect(await dest.getLease('prod')).toEqual({ holder: 'a', expiresAt: r.expiresAt }) + }) + + it('refuses a second process and tells it who holds the lease', async () => { + const first = await dest.acquireLease('prod', 'a', 60_000) + const second = await dest.acquireLease('prod', 'b', 60_000) + expect(second).toEqual({ acquired: false, holder: 'a', expiresAt: first.expiresAt }) + // The refusal changed nothing. + expect(await dest.getLease('prod')).toEqual({ holder: 'a', expiresAt: first.expiresAt }) + }) + + it('renews for the holder without changing hands', async () => { + const first = await dest.acquireLease('prod', 'a', 1_000) + const renewed = await dest.acquireLease('prod', 'a', 60_000) + expect(renewed.acquired).toBe(true) + expect(renewed.holder).toBe('a') + expect(Date.parse(renewed.expiresAt!)).toBeGreaterThan(Date.parse(first.expiresAt!)) + + // acquired_at marks the original takeover, not the latest heartbeat. + const doc = (await sync().findOne({ sync_id: 'default', env: 'prod' }))! + expect(doc.lease!.acquired_at.getTime()).toBeLessThan(doc.lease!.expires_at.getTime() - 30_000) + }) + + it('passes to a new process once it has expired', async () => { + // A TTL of zero expires on arrival: the holder has, in effect, crashed. + const stale = await dest.acquireLease('prod', 'a', 0) + expect(stale.acquired).toBe(true) + expect(await dest.getLease('prod')).toBeNull() + + const takeover = await dest.acquireLease('prod', 'b', 60_000) + expect(takeover.acquired).toBe(true) + expect(takeover.holder).toBe('b') + // And the original holder is now the one refused. + expect((await dest.acquireLease('prod', 'a', 60_000)).holder).toBe('b') + }) + + it('judges expiry by the database clock', async () => { + await dest.acquireLease('prod', 'a', 60_000) + await expire() + expect(await dest.getLease('prod')).toBeNull() + expect((await dest.acquireLease('prod', 'b', 60_000)).acquired).toBe(true) + }) + + it('is freed by release', async () => { + await dest.acquireLease('prod', 'a', 60_000) + await dest.releaseLease('prod', 'a') + expect(await dest.getLease('prod')).toBeNull() + expect((await dest.acquireLease('prod', 'b', 60_000)).acquired).toBe(true) + }) + + it('ignores a release from anyone but the holder', async () => { + const r = await dest.acquireLease('prod', 'a', 60_000) + await dest.releaseLease('prod', 'b') + await dest.releaseLease('prod', 'a-but-older') + expect(await dest.getLease('prod')).toEqual({ holder: 'a', expiresAt: r.expiresAt }) + }) + + it('is scoped to the sync id and the environment', async () => { + await dest.acquireLease('prod', 'a', 60_000) + // Same database, same environment label, a different sync: no contention. + const other = new MongoDestination({ client: t.client, database: t.name, syncId: 'reporting' }) + await other.init([streamByName('users')!]) + expect((await other.acquireLease('prod', 'b', 60_000)).acquired).toBe(true) + // Same sync, another environment: also independent. + expect((await dest.acquireLease('staging', 'c', 60_000)).acquired).toBe(true) + expect(await dest.getLease('prod')).toMatchObject({ holder: 'a' }) + }) + + it('survives two processes racing to create the document', async () => { + const results = await Promise.all(['a', 'b', 'c', 'd'].map((h) => dest.acquireLease('prod', h, 60_000))) + expect(results.filter((r) => r.acquired)).toHaveLength(1) + const holder = results.find((r) => r.acquired)!.holder + expect(results.every((r) => r.holder === holder)).toBe(true) + }) + + describe('fences the state write', () => { + const stored = () => dest.getState>('prod', '__tail__') + + it('writes for the holder, and refuses anyone else', async () => { + await dest.acquireLease('prod', 'a', 60_000) + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c1' }, 'b', null)).toBe(false) + expect(await stored()).toBeNull() + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c1' }, 'a', null)).toBe(true) + expect(await stored()).toEqual({ cursor: 'c1' }) + }) + + it('refuses a holder whose lease has expired or been taken over', async () => { + await dest.acquireLease('prod', 'a', 60_000) + await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c1' }, 'a', null) + await expire() + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c2' }, 'a', { cursor: 'c1' })).toBe(false) + // The replacement is now the writer; the former holder stays refused. + await dest.acquireLease('prod', 'b', 60_000) + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c2' }, 'a', { cursor: 'c1' })).toBe(false) + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c2' }, 'b', { cursor: 'c1' })).toBe(true) + expect(await stored()).toEqual({ cursor: 'c2' }) + }) + + it('refuses a write whose expected state is no longer the stored one', async () => { + await dest.acquireLease('prod', 'a', 60_000) + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c1' }, 'a', { cursor: 'c0' })).toBe(false) + expect(await stored()).toBeNull() + await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c1' }, 'a', null) + // A cursor appeared where none was expected, and a stale expectation: both + // are how a holder would rewind a position someone else moved. + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c2' }, 'a', null)).toBe(false) + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c2' }, 'a', { cursor: 'c0' })).toBe(false) + expect(await stored()).toEqual({ cursor: 'c1' }) + // Equality is semantic, not textual: key order does not matter. + await dest.setState('prod', '__tail__', { + cursor: 'c1', + lastEventAt: null, + nested: { b: 1, a: [2, { d: 1, c: 2 }] }, + }) + expect( + await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c2' }, 'a', { + nested: { a: [2, { c: 2, d: 1 }], b: 1 }, + lastEventAt: null, + cursor: 'c1', + }), + ).toBe(true) + expect(await stored()).toEqual({ cursor: 'c2' }) + }) + + it('keeps cursors per stream and per environment in the one document', async () => { + await dest.setState('prod', 'users', { kind: 'backfill', status: 'running' }) + await dest.setState('prod', '__reconcile__:users', { kind: 'reconcile', status: 'complete' }) + await dest.setState('staging', 'users', { kind: 'backfill', status: 'complete' }) + expect(await dest.getState('prod', 'users')).toEqual({ kind: 'backfill', status: 'running' }) + expect(await dest.getState('prod', '__reconcile__:users')).toEqual({ kind: 'reconcile', status: 'complete' }) + expect(await dest.getState('staging', 'users')).toEqual({ kind: 'backfill', status: 'complete' }) + expect(await dest.getState('prod', 'organizations')).toBeNull() + expect(await sync().countDocuments()).toBe(2) + // Setting the state again replaces it whole. + await dest.setState('prod', 'users', { kind: 'backfill', status: 'complete' }) + expect(await dest.getState('prod', 'users')).toEqual({ kind: 'backfill', status: 'complete' }) + }) + }) +}) + +describe('canonicalJson', () => { + it('sorts keys at every depth and drops undefined, so equal values serialise equally', () => { + expect(canonicalJson({ b: 1, a: { d: [{ f: 1, e: 2 }], c: null }, u: undefined })).toBe( + '{"a":{"c":null,"d":[{"e":2,"f":1}]},"b":1}', + ) + expect(canonicalJson([3, { b: 1, a: 2 }])).toBe('[3,{"a":2,"b":1}]') + expect(canonicalJson('x')).toBe('"x"') + }) +}) diff --git a/packages/sync-mongodb/test/migrate.test.ts b/packages/sync-mongodb/test/migrate.test.ts new file mode 100644 index 0000000..805095f --- /dev/null +++ b/packages/sync-mongodb/test/migrate.test.ts @@ -0,0 +1,242 @@ +/** + * Schema setup: collections, indexes and views derived from the catalog, with + * a fingerprint so an unchanged catalog costs one query. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { streamByName, STREAMS, selectStreams, type ColumnDef, type TableDef } from '@workos/sync-engine' +import { createTestDb, type TestDb } from './helpers/mongo.js' +import { captureLogger } from './helpers/faulty.js' +import { migrate, fingerprint, indexSpecs, indexName, DDL_VERSION } from '../src/schema/migrate.js' +import { ensureLiveView, LIVE_PIPELINE } from '../src/schema/views.js' +import { Names } from '../src/schema/names.js' +import { MongoDestination } from '../src/destination.js' + +const names = new Names() +const users = streamByName('users')! + +const withColumns = (stream: TableDef, extra: ColumnDef[]): TableDef => ({ + ...stream, + columns: [...stream.columns, ...extra], +}) + +describe('fingerprint', () => { + it('changes with the catalog, the prefix and the DDL version, and not with stream order', () => { + const a = fingerprint(STREAMS, 'workos_') + expect(a).toMatch(/^[0-9a-f]{16}$/) + expect(fingerprint([...STREAMS].reverse(), 'workos_')).toBe(a) + expect(fingerprint(STREAMS, 'mirror_')).not.toBe(a) + expect(fingerprint(selectStreams(['users']), 'workos_')).not.toBe(a) + expect( + fingerprint( + STREAMS.map((s) => + s.name === 'users' + ? withColumns(s, [{ name: 'region', type: 'text', path: ['metadata', 'region'], index: true }]) + : s, + ), + 'workos_', + ), + ).not.toBe(a) + expect(DDL_VERSION).toBeGreaterThanOrEqual(1) + }) +}) + +describe('indexSpecs', () => { + it('derives the key, the indexed columns and the engine’s own scans, under the default names', () => { + const specs = indexSpecs(users) + expect(specs[0]).toEqual({ key: { _env: 1, '_raw.id': 1 }, unique: true }) + expect(specs.map((s) => indexName(s.key))).toEqual([ + '_env_1__raw.id_1', + '_env_1__raw.email_1', + '_env_1__raw.external_id_1', + '_synced_at_1', + '_deleted_at_1', + ]) + expect(specs.at(-1)).toEqual({ + key: { _deleted_at: 1 }, + partialFilterExpression: { _deleted_at: { $type: 'date' } }, + }) + // Composite keys and nested paths. + const roles = streamByName('organization_roles')! + expect(indexSpecs(roles)[0]).toEqual({ key: { _env: 1, '_raw.organization_id': 1, '_raw.slug': 1 }, unique: true }) + // Append-only: no tombstone index. An indexed key column is not indexed twice. + const events = streamByName('events')! + expect(indexSpecs(events).some((s) => '_deleted_at' in s.key)).toBe(false) + expect( + indexSpecs(withColumns(users, [{ name: 'id', type: 'text', path: 'id', index: true }])).filter( + (s) => indexName(s.key) === '_env_1__raw.id_1', + ), + ).toHaveLength(1) + }) +}) + +describe('migrate', () => { + let t: TestDb + + beforeEach(async () => { + t = await createTestDb() + }) + afterEach(async () => { + await t.close() + }) + + // `system.views` appears once the first view exists; it is the server's, not ours. + const collections = async () => + (await t.db.listCollections().toArray()) + .filter((c) => !c.name.startsWith('system.')) + .map((c) => [c.name, c.type] as const) + .sort() + const indexNames = async (coll: string) => + (await t.db.collection(coll).listIndexes().toArray()).map((i) => i.name as string).sort() + + it('creates collections, indexes, views and bookkeeping once, then is a no-op', async () => { + const streams = selectStreams(['users', 'events']) + const first = await migrate(t.db, streams, names) + // users: key, email, external_id, _synced_at, _deleted_at; events: key, event, object_id, created_at, _synced_at. + expect(first).toMatchObject({ + applied: true, + collections: 2, + indexes: 5 + 5, + views: ['workos_users_live'], + orphans: [], + }) + + expect(await collections()).toEqual([ + ['workos__migrations', 'collection'], + ['workos__rejected', 'collection'], + ['workos__sync', 'collection'], + ['workos_events', 'collection'], + ['workos_users', 'collection'], + ['workos_users_live', 'view'], + ]) + expect(await indexNames('workos_users')).toEqual([ + '_deleted_at_1', + '_env_1__raw.email_1', + '_env_1__raw.external_id_1', + '_env_1__raw.id_1', + '_id_', + '_synced_at_1', + ]) + expect(await indexNames('workos__sync')).toEqual(['_id_', 'sync_id_1_env_1']) + expect(await indexNames('workos__rejected')).toEqual(['_id_', 'sync_id_1_env_1_event_id_1']) + const unique = (await t.db.collection('workos_users').listIndexes().toArray()).find( + (i) => i.name === '_env_1__raw.id_1', + ) + expect(unique?.unique).toBe(true) + + const second = await migrate(t.db, streams, names) + expect(second).toMatchObject({ applied: false, fingerprint: first.fingerprint }) + expect(await t.db.collection('workos__migrations').countDocuments()).toBe(1) + }) + + it('adds an index for a column promoted later and reports one the catalog dropped, without removing it', async () => { + await migrate(t.db, [users], names) + const promoted = withColumns(users, [{ name: 'region', type: 'text', path: ['metadata', 'region'], index: true }]) + const { logger, lines } = captureLogger() + const r = await migrate(t.db, [promoted], names, logger) + expect(r.applied).toBe(true) + expect(await indexNames('workos_users')).toContain('_env_1__raw.metadata.region_1') + expect(r.orphans).toEqual([]) + + // The column moves to another path: the old index lingers and is named, never dropped. + const moved = withColumns(users, [{ name: 'region', type: 'text', path: ['metadata', 'zone'], index: true }]) + const back = await migrate(t.db, [moved], names, logger) + expect(back.applied).toBe(true) + expect(back.orphans).toEqual([ + { + collection: 'workos_users', + index: '_env_1__raw.metadata.region_1', + key: { _env: 1, '_raw.metadata.region': 1 }, + }, + ]) + expect(await indexNames('workos_users')).toContain('_env_1__raw.metadata.region_1') + expect(await indexNames('workos_users')).toContain('_env_1__raw.metadata.zone_1') + expect(lines.find((l) => l.level === 'warn' && l.msg === 'orphan index')?.fields).toMatchObject({ + index: '_env_1__raw.metadata.region_1', + }) + // Every catalog seen was recorded; detection only runs when one is new. + expect(await t.db.collection('workos__migrations').countDocuments()).toBe(3) + expect((await migrate(t.db, [users], names, logger)).applied).toBe(false) + }) + + it('tolerates two inits racing', async () => { + const results = await Promise.all([ + migrate(t.db, [users], names), + migrate(t.db, [users], names), + migrate(t.db, [users], names), + ]) + expect(results.every((r) => r.fingerprint === results[0]!.fingerprint)).toBe(true) + expect(await t.db.collection('workos__migrations').countDocuments()).toBe(1) + expect(await collections()).toContainEqual(['workos_users_live', 'view']) + }) + + it('keeps prefixes apart in one database', async () => { + await migrate(t.db, [users], new Names('a_')) + await migrate(t.db, [users], new Names('b_')) + const all = (await collections()).map(([n]) => n) + expect(all).toContain('a_users') + expect(all).toContain('b_users') + expect(all).toContain('a__sync') + expect(all).toContain('b__sync') + }) + + describe('live views', () => { + it('hide tombstones and nothing else', async () => { + const dest = new MongoDestination({ client: t.client, database: t.name }) + await dest.init([users]) + const at = '2026-01-01T00:00:00.000Z' + await dest.upsert( + users, + 'prod', + [ + { data: { id: 'user_1', email: 'stays@example.com' }, updatedAt: at }, + { data: { id: 'user_2', email: 'gone@example.com' }, updatedAt: at }, + ], + 'replace', + ) + await dest.delete(users, 'prod', [{ data: { id: 'user_2' }, updatedAt: at }]) + const live = t.db.collection('workos_users_live') + expect(await live.countDocuments()).toBe(1) + expect((await live.findOne({}))!['_raw']).toMatchObject({ email: 'stays@example.com' }) + expect(await t.db.collection('workos_users').countDocuments()).toBe(2) + // A re-add surfaces again, since the pipeline matches null as well as missing. + const memberships = streamByName('directory_group_memberships')! + await dest.init([users, memberships]) + const pair = { directory_group_id: 'g', directory_user_id: 'u' } + await dest.unlink(memberships, 'prod', [{ data: pair, updatedAt: at }]) + expect(await t.db.collection('workos_directory_group_memberships_live').countDocuments()).toBe(0) + await dest.relink(memberships, 'prod', [{ data: pair, updatedAt: '2026-01-02T00:00:00.000Z' }]) + expect(await t.db.collection('workos_directory_group_memberships_live').countDocuments()).toBe(1) + }) + + it('are kept when already right, redefined only on request, and never clobber a stranger', async () => { + expect(await ensureLiveView(t.db, names, users)).toBe('created') + expect(await ensureLiveView(t.db, names, users)).toBe('kept') + + // Somebody changed the pipeline by hand. + await t.db.command({ + collMod: 'workos_users_live', + viewOn: 'workos_users', + pipeline: [{ $match: { _env: 'prod' } }], + }) + await expect(ensureLiveView(t.db, names, users)).rejects.toThrow(/init --repair/) + await expect(migrate(t.db, [users], names)).rejects.toThrow(/init --repair/) + const { logger, lines } = captureLogger() + expect(await ensureLiveView(t.db, names, users, logger, { repairViews: true })).toBe('rebuilt') + expect(lines.some((l) => l.msg === 'live view redefined to the generated pipeline')).toBe(true) + const info = (await t.db.listCollections({ name: 'workos_users_live' }, { nameOnly: false }).toArray())[0]! + expect(info.options?.['pipeline']).toEqual(LIVE_PIPELINE) + + // A view over something else, or a plain collection, is not ours to touch. + await t.db.dropCollection('workos_users_live') + await t.db.createCollection('workos_users_live', { viewOn: 'somebody_elses', pipeline: [] }) + await expect(ensureLiveView(t.db, names, users, undefined, { repairViews: true })).rejects.toThrow( + /not the generated view/, + ) + await t.db.dropCollection('workos_users_live') + await t.db.createCollection('workos_users_live') + await expect(ensureLiveView(t.db, names, users, undefined, { repairViews: true })).rejects.toThrow( + /not the generated view/, + ) + }) + }) +}) diff --git a/packages/sync-mongodb/test/otel.test.ts b/packages/sync-mongodb/test/otel.test.ts new file mode 100644 index 0000000..2b4d314 --- /dev/null +++ b/packages/sync-mongodb/test/otel.test.ts @@ -0,0 +1,219 @@ +/** + * The engine's own instruments driven by real ticks against a real MongoDB + * (the bridge and the client instruments are unit-tested in + * sync-engine/test/otel.test.ts), and the CLI's SDK bootstrap. + */ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { context, metrics, propagation, type Attributes } from '@opentelemetry/api' +import { + AggregationTemporality, + InMemoryMetricExporter, + MeterProvider, + PeriodicExportingMetricReader, + type MetricData, +} from '@opentelemetry/sdk-metrics' +import { SyncEngine, WorkOSClient, silentLogger, METER_NAME } from '@workos/sync-engine' +import { createTestDb, type TestDb } from './helpers/mongo.js' +import { FakeWorkOS, resetIds } from '../../sync-engine/test/helpers/fake-workos.js' +import { FaultyDestination } from './helpers/faulty.js' + +import { DEFAULT_SERVICE_NAME, otelEnabled, startOtel, type SdkModule } from '../src/otel.js' + +const STREAMS = ['users', 'organizations', 'events'] + +/** An SDK in memory, installed as the global meter provider until `uninstall()`. */ +function install() { + const exporter = new InMemoryMetricExporter(AggregationTemporality.CUMULATIVE) + const provider = new MeterProvider({ + readers: [new PeriodicExportingMetricReader({ exporter, exportIntervalMillis: 3_600_000 })], + }) + expect(metrics.setGlobalMeterProvider(provider)).toBe(true) + return { + async collect(): Promise { + exporter.reset() + await provider.forceFlush() + const rm = exporter.getMetrics().at(-1) + return rm?.scopeMetrics.find((s) => s.scope.name === METER_NAME)?.metrics ?? [] + }, + async uninstall() { + metrics.disable() + await provider.shutdown() + }, + } +} + +const metric = (ms: MetricData[], name: string) => ms.find((m) => m.descriptor.name === name) + +function point(m: MetricData | undefined, attrs: Attributes) { + return m?.dataPoints.find((p) => { + const keys = new Set([...Object.keys(p.attributes), ...Object.keys(attrs)]) + return [...keys].every((k) => p.attributes[k] === attrs[k]) + }) +} + +const value = (m: MetricData | undefined, attrs: Attributes = {}) => point(m, attrs)?.value as number | undefined +const histogram = (m: MetricData | undefined, attrs: Attributes) => + point(m, attrs)?.value as { count: number; sum: number } | undefined + +describe('engine metrics', () => { + let t: TestDb + let fake: FakeWorkOS + let sdk: ReturnType + + beforeEach(async () => { + t = await createTestDb() + resetIds() + fake = new FakeWorkOS() + sdk = install() + }) + afterEach(async () => { + await sdk.uninstall() + await t.close() + }) + + function makeEngine(destination = new FaultyDestination({ client: t.client, database: t.name })) { + return new SyncEngine( + { + destination, + environments: [{ name: 'prod', apiKey: 'sk_test_1' }], + streams: STREAMS, + hydrate: 'off', + logger: silentLogger, + lagWindowMs: 0, + }, + { + clientFor: () => + new WorkOSClient({ + apiKey: 'sk_test_1', + apiUrl: 'http://fake.workos.local', + fetchImpl: fake.fetchImpl, + maxRetries: 0, + pageSize: 100, + }), + }, + ) + } + + it("times each environment's poll by outcome, with the bridge and the client following along", async () => { + fake.createUser() + fake.createOrganization() + const engine = makeEngine() + await engine.backfill() + await engine.tick() // two events: ok + await engine.tick() // nothing new: empty + + const ms = await sdk.collect() + const duration = metric(ms, 'workos.sync.tick.duration_seconds') + expect(histogram(duration, { env: 'prod', outcome: 'ok' })?.count).toBe(1) + expect(histogram(duration, { env: 'prod', outcome: 'empty' })?.count).toBe(1) + expect(duration?.dataPoints).toHaveLength(2) + + // The observable series were made at construction and read live state. + expect(value(metric(ms, 'workos.sync.up'))).toBe(1) + expect(value(metric(ms, 'workos.sync.ticks'), { env: 'prod' })).toBe(2) + expect(value(metric(ms, 'workos.sync.events'), { env: 'prod' })).toBe(2) + expect(value(metric(ms, 'workos.sync.poll_age_seconds'), { env: 'prod' })).toBeGreaterThanOrEqual(0) + + // The client the engine built is instrumented too, by phase and by the + // rate-limit bucket the endpoint draws on: users are under /user_management. + const requests = metric(ms, 'workos.sync.api.request.duration_seconds') + const ok = (phase: string, rate_key: string) => + histogram(requests, { phase, rate_key, 'http.response.status_code': 200 })?.count ?? 0 + expect(ok('tail', 'global')).toBeGreaterThanOrEqual(2) + expect(ok('backfill', 'global')).toBe(1) + expect(ok('backfill', 'user_management')).toBe(1) + }) + + it('counts a failed poll as an error and the events a later one quarantines', async () => { + fake.createUser({ email: 'good@example.com' }) + const bad = fake.createUser({ email: 'bad@example.com' }) + const destination = new FaultyDestination({ client: t.client, database: t.name }) + destination.poison = bad['id'] as string + const engine = makeEngine(destination) + + await engine.tick() // first sighting: the poll fails and the cursor stays + await engine.tick() // second: the poison event is set aside, the rest written + + const ms = await sdk.collect() + const duration = metric(ms, 'workos.sync.tick.duration_seconds') + expect(histogram(duration, { env: 'prod', outcome: 'error' })?.count).toBe(1) + expect(histogram(duration, { env: 'prod', outcome: 'ok' })?.count).toBe(1) + expect(value(metric(ms, 'workos.sync.events.rejected'), { env: 'prod' })).toBe(1) + expect(value(metric(ms, 'workos.sync.errors'), { env: 'prod' })).toBe(1) + expect(value(metric(ms, 'workos.sync.ticks'), { env: 'prod' })).toBe(2) + }) +}) + +describe('WORKOS_SYNC_OTEL', () => { + afterEach(() => { + vi.unstubAllEnvs() + // The real SDK registers these and leaves them on shutdown. + metrics.disable() + context.disable() + propagation.disable() + }) + + it('is off unless asked, and refuses what it cannot read', async () => { + expect(otelEnabled({})).toBe(false) + expect(otelEnabled({ WORKOS_SYNC_OTEL: '' })).toBe(false) + expect(otelEnabled({ WORKOS_SYNC_OTEL: '0' })).toBe(false) + expect(otelEnabled({ WORKOS_SYNC_OTEL: 'false' })).toBe(false) + expect(otelEnabled({ WORKOS_SYNC_OTEL: '1' })).toBe(true) + expect(otelEnabled({ WORKOS_SYNC_OTEL: ' True ' })).toBe(true) + expect(() => otelEnabled({ WORKOS_SYNC_OTEL: 'maybe' })).toThrow(/invalid WORKOS_SYNC_OTEL/) + await expect(startOtel({})).resolves.toBeNull() + }) + + it('starts the SDK with a service name of its own unless OTEL_SERVICE_NAME is set, and stops it', async () => { + const configs: unknown[] = [] + let started = 0 + let stopped = 0 + const sdkNode: SdkModule = { + NodeSDK: class { + constructor(configuration?: { serviceName?: string }) { + configs.push(configuration) + } + start() { + started++ + } + async shutdown() { + stopped++ + } + }, + } + const load = async () => sdkNode + + const handle = await startOtel({ WORKOS_SYNC_OTEL: '1' }, load) + expect(configs).toEqual([{ serviceName: DEFAULT_SERVICE_NAME }]) + expect(started).toBe(1) + await handle!.shutdown() + expect(stopped).toBe(1) + + await startOtel({ WORKOS_SYNC_OTEL: 'yes', OTEL_SERVICE_NAME: 'mirror' }, load) + expect(configs[1]).toEqual({}) + }) + + it('says what is missing when the SDK is not installed, and passes other failures through', async () => { + const missing = async () => { + throw Object.assign(new Error("Cannot find package '@opentelemetry/sdk-node'"), { code: 'ERR_MODULE_NOT_FOUND' }) + } + await expect(startOtel({ WORKOS_SYNC_OTEL: '1' }, missing)).rejects.toThrow(/not installed/) + const broken = async () => { + throw new Error('boom') + } + await expect(startOtel({ WORKOS_SYNC_OTEL: '1' }, broken)).rejects.toThrow('boom') + }) + + it('installs the real SDK as the global meter provider, configured from OTEL_* variables', async () => { + vi.stubEnv('OTEL_METRICS_EXPORTER', 'console') + vi.stubEnv('OTEL_TRACES_EXPORTER', 'none') + vi.stubEnv('OTEL_LOGS_EXPORTER', 'none') + const before = metrics.getMeterProvider() + const handle = await startOtel({ WORKOS_SYNC_OTEL: '1' }) + expect(handle).not.toBeNull() + const provider = metrics.getMeterProvider() + expect(provider).not.toBe(before) + expect(typeof (provider as { forceFlush?: unknown }).forceFlush).toBe('function') + await handle!.shutdown() + }) +}) diff --git a/packages/sync-mongodb/test/reconcile.test.ts b/packages/sync-mongodb/test/reconcile.test.ts new file mode 100644 index 0000000..4cdc3e4 --- /dev/null +++ b/packages/sync-mongodb/test/reconcile.test.ts @@ -0,0 +1,183 @@ +/** + * The reconcile set difference, computed by the server rather than in Node. + * + * These exercise the database half on its own, with the remote side handed in + * as pages. The engine half — pagination, repair — is covered by the drift + * and repair cases in engine.test.ts. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { diffIdsInMemory, streamByName, STREAMS } from '@workos/sync-engine' +import type { Collection } from 'mongodb' +import { createTestDb, type TestDb } from './helpers/mongo.js' +import { migrate } from '../src/schema/migrate.js' +import { Names } from '../src/schema/names.js' +import { upsertRows, deleteRows, type MirrorDoc } from '../src/db/upsert.js' +import { diffIdsInMongo } from '../src/db/reconcile.js' +import { MongoDestination } from '../src/destination.js' + +const users = streamByName('users')! +const names = new Names() +const opts = {} + +// Zero-padded so lexical order (what the server sorts by) is numeric order. +const user = (n: number, over: Record = {}) => ({ + object: 'user', + id: `user_${String(n).padStart(5, '0')}`, + email: `u${n}@example.com`, + created_at: '2026-01-01T00:00:00.000Z', + updated_at: '2026-01-01T00:00:00.000Z', + ...over, +}) + +const range = (from: number, to: number) => Array.from({ length: to - from + 1 }, (_, i) => user(from + i)) + +async function* pages(rows: Record[], size = 100) { + for (let i = 0; i < rows.length; i += size) yield rows.slice(i, i + size) +} + +describe('diff in mongodb', () => { + let t: TestDb + let coll: Collection + + beforeEach(async () => { + t = await createTestDb() + await migrate(t.db, STREAMS, names) + coll = t.db.collection(names.mirror(users)) + }) + afterEach(async () => { + await t.close() + }) + + const seed = (rows: Record[], env = 'prod') => + upsertRows( + coll, + opts, + users, + env, + rows.map((data) => ({ data, updatedAt: data['updated_at'] as string })), + 'replace', + ) + + const stagingLeft = async () => (await t.db.listCollections({ name: /_reconcile_/ }).toArray()).map((c) => c.name) + + it('reports matched, localOnly and remoteOnly, with the payload intact', async () => { + await seed([user(1), user(2), user(3)]) + const four = user(4, { email: 'four@example.com', metadata: { plan: 'pro', 'a.b': '$x' } }) + + const diff = await diffIdsInMongo(t.db, names, opts, users, 'prod', pages([user(2), user(3), four])) + + expect(diff.matched).toBe(2) + expect(diff.localOnly).toEqual(['user_00001']) + expect(diff.remoteOnly).toEqual([four]) + expect(diff.skipped).toBe(0) + // The staging collections are gone once the answer is out. + expect(await stagingLeft()).toEqual([]) + }) + + it('compares complete composite keys on the server and in memory', async () => { + const destination = new MongoDestination({ client: t.client, database: t.name }) + await destination.init(STREAMS) + const roles = streamByName('organization_roles')! + const removed = { organization_id: 'org_1', slug: 'admin' } + const kept = { organization_id: 'org_1', slug: 'auditor' } + const elsewhere = { organization_id: 'org_2', slug: 'admin' } + const added = { organization_id: 'org_1', slug: 'member', permissions: ['read'] } + await destination.upsert( + roles, + 'prod', + [removed, kept, elsewhere].map((data) => ({ data, updatedAt: '2026-01-01T00:00:00.000Z' })), + 'replace', + ) + const remote = [ + kept, + elsewhere, + added, + { ...added, permissions: [] }, + { organization_id: 'org_1' }, + { slug: 'admin' }, + { organization_id: 'org_1', slug: '' }, + ] + + const diff = await diffIdsInMongo(t.db, names, opts, roles, 'prod', pages(remote, 1)) + + expect(diff).toEqual({ + matched: 2, + localOnly: ['["org_1","admin"]'], + remoteOnly: [added], + skipped: 3, + }) + expect(await destination.liveIds(roles, 'prod')).toEqual([ + '["org_1","admin"]', + '["org_1","auditor"]', + '["org_2","admin"]', + ]) + expect(await diffIdsInMemory(destination, roles, 'prod', pages(remote, 1))).toEqual({ + matched: diff.matched, + localOnly: diff.localOnly, + remoteOnly: diff.remoteOnly, + }) + }) + + it('ignores tombstoned rows and other environments, and keeps the first of a duplicated id', async () => { + await seed(range(1, 5)) + await seed([user(1, { email: 'staging@example.com' })], 'staging') + await deleteRows(coll, opts, users, 'prod', [{ data: user(5), updatedAt: '2026-01-02T00:00:00.000Z' }]) + + // user 3 arrives twice across a page boundary, as it would when a write + // moves the boundary under the listing; the first copy is what is kept. + const remote = [ + user(1), + user(2), + user(3, { email: 'first@example.com' }), + user(3, { email: 'second@example.com' }), + user(6), + ] + const diff = await diffIdsInMongo(t.db, names, opts, users, 'prod', pages(remote, 3)) + + expect(diff.matched).toBe(3) + expect(diff.localOnly).toEqual(['user_00004']) + expect(diff.remoteOnly).toEqual([user(6)]) + expect(remote.filter((r) => r['id'] === 'user_00003')).toHaveLength(2) + }) + + it('scales with the drift, not the stream, and holds pages one at a time', async () => { + await seed(range(1, 1_200)) + let served = 0 + async function* counted() { + for await (const page of pages( + range(1, 1_210).filter((u) => u.id !== 'user_00007'), + 100, + )) { + served++ + yield page + } + } + const diff = await diffIdsInMongo(t.db, names, { batchSize: 50 }, users, 'prod', counted()) + expect(served).toBe(13) + expect(diff.matched).toBe(1_199) + expect(diff.localOnly).toEqual(['user_00007']) + expect(diff.remoteOnly.map((r) => r['id'])).toEqual(range(1_201, 1_210).map((u) => u.id)) + }) + + it('cleans up its staging collections when the listing fails midway', async () => { + await seed(range(1, 3)) + async function* failing() { + yield [user(1)] + throw new Error('listing failed') + } + await expect(diffIdsInMongo(t.db, names, opts, users, 'prod', failing())).rejects.toThrow('listing failed') + expect(await stagingLeft()).toEqual([]) + }) + + it('reports an empty mirror and an empty listing without choking', async () => { + expect(await diffIdsInMongo(t.db, names, opts, users, 'prod', pages([]))).toEqual({ + matched: 0, + localOnly: [], + remoteOnly: [], + skipped: 0, + }) + expect(await diffIdsInMongo(t.db, names, opts, users, 'prod', pages([user(1)]))).toMatchObject({ + remoteOnly: [user(1)], + }) + }) +}) diff --git a/packages/sync-mongodb/test/rejected.test.ts b/packages/sync-mongodb/test/rejected.test.ts new file mode 100644 index 0000000..130806e --- /dev/null +++ b/packages/sync-mongodb/test/rejected.test.ts @@ -0,0 +1,229 @@ +/** + * The quarantine store on its own, plus the two things that reach into it + * from outside: `redact`, which must scrub it, and `freshness`, which the + * status command reports alongside its count. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { streamByName, STREAMS, type WorkOSEvent } from '@workos/sync-engine' +import { createTestDb, type TestDb } from './helpers/mongo.js' +import { MongoDestination } from '../src/destination.js' +import type { RejectedDoc } from '../src/db/rejected.js' + +const users = streamByName('users')! +const events = streamByName('events')! +const auth = streamByName('auth_attempts')! + +const event = (n: number, data: Record, over: Partial = {}): WorkOSEvent => ({ + id: `event_${String(n).padStart(3, '0')}`, + event: 'user.updated', + data, + created_at: '2026-01-01T00:00:00.000Z', + ...over, +}) + +describe('quarantine', () => { + let t: TestDb + let dest: MongoDestination + + beforeEach(async () => { + t = await createTestDb() + dest = new MongoDestination({ client: t.client, database: t.name }) + await dest.init(STREAMS) + }) + afterEach(async () => { + await t.close() + }) + + it('stores whole events, lists them oldest first, counts without reading them, and forgets on request', async () => { + const e2 = event(2, { id: 'user_2', name: '$ref', nested: { 'a.b': 1 } }) + const e1 = event(1, { id: 'user_1' }) + await dest.reject('prod', [ + { event: e2, error: 'boom' }, + { event: e1, error: 'bang' }, + ]) + await dest.reject('staging', [{ event: event(3, { id: 'user_3' }), error: 'elsewhere' }]) + + expect(await dest.rejectedCount('prod')).toBe(2) + const listed = await dest.rejected('prod') + expect(listed.map((r) => r.eventId)).toEqual(['event_001', 'event_002']) + expect(listed[1]).toMatchObject({ + eventId: 'event_002', + eventType: 'user.updated', + objectId: 'user_2', + createdAt: '2026-01-01T00:00:00.000Z', + error: 'boom', + raw: e2, + }) + expect(listed[1]!.rejectedAt).toMatch(/^\d{4}-\d{2}-\d{2}T/) + expect(await dest.rejected('prod', { ids: ['event_002'] })).toHaveLength(1) + expect(await dest.rejected('prod', { ids: [] })).toEqual([]) + + expect(await dest.clearRejected('prod', ['event_001', 'event_999'])).toBe(1) + expect(await dest.clearRejected('prod', [])).toBe(0) + expect(await dest.rejectedCount('prod')).toBe(1) + expect(await dest.rejectedCount('staging')).toBe(1) + }) + + it('replaces the error and the timestamp when an id is rejected again', async () => { + const e = event(1, { id: 'user_1' }) + await dest.reject('prod', [{ event: e, error: 'first' }]) + const before = (await dest.rejected('prod'))[0]! + await new Promise((r) => setTimeout(r, 5)) + await dest.reject('prod', [{ event: { ...e, data: { id: 'user_1', fixed: true } }, error: 'second' }]) + const after = (await dest.rejected('prod'))[0]! + expect(await dest.rejectedCount('prod')).toBe(1) + expect(after.error).toBe('second') + expect(after.raw.data).toEqual({ id: 'user_1', fixed: true }) + expect(Date.parse(after.rejectedAt)).toBeGreaterThanOrEqual(Date.parse(before.rejectedAt)) + }) + + it('never fails on the timestamp that may have caused the rejection', async () => { + const e = event(1, { id: 'user_1' }, { created_at: '2026-02-30T12:00:00Z' }) + await dest.reject('prod', [{ event: e, error: 'out of range' }]) + const stored = await t.db.collection('workos__rejected').findOne({ event_id: 'event_001' }) + expect(stored!.created_at).toBeNull() + expect((await dest.rejected('prod'))[0]!.createdAt).toBe('2026-02-30T12:00:00Z') + // A membership event has no data.id; object_id is null rather than invented. + await dest.reject('prod', [{ event: event(2, { group_id: 'g', organization_membership_id: 'om' }), error: 'x' }]) + expect((await dest.rejected('prod', { ids: ['event_002'] }))[0]!.objectId).toBeNull() + }) + + it('is scoped to the sync id', async () => { + await dest.reject('prod', [{ event: event(1, { id: 'user_1' }), error: 'x' }]) + const other = new MongoDestination({ client: t.client, database: t.name, syncId: 'reporting' }) + await other.init(STREAMS) + expect(await other.rejectedCount('prod')).toBe(0) + expect(await other.clearRejected('prod', ['event_001'])).toBe(0) + expect(await dest.rejectedCount('prod')).toBe(1) + }) +}) + +describe('redact', () => { + let t: TestDb + let dest: MongoDestination + + beforeEach(async () => { + t = await createTestDb() + dest = new MongoDestination({ client: t.client, database: t.name }) + await dest.init(STREAMS) + }) + afterEach(async () => { + await t.close() + }) + + it('erases a subject from the mirror, the archive and the quarantine, and reports each', async () => { + const at = '2026-01-01T00:00:00.000Z' + await dest.upsert( + users, + 'prod', + [ + { data: { id: 'user_1', email: 'forget@example.com' }, updatedAt: at }, + { data: { id: 'user_2', email: 'keep@example.com' }, updatedAt: at }, + ], + 'replace', + ) + await dest.upsert( + auth, + 'prod', + [ + { data: { event_id: 'event_a', user_id: 'user_1', status: 'failed' }, updatedAt: at }, + { data: { event_id: 'event_b', user_id: 'user_2', status: 'failed' }, updatedAt: at }, + ], + 'replace', + ) + await dest.upsert( + events, + 'prod', + [ + { data: event(1, { id: 'user_1' }) as unknown as Record, updatedAt: at }, + { + data: event(2, { id: 'session_1', user_id: 'user_1' }, { event: 'session.created' }) as unknown as Record< + string, + unknown + >, + updatedAt: at, + }, + { data: event(3, { id: 'user_2' }) as unknown as Record, updatedAt: at }, + ], + 'replace', + ) + await dest.reject('prod', [ + { event: event(4, { id: 'user_1' }), error: 'x' }, + { event: event(5, { id: 'om_1', user_id: 'user_1' }, { event: 'organization_membership.created' }), error: 'x' }, + { event: event(6, { id: 'user_2' }), error: 'x' }, + ]) + // Another environment's rows are not this environment's to erase. + await dest.upsert( + users, + 'staging', + [{ data: { id: 'user_1', email: 'staging@example.com' }, updatedAt: at }], + 'replace', + ) + + const removed = await dest.redact(STREAMS, 'prod', 'user_1') + expect(removed).toEqual( + expect.arrayContaining([ + { table: 'users', rows: 1 }, + { table: 'auth_attempts', rows: 1 }, + { table: 'events', rows: 2 }, + { table: '_rejected', rows: 2 }, + ]), + ) + expect(removed.find((r) => r.table === 'organizations')).toBeUndefined() + expect(await t.db.collection('workos_users').countDocuments({ _env: 'prod' })).toBe(1) + expect(await t.db.collection('workos_users').countDocuments({ _env: 'staging' })).toBe(1) + expect(await t.db.collection('workos_events').countDocuments()).toBe(1) + expect(await dest.rejectedCount('prod')).toBe(1) + expect(await dest.redact(STREAMS, 'prod', 'user_1')).toEqual([]) + }) +}) + +describe('freshness', () => { + let t: TestDb + let dest: MongoDestination + + beforeEach(async () => { + t = await createTestDb() + dest = new MongoDestination({ client: t.client, database: t.name }) + await dest.init(STREAMS) + }) + afterEach(async () => { + await t.close() + }) + + it('reports counts, tombstones, the last write and the lag inside the window', async () => { + expect(await dest.freshness(users, 'prod', 24)).toEqual({ + stream: 'users', + rows: 0, + deleted: 0, + lastSyncedAt: null, + stalenessSeconds: null, + lagSeconds: null, + lagSampleSize: 0, + }) + const recent = new Date(Date.now() - 30_000).toISOString() + await dest.upsert( + users, + 'prod', + [ + { data: { id: 'user_1' }, updatedAt: recent }, + { data: { id: 'user_2' }, updatedAt: '2025-01-01T00:00:00.000Z' }, + ], + 'replace', + ) + await dest.delete(users, 'prod', [{ data: { id: 'user_2' }, updatedAt: '2025-01-02T00:00:00.000Z' }]) + + const f = await dest.freshness(users, 'prod', 24) + expect(f).toMatchObject({ stream: 'users', rows: 2, deleted: 1, lagSampleSize: 1 }) + expect(Date.parse(f.lastSyncedAt!)).toBeGreaterThan(Date.now() - 60_000) + expect(f.stalenessSeconds).toBeGreaterThanOrEqual(29) + expect(f.stalenessSeconds).toBeLessThan(120) + // Only the recent row is in the lag average; the year-old one would make a current mirror look months behind. + expect(f.lagSeconds).toBeGreaterThanOrEqual(29) + expect(f.lagSeconds).toBeLessThan(120) + expect((await dest.freshness(users, 'prod', 0)).lagSeconds).toBeNull() + expect((await dest.freshness(users, 'staging', 24)).rows).toBe(0) + // Append-only streams have no tombstones to count. + expect((await dest.freshness(events, 'prod', 24)).deleted).toBe(0) + }) +}) diff --git a/packages/sync-mongodb/test/setup.ts b/packages/sync-mongodb/test/setup.ts new file mode 100644 index 0000000..49914df --- /dev/null +++ b/packages/sync-mongodb/test/setup.ts @@ -0,0 +1,5 @@ +/** Runs in every test file: gives the shared client back once the file is done. */ +import { afterAll } from 'vitest' +import { closeSharedClient } from './helpers/mongo.js' + +afterAll(() => closeSharedClient()) diff --git a/packages/sync-mongodb/test/timestamps.test.ts b/packages/sync-mongodb/test/timestamps.test.ts new file mode 100644 index 0000000..e5f8f60 --- /dev/null +++ b/packages/sync-mongodb/test/timestamps.test.ts @@ -0,0 +1,74 @@ +/** + * The strict ISO-8601 parser the staleness gate depends on. It has to agree + * with `parse_iso8601` in the Postgres destination about what is a date, so + * the same event is quarantined by both rather than stored by one. + */ +import { describe, it, expect } from 'vitest' +import { parseIso8601, parseInstant, parseInstantOrNull, InvalidTimestampError } from '../src/db/timestamps.js' + +const at = (iso: string) => { + const r = parseIso8601(iso) + return 'date' in r ? r.date.toISOString() : r.failure +} + +describe('parseIso8601', () => { + it('reads the shapes WorkOS emits', () => { + expect(at('2026-01-02T03:04:05.678Z')).toBe('2026-01-02T03:04:05.678Z') + expect(at('2026-01-02T03:04:05Z')).toBe('2026-01-02T03:04:05.000Z') + expect(at('2026-01-02 03:04:05.1z')).toBe('2026-01-02T03:04:05.100Z') + // Beyond millisecond precision is dropped, not rounded, like a BSON date. + expect(at('2026-01-02T03:04:05.6789999Z')).toBe('2026-01-02T03:04:05.678Z') + }) + + it('applies an offset, and reads no designator as UTC', () => { + expect(at('2026-01-02T03:04:05+02:00')).toBe('2026-01-02T01:04:05.000Z') + expect(at('2026-01-02T03:04:05-0530')).toBe('2026-01-02T08:34:05.000Z') + expect(at('2026-01-02T03:04:05')).toBe('2026-01-02T03:04:05.000Z') + }) + + it('refuses what is not an instant', () => { + expect(at('2026-01-02')).toBe('shape') + expect(at('yesterday')).toBe('shape') + expect(at('')).toBe('shape') + expect(at('1700000000')).toBe('shape') + }) + + it('refuses a well-shaped value whose fields are out of range, which V8 would roll over', () => { + expect(new Date('2026-02-30T12:00:00Z').getTime()).not.toBeNaN() + expect(at('2026-02-30T12:00:00Z')).toBe('range') + expect(at('2026-13-01T00:00:00Z')).toBe('range') + expect(at('2026-04-31T00:00:00Z')).toBe('range') + expect(at('2026-01-01T24:00:00Z')).toBe('range') + expect(at('2026-01-01T00:60:00Z')).toBe('range') + expect(at('0000-01-01T00:00:00Z')).toBe('range') + // Leap years are real, and so is the leap second. + expect(at('2024-02-29T00:00:00Z')).toBe('2024-02-29T00:00:00.000Z') + expect(at('2100-02-29T00:00:00Z')).toBe('range') + expect(at('2026-06-30T23:59:60Z')).toBe('2026-07-01T00:00:00.000Z') + }) + + it('keeps years below 100 as written', () => { + expect(at('0042-01-01T00:00:00Z')).toBe('0042-01-01T00:00:00.000Z') + }) +}) + +describe('parseInstant', () => { + it('throws a deterministic, coded error the engine can quarantine on', () => { + expect(() => parseInstant('2026-02-30T12:00:00Z')).toThrow(InvalidTimestampError) + expect(() => parseInstant('2026-02-30T12:00:00Z')).toThrow(/out of range/) + expect(() => parseInstant('soon')).toThrow(/not an ISO-8601 instant/) + try { + parseInstant('soon') + } catch (err) { + expect((err as InvalidTimestampError).code).toBe('INVALID_TIMESTAMP') + } + expect(parseInstant('2026-01-02T03:04:05.678Z').toISOString()).toBe('2026-01-02T03:04:05.678Z') + }) + + it('has a lenient twin for fields that must never fail a write', () => { + expect(parseInstantOrNull('2026-02-30T12:00:00Z')).toBeNull() + expect(parseInstantOrNull(undefined)).toBeNull() + expect(parseInstantOrNull(42)).toBeNull() + expect(parseInstantOrNull('2026-01-02T03:04:05Z')?.toISOString()).toBe('2026-01-02T03:04:05.000Z') + }) +}) diff --git a/packages/sync-mongodb/test/upsert.test.ts b/packages/sync-mongodb/test/upsert.test.ts new file mode 100644 index 0000000..57c7262 --- /dev/null +++ b/packages/sync-mongodb/test/upsert.test.ts @@ -0,0 +1,412 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { streamByName, STREAMS } from '@workos/sync-engine' +import type { Collection } from 'mongodb' +import { createTestDb, type TestDb } from './helpers/mongo.js' +import { migrate } from '../src/schema/migrate.js' +import { Names } from '../src/schema/names.js' +import { + upsertRows, + deleteRows, + relinkRows, + unlinkRows, + dedupeByKey, + wavesByKey, + keyValues, + keyFilter, + type MirrorDoc, +} from '../src/db/upsert.js' +import { InvalidTimestampError } from '../src/db/timestamps.js' + +const users = streamByName('users')! +const events = streamByName('events')! +const memberships = streamByName('directory_group_memberships')! +const names = new Names() +const opts = {} + +const userPayload = (over: Record = {}) => ({ + object: 'user', + id: 'user_1', + email: 'ada@example.com', + first_name: 'Ada', + created_at: '2026-01-01T00:00:00.000Z', + updated_at: '2026-01-01T00:00:00.000Z', + ...over, +}) + +describe('key extraction', () => { + it('reads slug-keyed streams that have no id in their event payload', () => { + // role.* events carry only a slug — no id at all. + const roles = streamByName('roles')! + expect(keyValues(roles, { slug: 'admin', permissions: [] })).toEqual(['admin']) + expect(keyValues(roles, { permissions: [] })).toBeUndefined() + }) + + it('reads composite keys, and names a row by every one of them', () => { + const orgRoles = streamByName('organization_roles')! + expect(keyValues(orgRoles, { organization_id: 'org_1', slug: 'admin' })).toEqual(['org_1', 'admin']) + expect(keyFilter(orgRoles, 'prod', { organization_id: 'org_1', slug: 'admin' })).toEqual({ + _env: 'prod', + '_raw.organization_id': 'org_1', + '_raw.slug': 'admin', + }) + // Nested paths are addressed through _raw too. + expect(keyFilter(users, 'prod', { id: 'user_1' })).toEqual({ _env: 'prod', '_raw.id': 'user_1' }) + }) + + it('collapses duplicate keys within a batch, keeping the newest', () => { + const { rows, dropped } = dedupeByKey(users, [ + { data: userPayload({ email: 'old@example.com' }), updatedAt: '2026-01-01T00:00:00.000Z' }, + { data: userPayload({ email: 'new@example.com' }), updatedAt: '2026-01-02T00:00:00.000Z' }, + { data: { email: 'no-id@example.com' }, updatedAt: '2026-01-03T00:00:00.000Z' }, + ]) + expect(rows).toHaveLength(1) + expect(rows[0]!.data['email']).toBe('new@example.com') + expect(dropped).toBe(1) + }) + + it('splits repeated keys into ordered waves and leaves distinct keys in one', () => { + const first = { data: userPayload({ external_id: 'ext_1' }), updatedAt: '2026-01-01T00:00:00.000Z' } + const other = { data: userPayload({ id: 'user_2' }), updatedAt: '2026-01-01T00:00:00.000Z' } + const second = { data: { id: 'user_1', first_name: 'Grace' }, updatedAt: '2026-01-02T00:00:00.000Z' } + const { waves, dropped } = wavesByKey(users, [ + first, + other, + second, + { data: { email: 'no-id@example.com' }, updatedAt: '2026-01-03T00:00:00.000Z' }, + ]) + expect(waves).toEqual([[first, other], [second]]) + expect(waves[0]![0]).toBe(first) + expect(dropped).toBe(1) + expect(wavesByKey(users, [first, other]).waves).toEqual([[first, other]]) + }) +}) + +describe('write path', () => { + let t: TestDb + let coll: Collection + + beforeEach(async () => { + t = await createTestDb() + await migrate(t.db, STREAMS, names) + coll = t.db.collection(names.mirror(users)) + }) + afterEach(async () => { + await t.close() + }) + + const row = (data: Record, updatedAt: string) => ({ data, updatedAt }) + const readUser = async (id = 'user_1') => coll.findOne({ _env: 'prod', '_raw.id': id }) + + it('inserts and reports created vs updated', async () => { + const first = await upsertRows( + coll, + opts, + users, + 'prod', + [row(userPayload(), '2026-01-01T00:00:00.000Z')], + 'replace', + ) + expect(first).toMatchObject({ created: 1, updated: 0, skipped: 0 }) + + const second = await upsertRows( + coll, + opts, + users, + 'prod', + [row(userPayload({ email: 'changed@example.com' }), '2026-01-02T00:00:00.000Z')], + 'replace', + ) + expect(second).toMatchObject({ created: 0, updated: 1, skipped: 0 }) + const doc = (await readUser())! + expect(doc._raw['email']).toBe('changed@example.com') + expect(doc._updated_at.toISOString()).toBe('2026-01-02T00:00:00.000Z') + expect(doc._synced_at.getTime()).toBeGreaterThan(Date.now() - 60_000) + expect(doc._env).toBe('prod') + }) + + it('rejects a write that is older than the stored row', async () => { + await upsertRows( + coll, + opts, + users, + 'prod', + [row(userPayload({ email: 'current@example.com' }), '2026-01-05T00:00:00.000Z')], + 'replace', + ) + // An out-of-order delivery arriving late must not win. + const stale = await upsertRows( + coll, + opts, + users, + 'prod', + [row(userPayload({ email: 'stale@example.com' }), '2026-01-01T00:00:00.000Z')], + 'replace', + ) + expect(stale).toMatchObject({ created: 0, updated: 0, skipped: 1 }) + expect((await readUser())!._raw['email']).toBe('current@example.com') + }) + + it('accepts a write with an identical timestamp', async () => { + // The gate is >= rather than >. Two events can share a timestamp, and with + // a strict comparison the second is dropped forever. + const ts = '2026-01-05T00:00:00.000Z' + await upsertRows(coll, opts, users, 'prod', [row(userPayload({ email: 'first@example.com' }), ts)], 'replace') + const same = await upsertRows( + coll, + opts, + users, + 'prod', + [row(userPayload({ email: 'second@example.com' }), ts)], + 'replace', + ) + expect(same.updated).toBe(1) + expect((await readUser())!._raw['email']).toBe('second@example.com') + }) + + it('skips a write with the same payload and timestamp', async () => { + await upsertRows(coll, opts, users, 'prod', [row(userPayload(), '2026-01-01T00:00:00.000Z')], 'replace') + const before = (await readUser())!._synced_at + const again = await upsertRows( + coll, + opts, + users, + 'prod', + [row(userPayload(), '2026-01-01T00:00:00.000Z')], + 'replace', + ) + // Rewriting an identical observation would move _synced_at for nothing. + expect(again).toMatchObject({ created: 0, updated: 0, skipped: 1 }) + expect((await readUser())!._synced_at).toEqual(before) + }) + + it('keeps the fields of a full snapshot under a later subset in one merge batch', async () => { + // A hydrated create and the update event that followed it share a poll. + const stats = await upsertRows( + coll, + opts, + users, + 'prod', + [ + row(userPayload({ external_id: 'ext_1', metadata: { plan: 'pro' } }), '2026-01-01T00:00:00.000Z'), + row( + { object: 'user', id: 'user_1', email: 'ada@example.com', first_name: 'Grace' }, + '2026-01-02T00:00:00.000Z', + ), + ], + 'merge', + ) + expect(stats).toMatchObject({ created: 1, updated: 1, skipped: 0 }) + const doc = (await readUser())! + expect(doc._raw).toMatchObject({ first_name: 'Grace', external_id: 'ext_1', metadata: { plan: 'pro' } }) + expect(doc._updated_at.toISOString()).toBe('2026-01-02T00:00:00.000Z') + }) + + it('merges shallowly: a nested object is replaced whole, a null is kept, a payload is never read as an expression', async () => { + await upsertRows( + coll, + opts, + users, + 'prod', + [row(userPayload({ metadata: { plan: 'pro', region: 'eu' }, locale: 'en' }), '2026-01-01T00:00:00.000Z')], + 'replace', + ) + const merged = await upsertRows( + coll, + opts, + users, + 'prod', + [ + row( + { + id: 'user_1', + // `$100` would be a field path in an expression; it must land as text. + first_name: '$100 plan', + // A key the event omits inside metadata is gone, as `jsonb ||` would make it. + metadata: { plan: 'free' }, + locale: null, + // Directory payloads carry attribute names with dots and dollars. + raw_attributes: { 'urn:ietf:params:scim:schemas:core:2.0:User': { 'a.b': 1, $ref: 'x' } }, + }, + '2026-01-02T00:00:00.000Z', + ), + ], + 'merge', + ) + expect(merged).toMatchObject({ updated: 1 }) + const doc = (await readUser())! + expect(doc._raw).toMatchObject({ + email: 'ada@example.com', + first_name: '$100 plan', + metadata: { plan: 'free' }, + locale: null, + raw_attributes: { 'urn:ietf:params:scim:schemas:core:2.0:User': { 'a.b': 1, $ref: 'x' } }, + }) + expect((doc._raw['metadata'] as Record)['region']).toBeUndefined() + }) + + it('replaces whole when told to, dropping what the payload omits', async () => { + await upsertRows( + coll, + opts, + users, + 'prod', + [row(userPayload({ external_id: 'ext_1' }), '2026-01-01T00:00:00.000Z')], + 'replace', + ) + await upsertRows( + coll, + opts, + users, + 'prod', + [row({ id: 'user_1', email: 'x@example.com' }, '2026-01-02T00:00:00.000Z')], + 'replace', + ) + expect((await readUser())!._raw).toEqual({ id: 'user_1', email: 'x@example.com' }) + }) + + it('tombstones terminally: a later update refreshes the payload but never resurrects the row', async () => { + await upsertRows(coll, opts, users, 'prod', [row(userPayload(), '2026-01-01T00:00:00.000Z')], 'replace') + const deleted = await deleteRows(coll, opts, users, 'prod', [row({ id: 'user_1' }, '2026-01-02T00:00:00.000Z')]) + expect(deleted).toMatchObject({ deleted: 1 }) + let doc = (await readUser())! + expect(doc._deleted_at?.toISOString()).toBe('2026-01-02T00:00:00.000Z') + expect(doc._raw['email']).toBe('ada@example.com') + + await upsertRows( + coll, + opts, + users, + 'prod', + [row({ id: 'user_1', email: 'late@example.com' }, '2026-01-03T00:00:00.000Z')], + 'merge', + ) + doc = (await readUser())! + expect(doc._raw['email']).toBe('late@example.com') + expect(doc._deleted_at?.toISOString()).toBe('2026-01-02T00:00:00.000Z') + + // A delete for a row never seen inserts the tombstone, and an older + // delete never moves a newer tombstone back. + expect( + await deleteRows(coll, opts, users, 'prod', [row({ id: 'user_9' }, '2026-01-01T00:00:00.000Z')]), + ).toMatchObject({ deleted: 1 }) + expect((await readUser('user_9'))!._deleted_at?.toISOString()).toBe('2026-01-01T00:00:00.000Z') + await deleteRows(coll, opts, users, 'prod', [row({ id: 'user_1' }, '2026-01-01T12:00:00.000Z')]) + expect((await readUser())!._deleted_at?.toISOString()).toBe('2026-01-02T00:00:00.000Z') + }) + + it('applies membership links and unlinks by time, in both directions', async () => { + const mcoll = t.db.collection(names.mirror(memberships)) + const pair = { directory_group_id: 'dg_1', directory_user_id: 'du_1', directory_id: 'dir_1' } + const at = (day: number) => `2026-01-${String(day).padStart(2, '0')}T00:00:00.000Z` + const read = () => + mcoll.findOne({ _env: 'prod', '_raw.directory_group_id': 'dg_1', '_raw.directory_user_id': 'du_1' }) + + // Added, then removed. + expect(await relinkRows(mcoll, opts, memberships, 'prod', [row(pair, at(1))])).toMatchObject({ created: 1 }) + expect(await unlinkRows(mcoll, opts, memberships, 'prod', [row(pair, at(3))])).toMatchObject({ deleted: 1 }) + expect((await read())!._deleted_at?.toISOString()).toBe(at(3)) + + // A re-add older than the removal is a replayed stale event: a no-op. + expect(await relinkRows(mcoll, opts, memberships, 'prod', [row(pair, at(2))])).toMatchObject({ + skipped: 1, + updated: 0, + }) + expect((await read())!._deleted_at?.toISOString()).toBe(at(3)) + + // A re-add at least as new as the removal clears the tombstone. + expect(await relinkRows(mcoll, opts, memberships, 'prod', [row(pair, at(3))])).toMatchObject({ updated: 1 }) + expect((await read())!._deleted_at).toBeNull() + + // A removal older than the re-add must not hide a membership active now. + expect(await unlinkRows(mcoll, opts, memberships, 'prod', [row(pair, at(2))])).toMatchObject({ + skipped: 1, + deleted: 0, + }) + expect((await read())!._deleted_at).toBeNull() + + // A removal that really is newer lands, and a reconcile tombstone (not gated) + // is compared against by the next re-add. + expect(await unlinkRows(mcoll, opts, memberships, 'prod', [row(pair, at(5))])).toMatchObject({ deleted: 1 }) + await deleteRows(mcoll, opts, memberships, 'prod', [row(pair, at(7))]) + expect((await read())!._deleted_at?.toISOString()).toBe(at(7)) + expect(await relinkRows(mcoll, opts, memberships, 'prod', [row(pair, at(6))])).toMatchObject({ skipped: 1 }) + expect(await relinkRows(mcoll, opts, memberships, 'prod', [row(pair, at(7))])).toMatchObject({ updated: 1 }) + expect((await read())!._deleted_at).toBeNull() + + // An unlink for a membership never seen inserts it tombstoned. + const other = { ...pair, directory_user_id: 'du_2' } + expect(await unlinkRows(mcoll, opts, memberships, 'prod', [row(other, at(1))])).toMatchObject({ deleted: 1 }) + expect((await mcoll.findOne({ '_raw.directory_user_id': 'du_2' }))!._deleted_at?.toISOString()).toBe(at(1)) + }) + + it('treats an append-only stream as an immutable log', async () => { + const ecoll = t.db.collection(names.mirror(events)) + const event = { + id: 'event_1', + event: 'user.created', + data: { id: 'user_1', name: '$x' }, + created_at: '2026-01-01T00:00:00.000Z', + } + expect(await upsertRows(ecoll, opts, events, 'prod', [row(event, event.created_at)], 'replace')).toMatchObject({ + created: 1, + }) + // Re-delivery, even with a changed payload or a newer timestamp, changes nothing. + expect( + await upsertRows( + ecoll, + opts, + events, + 'prod', + [row({ ...event, data: { id: 'other' } }, '2026-01-02T00:00:00.000Z')], + 'replace', + ), + ).toMatchObject({ created: 0, updated: 0, skipped: 1 }) + const doc = (await ecoll.findOne({ '_raw.id': 'event_1' }))! + expect(doc._raw).toEqual(event) + expect(doc._deleted_at).toBeUndefined() + }) + + it('keeps environments apart under the same key', async () => { + await upsertRows( + coll, + opts, + users, + 'prod', + [row(userPayload({ email: 'prod@example.com' }), '2026-01-01T00:00:00.000Z')], + 'replace', + ) + await upsertRows( + coll, + opts, + users, + 'staging', + [row(userPayload({ email: 'staging@example.com' }), '2026-01-01T00:00:00.000Z')], + 'replace', + ) + expect(await coll.countDocuments()).toBe(2) + expect((await coll.findOne({ _env: 'staging' }))!._raw['email']).toBe('staging@example.com') + }) + + it('counts rows it cannot key as skipped, and refuses a timestamp that is not an instant', async () => { + const stats = await upsertRows( + coll, + opts, + users, + 'prod', + [row({ email: 'no-id@example.com' }, '2026-01-01T00:00:00.000Z')], + 'replace', + ) + expect(stats).toMatchObject({ created: 0, skipped: 1 }) + + await expect( + upsertRows(coll, opts, users, 'prod', [row(userPayload(), '2026-02-30T00:00:00.000Z')], 'replace'), + ).rejects.toThrow(InvalidTimestampError) + expect(await coll.countDocuments()).toBe(0) + }) + + it('writes in batches of the size asked for', async () => { + const rows = Array.from({ length: 7 }, (_, i) => row(userPayload({ id: `user_${i}` }), '2026-01-01T00:00:00.000Z')) + expect(await upsertRows(coll, { batchSize: 3 }, users, 'prod', rows, 'replace')).toMatchObject({ created: 7 }) + expect(await coll.countDocuments({ _env: 'prod' })).toBe(7) + }) +}) diff --git a/packages/sync-mongodb/tsconfig.build.json b/packages/sync-mongodb/tsconfig.build.json new file mode 100644 index 0000000..f44fac8 --- /dev/null +++ b/packages/sync-mongodb/tsconfig.build.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "outDir": "dist", + "rootDir": "src", + "paths": {} + }, + "include": ["src/**/*"], + "exclude": ["test/**/*"] +} diff --git a/packages/sync-mongodb/tsconfig.json b/packages/sync-mongodb/tsconfig.json new file mode 100644 index 0000000..09ebd6a --- /dev/null +++ b/packages/sync-mongodb/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2023"], + "strict": true, + "noUncheckedIndexedAccess": true, + "exactOptionalPropertyTypes": false, + "esModuleInterop": true, + "skipLibCheck": true, + "declaration": true, + "sourceMap": true, + "noEmit": true, + "types": ["node"], + // Typecheck and editors see @workos/sync-engine source directly, so a change + // there never needs a rebuild to show up here. The build (tsconfig.build) + // clears this and resolves the real package. + "paths": { "@workos/sync-engine": ["../sync-engine/src/index.ts"] } + }, + "include": ["src/**/*", "test/**/*"] +} diff --git a/packages/sync-mongodb/vitest.config.ts b/packages/sync-mongodb/vitest.config.ts new file mode 100644 index 0000000..ddc872c --- /dev/null +++ b/packages/sync-mongodb/vitest.config.ts @@ -0,0 +1,16 @@ +import { fileURLToPath } from 'node:url' +import { defineConfig } from 'vitest/config' +export default defineConfig({ + resolve: { + // Tests run against @workos/sync-engine source, not its last build. + alias: { '@workos/sync-engine': fileURLToPath(new URL('../sync-engine/src/index.ts', import.meta.url)) }, + }, + test: { + include: ['test/**/*.test.ts'], + globalSetup: ['test/global-setup.ts'], + setupFiles: ['test/setup.ts'], + testTimeout: 60_000, + hookTimeout: 120_000, + pool: 'forks', + }, +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f2590ab..7b706fc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -254,6 +254,38 @@ importers: specifier: ^5.0.5 version: 5.0.5(@opentelemetry/api@1.10.0)(@types/node@26.6.5)(vite@8.3.4(@types/node@26.6.5)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1)) + packages/sync-mongodb: + dependencies: + '@workos/sync-engine': + specifier: workspace:* + version: link:../sync-engine + mongodb: + specifier: ^6.17.0 + version: 6.22.0 + devDependencies: + '@opentelemetry/api': + specifier: ^1.10.0 + version: 1.10.0 + '@opentelemetry/sdk-metrics': + specifier: ^3.0.0 + version: 3.0.0(@opentelemetry/api@1.10.0) + '@types/node': + specifier: ^26.6.5 + version: 26.6.5 + mongodb-memory-server: + specifier: ^10.1.4 + version: 10.4.3(bare-buffer@3.7.1) + typescript: + specifier: ^7.0.2 + version: 7.0.2 + vitest: + specifier: ^5.0.5 + version: 5.0.5(@opentelemetry/api@1.10.0)(@types/node@26.6.5)(vite@8.3.4(@types/node@26.6.5)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1)) + optionalDependencies: + '@opentelemetry/sdk-node': + specifier: ^0.300.0 + version: 0.300.0(@opentelemetry/api@1.10.0) + packages/sync-postgres: dependencies: '@workos/sync-engine': @@ -568,6 +600,9 @@ packages: '@js-sdsl/ordered-map@4.4.2': resolution: {integrity: sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==} + '@mongodb-js/saslprep@1.6.1': + resolution: {integrity: sha512-D4CQtk9IwjCLAvCf1xvHEod3960q8XMUzfmdRU9qjUNlHnesGuTOvJcOTaPm9koZ1caUKA2aTbszZiAkcmSgmg==} + '@opentelemetry/api@1.10.0': resolution: {integrity: sha512-lMkyes8iHrG+mk6WH7X70MdCLI2RS0FwNBY1PyQ8UemaijE2m5mv5A0Niy3HtbYTFrgkUEQjFMXmSAk2hQDuPQ==} engines: {node: '>=8.0.0'} @@ -1116,6 +1151,12 @@ packages: '@types/pg@8.23.1': resolution: {integrity: sha512-fKVHpikPdg4GKks3JuLEhvwSyvwzF23hnabPy6DD8ljVbC7+6J5dQzdv4arV6jqq57djnMgs1HKBxX4P8aBI3A==} + '@types/webidl-conversions@7.0.3': + resolution: {integrity: sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==} + + '@types/whatwg-url@11.0.5': + resolution: {integrity: sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==} + '@typescript/typescript-aix-ppc64@7.0.2': resolution: {integrity: sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==} engines: {node: '>=16.20.0'} @@ -1253,6 +1294,10 @@ packages: '@workos/openapi-spec@0.143.0': resolution: {integrity: sha512-7YrI/JSKHYkDVZtyiKsPJejMfvdZL1uJNOfU+W3Cva712ZaQvi0hw7KlcMfTXNBypsuj+U6T3BJvdDEXENI7oA==} + agent-base@7.1.4: + resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} + engines: {node: '>= 14'} + ajv@8.20.0: resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==} @@ -1283,10 +1328,70 @@ packages: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} + async-mutex@0.5.0: + resolution: {integrity: sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA==} + + b4a@1.9.0: + resolution: {integrity: sha512-dpfcF9fDNR6++cthXR67iyhgqWy9CBouAvIWhIntzBG6cvK/cnIPiZQjBwi/ZqjjBEDGfoNDtmB0kTjroOJ3pQ==} + peerDependencies: + react-native-b4a: '*' + peerDependenciesMeta: + react-native-b4a: + optional: true + + bare-buffer@3.7.1: + resolution: {integrity: sha512-cIjZnSO+y89ykVX9x96OVnA6tGORBn7vGJ/e3PCbBHMEyfMNG/ixzOS0lLa6saMlWcs69bSKjxFyDwjK152OMg==} + engines: {bare: '>=1.20.0'} + + bare-events@2.9.2: + resolution: {integrity: sha512-AIPKioV7/Y/8KfZ3AAhjPJxLLbY49S64Ym5DakZlUg75qQiTgUq9hEJoEwa4eUezPUlXRy/i5NpsKvo9jgKmoA==} + peerDependencies: + bare-abort-controller: '*' + peerDependenciesMeta: + bare-abort-controller: + optional: true + + bare-fs@4.8.4: + resolution: {integrity: sha512-MPC2eyrpbk3A873sK4y6rk+Iv4o2enxTjHZ8zH3HO0mjrpm5XjnguY2XmfidIsDV6vnjpbMBKUAJpV0ISD2SsQ==} + engines: {bare: '>=1.28.0'} + peerDependencies: + bare-buffer: '*' + peerDependenciesMeta: + bare-buffer: + optional: true + + bare-path@3.1.2: + resolution: {integrity: sha512-ZyKbsuuqK6Ag0K8pX6V5Txq6XeJRvY+wXucnFGRjiyVYP9YWDpIQugk/b+enRYrEYBJaqLzghRQpXPMR7341Nw==} + + bare-stream@2.13.4: + resolution: {integrity: sha512-PcrQ8lVLbiJscNm1Kez+Yp4Gy4AHGcN1lzwjvf5NybWen7VvEgUfyfnXYJ2zNqWnzOfCb1Abq6lH8ti0syQszA==} + peerDependencies: + bare-abort-controller: '*' + bare-buffer: '*' + bare-events: '*' + peerDependenciesMeta: + bare-abort-controller: + optional: true + bare-buffer: + optional: true + bare-events: + optional: true + + bare-url@2.5.6: + resolution: {integrity: sha512-iqkXHkW65De+BT14nHLiBFbNuJ/DdMrZaVNJ1IXPctpK6b1olUJ5Qmy8QBvwROajTiGztso2n82kFEvCsNG7Rg==} + + bson@6.10.4: + resolution: {integrity: sha512-WIsKqkSC0ABoBJuT1LEX+2HEvNmNKKgnTAyd0fL8qzK4SH2i9NXg+t08YtdZp/V9IZ33cxe3iV4yM0qg8lMQng==} + engines: {node: '>=16.20.1'} + callsites@3.1.0: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} engines: {node: '>=6'} + camelcase@6.3.0: + resolution: {integrity: sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==} + engines: {node: '>=10'} + chai@6.3.0: resolution: {integrity: sha512-XWAtwJ6OHO+tj0EKCs0Y2UamnyOxseZWltU4x2U2wh8g4AigdjwvtUjvLP2tqkA/avxHEtzxNaqGq/YGNwckKg==} engines: {node: '>=18'} @@ -1309,6 +1414,9 @@ packages: color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + commondir@1.0.1: + resolution: {integrity: sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==} + conventional-changelog-angular@9.4.0: resolution: {integrity: sha512-HdxRxuS8bBXVIuo4V82gvSwAXT0vYQUizrjs/izmPg5JdDstr8v8I5hduGL3iQbG+o310dUDxC4+LetuS5hu9w==} engines: {node: '>=22'} @@ -1378,6 +1486,9 @@ packages: estree-walker@3.0.3: resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + events-universal@1.0.1: + resolution: {integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==} + expect-type@1.4.0: resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} engines: {node: '>=12.0.0'} @@ -1385,6 +1496,9 @@ packages: fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + fast-fifo@1.3.2: + resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} + fast-uri@3.1.9: resolution: {integrity: sha512-hS/0Wt3Ya043oKXs0sIxrMpKQITXhMLIrVAKKXi4gozGC9v7YRWVnXIZzjNPTO02M9BdxLCWqK1yRXgNziVyJQ==} @@ -1397,6 +1511,23 @@ packages: picomatch: optional: true + find-cache-dir@3.3.2: + resolution: {integrity: sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig==} + engines: {node: '>=8'} + + find-up@4.1.0: + resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==} + engines: {node: '>=8'} + + follow-redirects@1.16.2: + resolution: {integrity: sha512-/kVP2U8z596Ia0gwFrwyIeo74ta5aVmSeU1FfAUMAmcEP5EP7zydmag3SLiN64NaRKef3lxsq7i/g7ltJQDGow==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -1414,6 +1545,10 @@ packages: resolution: {integrity: sha512-1pgFdhK3J2LeM+dVf2Pd424yHx2ou338lC0ErNP2hPx4j8eW1Sp0XqSjNxtk6Tc4Kr5wlWtSvz8cn2yb7/SG/w==} engines: {node: '>=20'} + https-proxy-agent@7.0.6: + resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} + engines: {node: '>= 14'} + husky@9.1.7: resolution: {integrity: sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA==} engines: {node: '>=18'} @@ -1536,6 +1671,10 @@ packages: lines-and-columns@1.2.4: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} + locate-path@5.0.0: + resolution: {integrity: sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==} + engines: {node: '>=8'} + lodash.camelcase@4.3.0: resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} @@ -1545,9 +1684,54 @@ packages: magic-string@1.4.3: resolution: {integrity: sha512-z12OxmaPGE0F4xlpGdjuAUckipLpQlTAuAmyGhNoD7ODpYUzvDfvtxUbjM/jwznyP178oju/KDdyi220wNJ0RQ==} + make-dir@3.1.0: + resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==} + engines: {node: '>=8'} + + memory-pager@1.5.0: + resolution: {integrity: sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==} + module-details-from-path@1.0.4: resolution: {integrity: sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==} + mongodb-connection-string-url@3.0.2: + resolution: {integrity: sha512-rMO7CGo/9BFwyZABcKAWL8UJwH/Kc2x0g72uhDWzG48URRax5TCIcJ7Rc3RZqffZzO/Gwff/jyKwCU9TN8gehA==} + + mongodb-memory-server-core@10.4.3: + resolution: {integrity: sha512-IPjlw73IoSYopnqBibQKxmAXMbOEPf5uGAOsBcaUiNH/TOI7V19WO+K7n5KYtnQ9FqzLGLpvwCGuPOTBSg4s5Q==} + engines: {node: '>=16.20.1'} + + mongodb-memory-server@10.4.3: + resolution: {integrity: sha512-CDZvFisXvGIigsIw5gqH6r9NI/zxGa/uRdutgUL/isuJh+inj0YXb7Ykw6oFMFzqgTJWb7x0I5DpzrqCstBWpg==} + engines: {node: '>=16.20.1'} + + mongodb@6.22.0: + resolution: {integrity: sha512-k9YPpPXZEVyT4kjglCen99LCDToxzM+R89x3cMp8kSWb1gnXDvrsEJQh3FQ4t4jS9+ZA4yBOojHNS1oAB5qwQg==} + engines: {node: '>=16.20.1'} + peerDependencies: + '@aws-sdk/credential-providers': ^3.188.0 + '@mongodb-js/zstd': ^1.1.0 || ^2.0.0 + gcp-metadata: ^5.2.0 + kerberos: ^2.0.1 + mongodb-client-encryption: '>=6.0.0 <7' + snappy: ^7.3.2 + socks: ^2.7.1 + peerDependenciesMeta: + '@aws-sdk/credential-providers': + optional: true + '@mongodb-js/zstd': + optional: true + gcp-metadata: + optional: true + kerberos: + optional: true + mongodb-client-encryption: + optional: true + snappy: + optional: true + socks: + optional: true + ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -1556,6 +1740,10 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true + new-find-package-json@2.0.0: + resolution: {integrity: sha512-lDcBsjBSMlj3LXH2v/FW3txlh2pYTjmbOXPYJD93HI5EwuLzI11tdHSIpUMmfq/IOsldj4Ps8M8flhm+pCK4Ew==} + engines: {node: '>=12.22.0'} + obug@3.0.0: resolution: {integrity: sha512-5vvB5+W7ePv+p3uqxi+RcW1XAzLW0/hxt3/4X4Lc4qHudzOhmBiBwOY6DRob4WnanAEGvNcLjF+KNOufrUoEQw==} engines: {node: '>=12.20.0'} @@ -1589,6 +1777,18 @@ packages: vite-plus: optional: true + p-limit@2.3.0: + resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==} + engines: {node: '>=6'} + + p-locate@4.1.0: + resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==} + engines: {node: '>=8'} + + p-try@2.2.0: + resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} + engines: {node: '>=6'} + parent-module@1.0.1: resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} engines: {node: '>=6'} @@ -1597,6 +1797,13 @@ packages: resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} engines: {node: '>=8'} + path-exists@4.0.0: + resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} + engines: {node: '>=8'} + + pend@1.2.0: + resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==} + pg-cloudflare@1.4.1: resolution: {integrity: sha512-6PQbsFWZcp9EmJEwy5cGQ2La+AMWpP46lgbb8X+U/XsHIUweYDNCpeuKck5RxL2MdVFi7krbbEi5nX4Zh7JhrQ==} @@ -1638,6 +1845,10 @@ packages: resolution: {integrity: sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==} engines: {node: '>=12'} + pkg-dir@4.2.0: + resolution: {integrity: sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==} + engines: {node: '>=8'} + postcss@8.5.31: resolution: {integrity: sha512-YUbWYCgErwnMAVkjUnYa+KlMBzGpHwiIt/DE7f0G33AuX2LH9UcTk6KgWxd3zRU+pN3u/C+Xl3EnMLM4Hea7Sg==} engines: {node: ^10 || ^12 || >=14} @@ -1667,6 +1878,10 @@ packages: resolution: {integrity: sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==} engines: {node: '>=12.0.0'} + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + require-directory@2.1.1: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} @@ -1692,6 +1907,10 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} hasBin: true + semver@6.3.2: + resolution: {integrity: sha512-GYrCADwpu6yV5OAvchkQ27Z5wwMxWAEMg5NNWjyHh/5DboFFqbdbvYNIvZouQpxXyeMPoTOh0vdPYKlf0ZswdQ==} + hasBin: true + semver@7.8.5: resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} engines: {node: '>=10'} @@ -1701,6 +1920,9 @@ packages: resolution: {integrity: sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==} engines: {node: '>=0.10.0'} + sparse-bitfield@3.0.3: + resolution: {integrity: sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==} + split2@4.2.0: resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} engines: {node: '>= 10.x'} @@ -1708,6 +1930,9 @@ packages: std-env@4.3.0: resolution: {integrity: sha512-OtU/EgQ1kIm5KwqQpBC6ZEMXrZRui11w8zgfTWp8cdO9B8OaPsbA8bTHO2P+HNo1VlUTGMVBwPhydu6poeXiag==} + streamx@2.28.1: + resolution: {integrity: sha512-zEzXb0s5Cds7tqMH6rhZ05lcJydCWiQPEwiNngVqzsxCc962vLY4Uw+mW7od8kDH258k2Uz/JrOkdIAAhSh9VA==} + string-width@4.2.3: resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} engines: {node: '>=8'} @@ -1728,6 +1953,15 @@ packages: resolution: {integrity: sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==} engines: {node: '>=12'} + tar-stream@3.2.2: + resolution: {integrity: sha512-+8NeqHRjQWH9nYlwo2gamAMImZCVzI4UoEgDpWorBt9OEfppiZn+uSkskzQKPWIZyji/C8fpWO7u69G0DX0tbg==} + + teex@1.0.1: + resolution: {integrity: sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==} + + text-decoder@1.2.7: + resolution: {integrity: sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==} + tinybench@6.2.2: resolution: {integrity: sha512-nIZ2N2dFpjBsg2hgitKu7mA+FvsIJuGJOwysRngmVRRmWgR/UyPjwJj8c/fT/dhm8EkHnF5EOogRf+0Z2qrimA==} engines: {node: '>=20.0.0'} @@ -1744,6 +1978,13 @@ packages: resolution: {integrity: sha512-jBrmx4lYmaC9k/mgPbylxs7kBUxHtD8256up+HjLaDFfXScKJQyil+SWXSvhAtT7XHo+yTVlpB81PGmHP8oLSQ==} engines: {node: ^20.0.0 || >=22.0.0} + tr46@5.1.1: + resolution: {integrity: sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==} + engines: {node: '>=18'} + + tslib@2.9.1: + resolution: {integrity: sha512-a4Ux9VKuIMHsOS7ENbEq6hK2ZGYIT27Rtl0E4eD9KqwxhKiQeMZE/9xb3IdYwWaSOyIBQLExhQgXDTYoiCxnWA==} + tsx@4.23.15: resolution: {integrity: sha512-Yiex1Ovn8z2xPpOWckIiysV1SSyRMY9BkLF++q0yKiDxCqRhosKfMg3janKkiLBwZ5c/YryloKwGZcrEmtwxKw==} engines: {node: '>=18.0.0'} @@ -1845,6 +2086,14 @@ packages: jsdom: optional: true + webidl-conversions@7.0.0: + resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} + engines: {node: '>=12'} + + whatwg-url@14.2.0: + resolution: {integrity: sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==} + engines: {node: '>=18'} + why-is-node-running@3.2.1: resolution: {integrity: sha512-Tb2FUhB4vUsGQlfSquQLYkApkuPAFQXGFzxWKHHumVz2dK+X1RUm/HnID4+TfIGYJ1kTcwOaCk/buYCEJr6YjQ==} engines: {node: '>=20.11'} @@ -1887,6 +2136,10 @@ packages: resolution: {integrity: sha512-9OpKOLeaoNFecEp7P6iYbzze/5CqWoH7N3SMs/6y1XF4nMvFMspEGZzJ6uFi9MmQwXhyzqYoSEKO3tvA3Z/o2w==} engines: {node: ^20.19.0 || ^22.12.0 || >=23} + yauzl@3.4.0: + resolution: {integrity: sha512-jIH9yLR9wqr0wOS0TpBvo/g/2UgZH5qePVbjgRliiF0BYvOZyaBknKsF+x9Iht0O6sqgnB93rCICdOZFecJuDw==} + engines: {node: '>=12'} + snapshots: '@babel/code-frame@7.29.7': @@ -2120,6 +2373,10 @@ snapshots: '@js-sdsl/ordered-map@4.4.2': optional: true + '@mongodb-js/saslprep@1.6.1': + dependencies: + sparse-bitfield: 3.0.3 + '@opentelemetry/api@1.10.0': {} '@opentelemetry/configuration@0.300.0(@opentelemetry/api@1.10.0)': @@ -2548,6 +2805,12 @@ snapshots: pg-protocol: 1.16.1 pg-types: 2.2.0 + '@types/webidl-conversions@7.0.3': {} + + '@types/whatwg-url@11.0.5': + dependencies: + '@types/webidl-conversions': 7.0.3 + '@typescript/typescript-aix-ppc64@7.0.2': optional: true @@ -2621,6 +2884,8 @@ snapshots: '@workos/openapi-spec@0.143.0': {} + agent-base@7.1.4: {} + ajv@8.20.0: dependencies: fast-deep-equal: 3.1.3 @@ -2646,8 +2911,53 @@ snapshots: assertion-error@2.0.1: {} + async-mutex@0.5.0: + dependencies: + tslib: 2.9.1 + + b4a@1.9.0: {} + + bare-buffer@3.7.1: {} + + bare-events@2.9.2: {} + + bare-fs@4.8.4(bare-buffer@3.7.1): + dependencies: + bare-events: 2.9.2 + bare-path: 3.1.2 + bare-stream: 2.13.4(bare-buffer@3.7.1)(bare-events@2.9.2) + bare-url: 2.5.6 + fast-fifo: 1.3.2 + optionalDependencies: + bare-buffer: 3.7.1 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + + bare-path@3.1.2: {} + + bare-stream@2.13.4(bare-buffer@3.7.1)(bare-events@2.9.2): + dependencies: + b4a: 1.9.0 + streamx: 2.28.1 + teex: 1.0.1 + optionalDependencies: + bare-buffer: 3.7.1 + bare-events: 2.9.2 + transitivePeerDependencies: + - react-native-b4a + + bare-url@2.5.6: + dependencies: + bare-buffer: 3.7.1 + bare-path: 3.1.2 + + bson@6.10.4: {} + callsites@3.1.0: {} + camelcase@6.3.0: {} + chai@6.3.0: {} cjs-module-lexer@2.3.0: @@ -2674,6 +2984,8 @@ snapshots: color-name@1.1.4: optional: true + commondir@1.0.1: {} + conventional-changelog-angular@9.4.0: dependencies: '@conventional-changelog/template': 1.4.0 @@ -2699,7 +3011,6 @@ snapshots: debug@4.4.3: dependencies: ms: 2.1.3 - optional: true detect-libc@2.1.2: {} @@ -2756,16 +3067,39 @@ snapshots: dependencies: '@types/estree': 1.0.9 + events-universal@1.0.1: + dependencies: + bare-events: 2.9.2 + transitivePeerDependencies: + - bare-abort-controller + expect-type@1.4.0: {} fast-deep-equal@3.1.3: {} + fast-fifo@1.3.2: {} + fast-uri@3.1.9: {} fdir@6.5.0(picomatch@4.0.7): optionalDependencies: picomatch: 4.0.7 + find-cache-dir@3.3.2: + dependencies: + commondir: 1.0.1 + make-dir: 3.1.0 + pkg-dir: 4.2.0 + + find-up@4.1.0: + dependencies: + locate-path: 5.0.0 + path-exists: 4.0.0 + + follow-redirects@1.16.2(debug@4.4.3): + optionalDependencies: + debug: 4.4.3 + fsevents@2.3.3: optional: true @@ -2777,6 +3111,13 @@ snapshots: dependencies: ini: 6.0.0 + https-proxy-agent@7.0.6: + dependencies: + agent-base: 7.1.4 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + husky@9.1.7: {} import-fresh@3.3.1: @@ -2863,6 +3204,10 @@ snapshots: lines-and-columns@1.2.4: {} + locate-path@5.0.0: + dependencies: + p-locate: 4.1.0 + lodash.camelcase@4.3.0: optional: true @@ -2873,14 +3218,80 @@ snapshots: dependencies: '@jridgewell/sourcemap-codec': 1.6.0 + make-dir@3.1.0: + dependencies: + semver: 6.3.2 + + memory-pager@1.5.0: {} + module-details-from-path@1.0.4: optional: true - ms@2.1.3: - optional: true + mongodb-connection-string-url@3.0.2: + dependencies: + '@types/whatwg-url': 11.0.5 + whatwg-url: 14.2.0 + + mongodb-memory-server-core@10.4.3(bare-buffer@3.7.1): + dependencies: + async-mutex: 0.5.0 + camelcase: 6.3.0 + debug: 4.4.3 + find-cache-dir: 3.3.2 + follow-redirects: 1.16.2(debug@4.4.3) + https-proxy-agent: 7.0.6 + mongodb: 6.22.0 + new-find-package-json: 2.0.0 + semver: 7.8.5 + tar-stream: 3.2.2(bare-buffer@3.7.1) + tslib: 2.9.1 + yauzl: 3.4.0 + transitivePeerDependencies: + - '@aws-sdk/credential-providers' + - '@mongodb-js/zstd' + - bare-abort-controller + - bare-buffer + - gcp-metadata + - kerberos + - mongodb-client-encryption + - react-native-b4a + - snappy + - socks + - supports-color + + mongodb-memory-server@10.4.3(bare-buffer@3.7.1): + dependencies: + mongodb-memory-server-core: 10.4.3(bare-buffer@3.7.1) + tslib: 2.9.1 + transitivePeerDependencies: + - '@aws-sdk/credential-providers' + - '@mongodb-js/zstd' + - bare-abort-controller + - bare-buffer + - gcp-metadata + - kerberos + - mongodb-client-encryption + - react-native-b4a + - snappy + - socks + - supports-color + + mongodb@6.22.0: + dependencies: + '@mongodb-js/saslprep': 1.6.1 + bson: 6.10.4 + mongodb-connection-string-url: 3.0.2 + + ms@2.1.3: {} nanoid@3.3.20: {} + new-find-package-json@2.0.0: + dependencies: + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + obug@3.0.0: {} oxfmt@0.75.0: @@ -2929,6 +3340,16 @@ snapshots: '@oxlint/binding-win32-ia32-msvc': 1.90.0 '@oxlint/binding-win32-x64-msvc': 1.90.0 + p-limit@2.3.0: + dependencies: + p-try: 2.2.0 + + p-locate@4.1.0: + dependencies: + p-limit: 2.3.0 + + p-try@2.2.0: {} + parent-module@1.0.1: dependencies: callsites: 3.1.0 @@ -2940,6 +3361,10 @@ snapshots: json-parse-even-better-errors: 2.3.1 lines-and-columns: 1.2.4 + path-exists@4.0.0: {} + + pend@1.2.0: {} + pg-cloudflare@1.4.1: optional: true @@ -2979,6 +3404,10 @@ snapshots: picomatch@4.0.7: {} + pkg-dir@4.2.0: + dependencies: + find-up: 4.1.0 + postcss@8.5.31: dependencies: nanoid: 3.3.20 @@ -3012,6 +3441,8 @@ snapshots: long: 5.3.2 optional: true + punycode@2.3.1: {} + require-directory@2.1.1: optional: true @@ -3050,14 +3481,29 @@ snapshots: '@rolldown/binding-win32-arm64-msvc': 1.2.14 '@rolldown/binding-win32-x64-msvc': 1.2.14 + semver@6.3.2: {} + semver@7.8.5: {} source-map-js@1.2.2: {} + sparse-bitfield@3.0.3: + dependencies: + memory-pager: 1.5.0 + split2@4.2.0: {} std-env@4.3.0: {} + streamx@2.28.1: + dependencies: + events-universal: 1.0.1 + fast-fifo: 1.3.2 + text-decoder: 1.2.7 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + string-width@4.2.3: dependencies: emoji-regex: 8.0.0 @@ -3085,6 +3531,30 @@ snapshots: dependencies: ansi-regex: 6.5.0 + tar-stream@3.2.2(bare-buffer@3.7.1): + dependencies: + b4a: 1.9.0 + bare-fs: 4.8.4(bare-buffer@3.7.1) + fast-fifo: 1.3.2 + streamx: 2.28.1 + transitivePeerDependencies: + - bare-abort-controller + - bare-buffer + - react-native-b4a + + teex@1.0.1: + dependencies: + streamx: 2.28.1 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + + text-decoder@1.2.7: + dependencies: + b4a: 1.9.0 + transitivePeerDependencies: + - react-native-b4a + tinybench@6.2.2: {} tinyexec@1.3.1: {} @@ -3096,6 +3566,12 @@ snapshots: tinypool@2.2.0: {} + tr46@5.1.1: + dependencies: + punycode: 2.3.1 + + tslib@2.9.1: {} + tsx@4.23.15: dependencies: esbuild: 0.28.2 @@ -3166,6 +3642,13 @@ snapshots: transitivePeerDependencies: - msw + webidl-conversions@7.0.0: {} + + whatwg-url@14.2.0: + dependencies: + tr46: 5.1.1 + webidl-conversions: 7.0.0 + why-is-node-running@3.2.1: {} wrap-ansi@7.0.0: @@ -3212,3 +3695,7 @@ snapshots: string-width: 8.3.0 y18n: 5.0.8 yargs-parser: 22.0.0 + + yauzl@3.4.0: + dependencies: + pend: 1.2.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 9fb24c7..8bdc89c 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -5,5 +5,9 @@ allowBuilds: esbuild: true # Pulled in by @opentelemetry/sdk-node; its postinstall only prints a notice. protobufjs: false + # Its postinstall downloads a mongod binary. Skipped here so that `pnpm + # install` (and the Docker build) stays network-light; the test helper + # downloads it on first use instead. + mongodb-memory-server: false minimumReleaseAgeExclude: - '@workos/openapi-spec@0.143.0' diff --git a/release-please-config.json b/release-please-config.json index b019aef..fbb0c7e 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -15,6 +15,10 @@ "packages/sync-postgres": { "release-type": "node", "component": "sync-postgres" + }, + "packages/sync-mongodb": { + "release-type": "node", + "component": "sync-mongodb" } } } From bbd191a5992a8b94cffe0b5a4d694e410301fff6 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 5 Nov 2026 13:55:01 -0500 Subject: [PATCH 2/8] chore(mongodb): drop a worktree directory added by mistake .claude/worktrees/docs-neon-guide belongs to another checkout and was swept into the previous commit as an embedded repository. --- .claude/worktrees/docs-neon-guide | 1 - 1 file changed, 1 deletion(-) delete mode 160000 .claude/worktrees/docs-neon-guide diff --git a/.claude/worktrees/docs-neon-guide b/.claude/worktrees/docs-neon-guide deleted file mode 160000 index 0c969a7..0000000 --- a/.claude/worktrees/docs-neon-guide +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0c969a7736624f52af179b3ed1cd9d7cc43403eb From f3e3178aff476c1f76b2306121ffda9761b831d3 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 5 Nov 2026 15:21:19 -0500 Subject: [PATCH 3/8] fix(mongodb): run the installed command, repair recorded views, store configured strings as text Six review findings on the MongoDB destination, each with a test: - The entrypoint check compared `process.argv[1]` against `cli.js`, but npm installs the `bin` as a symlink named `workos-sync-mongodb`, so the installed command imported `main()` and exited. Both sides are now resolved to real paths and compared. - `migrate` returned early on a recorded fingerprint before reaching `ensureLiveView`, so `init --repair` with unchanged streams never redefined a view changed by hand. A repair now always makes the (still idempotent) full pass. - In the lease's update pipeline and the mirror writes' pipelines a sync id, environment label, holder or `_env` that starts with `$` was read as a field path. They are wrapped in `$literal`. - `keyFilter` copied a row's key value straight into the query, so a malformed row with `id: { $ne: null }` could match another row. The filter is an explicit `$eq`, and `keyValues` treats an object as no key. - `chunk` looped forever on a batch size of zero or less; it now refuses anything but a positive integer. - The README's `$lookup` example joins on `_env` as well as the user id. --- packages/sync-mongodb/README.md | 3 ++ packages/sync-mongodb/src/cli.ts | 22 +++++++-- packages/sync-mongodb/src/db/lease.ts | 17 ++++--- packages/sync-mongodb/src/db/upsert.ts | 32 +++++++++---- packages/sync-mongodb/src/schema/migrate.ts | 9 +++- packages/sync-mongodb/test/cli.test.ts | 50 ++++++++++++++++++++- packages/sync-mongodb/test/lease.test.ts | 16 +++++++ packages/sync-mongodb/test/migrate.test.ts | 17 +++++++ packages/sync-mongodb/test/upsert.test.ts | 43 ++++++++++++++++-- 9 files changed, 186 insertions(+), 23 deletions(-) diff --git a/packages/sync-mongodb/README.md b/packages/sync-mongodb/README.md index 0c9e0e1..c4946ce 100644 --- a/packages/sync-mongodb/README.md +++ b/packages/sync-mongodb/README.md @@ -16,6 +16,9 @@ db.workos_organization_memberships_live.aggregate([ from: "workos_users_live", localField: "_raw.user_id", foreignField: "_raw.id", + // Ids repeat across environments, so the join is pinned to this one. + let: { env: "$_env" }, + pipeline: [{ $match: { $expr: { $eq: ["$_env", "$$env"] } } }], as: "user", }, }, diff --git a/packages/sync-mongodb/src/cli.ts b/packages/sync-mongodb/src/cli.ts index ea1f455..a53ef1b 100644 --- a/packages/sync-mongodb/src/cli.ts +++ b/packages/sync-mongodb/src/cli.ts @@ -30,6 +30,8 @@ import { type SyncEngineConfig, } from '@workos/sync-engine' import { X509Certificate } from 'node:crypto' +import { realpathSync } from 'node:fs' +import { fileURLToPath } from 'node:url' import type { MongoClient } from 'mongodb' import { MongoDestination } from './destination.js' import { healthOptionsFromEnv } from './health.js' @@ -477,9 +479,23 @@ async function runCommand(command: string, flags: Record process.exit(code), (err) => { diff --git a/packages/sync-mongodb/src/db/lease.ts b/packages/sync-mongodb/src/db/lease.ts index efd82d9..fda94f5 100644 --- a/packages/sync-mongodb/src/db/lease.ts +++ b/packages/sync-mongodb/src/db/lease.ts @@ -22,6 +22,11 @@ * expired. Expiry is judged by `$$NOW`, the server's clock, never the * client's, so two hosts with skewed clocks still agree on who holds it. * + * Inside that pipeline a string that starts with `$` is a field path, so the + * sync id, the environment and the holder — configured strings, any of which + * could start with one — are wrapped in `$literal` wherever the pipeline + * stores or compares them. + * * Cursor state is stored as canonical JSON — keys sorted, recursively — * because MongoDB compares embedded documents field by field *in order*, and * the engine compares cursors by value: `{a, b}` and `{b, a}` are the same @@ -90,22 +95,24 @@ export async function acquireLease( const free = { $or: [ { $in: [{ $type: '$lease' }, ['missing', 'null']] }, - { $eq: ['$lease.holder', holder] }, + { $eq: ['$lease.holder', { $literal: holder }] }, { $lte: ['$lease.expires_at', '$$NOW'] }, ], } const update: Document[] = [ { $set: { - sync_id: syncId, - env, + sync_id: { $literal: syncId }, + env: { $literal: env }, lease: { $cond: [ free, { - holder, + holder: { $literal: holder }, // A renewal keeps the original takeover time; a takeover starts a new one. - acquired_at: { $cond: [{ $eq: ['$lease.holder', holder] }, '$lease.acquired_at', '$$NOW'] }, + acquired_at: { + $cond: [{ $eq: ['$lease.holder', { $literal: holder }] }, '$lease.acquired_at', '$$NOW'], + }, expires_at: { $add: ['$$NOW', ttl] }, }, '$lease', diff --git a/packages/sync-mongodb/src/db/upsert.ts b/packages/sync-mongodb/src/db/upsert.ts index a0b66f4..049c417 100644 --- a/packages/sync-mongodb/src/db/upsert.ts +++ b/packages/sync-mongodb/src/db/upsert.ts @@ -65,12 +65,16 @@ export function readPath(data: Record, path: string | string[]) return cur } -/** Key values for a row, in `stream.key` order. Undefined if any part is missing. */ +/** + * Key values for a row, in `stream.key` order. Undefined if any part is + * missing, or is not a scalar: an object where an id should be is a malformed + * row, not a key, and is never let near a filter. + */ export function keyValues(stream: TableDef, data: Record): string[] | undefined { const out: string[] = [] for (const k of stream.key) { const v = readPath(data, keyPath(stream, k)) - if (v == null || v === '') return undefined + if (v == null || v === '' || typeof v === 'object') return undefined out.push(String(v)) } return out @@ -138,6 +142,11 @@ export function wavesByKey(stream: TableDef, rows: Row[]): { waves: Row[][]; dro } export const chunk = (items: T[], size: number): T[][] => { + // A size that never advances the loop would grow `out` until the process + // ran out of memory; a bad setting gets an error instead. + if (!Number.isInteger(size) || size < 1) { + throw new Error(`batchSize must be a positive integer, got ${String(size)}`) + } const out: T[][] = [] for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size)) return out @@ -150,12 +159,17 @@ export interface WriteOptions { const DEFAULT_BATCH_SIZE = 500 -/** The filter that names one row: its environment and every key column at its catalog path. */ +/** + * The filter that names one row: its environment and every key column at its + * catalog path. Each value is an explicit `$eq`, so a payload whose key is an + * object — `{ $ne: null }`, say — is compared as a value rather than read as + * an operator that could match somebody else's row. + */ export function keyFilter(stream: TableDef, env: string, data: Record): Filter { const filter: Record = { _env: env } for (const k of stream.key) { const path = keyPath(stream, k) - filter[fieldPath(path)] = readPath(data, path) + filter[fieldPath(path)] = { $eq: readPath(data, path) } } return filter as Filter } @@ -186,7 +200,7 @@ function upsertOp(stream: TableDef, env: string, row: Row, mode: WriteMode): Op update: [ { $set: { - _env: env, + _env: { $literal: env }, _raw: { $cond: [inserting, { $literal: row.data }, '$_raw'] }, _updated_at: { $ifNull: ['$_updated_at', at] }, _synced_at: { $ifNull: ['$_synced_at', '$$NOW'] }, @@ -211,7 +225,7 @@ function upsertOp(stream: TableDef, env: string, row: Row, mode: WriteMode): Op update: [ { $set: { - _env: env, + _env: { $literal: env }, _raw: { $cond: [passes, newRaw, '$_raw'] }, _updated_at: { $cond: [passes, at, '$_updated_at'] }, _synced_at: synced(changed), @@ -276,7 +290,7 @@ export async function deleteRows( update: [ { $set: { - _env: env, + _env: { $literal: env }, _raw: merged(row.data), _updated_at: { $ifNull: ['$_updated_at', at] }, _deleted_at: { $max: ['$_deleted_at', at] }, @@ -328,7 +342,7 @@ export async function relinkRows( update: [ { $set: { - _env: env, + _env: { $literal: env }, _raw: { $cond: [passes, merged(row.data), '$_raw'] }, _updated_at: { $cond: [passes, at, '$_updated_at'] }, _deleted_at: { $cond: [passes, null, '$_deleted_at'] }, @@ -381,7 +395,7 @@ export async function unlinkRows( update: [ { $set: { - _env: env, + _env: { $literal: env }, _raw: { $cond: [passes, merged(row.data), '$_raw'] }, _updated_at: { $cond: [passes, at, '$_updated_at'] }, _deleted_at: { $cond: [passes, { $max: ['$_deleted_at', at] }, '$_deleted_at'] }, diff --git a/packages/sync-mongodb/src/schema/migrate.ts b/packages/sync-mongodb/src/schema/migrate.ts index 8fc503d..4237396 100644 --- a/packages/sync-mongodb/src/schema/migrate.ts +++ b/packages/sync-mongodb/src/schema/migrate.ts @@ -12,7 +12,9 @@ * `createIndexes` is a no-op for an index that already exists with the same * specification, a view is compared before it is touched — so two processes * running `init` at once need no lock. A fingerprint of the catalog gates the - * whole pass, so an unchanged catalog costs one query on every start. + * whole pass, so an unchanged catalog costs one query on every start — except + * when a repair is asked for, since what a repair fixes (a view changed by + * hand) is exactly what a hash of the catalog cannot see. * * There are no typed columns to drift, so there is no repair pass. What can * linger is an index for a column the catalog no longer indexes; it is @@ -147,7 +149,10 @@ export async function migrate( await ensureBookkeeping(db, names) const migrations = db.collection(names.migrations) - if (await migrations.findOne({ _id: fp }, { projection: { _id: 1 } })) { + // `init --repair` is run because something the fingerprint vouches for is + // wrong, so it always makes the full pass; every step is idempotent, and + // an unchanged database costs it only the queries. + if (!options.repairViews && (await migrations.findOne({ _id: fp }, { projection: { _id: 1 } }))) { logger?.debug('schema up to date', { fingerprint: fp }) return { applied: false, fingerprint: fp, collections: 0, indexes: 0, views: [], orphans: [] } } diff --git a/packages/sync-mongodb/test/cli.test.ts b/packages/sync-mongodb/test/cli.test.ts index 58ac456..f283af6 100644 --- a/packages/sync-mongodb/test/cli.test.ts +++ b/packages/sync-mongodb/test/cli.test.ts @@ -1,12 +1,60 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { pathToFileURL } from 'node:url' import { SyncEngine, WorkOSClient, silentLogger } from '@workos/sync-engine' -import { main, parseHydrate, parseLeaseTtl, parseMaxPages, parseSslCa, repairRequested } from '../src/cli.js' +import { + invokedDirectly, + main, + parseHydrate, + parseLeaseTtl, + parseMaxPages, + parseSslCa, + repairRequested, +} from '../src/cli.js' import { createTestDb, type TestDb } from './helpers/mongo.js' import { FaultyDestination } from './helpers/faulty.js' import { MongoDestination } from '../src/destination.js' import { TEST_CA_PEM } from './helpers/ca.js' import { FakeWorkOS, resetIds } from '../../sync-engine/test/helpers/fake-workos.js' +describe('entrypoint', () => { + let dir: string + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'workos-sync-mongodb-bin-')) + }) + afterEach(() => { + rmSync(dir, { recursive: true, force: true }) + }) + + it('recognises the script itself and the symlink npm installs as the command', () => { + const script = join(dir, 'cli.js') + writeFileSync(script, '') + const url = pathToFileURL(script).href + expect(invokedDirectly(script, url)).toBe(true) + + // `npm install` links `node_modules/.bin/workos-sync-mongodb` at dist/cli.js, + // so argv[1] is the link, not the file. + const link = join(dir, 'workos-sync-mongodb') + symlinkSync(script, link) + expect(invokedDirectly(link, url)).toBe(true) + }) + + it('is false for an import, a missing path, or another script', () => { + const script = join(dir, 'cli.js') + writeFileSync(script, '') + const url = pathToFileURL(script).href + expect(invokedDirectly(undefined, url)).toBe(false) + expect(invokedDirectly(join(dir, 'missing'), url)).toBe(false) + const other = join(dir, 'vitest.mjs') + writeFileSync(other, '') + expect(invokedDirectly(other, url)).toBe(false) + // This test file was started by the test runner, not by the CLI. + expect(invokedDirectly(process.argv[1], url)).toBe(false) + }) +}) + describe('WORKOS_SYNC_HYDRATE', () => { it('leaves the engine default in place when unset', () => { expect(parseHydrate(undefined)).toBeUndefined() diff --git a/packages/sync-mongodb/test/lease.test.ts b/packages/sync-mongodb/test/lease.test.ts index 228b5cf..11af5e6 100644 --- a/packages/sync-mongodb/test/lease.test.ts +++ b/packages/sync-mongodb/test/lease.test.ts @@ -98,6 +98,22 @@ describe('events tail lease', () => { expect(await dest.getLease('prod')).toMatchObject({ holder: 'a' }) }) + it('stores a sync id, an environment and a holder that start with "$" as text', async () => { + // Inside the update pipeline a bare "$reporting" would be read as a + // field path, and the document would be keyed by nothing. + const dollar = new MongoDestination({ client: t.client, database: t.name, syncId: '$reporting' }) + await dollar.init([streamByName('users')!]) + const r = await dollar.acquireLease('$production', '$host', 60_000) + expect(r).toMatchObject({ acquired: true, holder: '$host' }) + expect(await dollar.getLease('$production')).toMatchObject({ holder: '$host' }) + const doc = await sync().findOne({ sync_id: '$reporting', env: '$production' }) + expect(doc?.lease?.holder).toBe('$host') + // A renewal still recognises the holder, and the fenced write finds the document. + expect((await dollar.acquireLease('$production', '$host', 60_000)).acquired).toBe(true) + expect(await dollar.setStateIfHeld('$production', '__tail__', { cursor: 'c1' }, '$host', null)).toBe(true) + expect(await dollar.getState('$production', '__tail__')).toEqual({ cursor: 'c1' }) + }) + it('survives two processes racing to create the document', async () => { const results = await Promise.all(['a', 'b', 'c', 'd'].map((h) => dest.acquireLease('prod', h, 60_000))) expect(results.filter((r) => r.acquired)).toHaveLength(1) diff --git a/packages/sync-mongodb/test/migrate.test.ts b/packages/sync-mongodb/test/migrate.test.ts index 805095f..4c00da8 100644 --- a/packages/sync-mongodb/test/migrate.test.ts +++ b/packages/sync-mongodb/test/migrate.test.ts @@ -208,6 +208,23 @@ describe('migrate', () => { expect(await t.db.collection('workos_directory_group_memberships_live').countDocuments()).toBe(1) }) + it('are redefined by a repair even though the fingerprint is already recorded', async () => { + const dest = new MongoDestination({ client: t.client, database: t.name }) + await dest.init([users]) + expect(await migrate(t.db, [users], names)).toMatchObject({ applied: false }) + + // Somebody changed the pipeline by hand after init recorded its fingerprint. + await t.db.command({ collMod: 'workos_users_live', viewOn: 'workos_users', pipeline: [] }) + // Same streams, same prefix: the fingerprint matches, and a repair must not trust it. + const r = await migrate(t.db, [users], names, undefined, { repairViews: true }) + expect(r).toMatchObject({ applied: true, views: ['workos_users_live'] }) + const info = (await t.db.listCollections({ name: 'workos_users_live' }, { nameOnly: false }).toArray())[0]! + expect(info.options?.['pipeline']).toEqual(LIVE_PIPELINE) + // The fingerprint was recorded once; the repair did not duplicate it. + expect(await t.db.collection(names.migrations).countDocuments()).toBe(1) + expect(await migrate(t.db, [users], names)).toMatchObject({ applied: false }) + }) + it('are kept when already right, redefined only on request, and never clobber a stranger', async () => { expect(await ensureLiveView(t.db, names, users)).toBe('created') expect(await ensureLiveView(t.db, names, users)).toBe('kept') diff --git a/packages/sync-mongodb/test/upsert.test.ts b/packages/sync-mongodb/test/upsert.test.ts index 57c7262..e8cd89b 100644 --- a/packages/sync-mongodb/test/upsert.test.ts +++ b/packages/sync-mongodb/test/upsert.test.ts @@ -13,6 +13,7 @@ import { wavesByKey, keyValues, keyFilter, + chunk, type MirrorDoc, } from '../src/db/upsert.js' import { InvalidTimestampError } from '../src/db/timestamps.js' @@ -46,11 +47,26 @@ describe('key extraction', () => { expect(keyValues(orgRoles, { organization_id: 'org_1', slug: 'admin' })).toEqual(['org_1', 'admin']) expect(keyFilter(orgRoles, 'prod', { organization_id: 'org_1', slug: 'admin' })).toEqual({ _env: 'prod', - '_raw.organization_id': 'org_1', - '_raw.slug': 'admin', + '_raw.organization_id': { $eq: 'org_1' }, + '_raw.slug': { $eq: 'admin' }, }) // Nested paths are addressed through _raw too. - expect(keyFilter(users, 'prod', { id: 'user_1' })).toEqual({ _env: 'prod', '_raw.id': 'user_1' }) + expect(keyFilter(users, 'prod', { id: 'user_1' })).toEqual({ _env: 'prod', '_raw.id': { $eq: 'user_1' } }) + }) + + it('refuses an object where a key should be, and never lets one into a filter as an operator', () => { + expect(keyValues(users, { id: { $ne: null } })).toBeUndefined() + expect(keyValues(users, { id: ['user_1'] })).toBeUndefined() + // Were a row to reach the filter anyway, the object is a value to equal, not a query. + expect(keyFilter(users, 'prod', { id: { $ne: null } })).toEqual({ _env: 'prod', '_raw.id': { $eq: { $ne: null } } }) + }) + + it('splits into batches of a positive whole size only', () => { + expect(chunk([1, 2, 3, 4, 5], 2)).toEqual([[1, 2], [3, 4], [5]]) + expect(chunk([], 2)).toEqual([]) + for (const bad of [0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY]) { + expect(() => chunk([1], bad)).toThrow(/batchSize must be a positive integer/) + } }) it('collapses duplicate keys within a batch, keeping the newest', () => { @@ -387,6 +403,27 @@ describe('write path', () => { expect((await coll.findOne({ _env: 'staging' }))!._raw['email']).toBe('staging@example.com') }) + it('leaves every other row alone when a key is an object', async () => { + await upsertRows(coll, opts, users, 'prod', [row(userPayload(), '2026-01-01T00:00:00.000Z')], 'replace') + const malformed = [row(userPayload({ id: { $ne: null } }), '2026-02-01T00:00:00.000Z')] + expect(await upsertRows(coll, opts, users, 'prod', malformed, 'replace')).toMatchObject({ skipped: 1 }) + expect(await deleteRows(coll, opts, users, 'prod', malformed)).toMatchObject({ skipped: 1 }) + expect(await coll.countDocuments()).toBe(1) + const stored = (await readUser())! + expect(stored._raw['updated_at']).toBe('2026-01-01T00:00:00.000Z') + expect(stored._deleted_at ?? null).toBeNull() + // And the filter itself, given such a row, matches nothing rather than everything. + expect(await coll.countDocuments(keyFilter(users, 'prod', { id: { $ne: null } }))).toBe(0) + }) + + it('stores an environment label that starts with "$" as text', async () => { + const ecoll = t.db.collection(names.mirror(events)) + await upsertRows(coll, opts, users, '$staging', [row(userPayload(), '2026-01-01T00:00:00.000Z')], 'merge') + await upsertRows(ecoll, opts, events, '$staging', [row({ id: 'event_1' }, '2026-01-01T00:00:00.000Z')], 'replace') + expect((await coll.findOne({ '_raw.id': 'user_1' }))!._env).toBe('$staging') + expect((await ecoll.findOne({ '_raw.id': 'event_1' }))!._env).toBe('$staging') + }) + it('counts rows it cannot key as skipped, and refuses a timestamp that is not an instant', async () => { const stats = await upsertRows( coll, From b29dd5e4f0f496006f7318126e01772627afc344 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 5 Nov 2026 15:38:54 -0500 Subject: [PATCH 4/8] fix(mongodb): dedupe by instant and refuse out-of-range offsets Two more review findings: - `dedupeByKey` compared `updatedAt` as text, so a batch mixing offset and UTC forms could keep the older row. It now compares the parsed instants, and a timestamp that is not an instant fails there with the same error the write would have raised. - `parseIso8601` range-checked the clock but not the offset, so `+00:99` was accepted and applied. An offset's hours and minutes are now checked like the clock's. --- packages/sync-mongodb/src/db/timestamps.ts | 2 ++ packages/sync-mongodb/src/db/upsert.ts | 12 +++++++++--- packages/sync-mongodb/test/timestamps.test.ts | 4 ++++ packages/sync-mongodb/test/upsert.test.ts | 11 +++++++++++ 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/packages/sync-mongodb/src/db/timestamps.ts b/packages/sync-mongodb/src/db/timestamps.ts index b62ecf7..81a4d38 100644 --- a/packages/sync-mongodb/src/db/timestamps.ts +++ b/packages/sync-mongodb/src/db/timestamps.ts @@ -34,6 +34,8 @@ export function parseIso8601(iso: string): { date: Date } | { failure: ParseFail if (year < 1 || month < 1 || month > 12) return { failure: 'range' } if (day < 1 || day > daysInMonth(year, month)) return { failure: 'range' } if (hour > 23 || minute > 59 || second > 60) return { failure: 'range' } + // An offset is hours and minutes too; `+00:99` names no zone. + if (sign && (Number(oh) > 23 || Number(om) > 59)) return { failure: 'range' } // Millisecond precision is what BSON dates carry; further digits are dropped. const ms = frac ? Number(frac.slice(0, 3).padEnd(3, '0')) : 0 // Built in two steps so that years below 100 are not read as 19xx. diff --git a/packages/sync-mongodb/src/db/upsert.ts b/packages/sync-mongodb/src/db/upsert.ts index 049c417..c53b637 100644 --- a/packages/sync-mongodb/src/db/upsert.ts +++ b/packages/sync-mongodb/src/db/upsert.ts @@ -84,9 +84,14 @@ export function keyValues(stream: TableDef, data: Record): stri * Collapse rows that share a primary key, keeping the newest. It serves the * delete, relink and unlink paths, whose rows are whole; upserts use * `wavesByKey`. + * + * "Newest" is judged by the instant, not the text: `01:00:00+01:00` sorts + * after `00:30:00Z` as a string and is half an hour older as a time. A + * timestamp that is not an instant fails here with the same error the write + * would have raised for it. */ export function dedupeByKey(stream: TableDef, rows: Row[]): { rows: Row[]; dropped: number } { - const byKey = new Map() + const byKey = new Map() let dropped = 0 for (const row of rows) { const key = keyValues(stream, row.data) @@ -95,10 +100,11 @@ export function dedupeByKey(stream: TableDef, rows: Row[]): { rows: Row[]; dropp continue } const k = key.join('\0') + const at = parseInstant(row.updatedAt).getTime() const existing = byKey.get(k) - if (!existing || row.updatedAt >= existing.updatedAt) byKey.set(k, row) + if (!existing || at >= existing.at) byKey.set(k, { row, at }) } - return { rows: [...byKey.values()], dropped } + return { rows: [...byKey.values()].map((e) => e.row), dropped } } /** diff --git a/packages/sync-mongodb/test/timestamps.test.ts b/packages/sync-mongodb/test/timestamps.test.ts index e5f8f60..8abc795 100644 --- a/packages/sync-mongodb/test/timestamps.test.ts +++ b/packages/sync-mongodb/test/timestamps.test.ts @@ -45,6 +45,10 @@ describe('parseIso8601', () => { expect(at('2024-02-29T00:00:00Z')).toBe('2024-02-29T00:00:00.000Z') expect(at('2100-02-29T00:00:00Z')).toBe('range') expect(at('2026-06-30T23:59:60Z')).toBe('2026-07-01T00:00:00.000Z') + // The offset's fields are checked like the clock's. + expect(at('2026-01-01T00:00:00+00:99')).toBe('range') + expect(at('2026-01-01T00:00:00-24:00')).toBe('range') + expect(at('2026-01-01T00:00:00+23:59')).toBe('2025-12-31T00:01:00.000Z') }) it('keeps years below 100 as written', () => { diff --git a/packages/sync-mongodb/test/upsert.test.ts b/packages/sync-mongodb/test/upsert.test.ts index e8cd89b..fd7c99f 100644 --- a/packages/sync-mongodb/test/upsert.test.ts +++ b/packages/sync-mongodb/test/upsert.test.ts @@ -80,6 +80,17 @@ describe('key extraction', () => { expect(dropped).toBe(1) }) + it('judges the newest row by its instant, not by the text of its timestamp', () => { + // "01:00:00+01:00" sorts after "00:30:00Z" as text and is half an hour earlier as a time. + const { rows } = dedupeByKey(users, [ + { data: userPayload({ email: 'older@example.com' }), updatedAt: '2026-01-01T01:00:00+01:00' }, + { data: userPayload({ email: 'newer@example.com' }), updatedAt: '2026-01-01T00:30:00Z' }, + ]) + expect(rows.map((r) => r.data['email'])).toEqual(['newer@example.com']) + // A timestamp that is not an instant is refused here, as the write would refuse it. + expect(() => dedupeByKey(users, [{ data: userPayload(), updatedAt: 'yesterday' }])).toThrow(InvalidTimestampError) + }) + it('splits repeated keys into ordered waves and leaves distinct keys in one', () => { const first = { data: userPayload({ external_id: 'ext_1' }), updatedAt: '2026-01-01T00:00:00.000Z' } const other = { data: userPayload({ id: 'user_2' }), updatedAt: '2026-01-01T00:00:00.000Z' } From 90412dd630f3afcdbe3631212a42fafed9232820 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 5 Nov 2026 15:39:52 -0500 Subject: [PATCH 5/8] test(mongodb): check the reconcile diff before its exit code A failed exit-code assertion shows only the code; checking the diff first shows the whole result when something is off. --- packages/sync-mongodb/test/cli.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/sync-mongodb/test/cli.test.ts b/packages/sync-mongodb/test/cli.test.ts index f283af6..94910a0 100644 --- a/packages/sync-mongodb/test/cli.test.ts +++ b/packages/sync-mongodb/test/cli.test.ts @@ -392,14 +392,16 @@ describe('commands', () => { diffs: Record } ).diffs['prod']! - // A slice that stops short is the outcome asked for, not an error, but drift is. - expect(await main(['reconcile', '--max-pages', '2'], { engine })).toBe(2) + // The diff is checked before the exit code, so a failure shows the whole result. + const code = await main(['reconcile', '--max-pages', '2'], { engine }) expect(diffs().find((d) => d.stream === 'users')).toMatchObject({ pages: 3, complete: true, localOnly: [gone['id']], }) expect(diffs().find((d) => d.stream === 'invitations')).toMatchObject({ pages: 0, complete: false }) + // A slice that stops short is the outcome asked for, not an error, but drift is. + expect(code).toBe(2) expect(await main(['reconcile', '--repair'], { engine })).toBe(0) expect(await main(['reconcile'], { engine })).toBe(0) From 33d7529b876bcd947b808871027a5f4de3c7c447 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 5 Nov 2026 15:46:32 -0500 Subject: [PATCH 6/8] test(mongodb): assert the reconciled stream did not fail A stream whose diff throws is reported with a skippedReason, not thrown; asserting it is absent shows the failure's text when it is not. --- packages/sync-mongodb/test/cli.test.ts | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/packages/sync-mongodb/test/cli.test.ts b/packages/sync-mongodb/test/cli.test.ts index 94910a0..48c208b 100644 --- a/packages/sync-mongodb/test/cli.test.ts +++ b/packages/sync-mongodb/test/cli.test.ts @@ -386,19 +386,21 @@ describe('commands', () => { await engine.backfill() fake.users.delete(gone['id'] as string) - const diffs = () => - ( - lastJson() as { - diffs: Record - } - ).diffs['prod']! + interface Diff { + stream: string + pages: number + complete: boolean + localOnly: string[] + remoteOnly: string[] + skippedReason?: string + } + const diffs = () => (lastJson() as { diffs: Record }).diffs['prod']! // The diff is checked before the exit code, so a failure shows the whole result. const code = await main(['reconcile', '--max-pages', '2'], { engine }) - expect(diffs().find((d) => d.stream === 'users')).toMatchObject({ - pages: 3, - complete: true, - localOnly: [gone['id']], - }) + const users = diffs().find((d) => d.stream === 'users')! + // A stream whose diff threw is reported with the error here, not thrown. + expect(users.skippedReason).toBeUndefined() + expect(users).toMatchObject({ pages: 3, complete: true, localOnly: [gone['id']], remoteOnly: [] }) expect(diffs().find((d) => d.stream === 'invitations')).toMatchObject({ pages: 0, complete: false }) // A slice that stops short is the outcome asked for, not an error, but drift is. expect(code).toBe(2) From 9e944f72e54bdcdd0fa10044527b2172863b54b9 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 5 Nov 2026 16:07:46 -0500 Subject: [PATCH 7/8] feat(mongodb): abandon oversized reconcile streams, add --exclude-streams Main's #45 taught the engine to abandon a stream whose listing alone passes the reconcile page budget, and gave the Postgres CLI `--exclude-streams`. This brings the MongoDB CLI, its README and its tests to the same place: the reconcile help and table describe the oversized verdict, `--exclude-streams` is parsed the same way, and the drift test budgets the three pages users needs rather than two. --- packages/sync-mongodb/README.md | 24 ++++++------ packages/sync-mongodb/src/cli.ts | 32 +++++++++++++-- packages/sync-mongodb/test/cli.test.ts | 54 ++++++++++++++++++++------ 3 files changed, 84 insertions(+), 26 deletions(-) diff --git a/packages/sync-mongodb/README.md b/packages/sync-mongodb/README.md index c4946ce..5f874d0 100644 --- a/packages/sync-mongodb/README.md +++ b/packages/sync-mongodb/README.md @@ -56,18 +56,18 @@ npx workos-sync-mongodb backfill # load everything that already exists npx workos-sync-mongodb listen # tail the Events API, forever ``` -| Command | What it does | -| --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `init [--repair]` | Create the collections, indexes, `_live` views and bookkeeping. Idempotent; safe on every deploy. Fails if a generated `_live` view has a pipeline that is not the generated one; `--repair` redefines it. Reports indexes the catalog no longer calls for, and never drops them. | -| `backfill [--resume] [--max-pages ]` | Load current state from the REST list endpoints. `--resume` continues each stream from its checkpoint and skips streams already complete. `--max-pages` stops after `n` pages across all streams, for a host with a time limit; run again with `--resume` until every stream reports `complete`. | -| `listen [--health-port ]` | Poll the Events API continuously as the sole writer; waits if another listener holds the lease. Finishes the current batch on `SIGINT` or `SIGTERM`. With a port, serves `GET /healthz` and `/metrics` on it. | -| `tick` | One poll, then exit. For a cron job instead of a long-running process. Exits 3 if a listener holds the lease. | -| `reconcile [--repair] [--resume] [--max-pages ]` | Diff ids against the API. Exits 2 on drift; `--repair` tombstones missed deletes and inserts missed creates. `--max-pages` stops starting streams once `n` pages have been fetched, for a host with a time limit; a stream that starts runs to its last page, since a diff needs the whole listing. Run again with `--resume` until every stream reports `complete`. | -| `refresh [--resume] [--max-pages ]` | Re-list top-level streams from the REST API, replacing each document; `--max-pages` and `--resume` slice it like `backfill`, with which it shares checkpoints. Re-run `backfill` for per-parent streams. | -| `status` | Cursor, time since the last successful poll, and per-stream document counts, tombstones and lag. | -| `redact --user ` | Erase one subject from the mirror, the event archive and the quarantine. | -| `rejected [--replay] [--id ]` | Show events the tail quarantined; exits 2 if there are any. `--replay` re-applies them (one, with `--id`) once the cause is fixed. | -| `streams` | List what this build can mirror, with group, mode, and whether it is on by default. | +| Command | What it does | +| ----------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `init [--repair]` | Create the collections, indexes, `_live` views and bookkeeping. Idempotent; safe on every deploy. Fails if a generated `_live` view has a pipeline that is not the generated one; `--repair` redefines it. Reports indexes the catalog no longer calls for, and never drops them. | +| `backfill [--resume] [--max-pages ]` | Load current state from the REST list endpoints. `--resume` continues each stream from its checkpoint and skips streams already complete. `--max-pages` stops after `n` pages across all streams, for a host with a time limit; run again with `--resume` until every stream reports `complete`. | +| `listen [--health-port ]` | Poll the Events API continuously as the sole writer; waits if another listener holds the lease. Finishes the current batch on `SIGINT` or `SIGTERM`. With a port, serves `GET /healthz` and `/metrics` on it. | +| `tick` | One poll, then exit. For a cron job instead of a long-running process. Exits 3 if a listener holds the lease. | +| `reconcile [--repair] [--resume] [--max-pages ] [--exclude-streams ]` | Diff ids against the API. Exits 2 on drift; `--repair` tombstones missed deletes and inserts missed creates. `--max-pages` stops starting streams once `n` pages have been fetched, for a host with a time limit; a stream that starts runs to its last page, since a diff needs the whole listing, unless it alone needs more than `n`: it is then abandoned, reported `oversized`, and skipped by `--resume` until a run with a larger budget or none. Run again with `--resume` until every stream reports `complete`. `--exclude-streams` leaves streams out, for one that has to run somewhere without a time limit. | +| `refresh [--resume] [--max-pages ]` | Re-list top-level streams from the REST API, replacing each document; `--max-pages` and `--resume` slice it like `backfill`, with which it shares checkpoints. Re-run `backfill` for per-parent streams. | +| `status` | Cursor, time since the last successful poll, streams `reconcile` has abandoned as oversized, and per-stream document counts, tombstones and lag. | +| `redact --user ` | Erase one subject from the mirror, the event archive and the quarantine. | +| `rejected [--replay] [--id ]` | Show events the tail quarantined; exits 2 if there are any. `--replay` re-applies them (one, with `--id`) once the cause is fixed. | +| `streams` | List what this build can mirror, with group, mode, and whether it is on by default. | Flags every command takes: diff --git a/packages/sync-mongodb/src/cli.ts b/packages/sync-mongodb/src/cli.ts index a53ef1b..108efd8 100644 --- a/packages/sync-mongodb/src/cli.ts +++ b/packages/sync-mongodb/src/cli.ts @@ -219,6 +219,27 @@ export function parseMaxPages(value: string | boolean | undefined): number | und return n } +/** + * `--exclude-streams ` names streams `reconcile` leaves out, for one + * whose listing has to run somewhere without a time limit. A bare flag is + * refused here: excluding nothing by accident is the mistake the flag exists + * to prevent. The names themselves are checked by the engine. + */ +export function parseExcludeStreams(value: string | boolean | undefined): string[] | undefined { + if (value === undefined || value === false) return undefined + const names = + typeof value === 'string' + ? value + .split(',') + .map((s) => s.trim()) + .filter(Boolean) + : [] + if (names.length === 0) { + throw new Error(`invalid --exclude-streams "${String(value)}": expected a comma-separated list of stream names`) + } + return names +} + function configFromEnv( flags: Record, destination: MongoDestination, @@ -260,12 +281,16 @@ Commands with a port, serve GET /healthz and /metrics on it tick One poll of the Events API, then exit (for a cron job); exits 3 if a listener holds the lease - reconcile [--repair] [--resume] [--max-pages ] + reconcile [--repair] [--resume] [--max-pages ] [--exclude-streams ] Compare the mirror against the API; --repair fixes drift. --max-pages stops starting streams once n pages have been fetched (a stream that starts runs to its last page, since - a diff needs the whole listing); run again with --resume - until every stream reports complete + a diff needs the whole listing, unless it alone needs more + than n: it is then abandoned and reported oversized until + a run with a larger budget or none); run again with + --resume until every stream reports complete. + --exclude-streams leaves streams out, for one that has to + run somewhere without a time limit refresh [--resume] [--max-pages ] Re-list top-level streams, replacing each document; --max-pages and --resume slice it like backfill, with @@ -426,6 +451,7 @@ async function runCommand(command: string, flags: Record { }) const lastJson = () => JSON.parse(out[out.length - 1]!) as Record + interface Diff { + stream: string + pages: number + complete: boolean + oversized?: boolean + localOnly: string[] + remoteOnly: string[] + skippedReason?: string + } + const diffs = () => (lastJson() as { diffs: Record }).diffs['prod']! const engineOver = (destination: MongoDestination, streams: string[]) => new SyncEngine( { @@ -386,17 +397,9 @@ describe('commands', () => { await engine.backfill() fake.users.delete(gone['id'] as string) - interface Diff { - stream: string - pages: number - complete: boolean - localOnly: string[] - remoteOnly: string[] - skippedReason?: string - } - const diffs = () => (lastJson() as { diffs: Record }).diffs['prod']! - // The diff is checked before the exit code, so a failure shows the whole result. - const code = await main(['reconcile', '--max-pages', '2'], { engine }) + // Users needs exactly the three pages budgeted: the stream completes and + // spends the budget, so invitations is left for a resume. + const code = await main(['reconcile', '--max-pages', '3'], { engine }) const users = diffs().find((d) => d.stream === 'users')! // A stream whose diff threw is reported with the error here, not thrown. expect(users.skippedReason).toBeUndefined() @@ -410,6 +413,35 @@ describe('commands', () => { expect(await t.db.collection('workos_users').countDocuments({ _deleted_at: { $type: 'date' } })).toBe(1) }) + it('abandons a stream whose listing passes --max-pages and reports it oversized, still exiting 0', async () => { + for (let i = 0; i < 250; i++) fake.createUser() + const engine = engineOver(new MongoDestination({ client: t.client, database: t.name }), ['users']) + await engine.backfill() + + expect(await main(['reconcile', '--max-pages', '2'], { engine })).toBe(0) + expect(diffs().find((d) => d.stream === 'users')).toMatchObject({ pages: 3, complete: false, oversized: true }) + expect(await main(['status'], { engine })).toBe(0) + expect(lastJson()).toMatchObject({ prod: { oversized: ['users'] } }) + }) + + it('leaves out --exclude-streams, and refuses a bare flag or an unknown name', async () => { + fake.createUser() + const engine = engineOver(new MongoDestination({ client: t.client, database: t.name }), ['users', 'invitations']) + await engine.backfill() + + expect(await main(['reconcile', '--exclude-streams', 'users'], { engine })).toBe(0) + const names = diffs().map((d) => d.stream) + expect(names).not.toContain('users') + expect(names).toContain('invitations') + + await expect(main(['reconcile', '--exclude-streams', 'customers'], { engine })).rejects.toThrow( + /unknown stream: customers/, + ) + // A bare flag parses as `true`; it must not become an exclusion of nothing. + expect(() => parseExcludeStreams(true)).toThrow(/invalid --exclude-streams/) + expect(parseExcludeStreams('users, invitations')).toEqual(['users', 'invitations']) + }) + it('refresh slices with --max-pages and converges with --resume', async () => { for (let i = 0; i < 250; i++) fake.createUser() const engine = engineOver(new MongoDestination({ client: t.client, database: t.name }), ['users']) From 1cef1d029f506c76908e4709706dd6f95c629380 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 5 Nov 2026 18:01:18 -0500 Subject: [PATCH 8/8] ci(mongodb): publish sync-mongodb releases release-please knows the package, but the workflow that publishes a component once its release is cut listed only the engine, postgres and (since #47) mysql. The MongoDB package gets the same job, so a release tag reaches npm and the image registry rather than stopping at GitHub. --- .github/workflows/release-please.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 6f8982b..7bd2c70 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -37,6 +37,8 @@ jobs: sync-postgres--tag_name: ${{ steps.release.outputs['packages/sync-postgres--tag_name'] }} sync-mysql--release_created: ${{ steps.release.outputs['packages/sync-mysql--release_created'] }} sync-mysql--tag_name: ${{ steps.release.outputs['packages/sync-mysql--tag_name'] }} + sync-mongodb--release_created: ${{ steps.release.outputs['packages/sync-mongodb--release_created'] }} + sync-mongodb--tag_name: ${{ steps.release.outputs['packages/sync-mongodb--tag_name'] }} steps: # Release PRs and tags come from the SDK Automation Bot, as in workos-node. # Events raised with GITHUB_TOKEN start no workflow runs, so a release PR @@ -94,3 +96,15 @@ jobs: tag_name: ${{ needs.release-please.outputs.sync-mysql--tag_name }} secrets: PUBLIC_SOCKET_FIREWALL_TOKEN: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }} + + publish-sync-mongodb: + needs: [release-please, publish-sync-engine] + if: >- + ${{ !cancelled() + && needs.publish-sync-engine.result != 'failure' + && needs.release-please.outputs.sync-mongodb--release_created == 'true' }} + uses: ./.github/workflows/release.yml + with: + tag_name: ${{ needs.release-please.outputs.sync-mongodb--tag_name }} + secrets: + PUBLIC_SOCKET_FIREWALL_TOKEN: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }}