From 04fb0f7b9e4f2c7c256eccd734fbe095094b6596 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 5 Nov 2026 16:02:42 -0500 Subject: [PATCH 01/10] feat(clickhouse): add the ClickHouse destination package Mirror streams are ReplacingMergeTree tables with tombstones and _live views; append streams are plain MergeTree. The lease, cursors and schema bookkeeping live in KeeperMap tables, the one compare-and-swap ClickHouse offers, so a Keeper and keeper_map_path_prefix are required. Projected columns are DEFAULT expressions over _raw, materialized when added to an existing table and recomputed on repair. Wires the package into CI (a ClickHouse container with an embedded Keeper), the Docker matrix, release-please and the READMEs. --- .github/workflows/ci.yml | 50 ++ .github/workflows/docker.yml | 1 + .release-please-manifest.json | 3 +- README.md | 12 +- packages/sync-clickhouse/LICENSE.txt | 21 + packages/sync-clickhouse/README.md | 246 +++++++ packages/sync-clickhouse/package.json | 50 ++ packages/sync-clickhouse/src/cli.ts | 551 +++++++++++++++ packages/sync-clickhouse/src/db/client.ts | 144 ++++ packages/sync-clickhouse/src/db/errors.ts | 72 ++ packages/sync-clickhouse/src/db/lease.ts | 272 +++++++ packages/sync-clickhouse/src/db/reconcile.ts | 99 +++ packages/sync-clickhouse/src/db/rejected.ts | 116 +++ packages/sync-clickhouse/src/db/upsert.ts | 445 ++++++++++++ packages/sync-clickhouse/src/destination.ts | 299 ++++++++ packages/sync-clickhouse/src/health.ts | 39 + packages/sync-clickhouse/src/index.ts | 53 ++ packages/sync-clickhouse/src/otel.ts | 75 ++ packages/sync-clickhouse/src/schema/ddl.ts | 264 +++++++ packages/sync-clickhouse/src/schema/drift.ts | 194 +++++ .../sync-clickhouse/src/schema/identifiers.ts | 27 + .../sync-clickhouse/src/schema/migrate.ts | 214 ++++++ packages/sync-clickhouse/src/schema/views.ts | 91 +++ packages/sync-clickhouse/test/cli.test.ts | 298 ++++++++ .../test/clickhouse/config.xml | 30 + .../sync-clickhouse/test/clickhouse/users.xml | 8 + packages/sync-clickhouse/test/client.test.ts | 79 +++ .../sync-clickhouse/test/destination.test.ts | 32 + packages/sync-clickhouse/test/drift.test.ts | 288 ++++++++ packages/sync-clickhouse/test/engine.test.ts | 665 ++++++++++++++++++ packages/sync-clickhouse/test/errors.test.ts | 49 ++ packages/sync-clickhouse/test/helpers/ca.ts | 17 + .../test/helpers/clickhouse.ts | 49 ++ .../sync-clickhouse/test/helpers/faulty.ts | 65 ++ packages/sync-clickhouse/test/lease.test.ts | 203 ++++++ .../sync-clickhouse/test/reconcile.test.ts | 232 ++++++ .../sync-clickhouse/test/rejected.test.ts | 100 +++ packages/sync-clickhouse/test/schema.test.ts | 344 +++++++++ packages/sync-clickhouse/test/upsert.test.ts | 596 ++++++++++++++++ packages/sync-clickhouse/tsconfig.build.json | 11 + packages/sync-clickhouse/tsconfig.json | 22 + packages/sync-clickhouse/vitest.config.ts | 14 + packages/sync-engine/README.md | 9 +- pnpm-lock.yaml | 35 + release-please-config.json | 4 + 45 files changed, 6478 insertions(+), 10 deletions(-) create mode 100644 packages/sync-clickhouse/LICENSE.txt create mode 100644 packages/sync-clickhouse/README.md create mode 100644 packages/sync-clickhouse/package.json create mode 100644 packages/sync-clickhouse/src/cli.ts create mode 100644 packages/sync-clickhouse/src/db/client.ts create mode 100644 packages/sync-clickhouse/src/db/errors.ts create mode 100644 packages/sync-clickhouse/src/db/lease.ts create mode 100644 packages/sync-clickhouse/src/db/reconcile.ts create mode 100644 packages/sync-clickhouse/src/db/rejected.ts create mode 100644 packages/sync-clickhouse/src/db/upsert.ts create mode 100644 packages/sync-clickhouse/src/destination.ts create mode 100644 packages/sync-clickhouse/src/health.ts create mode 100644 packages/sync-clickhouse/src/index.ts create mode 100644 packages/sync-clickhouse/src/otel.ts create mode 100644 packages/sync-clickhouse/src/schema/ddl.ts create mode 100644 packages/sync-clickhouse/src/schema/drift.ts create mode 100644 packages/sync-clickhouse/src/schema/identifiers.ts create mode 100644 packages/sync-clickhouse/src/schema/migrate.ts create mode 100644 packages/sync-clickhouse/src/schema/views.ts create mode 100644 packages/sync-clickhouse/test/cli.test.ts create mode 100644 packages/sync-clickhouse/test/clickhouse/config.xml create mode 100644 packages/sync-clickhouse/test/clickhouse/users.xml create mode 100644 packages/sync-clickhouse/test/client.test.ts create mode 100644 packages/sync-clickhouse/test/destination.test.ts create mode 100644 packages/sync-clickhouse/test/drift.test.ts create mode 100644 packages/sync-clickhouse/test/engine.test.ts create mode 100644 packages/sync-clickhouse/test/errors.test.ts create mode 100644 packages/sync-clickhouse/test/helpers/ca.ts create mode 100644 packages/sync-clickhouse/test/helpers/clickhouse.ts create mode 100644 packages/sync-clickhouse/test/helpers/faulty.ts create mode 100644 packages/sync-clickhouse/test/lease.test.ts create mode 100644 packages/sync-clickhouse/test/reconcile.test.ts create mode 100644 packages/sync-clickhouse/test/rejected.test.ts create mode 100644 packages/sync-clickhouse/test/schema.test.ts create mode 100644 packages/sync-clickhouse/test/upsert.test.ts create mode 100644 packages/sync-clickhouse/tsconfig.build.json create mode 100644 packages/sync-clickhouse/tsconfig.json create mode 100644 packages/sync-clickhouse/vitest.config.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a6da4c..702f8d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,3 +151,53 @@ jobs: - name: Build run: pnpm --filter "@workos/sync-mysql..." build + + clickhouse: + name: ClickHouse Node ${{ matrix.node }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + node: [22, 24] + env: + # There is no embedded ClickHouse, so the whole suite runs against the + # container; without this variable the database-backed tests skip. + TEST_CLICKHOUSE_URL: http://default@127.0.0.1:8123 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # Not a service container: those start before the checkout, and the + # server needs the config in test/clickhouse/ — an embedded Keeper for + # the KeeperMap bookkeeping tables and `keeper_map_path_prefix`. + - name: Start ClickHouse + run: | + docker run -d --name clickhouse -p 127.0.0.1:8123:8123 \ + -v "$PWD/packages/sync-clickhouse/test/clickhouse/config.xml:/etc/clickhouse-server/config.d/sync.xml:ro" \ + -v "$PWD/packages/sync-clickhouse/test/clickhouse/users.xml:/etc/clickhouse-server/users.d/sync.xml:ro" \ + -e CLICKHOUSE_SKIP_USER_SETUP=1 \ + --ulimit nofile=262144:262144 \ + clickhouse/clickhouse-server:26.10 + for _ in $(seq 1 60); do + if curl -fsS http://127.0.0.1:8123/ping >/dev/null 2>&1; then exit 0; fi + sleep 1 + done + echo "ClickHouse did not become ready" >&2 + docker logs clickhouse >&2 + exit 1 + + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: ${{ matrix.node }} + + - name: Install + run: pnpm install --frozen-lockfile + + - name: Test + run: pnpm --filter @workos/sync-clickhouse test:database + + - name: Build + run: pnpm --filter "@workos/sync-clickhouse..." build diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2708a10..ff890e3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -23,6 +23,7 @@ jobs: destination: - postgres - mysql + - clickhouse runner: [ubuntu-latest, ubuntu-24.04-arm] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 2e4d4b4..8ac5505 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,5 +1,6 @@ { "packages/sync-engine": "0.1.0", "packages/sync-postgres": "0.1.0", - "packages/sync-mysql": "0.0.0" + "packages/sync-mysql": "0.0.0", + "packages/sync-clickhouse": "0.0.0" } diff --git a/README.md b/README.md index 40645b4..5a5c4ca 100644 --- a/README.md +++ b/README.md @@ -31,13 +31,15 @@ base tables keep them as tombstones for when you want to ask who left. One engine, one package per database. Each ships a CLI with the same commands and a container image under the same name: -| Destination | Package | Mirror | -| ----------- | ------------------------------------------------- | ---------------------------------------------------------------------- | -| Postgres | [`@workos/sync-postgres`](packages/sync-postgres) | A `workos` schema of tables with generated columns and `_live` views | -| MySQL | [`@workos/sync-mysql`](packages/sync-mysql) | A `workos` database of tables with generated columns and `_live` views | +| Destination | Package | Mirror | +| ----------- | ----------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Postgres | [`@workos/sync-postgres`](packages/sync-postgres) | A `workos` schema of tables with generated columns and `_live` views | +| MySQL | [`@workos/sync-mysql`](packages/sync-mysql) | A `workos` database of tables with generated columns and `_live` views | +| ClickHouse | [`@workos/sync-clickhouse`](packages/sync-clickhouse) | A `workos` database of `ReplacingMergeTree` tables with `_live` views, plus append-only tables for the sign-in log and events; for history that outlives the 90-day retention | The rest of this page shows Postgres; every command and option below has a -MySQL counterpart, and each package's README is the reference for it. +MySQL and a ClickHouse counterpart, and each package's README is the +reference for it. ## Try it in 30 seconds diff --git a/packages/sync-clickhouse/LICENSE.txt b/packages/sync-clickhouse/LICENSE.txt new file mode 100644 index 0000000..c0cf745 --- /dev/null +++ b/packages/sync-clickhouse/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 WorkOS + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/sync-clickhouse/README.md b/packages/sync-clickhouse/README.md new file mode 100644 index 0000000..50efccb --- /dev/null +++ b/packages/sync-clickhouse/README.md @@ -0,0 +1,246 @@ +# @workos/sync-clickhouse + +Keep your WorkOS sign-in and event history in ClickHouse for as long as you +like, with the users and organizations beside it for the joins. + +WorkOS keeps events for 90 days, and the `authentication.*` events — every +sign-in, success or failure — have no REST endpoint at all. This package tails +the Events API from a durable cursor into append-only ClickHouse tables, so +"failed sign-ins per organization per hour, last 18 months" is a `GROUP BY` +over data you own: + +```sql +SELECT o.name AS organization, toStartOfHour(a.created_at) AS hour, count() AS failures +FROM workos.auth_attempts a +JOIN workos.users_live u ON u.id = a.user_id AND u._env = a._env +JOIN workos.organization_memberships_live m ON m.user_id = u.id AND m._env = u._env +JOIN workos.organizations_live o ON o.id = m.organization_id AND o._env = m._env +WHERE a.status = 'failed' AND a.created_at > now() - INTERVAL 7 DAY +GROUP BY organization, hour +ORDER BY failures DESC; +``` + +It also mirrors the identity objects (users, organizations, memberships, +directories, connections, roles, …) the way `@workos/sync-postgres` does, as +versioned tables with `_live` views, so the join above works in one database. +Start with `--streams auth_attempts` if the logs are what you came for; add +the mirror streams when you want to join against them. + +## Requirements + +- ClickHouse 23.3 or later, self-hosted or ClickHouse Cloud. Tested against + 26.x in CI and against ClickHouse Cloud. +- A ClickHouse Keeper (or ZooKeeper) reachable by the server, with + `keeper_map_path_prefix` set. The single-writer lease and the cursors live + in a [KeeperMap](https://clickhouse.com/docs/engines/table-engines/special/keeper-map) + table: it is the one place ClickHouse offers an atomic compare-and-swap. + Cloud and any replicated cluster have this already. A single self-hosted + node needs the embedded Keeper enabled; the few lines are in + [`test/clickhouse/config.xml`](test/clickhouse/config.xml). +- The HTTP interface (port 8123, or 8443 with TLS). The package uses the + official `@clickhouse/client`. + +## Install + +```bash +npm install @workos/sync-clickhouse +``` + +This package ships the ClickHouse destination, the `workos-sync-clickhouse` +command, and re-exports the engine from +[`@workos/sync-engine`](https://www.npmjs.com/package/@workos/sync-engine), +so it is the only package you need for ClickHouse. + +## Command line + +```bash +export DATABASE_URL=https://default:password@abc123.us-east-1.aws.clickhouse.cloud:8443 +export WORKOS_API_KEY=sk_live_... + +npx workos-sync-clickhouse init # create the database, tables and views +npx workos-sync-clickhouse backfill --streams auth_attempts # nothing to list for logs; sets the cursor +npx workos-sync-clickhouse listen --streams auth_attempts # tail the Events API, forever +``` + +Auth history begins the day you turn the tail on: there is no endpoint to +list past sign-ins, so `backfill` cannot recover them. That is the argument +for starting the tail early. + +| Command | What it does | +| ----------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `init [--repair] [--print-sql]` | Create the database, tables, `_live` views and bookkeeping. Idempotent; safe on every deploy. Exits 2 if a projected column has drifted from the catalog, and fails if a view under a generated `_live` name is not the generated shape; `--repair` redefines columns and replaces such views. `--print-sql` prints the DDL instead of running it. | +| `backfill [--resume] [--max-pages ]` | Load current state from the REST list endpoints. `--resume` continues each stream from its checkpoint and skips streams already complete. `--max-pages` stops after `n` pages across all streams, for a host with a time limit; run again with `--resume` until every stream reports `complete`. | +| `listen [--health-port ]` | Poll the Events API continuously as the sole writer; waits if another listener holds the lease. Finishes the current batch on `SIGINT` or `SIGTERM`. With a port, serves `GET /healthz` and `/metrics` on it. | +| `tick` | One poll, then exit. For a cron job instead of a long-running process. Exits 3 if a listener holds the lease. | +| `reconcile [--repair] [--resume] [--max-pages ] [--exclude-streams ]` | Diff ids against the API. Exits 2 on drift; `--repair` tombstones missed deletes and inserts missed creates. `--max-pages` stops starting streams once `n` pages have been fetched, for a host with a time limit; a stream that starts runs to its last page, since a diff needs the whole listing, unless it alone needs more than `n`: it is then abandoned, reported `oversized`, and skipped by `--resume` until a run with a larger budget or none. Run again with `--resume` until every stream reports `complete`. `--exclude-streams` leaves streams out, for one that has to run somewhere without a time limit. | +| `refresh [--resume] [--max-pages ]` | Re-list top-level streams from the REST API, replacing each row; `--max-pages` and `--resume` slice it like `backfill`, with which it shares checkpoints. Re-run `backfill` for per-parent streams. | +| `status` | Cursor, time since the last successful poll, streams `reconcile` has abandoned as oversized, and per-stream row counts, tombstones and lag. | +| `redact --user ` | Erase one subject from the mirror, the sign-in log and the event archive, with a lightweight `DELETE`. | +| `rejected [--replay] [--id ]` | Show events the tail quarantined; exits 2 if there are any. `--replay` re-applies them (one, with `--id`) once the cause is fixed. | +| `streams` | List what this build can mirror, with group, mode, and whether it is on by default. | + +Flags every command takes: + +| Flag | Overrides | What it does | +| ---------------------- | -------------------- | ---------------------------------------------------------- | +| `--database-url ` | `DATABASE_URL` | The HTTP interface, `https://user:password@host:8443`. | +| `--schema ` | `WORKOS_SYNC_SCHEMA` | The database the mirror lives in. | +| `--streams ` | | Restrict to specific streams. `events` is always included. | +| `--help` | | Print usage. | + +Rows are labelled `_env = 'production'` by default; set +`WORKOS_ENVIRONMENT_NAME` if the key is for something else. The label is +written to every row and must never change once you have data. To mirror +several environments into one database, run one process per key with a +different label each; cursors are kept per label, so they never collide. + +The command reads its settings from the environment; where a flag exists for +the same setting, the flag wins. + +| Variable | Default | What it does | +| ------------------------------------ | ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `DATABASE_URL` | required | The HTTP interface with credentials. A path in it is ignored; see `WORKOS_SYNC_SCHEMA`. | +| `WORKOS_API_KEY` | required | API key for the environment to mirror. One key, one environment, one process; dedicate it to the sync (see Operations). | +| `WORKOS_ENVIRONMENT_NAME` | `production` | Label written to `_env` on every row. Must not change once you have data. Run one process per environment to mirror several. | +| `WORKOS_API_URL` | `https://api.workos.com` | API base URL. | +| `WORKOS_SYNC_SCHEMA` | `workos` | The database the mirror lives in, created if missing. Every statement qualifies its tables with it. | +| `WORKOS_SYNC_ID` | `default` | Identifier when several syncs share one database. Each keeps its own cursors. | +| `WORKOS_SYNC_HYDRATE` | `off` | Re-fetch objects when their events arrive: `off`, `on_create`, `always`, or per stream as `users=always,organizations=off`. | +| `WORKOS_SYNC_POLL_MS` | `5000` | Idle wait between polls, in ms, when the last poll returned nothing. | +| `WORKOS_SYNC_LAG_MS` | `5000` | Hold back events younger than this, in ms, before advancing the cursor. | +| `WORKOS_SYNC_MAX_EVENTS_PER_TICK` | one page (100) | Events to apply per poll, fetched across as many pages as needed. Raise it: each poll is a few inserts, and ClickHouse prefers fewer, larger ones. | +| `WORKOS_SYNC_HEALTH_PORT` | off (`8080` in the container image) | `listen` only: serve `GET /healthz` and `GET /metrics` on this port, from in-memory state, with no database query. | +| `WORKOS_SYNC_HEALTH_HOST` | `0.0.0.0` | Interface the health server binds. | +| `WORKOS_SYNC_HEALTH_MAX_POLL_AGE_MS` | `60000` | `/healthz` returns 503 once any environment's last successful poll is older than this. | +| `WORKOS_SYNC_OTEL` | off | `1` starts the OpenTelemetry Node SDK for the command, configured by the standard `OTEL_*` variables, and flushes it on exit. Every command, `tick` included. | +| `WORKOS_SYNC_LEASE_TTL_MS` | `60000` | How long the tail lease outlives a crashed listener, in ms; at least 1. The most lag a crash can add before a replacement takes over. | +| `WORKOS_SYNC_CLICKHOUSE_TLS_CA` | system trust store | CA certificate the server's certificate must chain to, as PEM or base64 of it. Only with an `https://` URL. | +| `LOG_LEVEL` | `info` | `debug`, `info`, `warn`, `error` or `silent`. Logs are JSON lines on stdout. | + +The user the sync connects as needs `CREATE DATABASE` the first time, or an +existing database it may create tables in, plus `CREATE TABLE`, `CREATE VIEW`, +`ALTER`, `INSERT`, `SELECT` and `ALTER DELETE` on that database. + +## Embed it + +```ts +import { SyncEngine, ClickHouseDestination } from "@workos/sync-clickhouse"; + +const engine = new SyncEngine({ + destination: new ClickHouseDestination({ + databaseUrl: process.env.DATABASE_URL, + }), + environments: [{ name: "production", apiKey: process.env.WORKOS_API_KEY! }], + streams: ["auth_attempts", "users", "organizations"], +}); + +await engine.backfill(); +await engine.listen(); // resolves when you call engine.stop() +``` + +To reuse an existing `@clickhouse/client` instance instead of a connection +string, wrap it with `fromClient` and pass it as `db`. Give it the settings +in `DEFAULT_SETTINGS`: the engine relies on waited async inserts, synchronous +mutations and sequentially consistent reads. + +`ClickHouseDestination` options: + +| Option | Default | | +| --------------- | --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `databaseUrl` | — | The HTTP interface with credentials. Either this or `db`. | +| `sslCa` | — | PEM CA certificate the server's certificate must chain to. With an `https://` `databaseUrl` only. | +| `db` | — | A pre-built client, e.g. `fromClient(client)`. Not closed by the engine. | +| `schema` | `workos` | The database the mirror lives in. Created if missing. | +| `syncId` | `default` | Identifier when several syncs share one database. | +| `batchSize` | `500` | Rows per statement. | +| `repairViews` | `false` | On `init()`, replace a view under a generated `_live` name that is not the generated shape. Off, `init()` fails and names it. | +| `repairColumns` | `false` | On `init()`, redefine projected columns that no longer match the catalog and recompute them. Off, drift is reported and `init` withholds the schema fingerprint. | + +Engine options (`environments`, `streams`, `extraColumns`, `hydrate`, +`pollIntervalMs`, `lagWindowMs`, `maxEventsPerTick`, …) are documented in +[`@workos/sync-engine`](https://www.npmjs.com/package/@workos/sync-engine). + +## The schema + +The engine writes one column, `_raw`, and computes the rest from it with +`DEFAULT` expressions: + +```sql +CREATE TABLE workos.auth_attempts ( + _env String, + _raw String, -- the event payload, the only column the engine writes + _updated_at DateTime64(3, 'UTC'), -- the event's own time + _synced_at DateTime64(6, 'UTC') DEFAULT now64(6), + event_id String DEFAULT JSONExtractString(_raw, 'event_id'), + status Nullable(String) DEFAULT JSONExtract(_raw, 'status', 'Nullable(String)'), + user_id Nullable(String) DEFAULT JSONExtract(_raw, 'user_id', 'Nullable(String)'), + created_at Nullable(DateTime64(3, 'UTC')) DEFAULT parseDateTime64BestEffortOrNull(JSONExtractString(_raw, 'created_at'), 3, 'UTC'), + INDEX auth_attempts_user_id_idx user_id TYPE bloom_filter GRANULARITY 4, + INDEX auth_attempts_created_at_idx created_at TYPE minmax GRANULARITY 1 +) ENGINE = MergeTree ORDER BY (_env, event_id); +``` + +Append-only streams (`auth_attempts`, `events`) are plain `MergeTree`: one row +per event, never updated, the shape every ClickHouse query expects. + +Mirror streams are `ReplacingMergeTree` versioned by `_updated_at`. Every +write inserts the row's next state, and the newest source timestamp wins at +merge time and under `FINAL`; that is the staleness gate, enforced by the +table whatever order writes land in. Each mirror table has a generated +`_live` view — `workos.users_live`, `workos.organizations_live`, and so +on — that reads with `FINAL` and hides tombstoned rows. **Query the views.** +The base table shows every version a row ever had, plus its tombstones +(`_deleted_at`), for when you want to ask who left. + +A field added upstream can never break ingest — it is queryable as +`JSONExtractString(_raw, 'field')` immediately, and promoting it to a typed +column is pure DDL, computed for the rows already there. A value a typed +column cannot cast projects as NULL, with the value still in `_raw`. Columns +marked for an index get a data-skipping index: a bloom filter for text, a +min-max index for timestamps and numbers. + +## What it mirrors + +Users, organizations, memberships, domains, invitations and sessions; +directories, directory users and groups; SSO connections; AuthKit groups, +roles, permissions and feature flags. Every event is archived verbatim to +`workos.events`, including event types this version has never heard of; the +`authentication.*` events are projected into `auth_attempts`. + +## What it is not + +The mirror is a **read model**: eventually consistent, usually within seconds. +Read from it for analytics, dashboards, lists and agents. Keep writes, +authorization decisions and post-signup checks on the WorkOS API. The sync is +one-directional; there is no write-back. ClickHouse is not where the mirror +tables are at their best: for an application that reads single rows by key, +use `@workos/sync-postgres` or `@workos/sync-mysql`, and if you want both, run +both; the cursors are kept per package. + +## Testing + +`pnpm test` runs the pure tests without a server. The database-backed suite +needs a ClickHouse with a Keeper, started from the config in +`test/clickhouse/`: + +```sh +docker run -d --name clickhouse -p 127.0.0.1:8123:8123 \ + -v "$PWD/packages/sync-clickhouse/test/clickhouse/config.xml:/etc/clickhouse-server/config.d/sync.xml:ro" \ + -v "$PWD/packages/sync-clickhouse/test/clickhouse/users.xml:/etc/clickhouse-server/users.d/sync.xml:ro" \ + -e CLICKHOUSE_SKIP_USER_SETUP=1 clickhouse/clickhouse-server:latest +TEST_CLICKHOUSE_URL=http://default@127.0.0.1:8123 pnpm test +``` + +Each test creates a database of its own and drops it on completion. +`pnpm test:database` fails when the URL is missing instead of skipping. + +## Documentation + +- [Getting started](https://github.com/workos/sync-engine/blob/main/docs/getting-started.md) +- [How it works](https://github.com/workos/sync-engine/blob/main/docs/how-it-works.md) — backfill, tail, reconcile, and why all three +- [Operations](https://github.com/workos/sync-engine/blob/main/docs/operations.md) — deployment, alerting, the 90-day retention cliff +- [Consistency](https://github.com/workos/sync-engine/blob/main/docs/consistency.md) — what the mirror does and does not guarantee + +## License + +MIT diff --git a/packages/sync-clickhouse/package.json b/packages/sync-clickhouse/package.json new file mode 100644 index 0000000..53238a4 --- /dev/null +++ b/packages/sync-clickhouse/package.json @@ -0,0 +1,50 @@ +{ + "name": "@workos/sync-clickhouse", + "version": "0.0.0", + "description": "Mirror your WorkOS data into ClickHouse. Backfill, tail the Events API, and reconcile drift.", + "type": "module", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/workos/sync-engine.git", + "directory": "packages/sync-clickhouse" + }, + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "bin": { + "workos-sync-clickhouse": "./dist/cli.js" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "files": [ + "dist" + ], + "scripts": { + "build": "tsc -p tsconfig.build.json", + "test": "vitest run", + "test:database": "node -e \"if (!process.env.TEST_CLICKHOUSE_URL) { console.error('TEST_CLICKHOUSE_URL is required'); process.exit(1) }\" && vitest run", + "test:watch": "vitest", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@workos/sync-engine": "workspace:*", + "@clickhouse/client": "^1.11.0" + }, + "optionalDependencies": { + "@opentelemetry/sdk-node": "^0.300.0" + }, + "devDependencies": { + "@opentelemetry/api": "^1.10.0", + "@opentelemetry/sdk-metrics": "^3.0.0", + "@types/node": "^26.6.5", + "typescript": "^7.0.2", + "vitest": "^5.0.5" + }, + "engines": { + "node": ">=22" + } +} diff --git a/packages/sync-clickhouse/src/cli.ts b/packages/sync-clickhouse/src/cli.ts new file mode 100644 index 0000000..d918dc0 --- /dev/null +++ b/packages/sync-clickhouse/src/cli.ts @@ -0,0 +1,551 @@ +#!/usr/bin/env node +/** + * workos-sync-clickhouse — command line interface. + * + * workos-sync-clickhouse init [--repair] [--print-sql] + * create the schema; --repair rebuilds drifted columns and replaces + * foreign _live views; --print-sql prints the DDL instead of running it + * workos-sync-clickhouse backfill [--resume] [--max-pages ] + * page the list endpoints, optionally n pages at a time + * workos-sync-clickhouse listen tail the Events API + * workos-sync-clickhouse tick one poll, then exit (for cron) + * workos-sync-clickhouse reconcile [--repair] [--exclude-streams ] detect (and optionally fix) drift + * workos-sync-clickhouse refresh re-list to refresh partial event fields + * workos-sync-clickhouse status cursor age and per-stream freshness + * workos-sync-clickhouse redact --user erase one subject + * workos-sync-clickhouse rejected [--replay] quarantined events; replay them once fixed + * workos-sync-clickhouse streams list what this build can mirror + */ +import { + SyncEngine, + LeaseHeldError, + STREAMS, + DEFAULT_HYDRATE, + defaultStreams, + selectStreams, + createLogger, + startHealthServer, + type EnvironmentConfig, + type HydrateMode, + type Logger, + type LogLevel, + type SyncEngineConfig, +} from '@workos/sync-engine' +import { X509Certificate } from 'node:crypto' +import { ClickHouseDestination } from './destination.js' +import { buildSchemaSql } from './schema/ddl.js' +import { fingerprint } from './schema/migrate.js' +import { healthOptionsFromEnv } from './health.js' +import { startOtel } from './otel.js' +import type { ChClient } from './db/client.js' + +interface Args { + command: string + flags: Record +} + +function parseArgs(argv: string[]): Args { + const [command = 'help', ...rest] = argv + const flags: Record = {} + for (let i = 0; i < rest.length; i++) { + const arg = rest[i]! + if (!arg.startsWith('--')) continue + const key = arg.slice(2) + const next = rest[i + 1] + if (next && !next.startsWith('--')) { + flags[key] = next + i++ + } else { + flags[key] = true + } + } + return { command, flags } +} + +/** + * One key, one environment, one process. `WORKOS_ENVIRONMENT_NAME` is the label + * written to every row as `_env`. It is yours to choose because no API call + * reports which environment a key belongs to, and it must stay stable across + * key rotation. To mirror several environments into one database, run one + * process per key with a different label each; cursors are kept per label. + */ +function environmentsFromEnv(): EnvironmentConfig[] { + const apiKey = process.env['WORKOS_API_KEY'] + if (!apiKey) throw new Error('set WORKOS_API_KEY') + return [ + { + name: process.env['WORKOS_ENVIRONMENT_NAME'] ?? 'production', + apiKey, + apiUrl: process.env['WORKOS_API_URL'], + }, + ] +} + +const HYDRATE_MODES: HydrateMode[] = ['off', 'on_create', 'always'] + +/** + * `WORKOS_SYNC_HYDRATE` is either one mode for every stream (`off`, + * `on_create`, `always`) or a per-stream list (`users=always,organizations=off`). + * Unset means the engine default: off. + */ +export function parseHydrate(value: string | undefined): SyncEngineConfig['hydrate'] { + if (!value || !value.trim()) return undefined + const v = value.trim() + if ((HYDRATE_MODES as string[]).includes(v)) return v as HydrateMode + const out: Record = { ...DEFAULT_HYDRATE } + for (const pair of v.split(',')) { + const idx = pair.indexOf('=') + const stream = pair.slice(0, idx).trim() + const mode = pair.slice(idx + 1).trim() + if (idx < 1 || !(HYDRATE_MODES as string[]).includes(mode)) { + throw new Error( + `invalid WORKOS_SYNC_HYDRATE entry "${pair}": expected off | on_create | always, or stream=mode,...`, + ) + } + out[stream] = mode as HydrateMode + } + return out +} + +/** + * `--repair` on `init` means rebuilding drifted columns from `_raw` and + * replacing foreign views under generated `_live` names; on `reconcile` it + * means repairing rows. Sharing the flag name is deliberate — it is "fix what + * you found" in both — but each command reads only its own meaning. + */ +export function repairRequested(command: string, flags: Record): boolean { + return command === 'init' && Boolean(flags['repair']) +} + +const PEM_CERTIFICATE = /-----BEGIN CERTIFICATE-----[\s\S]*?-----END CERTIFICATE-----/g + +/** True when `text` holds at least one PEM certificate and every one of them parses. */ +function isCertificateBundle(text: string): boolean { + const blocks = text.match(PEM_CERTIFICATE) + if (!blocks) return false + try { + for (const block of blocks) new X509Certificate(block) + return true + } catch { + return false + } +} + +/** + * `WORKOS_SYNC_CLICKHOUSE_TLS_CA` is the CA certificate the database's certificate + * must chain to: PEM, or the PEM encoded as base64 for a secret store that + * cannot hold a multi-line value. A PEM whose newlines arrived as the two + * characters `\n` is straightened out, since that is how some hosts hand + * back a pasted certificate. Unset or empty means the driver's own defaults. + * Every certificate in the value is parsed, so a truncated or damaged one + * is refused here rather than handed to the driver, where it would surface + * as a handshake failure against the server and read as the server's fault + * rather than the configuration's. + */ +export function parseSslCa(value: string | undefined): string | undefined { + if (!value || !value.trim()) return undefined + const v = value.trim().replace(/\\n/g, '\n') + if (isCertificateBundle(v)) return v + const decoded = Buffer.from(v, 'base64').toString('utf8') + if (isCertificateBundle(decoded)) return decoded + throw new Error( + 'invalid WORKOS_SYNC_CLICKHOUSE_TLS_CA: expected one or more PEM certificates, or them encoded as base64', + ) +} + +function schemaFromFlags(flags: Record): string { + return (flags['schema'] as string) ?? process.env['WORKOS_SYNC_SCHEMA'] ?? 'workos' +} + +function streamsFromFlags(flags: Record): string[] | undefined { + return typeof flags['streams'] === 'string' ? flags['streams'].split(',') : undefined +} + +/** + * `init --print-sql`: the DDL `init` would run, as one script for + * clickhouse-client or a migration file, instead of running it. + * Nothing is touched, so no database or API key is needed; the stream + * selection is the engine's own. + * + * The header names the catalog fingerprint but the script deliberately does + * not record it. A recorded fingerprint means "the live columns were checked + * against this catalog" (see `migrate`), and only `init` can check. So after + * the script is applied, `init` still has to run once: it finds the DDL in + * place, runs that check, and records the fingerprint, after which later + * runs are no-ops. Every statement in it is idempotent, so it can be applied + * to a database in any state. The schema name is JSON-quoted in the header + * so that a line break in it cannot end the comment early; the DDL itself + * quotes it as an identifier. + */ +export function printSchemaSql(streamNames: string[] | undefined, schema: string): string { + const streams = selectStreams(streamNames) + const header = [ + '-- Generated by workos-sync-clickhouse init --print-sql.', + `-- Schema ${JSON.stringify(schema)}, catalog fingerprint ${fingerprint(streams, schema)}.`, + '-- Apply this, then run `init` once: it records the fingerprint after', + '-- checking the live columns against the catalog, which this script cannot do.', + ].join('\n') + return `${header}\n\n${buildSchemaSql(streams, schema) + .map((s) => `${s};`) + .join('\n\n')}\n` +} + +/** + * Built apart from the rest of the config because `init` reads the drift + * report back off it after the engine has run; the engine's own `init()` + * returns nothing. `db` is a test seam: a pre-built client in place + * of a connection string, so the CLI can be exercised end to end. + */ +function destinationFromEnv( + flags: Record, + repair: boolean, + db?: ChClient, +): ClickHouseDestination { + const databaseUrl = (flags['database-url'] as string) ?? process.env['DATABASE_URL'] + if (!db && !databaseUrl) throw new Error('set DATABASE_URL (or pass --database-url)') + return new ClickHouseDestination({ + databaseUrl, + sslCa: parseSslCa(process.env['WORKOS_SYNC_CLICKHOUSE_TLS_CA']), + db, + schema: schemaFromFlags(flags), + syncId: process.env['WORKOS_SYNC_ID'] ?? 'default', + repairColumns: repair, + repairViews: repair, + }) +} + +/** + * `WORKOS_SYNC_LEASE_TTL_MS` is a number of milliseconds, at least 1. Unset + * or empty means the engine default. Anything else is refused here rather + * than further down: an empty value would otherwise become zero, and a lease + * that expires on arrival excludes nobody. + */ +export function parseLeaseTtl(value: string | undefined): number | undefined { + if (!value || !value.trim()) return undefined + const n = Number(value) + if (!Number.isFinite(n) || Math.floor(n) < 1) { + throw new Error(`invalid WORKOS_SYNC_LEASE_TTL_MS "${value}": expected a number of milliseconds of at least 1`) + } + return n +} + +/** + * `--max-pages ` is a positive integer. Absent means unbounded. A bare + * `--max-pages` with no value, zero, or a fraction is refused here, before the + * engine connects: a budget of nothing would stop before the first page and + * look like a backfill that found nothing. + */ +export function parseMaxPages(value: string | boolean | undefined): number | undefined { + if (value === undefined || value === false) return undefined + const n = typeof value === 'string' ? Number(value.trim()) : NaN + if (typeof value !== 'string' || !value.trim() || !Number.isInteger(n) || n < 1) { + throw new Error(`invalid --max-pages "${String(value)}": expected a positive integer`) + } + return n +} + +/** + * `--exclude-streams ` names streams `reconcile` leaves out, for one + * whose listing has to run somewhere without a time limit. A bare flag is + * refused here: excluding nothing by accident is the mistake the flag exists + * to prevent. The names themselves are checked by the engine. + */ +export function parseExcludeStreams(value: string | boolean | undefined): string[] | undefined { + if (value === undefined || value === false) return undefined + const names = + typeof value === 'string' + ? value + .split(',') + .map((s) => s.trim()) + .filter(Boolean) + : [] + if (names.length === 0) { + throw new Error(`invalid --exclude-streams "${String(value)}": expected a comma-separated list of stream names`) + } + return names +} + +function configFromEnv( + flags: Record, + destination: ClickHouseDestination, +): SyncEngineConfig & { logger: Logger } { + const streams = streamsFromFlags(flags) + return { + destination, + environments: environmentsFromEnv(), + streams, + hydrate: parseHydrate(process.env['WORKOS_SYNC_HYDRATE']), + pollIntervalMs: Number(process.env['WORKOS_SYNC_POLL_MS'] ?? 5000), + lagWindowMs: Number(process.env['WORKOS_SYNC_LAG_MS'] ?? 5000), + maxEventsPerTick: process.env['WORKOS_SYNC_MAX_EVENTS_PER_TICK'] + ? Number(process.env['WORKOS_SYNC_MAX_EVENTS_PER_TICK']) + : undefined, + leaseTtlMs: parseLeaseTtl(process.env['WORKOS_SYNC_LEASE_TTL_MS']), + logger: createLogger((process.env['LOG_LEVEL'] as LogLevel) ?? 'info'), + } +} + +const HELP = `workos-sync-clickhouse — mirror WorkOS data into ClickHouse + +Usage + workos-sync-clickhouse [flags] + +Commands + init [--repair] [--print-sql] + Create the database and bookkeeping tables. Exits 2 if a generated + column has drifted from the catalog, and fails if a view under a + generated _live name is not the generated shape; --repair rebuilds + columns from _raw and replaces such views. --print-sql prints the + DDL instead of running it, with no database + needed; apply it, then run init once so it records the fingerprint + backfill [--resume] [--max-pages ] + Load current state from the REST list endpoints. + --resume continues each stream from its checkpoint and + skips streams already complete; --max-pages stops after + n pages across all streams, for hosts with a time limit + (run again with --resume until every stream is complete) + listen [--health-port ] + Tail the Events API and apply changes continuously + (waits while another listener holds the lease); + with a port, serve GET /healthz and /metrics on it + tick One poll of the Events API, then exit (for a cron job); + exits 3 if a listener holds the lease + reconcile [--repair] [--resume] [--max-pages ] [--exclude-streams ] + Compare the mirror against the API; --repair fixes drift. + --max-pages stops starting streams once n pages have been + fetched (a stream that starts runs to its last page, since + a diff needs the whole listing, unless it alone needs more + than n: it is then abandoned and reported oversized until + a run with a larger budget or none); run again with + --resume until every stream reports complete. + --exclude-streams leaves streams out, for one that has to + run somewhere without a time limit + refresh [--resume] [--max-pages ] + Re-list top-level streams, replacing each row; + --max-pages and --resume slice it like backfill, with + which it shares checkpoints + status Cursor position, poll age, and per-stream freshness + redact --user Delete one subject from the mirror and event archive + rejected Show events quarantined by the tail; exits 2 if there are any + --replay Re-apply them all (or one with --id ) once the cause is fixed + streams Show the streams this build can mirror + +Flags + --database-url ClickHouse HTTP URL, https://user:password@host:8443 (or DATABASE_URL) + --schema Target database (or WORKOS_SYNC_SCHEMA; default: workos) + --streams Restrict to specific streams (events is always included) + +Environment + DATABASE_URL ClickHouse HTTP URL + WORKOS_SYNC_CLICKHOUSE_TLS_CA CA certificate the database's certificate must chain to, as PEM + or base64 of it; TLS is then required and verified against it + (default: the system trust store) + WORKOS_API_KEY API key for the environment to mirror + WORKOS_ENVIRONMENT_NAME Label written to _env on every row (default: production) + WORKOS_API_URL API base URL (default: https://api.workos.com) + WORKOS_SYNC_SCHEMA Target database (default: workos) + WORKOS_SYNC_ID Sync identifier when several syncs share a database (default: default) + WORKOS_SYNC_HYDRATE off | on_create | always, or users=always,organizations=off + (default: off) + WORKOS_SYNC_POLL_MS Idle wait between polls (default: 5000) + WORKOS_SYNC_LAG_MS Hold back events younger than this (default: 5000) + WORKOS_SYNC_MAX_EVENTS_PER_TICK Events to apply per poll (default: one page) + WORKOS_SYNC_LEASE_TTL_MS How long the tail lease outlives a crashed listener, at least 1 (default: 60000) + WORKOS_SYNC_HEALTH_PORT listen only: serve /healthz and /metrics on this port (default: off) + WORKOS_SYNC_HEALTH_HOST Interface the health server binds (default: 0.0.0.0) + WORKOS_SYNC_HEALTH_MAX_POLL_AGE_MS + /healthz returns 503 once any environment's last successful + poll is older than this (default: 60000) + WORKOS_SYNC_OTEL 1 starts the OpenTelemetry Node SDK for the command, configured by + the OTEL_* variables, and flushes it on exit (default: off) + LOG_LEVEL debug | info | warn | error | silent +` + +export interface MainDeps { + /** A pre-built client in place of `DATABASE_URL`, so tests can run the CLI against a test server. */ + db?: ChClient + /** A whole engine in place of one built from the environment, for tests of commands that need faults injected. */ + engine?: SyncEngine +} + +export async function main(argv = process.argv.slice(2), deps: MainDeps = {}): Promise { + const { command, flags } = parseArgs(argv) + + if (command === 'help' || flags['help']) { + process.stdout.write(HELP) + return 0 + } + + if (command === 'streams') { + const defaults = new Set(defaultStreams()) + const rows = STREAMS.map((s) => ({ + stream: s.name, + group: s.group, + mode: s.mode, + default: defaults.has(s.name) ? 'yes' : 'no (opt-in)', + backfill: s.list ? 'yes' : s.name === 'organization_domains' ? 'from organizations' : 'no', + events: (s.events?.upsert?.length ?? 0) + (s.events?.delete?.length ?? 0), + })) + process.stdout.write(JSON.stringify(rows, null, 2) + '\n') + return 0 + } + + // The DDL alone, before anything that would need a database or a key. + if (command === 'init' && flags['print-sql']) { + process.stdout.write(printSchemaSql(streamsFromFlags(flags), schemaFromFlags(flags))) + return 0 + } + + // The SDK, when asked for, goes up before the engine is built, so that the + // instruments the engine makes at construction are real ones, and comes + // down after the command returns, which is what exports a one-shot + // command's metrics at all. + const otel = await startOtel() + try { + return await runCommand(command, flags, deps) + } finally { + await otel?.shutdown() + } +} + +async function runCommand(command: string, flags: Record, deps: MainDeps): Promise { + // With an injected engine there is no destination of ours to read the + // drift report from, and nothing to build from the environment. + const destination = deps.engine ? undefined : destinationFromEnv(flags, repairRequested(command, flags), deps.db) + const config = destination ? configFromEnv(flags, destination) : undefined + const engine = deps.engine ?? new SyncEngine(config!) + const logger = config?.logger ?? createLogger((process.env['LOG_LEVEL'] as LogLevel) ?? 'info') + try { + switch (command) { + case 'init': { + await engine.init() + const drift = destination?.lastMigration?.drift ?? [] + const repaired = destination?.lastMigration?.repaired ?? [] + if (drift.length > 0 || repaired.length > 0) { + process.stdout.write(JSON.stringify({ repaired, drift }, null, 2) + '\n') + } + // Orphans are advisory. Anything else means a projection is wrong and + // the fingerprint was withheld, so exit 2 the way `reconcile` does. + const unresolved = drift.some((d) => d.kind !== 'orphan') + if (!unresolved) process.stdout.write('schema ready\n') + return unresolved ? 2 : 0 + } + case 'backfill': { + const results = await engine.backfill({ + resume: Boolean(flags['resume']), + maxPages: parseMaxPages(flags['max-pages']), + }) + process.stdout.write(JSON.stringify(results, null, 2) + '\n') + // A budget that stops short is the outcome asked for, not an error: + // the JSON says which streams are complete, and the next run resumes. + return 0 + } + case 'listen': { + // Up before the engine connects, so a probe during a slow schema init + // sees "starting" rather than a refused connection. + const healthOptions = healthOptionsFromEnv(flags) + const health = healthOptions ? await startHealthServer(engine, healthOptions) : null + if (healthOptions && health) { + logger.info('health server listening', { + host: healthOptions.host, + port: health.port, + maxPollAgeMs: healthOptions.maxPollAgeMs, + }) + } + const shutdown = () => { + engine.stop() + } + process.on('SIGINT', shutdown) + process.on('SIGTERM', shutdown) + try { + await engine.listen() + } finally { + await health?.close() + } + return 0 + } + case 'tick': { + try { + const r = await engine.tick() + process.stdout.write(JSON.stringify(r, null, 2) + '\n') + return 0 + } catch (err) { + // Distinct from a failure (1) and from drift (2): the sync is fine, + // something else is tailing it. Usually a `listen` left running. + if (err instanceof LeaseHeldError) { + process.stderr.write(`${err.message}; nothing to do\n`) + return 3 + } + throw err + } + } + case 'reconcile': { + const r = await engine.reconcile({ + repair: Boolean(flags['repair']), + resume: Boolean(flags['resume']), + maxPages: parseMaxPages(flags['max-pages']), + excludeStreams: parseExcludeStreams(flags['exclude-streams']), + }) + process.stdout.write(JSON.stringify(r, null, 2) + '\n') + // A non-zero exit makes this usable as a scheduled check. + const drift = Object.values(r.diffs) + .flat() + .some((d) => d.localOnly.length > 0 || d.remoteOnly.length > 0) + return drift && !flags['repair'] ? 2 : 0 + } + case 'refresh': { + const r = await engine.refresh({ + resume: Boolean(flags['resume']), + maxPages: parseMaxPages(flags['max-pages']), + }) + process.stdout.write(JSON.stringify(r, null, 2) + '\n') + return 0 + } + case 'status': { + const r = await engine.status() + process.stdout.write(JSON.stringify(r, null, 2) + '\n') + return 0 + } + case 'redact': { + const user = flags['user'] + if (typeof user !== 'string') throw new Error('redact requires --user ') + const r = await engine.redact(user) + process.stdout.write(JSON.stringify(r, null, 2) + '\n') + return 0 + } + case 'rejected': { + const id = flags['id'] + if (id !== undefined && typeof id !== 'string') throw new Error('--id requires an event id') + const only = typeof id === 'string' ? { ids: [id] } : {} + if (flags['replay']) { + const r = await engine.replayRejected(only) + process.stdout.write(JSON.stringify(r, null, 2) + '\n') + // Anything still rejected after a replay is still a problem. + return r.stillRejected.length > 0 ? 2 : 0 + } + const r = await engine.rejected(only) + process.stdout.write(JSON.stringify(r, null, 2) + '\n') + // Like `reconcile`: a non-zero exit makes this a scheduled check. + const any = Object.values(r).some((rows) => rows.length > 0) + return any ? 2 : 0 + } + default: + process.stderr.write(`unknown command: ${command}\n\n${HELP}`) + return 1 + } + } finally { + await engine.close() + } +} + +// Run when invoked directly, not when imported by a test. +const invokedDirectly = process.argv[1] && (process.argv[1].endsWith('cli.js') || process.argv[1].endsWith('cli.ts')) +if (invokedDirectly) { + main().then( + (code) => process.exit(code), + (err) => { + process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`) + process.exit(1) + }, + ) +} diff --git a/packages/sync-clickhouse/src/db/client.ts b/packages/sync-clickhouse/src/db/client.ts new file mode 100644 index 0000000..47ca3c5 --- /dev/null +++ b/packages/sync-clickhouse/src/db/client.ts @@ -0,0 +1,144 @@ +/** + * A minimal ClickHouse interface over the official HTTP client, so the rest + * of the package reads like its SQL siblings: `query`, `command`, `insert`, + * `close`. Values travel as query parameters (`{name:Type}`), never spliced + * into SQL; rows travel as JSONEachRow. + * + * Every connection carries a few settings the engine's arithmetic depends on: + * + * - `async_insert = 1` with `wait_for_async_insert = 1`. The tail writes a + * small batch per stream every few seconds, and a MergeTree part per batch + * would outrun the merges on a busy mirror. Async inserts let the server + * batch them, and waiting for the flush keeps the engine's promise that a + * write is durable when the call returns. KeeperMap statements turn it off + * again: a batched insert collapses duplicate keys silently where a strict + * one must refuse them. + * - `mutations_sync = 2`. Lightweight deletes and KeeperMap updates return + * once every replica has them, so a redaction or a cursor is visible to + * the next query. + * - `select_sequential_consistency = 1`. On a replicated deployment a read + * may otherwise land on a replica the last insert has not reached, and the + * write path reads before it writes. + * - `date_time_input_format = 'best_effort'`. Timestamps are still sent in + * the plain `YYYY-MM-DD hh:mm:ss.fff` form, so this only widens what the + * server accepts; it never changes how a value is read. + */ +import type { Logger } from '@workos/sync-engine' +import { createClient, type ClickHouseClient, type ClickHouseSettings } from '@clickhouse/client' + +/** + * How the engine's connections introduce themselves: the `http_user_agent` + * the server records in `system.query_log`, the ClickHouse counterpart of + * Postgres's `application_name`. Kept a constant rather than carrying a + * version so that filters written against it survive a release. + */ +export const APPLICATION_NAME = 'workos-sync-engine' + +/** Settings every statement runs with unless it says otherwise. */ +export const DEFAULT_SETTINGS: ClickHouseSettings = { + async_insert: 1, + wait_for_async_insert: 1, + // 64-bit settings travel as strings in the client's types. + mutations_sync: '2', + select_sequential_consistency: '1', + date_time_input_format: 'best_effort', + keeper_map_strict_mode: 1, +} + +/** What a KeeperMap statement adds: strict mode is already on, async inserts must be off. */ +export const KEEPER_SETTINGS: ClickHouseSettings = { async_insert: 0 } + +export type Params = Record + +export interface ChClient { + /** The rows of a `SELECT`, parsed from JSONEachRow. */ + query>(sql: string, params?: Params, settings?: ClickHouseSettings): Promise + /** A statement with no result: DDL, `ALTER`, `DELETE`. */ + command(sql: string, params?: Params, settings?: ClickHouseSettings): Promise + /** Rows into `table` (fully qualified) as JSONEachRow. */ + insert(table: string, rows: Record[], settings?: ClickHouseSettings): Promise + close(): Promise +} + +/** Wrap a pre-built `@clickhouse/client` instance. */ +export function fromClient(client: ClickHouseClient): ChClient { + return { + async query(sql: string, params?: Params, settings?: ClickHouseSettings) { + const rs = await client.query({ + query: sql, + query_params: params, + format: 'JSONEachRow', + clickhouse_settings: settings, + }) + return rs.json() + }, + async command(sql: string, params?: Params, settings?: ClickHouseSettings) { + await client.command({ query: sql, query_params: params, clickhouse_settings: settings }) + }, + async insert(table: string, rows: Record[], settings?: ClickHouseSettings) { + if (rows.length === 0) return + await client.insert({ table, values: rows, format: 'JSONEachRow', clickhouse_settings: settings }) + }, + async close() { + await client.close() + }, + } +} + +export interface ConnectOptions { + /** + * PEM-encoded CA certificate(s) the server's certificate must chain to. + * TLS is then required and the certificate verified against this CA and + * the host name. + */ + sslCa?: string + /** Reserved for connection notices; the HTTP client has no idle sockets to lose. */ + logger?: Logger +} + +/** + * What `createClient(...)` is given for a connection string. The URL's user + * info becomes the credentials and its path is dropped: every statement the + * engine runs qualifies its tables with the configured schema, and a path + * would name a default database that `init` may not have created yet. + */ +export function clientConfig(databaseUrl: string, options: ConnectOptions = {}): Record { + const url = new URL(databaseUrl) + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + throw new Error(`unsupported ClickHouse URL scheme ${url.protocol}: use http:// or https:// (the HTTP interface)`) + } + const config: Record = { + url: `${url.protocol}//${url.host}`, + application: APPLICATION_NAME, + clickhouse_settings: DEFAULT_SETTINGS, + request_timeout: 120_000, + } + if (url.username) config['username'] = decodeURIComponent(url.username) + if (url.password) config['password'] = decodeURIComponent(url.password) + if (options.sslCa !== undefined) { + if (url.protocol !== 'https:') { + throw new Error('a CA certificate was given but the connection string is http://, which has no TLS: use https://') + } + config['tls'] = { ca_cert: Buffer.from(options.sslCa) } + } + return config +} + +export async function connect(databaseUrl: string, options: ConnectOptions = {}): Promise { + return fromClient(createClient(clientConfig(databaseUrl, options) as never)) +} + +/** `YYYY-MM-DD hh:mm:ss.fff` in UTC, the form every DateTime64(3) value is sent and received in. */ +export function toCh(d: Date): string { + return d.toISOString().replace('T', ' ').replace('Z', '') +} + +/** A DateTime64 as the server returns it in JSONEachRow, read as UTC. */ +export function fromCh(value: string): Date { + return new Date(value.replace(' ', 'T') + 'Z') +} + +/** ISO-8601 with millisecond precision, from a value the server returned. */ +export function isoOf(value: string): string { + return fromCh(value).toISOString() +} diff --git a/packages/sync-clickhouse/src/db/errors.ts b/packages/sync-clickhouse/src/db/errors.ts new file mode 100644 index 0000000..8db1284 --- /dev/null +++ b/packages/sync-clickhouse/src/db/errors.ts @@ -0,0 +1,72 @@ +/** + * Classifying ClickHouse errors for the quarantine. + * + * The engine only ever sets an event aside for an error that will not go + * away on retry. The server reports every error with a numeric code and a + * symbolic type (`999 KEEPER_EXCEPTION`), which the client surfaces as + * `code` (a string) and `type`. The list below names the codes that + * describe the server's state rather than the row's: timeouts, lost + * connections, resource pressure, a replica mid-failover, a Keeper + * transaction that lost a race. Not in it, on purpose: a value the server + * cannot parse, a missing table or column, a syntax error, a Keeper node + * that already exists. Those are verdicts on the statement and come back + * identically however many times it is retried. + */ + +/** Server error codes about timing, connections or resources. */ +const TRANSIENT_CODES = new Set([ + '159', // TIMEOUT_EXCEEDED + '160', // TOO_SLOW + '202', // TOO_MANY_SIMULTANEOUS_QUERIES + '203', // NO_FREE_CONNECTION + '209', // SOCKET_TIMEOUT + '210', // NETWORK_ERROR + '236', // ABORTED + '241', // MEMORY_LIMIT_EXCEEDED + '242', // TABLE_IS_READ_ONLY + '243', // NOT_ENOUGH_SPACE + '252', // TOO_MANY_PARTS + '290', // LIMIT_EXCEEDED + '319', // UNKNOWN_STATUS_OF_INSERT + '394', // QUERY_WAS_CANCELLED + '999', // KEEPER_EXCEPTION: a lost race or a lost session, both retried + '1002', // UNKNOWN_EXCEPTION: the server could not say, which is not a verdict on the row +]) + +/** Node socket errors the client surfaces as-is, with the errno name in `code`. */ +const TRANSIENT_SOCKET_CODES = new Set([ + 'ECONNRESET', + 'ECONNREFUSED', + 'ETIMEDOUT', + 'EPIPE', + 'EHOSTUNREACH', + 'ENETUNREACH', + 'EAI_AGAIN', +]) + +/** Message fragments of client-side failures that carry no code. */ +const TRANSIENT_MESSAGES = ['socket hang up', 'Timeout error', 'ECONNRESET', 'fetch failed', 'network error'] + +/** + * Whether `err` is a ClickHouse or socket error that a retry could fix. + * Anything without a recognisable code is reported as deterministic, which + * is the conservative answer: the engine's two-sightings rule covers the gap. + */ +export function isTransientClickHouseError(err: unknown): boolean { + if (typeof err !== 'object' || err === null) return false + const { code, message } = err as { code?: unknown; message?: unknown } + if (typeof code === 'string' && (TRANSIENT_CODES.has(code) || TRANSIENT_SOCKET_CODES.has(code))) return true + if (typeof message === 'string' && TRANSIENT_MESSAGES.some((m) => message.includes(m))) return true + return false +} + +/** Whether `err` is the server's refusal of a KeeperMap write that lost a race or found its key taken. */ +export function isKeeperConflict(err: unknown): boolean { + if (typeof err !== 'object' || err === null) return false + const { code, message } = err as { code?: unknown; message?: unknown } + return ( + code === '999' && + typeof message === 'string' && + (message.includes('Node exists') || message.includes('Bad version')) + ) +} diff --git a/packages/sync-clickhouse/src/db/lease.ts b/packages/sync-clickhouse/src/db/lease.ts new file mode 100644 index 0000000..ad963d1 --- /dev/null +++ b/packages/sync-clickhouse/src/db/lease.ts @@ -0,0 +1,272 @@ +/** + * The single-writer lease and the cursors, as one KeeperMap row. + * + * KeeperMap is a key-value table held in ClickHouse Keeper: linearizable + * writes, and under `keeper_map_strict_mode` an `INSERT` of an existing key + * is refused and an `ALTER ... UPDATE` only lands if the row did not change + * between its read and its write. That is a compare-and-swap, and it is the + * only one ClickHouse offers; a MergeTree row has no such thing. So the + * lease and every cursor for an environment share one row in `_sync`, keyed + * by `(sync_id, env)`: + * + * k = sync_id ‹SEP› env + * holder, acquired_at, expires_at the lease; holder '' when free + * states = {"__tail__": "", "users": "...", ...} + * + * Taking the lease is an `UPDATE` whose `WHERE` names the row as it must be + * for the takeover to be allowed — ours, free, or expired by the server's + * clock (`now64`, never the client's) — so either it lands or nothing + * happens. A checkpoint is an `UPDATE` whose `WHERE` names the holder, the + * unexpired lease and the whole `states` value as it was read: if anyone + * moved a cursor or took the lease in between, the value differs and the + * write is refused by the database. Each write is followed by a read, since + * a mutation reports no row count; strict mode's own race verdict ("Bad + * version") is retried, because the re-read then judges the new state. + * + * Cursor state is stored as canonical JSON — keys sorted, recursively — so + * that equal cursors compare equal as text, as they do as jsonb. + */ +import type { Lease, LeaseResult } from '@workos/sync-engine' +import { isoOf, KEEPER_SETTINGS, type ChClient } from './client.js' +import { isKeeperConflict } from './errors.js' +import { quoteIdent } from '../schema/ddl.js' + +const SEP = '\u0001' +/** + * How often a read-then-swap is retried when the row changed in between. + * Contention is rare in use — one tail per environment, one backfill at a + * time — but each retry is cheap, and a lost race must never be reported as + * a failure to a caller who did nothing wrong. + */ +const ATTEMPTS = 16 + +/** A short, jittered pause between attempts, so contenders do not retry in lockstep. */ +const backoff = () => new Promise((r) => setTimeout(r, 5 + Math.random() * 20)) + +export interface SyncRow { + holder: string + acquired_at: string + expires_at: string + states: string + /** The server's clock at the moment of the read, for judging expiry. */ + now: string +} + +/** `value` as JSON with every object's keys sorted, so equal values serialise equally. */ +export function canonicalJson(value: unknown): string { + return JSON.stringify(sortKeys(value)) +} + +function sortKeys(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortKeys) + if (value !== null && typeof value === 'object') { + const out: Record = {} + for (const key of Object.keys(value as Record).sort()) { + const v = (value as Record)[key] + if (v !== undefined) out[key] = sortKeys(v) + } + return out + } + return value +} + +export const syncKey = (syncId: string, env: string) => `${syncId}${SEP}${env}` + +const table = (schema: string) => `${quoteIdent(schema)}.\`_sync\`` + +export async function readSync(db: ChClient, schema: string, syncId: string, env: string): Promise { + const rows = await db.query( + `SELECT holder, acquired_at, expires_at, states, now64(3, 'UTC') AS now FROM ${table(schema)} WHERE k = {k:String}`, + { k: syncKey(syncId, env) }, + ) + return rows[0] ?? null +} + +/** Insert the row for `(syncId, env)` if it does not exist. False when it already did. */ +async function insertSync(db: ChClient, schema: string, syncId: string, env: string, states: string): Promise { + try { + await db.command( + `INSERT INTO ${table(schema)} (k, holder, acquired_at, expires_at, states) + VALUES ({k:String}, '', toDateTime64(0, 3, 'UTC'), toDateTime64(0, 3, 'UTC'), {states:String})`, + { k: syncKey(syncId, env), states }, + KEEPER_SETTINGS, + ) + return true + } catch (err) { + if (isKeeperConflict(err)) return false + throw err + } +} + +const parseStates = (row: SyncRow | null): Record => + row ? (JSON.parse(row.states) as Record) : {} + +const isHeld = (row: SyncRow | null, holder: string) => + row !== null && row.holder === holder && row.expires_at > row.now + +export async function acquireLease( + db: ChClient, + schema: string, + syncId: string, + env: string, + holder: string, + ttlMs: number, +): Promise { + const k = syncKey(syncId, env) + const ttl = Math.max(0, Math.floor(ttlMs)) + for (let attempt = 0; attempt < ATTEMPTS; attempt++) { + const row = await readSync(db, schema, syncId, env) + if (!row) { + await insertSync(db, schema, syncId, env, '{}') + continue + } + if (attempt > 0) await backoff() + try { + // Free when it is ours, unheld, or expired by the server's clock; + // anything else leaves the row exactly as it was. + await db.command( + `ALTER TABLE ${table(schema)} UPDATE + acquired_at = if(holder = {holder:String}, acquired_at, now64(3, 'UTC')), + expires_at = now64(3, 'UTC') + toIntervalMillisecond({ttl:UInt64}), + holder = {holder:String} + WHERE k = {k:String} AND (holder = {holder:String} OR holder = '' OR expires_at <= now64(3, 'UTC'))`, + { k, holder, ttl }, + KEEPER_SETTINGS, + ) + } catch (err) { + // Somebody else wrote the row between the read and the write: look again. + if (!isKeeperConflict(err)) throw err + continue + } + const after = await readSync(db, schema, syncId, env) + if (after && after.holder === holder) return { acquired: true, holder, expiresAt: isoOf(after.expires_at) } + const current = after && after.holder !== '' && after.expires_at > after.now ? after : null + return { acquired: false, holder: current?.holder ?? null, expiresAt: current ? isoOf(current.expires_at) : null } + } + throw new Error(`could not take the lease for ${env}: the row kept changing under ${ATTEMPTS} attempts`) +} + +export async function releaseLease( + db: ChClient, + schema: string, + syncId: string, + env: string, + holder: string, +): Promise { + // Clearing the holder rather than expiring it keeps `status` honest: a + // released lease reads as free, not as a stale holder with a past date. + // The cursors share the row, so the row itself stays. + try { + await db.command( + `ALTER TABLE ${table(schema)} UPDATE holder = '', expires_at = toDateTime64(0, 3, 'UTC') + WHERE k = {k:String} AND holder = {holder:String}`, + { k: syncKey(syncId, env), holder }, + KEEPER_SETTINGS, + ) + } catch (err) { + // A race with a takeover: the lease is no longer ours to release. + if (!isKeeperConflict(err)) throw err + } +} + +export async function getLease(db: ChClient, schema: string, syncId: string, env: string): Promise { + const row = await readSync(db, schema, syncId, env) + if (!row || row.holder === '' || row.expires_at <= row.now) return null + return { holder: row.holder, expiresAt: isoOf(row.expires_at) } +} + +export async function getState( + db: ChClient, + schema: string, + syncId: string, + env: string, + stream: string, +): Promise { + const stored = parseStates(await readSync(db, schema, syncId, env))[stream] + return stored === undefined ? null : (JSON.parse(stored) as T) +} + +/** Replace `states` with `next` only if it still reads `previous`. Whether it did. */ +async function swapStates( + db: ChClient, + schema: string, + syncId: string, + env: string, + previous: string, + next: string, + guard = '', + params: Record = {}, +): Promise { + try { + await db.command( + `ALTER TABLE ${table(schema)} UPDATE states = {next:String} + WHERE k = {k:String} AND states = {previous:String}${guard}`, + { ...params, k: syncKey(syncId, env), previous, next }, + KEEPER_SETTINGS, + ) + } catch (err) { + if (!isKeeperConflict(err)) throw err + return false + } + const after = await readSync(db, schema, syncId, env) + return after !== null && after.states === next +} + +export async function setState( + db: ChClient, + schema: string, + syncId: string, + env: string, + stream: string, + state: unknown, +): Promise { + const value = canonicalJson(state) + for (let attempt = 0; attempt < ATTEMPTS; attempt++) { + const row = await readSync(db, schema, syncId, env) + if (!row) { + if (await insertSync(db, schema, syncId, env, JSON.stringify({ [stream]: value }))) return + continue + } + const states = parseStates(row) + if (states[stream] === value) return + const next = JSON.stringify({ ...states, [stream]: value }) + if (await swapStates(db, schema, syncId, env, row.states, next)) return + await backoff() + } + throw new Error(`could not save the ${stream} state for ${env}: the row kept changing under ${ATTEMPTS} attempts`) +} + +/** + * One read, one conditional write: the holder, the unexpired lease and the + * cursor as it was expected to be are all judged on the row as read, and + * the write names that exact row, so it lands only if nothing changed. + */ +export async function setStateIfHeld( + db: ChClient, + schema: string, + syncId: string, + env: string, + stream: string, + state: unknown, + holder: string, + expected: unknown, +): Promise { + const row = await readSync(db, schema, syncId, env) + if (!isHeld(row, holder)) return false + const states = parseStates(row) + const stored = states[stream] + if (expected == null ? stored !== undefined : stored !== canonicalJson(expected)) return false + const next = JSON.stringify({ ...states, [stream]: canonicalJson(state) }) + // Expiry is judged again by the server, in the write itself: the read + // above may have waited, and a lease that ran out meanwhile must refuse. + return swapStates( + db, + schema, + syncId, + env, + row!.states, + next, + ` AND holder = {holder:String} AND expires_at > now64(3, 'UTC')`, + { holder }, + ) +} diff --git a/packages/sync-clickhouse/src/db/reconcile.ts b/packages/sync-clickhouse/src/db/reconcile.ts new file mode 100644 index 0000000..df7c9df --- /dev/null +++ b/packages/sync-clickhouse/src/db/reconcile.ts @@ -0,0 +1,99 @@ +/** + * The reconcile set difference, computed in ClickHouse. + * + * The live keys are copied into one staging table by the server, in a + * single `INSERT ... SELECT`, before the first page is requested; the + * remote pages are written into another as they arrive; then three queries + * produce the answer: a count for `matched`, an anti-join from the local + * side for `localOnly`, an anti-join from the remote side for `remoteOnly`. + * Node holds one page of remote rows at a time, plus the drift; the + * Postgres destination explains why that bound matters. + * + * The staging tables are ordinary `MergeTree` tables with a random suffix, + * not temporary ones: a temporary table lives in one HTTP session on one + * replica, and on a replicated deployment the next request may land + * elsewhere. They are dropped when the diff ends, however it ends. + */ +import { encodeKey, type IdDiff, type TableDef } from '@workos/sync-engine' +import { randomBytes } from 'node:crypto' +import type { ChClient } from './client.js' +import { chunk, keyExpression, keyValues, tableRef, KEY_SEPARATOR, type WriteOptions } from './upsert.js' +import { quoteIdent, ENV } from '../schema/ddl.js' + +export interface ChDiff extends IdDiff { + /** Remote rows with no value at the key path. Counted rather than thrown. */ + skipped: number +} + +const DEFAULT_BATCH_SIZE = 500 + +export async function diffIdsInClickHouse( + db: ChClient, + opts: WriteOptions, + stream: TableDef, + env: string, + remote: AsyncIterable[]>, +): Promise { + const token = randomBytes(6).toString('hex') + const s = quoteIdent(opts.schema) + const LOCAL = `${s}.\`_reconcile_local_${token}\`` + const STAGING = `${s}.\`_reconcile_remote_${token}\`` + const decode = (k: string) => encodeKey(stream.key.length === 1 ? [k] : k.split(KEY_SEPARATOR)) + + const drop = async () => { + await db.command(`DROP TABLE IF EXISTS ${LOCAL}`) + await db.command(`DROP TABLE IF EXISTS ${STAGING}`) + } + try { + await db.command(`CREATE TABLE ${LOCAL} (k String) ENGINE = MergeTree ORDER BY k`) + await db.command(`CREATE TABLE ${STAGING} (k String, seq UInt64, raw String) ENGINE = MergeTree ORDER BY (k, seq)`) + + // The live keys are frozen before the first page is requested: a row the + // tail inserts after the listing has passed that object's position is + // simply not a candidate, so `--repair` can never tombstone an object + // that exists upstream. One statement, one snapshot. + const live = stream.mode === 'append' ? '' : ` AND \`_deleted_at\` IS NULL` + const final = stream.mode === 'append' ? '' : ' FINAL' + await db.command( + `INSERT INTO ${LOCAL} (k) SELECT ${keyExpression(stream)} FROM ${tableRef(opts.schema, stream)}${final} + WHERE ${quoteIdent(ENV)} = {env:String}${live}`, + { env }, + ) + + let skipped = 0 + let seq = 0 + for await (const page of remote) { + const keyed: { k: string; seq: number; raw: string }[] = [] + for (const data of page) { + const id = keyValues(stream, data) + if (!id) { + skipped++ + continue + } + keyed.push({ k: id.join(KEY_SEPARATOR), seq: seq++, raw: JSON.stringify(data) }) + } + for (const batch of chunk(keyed, opts.batchSize ?? DEFAULT_BATCH_SIZE)) await db.insert(STAGING, batch) + } + + // An id the API returns twice (a page boundary moving under a concurrent + // write) keeps its first copy, by `seq`. Nothing is lost that the next + // reconcile would not find again. + const [matched, localOnly, remoteOnly] = await Promise.all([ + db.query<{ n: string }>( + `SELECT toString(count(DISTINCT k)) AS n FROM ${STAGING} WHERE k IN (SELECT k FROM ${LOCAL})`, + ), + db.query<{ k: string }>(`SELECT k FROM ${LOCAL} WHERE k NOT IN (SELECT k FROM ${STAGING}) ORDER BY k`), + db.query<{ raw: string }>( + `SELECT argMin(raw, seq) AS raw FROM ${STAGING} WHERE k NOT IN (SELECT k FROM ${LOCAL}) GROUP BY k ORDER BY k`, + ), + ]) + return { + matched: Number(matched[0]?.n ?? 0), + localOnly: localOnly.map((r) => decode(r.k)), + remoteOnly: remoteOnly.map((r) => JSON.parse(r.raw) as Record), + skipped, + } + } finally { + await drop() + } +} diff --git a/packages/sync-clickhouse/src/db/rejected.ts b/packages/sync-clickhouse/src/db/rejected.ts new file mode 100644 index 0000000..258944b --- /dev/null +++ b/packages/sync-clickhouse/src/db/rejected.ts @@ -0,0 +1,116 @@ +/** + * The quarantine store: `_rejected`, one row per event the tail set aside. + * + * Keyed by `(sync_id, env, event_id)` like the rest of the bookkeeping, so + * several syncs sharing a database never see each other's rejects. The whole + * event is stored verbatim in `_raw`, which is what makes a replay possible + * and is also why `redact` has to reach in here as well as into `events`. + * + * The table is a `ReplacingMergeTree` versioned by `rejected_at`, so a + * rejection written again is a new version that wins, and reads use + * `FINAL`. Forgetting an event is a lightweight `DELETE`, which the + * client's `mutations_sync` makes visible before the call returns. + */ +import type { RejectedEvent, WorkOSEvent } from '@workos/sync-engine' +import { isoOf, toCh, type ChClient } from './client.js' +import { quoteIdent } from '../schema/ddl.js' + +export interface RejectedOptions { + schema: string + syncId: string +} + +const table = (schema: string) => `${quoteIdent(schema)}.\`_rejected\`` + +/** The event's `created_at` as the server stores it, or null: a quarantine write must never fail on the data it is quarantining. */ +function createdAt(value: unknown): string | null { + if (typeof value !== 'string') return null + const d = new Date(value) + return Number.isNaN(d.getTime()) ? null : toCh(d) +} + +export async function rejectEvents( + db: ChClient, + opts: RejectedOptions, + env: string, + rows: { event: WorkOSEvent; error: string }[], +): Promise { + if (rows.length === 0) return + // A replay that fails again lands here a second time, as a newer version: + // `rejected` then reports what is wrong now, not what was. + await db.insert( + table(opts.schema), + rows.map(({ event, error }) => { + const objectId = (event.data as Record | undefined)?.['id'] + return { + sync_id: opts.syncId, + env, + event_id: event.id, + event_type: event.event, + object_id: typeof objectId === 'string' ? objectId : null, + created_at: createdAt(event.created_at), + error, + _raw: JSON.stringify(event), + } + }), + ) +} + +export async function listRejected( + db: ChClient, + opts: RejectedOptions, + env: string, + ids?: string[], +): Promise { + if (ids && ids.length === 0) return [] + const rows = await db.query<{ + event_id: string + event_type: string + object_id: string | null + rejected_at: string + error: string + raw: string + }>( + `SELECT event_id, event_type, object_id, rejected_at, error, \`_raw\` AS raw + FROM ${table(opts.schema)} FINAL + WHERE sync_id = {syncId:String} AND env = {env:String}${ids ? ' AND event_id IN ({ids:Array(String)})' : ''} + ORDER BY event_id`, + { syncId: opts.syncId, env, ids: ids ?? [] }, + ) + return rows.map((row) => { + const raw = JSON.parse(row.raw) as WorkOSEvent + return { + eventId: row.event_id, + eventType: row.event_type, + objectId: row.object_id, + // The event's own timestamp, as it arrived; the column is only for SQL. + createdAt: raw.created_at, + rejectedAt: isoOf(row.rejected_at), + error: row.error, + raw, + } + }) +} + +/** How many events are quarantined, without reading their payloads. */ +export async function countRejected(db: ChClient, opts: RejectedOptions, env: string): Promise { + const rows = await db.query<{ n: string }>( + `SELECT toString(count()) AS n FROM ${table(opts.schema)} FINAL WHERE sync_id = {syncId:String} AND env = {env:String}`, + { syncId: opts.syncId, env }, + ) + return Number(rows[0]?.n ?? 0) +} + +export async function clearRejected(db: ChClient, opts: RejectedOptions, env: string, ids: string[]): Promise { + if (ids.length === 0) return 0 + const params = { syncId: opts.syncId, env, ids } + const where = `sync_id = {syncId:String} AND env = {env:String} AND event_id IN ({ids:Array(String)})` + // A delete reports no count, so count first; the quarantine has one writer. + const rows = await db.query<{ n: string }>( + `SELECT toString(count(DISTINCT event_id)) AS n FROM ${table(opts.schema)} FINAL WHERE ${where}`, + params, + ) + const n = Number(rows[0]?.n ?? 0) + if (n > 0) await db.command(`DELETE FROM ${table(opts.schema)} WHERE ${where}`, params) + return n +} diff --git a/packages/sync-clickhouse/src/db/upsert.ts b/packages/sync-clickhouse/src/db/upsert.ts new file mode 100644 index 0000000..3c77de2 --- /dev/null +++ b/packages/sync-clickhouse/src/db/upsert.ts @@ -0,0 +1,445 @@ +/** + * Write path. + * + * The rules are the Postgres destination's, and for the same reasons (see + * `db/upsert.ts` there): the staleness gate is `>=` and never `>`, event + * writes merge where REST writes replace, deletion is terminal except for + * memberships, which relink and unlink only when the write is at least as + * new as the row. + * + * ClickHouse has no row update, so a write is an insert of the row's next + * state into a `ReplacingMergeTree` versioned by `_updated_at`. Each batch + * reads the current state of its rows (`FINAL`), decides every row in Node + * — the gate, the shallow merge, whether anything changed, which tombstone + * to carry forward — and inserts only the rows that passed. The table then + * keeps, for each key, the version with the newest `_updated_at`, and the + * last inserted among equals; so even two writers racing resolve the way + * the gate would have, and a refused row, never inserted, changes nothing. + * Append-only streams are plain `MergeTree`: the batch reads which ids are + * already there and inserts the rest, which is what makes re-delivery a + * no-op. + */ +import { emptyStats, type Row, type TableDef, type WriteMode, type WriteStats } from '@workos/sync-engine' +import { fromCh, toCh, type ChClient } from './client.js' +import { quoteIdent, safeIdentifier, ENV, RAW } from '../schema/ddl.js' + +/** Joins the components of a composite key, on both sides of every lookup. */ +export const KEY_SEPARATOR = '\u0001' + +export function readPath(data: Record, path: string | string[]): unknown { + const parts = Array.isArray(path) ? path : [path] + let cur: unknown = data + for (const p of parts) { + if (cur == null || typeof cur !== 'object') return undefined + cur = (cur as Record)[p] + } + return cur +} + +/** Key values for a row, in `stream.key` order. Undefined if any part is missing. */ +export function keyValues(stream: TableDef, data: Record): string[] | undefined { + const out: string[] = [] + for (const k of stream.key) { + const col = stream.columns.find((c) => c.name === k) + const v = readPath(data, col ? col.path : k) + if (v == null || v === '') return undefined + out.push(String(v)) + } + return out +} + +/** + * Collapse rows that share a primary key, keeping the newest. It serves the + * delete, relink and unlink paths, whose rows are whole; upserts use + * `wavesByKey`. + */ +export function dedupeByKey(stream: TableDef, rows: Row[]): { rows: Row[]; dropped: number } { + const byKey = new Map() + let dropped = 0 + for (const row of rows) { + const key = keyValues(stream, row.data) + if (!key) { + dropped++ + continue + } + const k = key.join(KEY_SEPARATOR) + const existing = byKey.get(k) + if (!existing || row.updatedAt >= existing.updatedAt) byKey.set(k, row) + } + return { rows: [...byKey.values()], dropped } +} + +/** + * Split a batch into waves in which no key appears twice, keeping order: the + * n-th row for a key goes in wave n. Written one after another, the waves + * are exactly the rows written one at a time — each row meets the staleness + * gate against what the row before it left stored, and a merge overlays it + * on that. See the Postgres destination for why keeping only the newest row + * would be wrong for a merge. + */ +export function wavesByKey(stream: TableDef, rows: Row[]): { waves: Row[][]; dropped: number } { + const waves: Row[][] = [] + const seen = new Map() + let dropped = 0 + for (const row of rows) { + const key = keyValues(stream, row.data) + if (!key) { + dropped++ + continue + } + const k = key.join(KEY_SEPARATOR) + const n = seen.get(k) ?? 0 + seen.set(k, n + 1) + const wave = waves[n] ?? (waves[n] = []) + wave.push(row) + } + return { waves, dropped } +} + +export const chunk = (items: T[], size: number): T[][] => { + const out: T[][] = [] + for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size)) + return out +} + +/** + * `Row.updatedAt` as a `Date`. Postgres casts the string in SQL and a bad one + * fails the statement; here the same verdict is reached before the + * statement, with a code the error classifier reports as deterministic, so + * the event is quarantined rather than retried forever. + */ +export function parseUpdatedAt(value: string): Date { + const d = new Date(value) + if (typeof value !== 'string' || Number.isNaN(d.getTime())) { + throw Object.assign(new Error(`invalid updatedAt timestamp: ${JSON.stringify(value)}`), { + code: 'INVALID_TIMESTAMP', + }) + } + return d +} + +/** Structural equality with key order ignored, which is what `is distinct from` means for jsonb. */ +export function jsonEqual(a: unknown, b: unknown): boolean { + if (a === b) return true + if (typeof a !== typeof b || a === null || b === null || typeof a !== 'object') return false + if (Array.isArray(a) !== Array.isArray(b)) return false + if (Array.isArray(a)) { + const bb = b as unknown[] + return a.length === bb.length && a.every((v, i) => jsonEqual(v, bb[i])) + } + const ao = a as Record + const bo = b as Record + const keys = Object.keys(ao) + if (keys.length !== Object.keys(bo).length) return false + return keys.every((k) => Object.hasOwn(bo, k) && jsonEqual(ao[k], bo[k])) +} + +/** Shallow merge: the jsonb `||` of the Postgres destination. */ +const merge = (stored: Record, incoming: Record): Record => ({ + ...stored, + ...incoming, +}) + +const later = (a: Date | null, b: Date): Date => (a !== null && a.getTime() > b.getTime() ? a : b) + +export interface WriteOptions { + schema: string + batchSize?: number +} + +const DEFAULT_BATCH_SIZE = 500 + +export const tableRef = (schema: string, stream: TableDef) => + `${quoteIdent(schema)}.${quoteIdent(safeIdentifier(stream.name))}` + +/** The whole key as one SQL expression: the column, or the components joined by the separator. */ +export function keyExpression(stream: TableDef): string { + const cols = stream.key.map((k) => quoteIdent(safeIdentifier(k))) + if (cols.length === 1) return cols[0]! + const parts: string[] = [] + for (const [i, col] of cols.entries()) { + if (i > 0) parts.push('char(1)') + parts.push(col) + } + return `concat(${parts.join(', ')})` +} + +interface StoredRow { + raw: Record + updatedAt: Date + deletedAt: Date | null +} + +interface Pending { + key: string[] + raw: Record + updatedAt: Date + deletedAt: Date | null +} + +/** The current state of the rows for `keys`, by joined key; keys with no row are absent. */ +async function readRows( + db: ChClient, + opts: WriteOptions, + stream: TableDef, + env: string, + keys: string[][], +): Promise> { + const rows = await db.query<{ k: string; raw: string; updated_at: string; deleted_at: string | null }>( + `SELECT ${keyExpression(stream)} AS k, ${quoteIdent(RAW)} AS raw, \`_updated_at\` AS updated_at, \`_deleted_at\` AS deleted_at + FROM ${tableRef(opts.schema, stream)} FINAL + WHERE ${quoteIdent(ENV)} = {env:String} AND ${keyExpression(stream)} IN ({keys:Array(String)})`, + { env, keys: keys.map((k) => k.join(KEY_SEPARATOR)) }, + ) + const out = new Map() + for (const row of rows) { + out.set(row.k, { + raw: JSON.parse(row.raw) as Record, + updatedAt: fromCh(row.updated_at), + deletedAt: row.deleted_at ? fromCh(row.deleted_at) : null, + }) + } + return out +} + +/** Which of `keys` already have a row, for append-only tables. */ +async function existingKeys( + db: ChClient, + opts: WriteOptions, + stream: TableDef, + env: string, + keys: string[][], +): Promise> { + const rows = await db.query<{ k: string }>( + `SELECT DISTINCT ${keyExpression(stream)} AS k FROM ${tableRef(opts.schema, stream)} + WHERE ${quoteIdent(ENV)} = {env:String} AND ${keyExpression(stream)} IN ({keys:Array(String)})`, + { env, keys: keys.map((k) => k.join(KEY_SEPARATOR)) }, + ) + return new Set(rows.map((r) => r.k)) +} + +/** Insert the decided rows as their next version. `_synced_at` is the server's clock. */ +async function writeRows(db: ChClient, opts: WriteOptions, stream: TableDef, env: string, rows: Pending[]) { + if (rows.length === 0) return + const hasDeleted = stream.mode !== 'append' + await db.insert( + tableRef(opts.schema, stream), + rows.map((r) => ({ + [ENV]: env, + [RAW]: JSON.stringify(r.raw), + _updated_at: toCh(r.updatedAt), + ...(hasDeleted ? { _deleted_at: r.deletedAt ? toCh(r.deletedAt) : null } : {}), + })), + ) +} + +/** One batch of keyed rows with their parsed timestamps, ready to be decided. */ +interface Keyed { + key: string[] + data: Record + updatedAt: Date +} + +const keyed = (stream: TableDef, rows: Row[]): Keyed[] => + rows.map((r) => ({ key: keyValues(stream, r.data)!, data: r.data, updatedAt: parseUpdatedAt(r.updatedAt) })) + +/** + * Read, decide, write: the shape of every mirror write. `decide` is handed + * each row with what is stored for it and returns what to write, or null to + * leave the row alone. + */ +async function applyBatch( + db: ChClient, + opts: WriteOptions, + stream: TableDef, + env: string, + batch: Keyed[], + decide: (row: Keyed, stored: StoredRow | undefined) => Pending | null, +): Promise { + const stored = await readRows( + db, + opts, + stream, + env, + batch.map((r) => r.key), + ) + const pending: Pending[] = [] + for (const row of batch) { + const out = decide(row, stored.get(row.key.join(KEY_SEPARATOR))) + if (out) pending.push(out) + } + await writeRows(db, opts, stream, env, pending) +} + +export async function upsertRows( + db: ChClient, + opts: WriteOptions, + stream: TableDef, + env: string, + rows: Row[], + mode: WriteMode, +): Promise { + const stats = emptyStats() + if (rows.length === 0) return stats + const { waves, dropped } = wavesByKey(stream, rows) + // Rows we could not key (a payload with no id and no slug) are counted as + // skipped rather than silently vanishing. + stats.skipped += dropped + for (const wave of waves) { + for (const batch of chunk(keyed(stream, wave), opts.batchSize ?? DEFAULT_BATCH_SIZE)) { + if (stream.mode === 'append') { + // Immutable log. Re-delivery is a no-op, which is the whole idempotency story. + const present = await existingKeys( + db, + opts, + stream, + env, + batch.map((r) => r.key), + ) + const fresh = batch.filter((r) => !present.has(r.key.join(KEY_SEPARATOR))) + await writeRows( + db, + opts, + stream, + env, + fresh.map((r) => ({ key: r.key, raw: r.data, updatedAt: r.updatedAt, deletedAt: null })), + ) + stats.created += fresh.length + stats.skipped += batch.length - fresh.length + continue + } + await applyBatch(db, opts, stream, env, batch, (row, stored) => { + if (!stored) { + stats.created++ + return { key: row.key, raw: row.data, updatedAt: row.updatedAt, deletedAt: null } + } + if (row.updatedAt.getTime() < stored.updatedAt.getTime()) { + stats.skipped++ + return null + } + const raw = mode === 'merge' ? merge(stored.raw, row.data) : row.data + // An equal timestamp with nothing new is a re-delivery: no new version, + // so `_synced_at` stays and lag is not measured against a no-op. + if (row.updatedAt.getTime() === stored.updatedAt.getTime() && jsonEqual(stored.raw, raw)) { + stats.skipped++ + return null + } + stats.updated++ + // The tombstone is carried forward: a late update never resurrects a row. + return { key: row.key, raw, updatedAt: row.updatedAt, deletedAt: stored.deletedAt } + }) + } + } + return stats +} + +export async function deleteRows( + db: ChClient, + opts: WriteOptions, + stream: TableDef, + env: string, + rows: Row[], +): Promise { + const stats = emptyStats() + if (rows.length === 0) return stats + const { rows: deduped, dropped } = dedupeByKey(stream, rows) + stats.skipped += dropped + for (const batch of chunk(keyed(stream, deduped), opts.batchSize ?? DEFAULT_BATCH_SIZE)) { + await applyBatch(db, opts, stream, env, batch, (row, stored) => { + stats.deleted++ + if (!stored) return { key: row.key, raw: row.data, updatedAt: row.updatedAt, deletedAt: row.updatedAt } + // The *latest* removal is kept, not the first: a membership removed, + // re-added and removed again must carry the last removal, because that + // is what `relinkRows` compares a re-add against. `_updated_at` stays, + // so the version is a tie and the tombstone, inserted later, wins it. + return { + key: row.key, + raw: merge(stored.raw, row.data), + updatedAt: stored.updatedAt, + deletedAt: later(stored.deletedAt, row.updatedAt), + } + }) + } + return stats +} + +/** Restore a tombstoned membership row (a re-add after a removal). */ +export async function relinkRows( + db: ChClient, + opts: WriteOptions, + stream: TableDef, + env: string, + rows: Row[], +): Promise { + const stats = emptyStats() + if (rows.length === 0) return stats + const { rows: deduped, dropped } = dedupeByKey(stream, rows) + stats.skipped += dropped + for (const batch of chunk(keyed(stream, deduped), opts.batchSize ?? DEFAULT_BATCH_SIZE)) { + await applyBatch(db, opts, stream, env, batch, (row, stored) => { + if (!stored) { + stats.created++ + return { key: row.key, raw: row.data, updatedAt: row.updatedAt, deletedAt: null } + } + // A membership is a fact that can legitimately come back: added, + // removed, added again. Unlike object deletes, a re-link clears the + // tombstone — but only a re-link at least as new as the removal. + // `_updated_at` alone is not enough: a tombstone written by `deleteRows` + // (a reconcile repair) leaves it untouched, so a link replayed from + // quarantine after that removal would pass the staleness gate and + // resurrect the membership. + const t = row.updatedAt.getTime() + if (t < stored.updatedAt.getTime() || (stored.deletedAt !== null && t < stored.deletedAt.getTime())) { + stats.skipped++ + return null + } + stats.updated++ + return { key: row.key, raw: merge(stored.raw, row.data), updatedAt: row.updatedAt, deletedAt: null } + }) + } + return stats +} + +/** + * Tombstone membership rows (a removal). The counterpart of `relinkRows`, + * and staleness-gated like it, which is what sets it apart from `deleteRows`: + * a membership can come back, so a removal is a fact with a time, not a + * terminal state, and one replayed from quarantine after a newer re-add has + * landed must not hide a membership that is active now. + */ +export async function unlinkRows( + db: ChClient, + opts: WriteOptions, + stream: TableDef, + env: string, + rows: Row[], +): Promise { + const stats = emptyStats() + if (rows.length === 0) return stats + const { rows: deduped, dropped } = dedupeByKey(stream, rows) + stats.skipped += dropped + for (const batch of chunk(keyed(stream, deduped), opts.batchSize ?? DEFAULT_BATCH_SIZE)) { + await applyBatch(db, opts, stream, env, batch, (row, stored) => { + if (!stored) { + stats.deleted++ + return { key: row.key, raw: row.data, updatedAt: row.updatedAt, deletedAt: row.updatedAt } + } + if (row.updatedAt.getTime() < stored.updatedAt.getTime()) { + stats.skipped++ + return null + } + // `_updated_at` moves to the removal time, so the next re-add is gated + // against the removal by the same comparison every membership write + // uses. `_deleted_at` keeps the latest removal for the same reason + // `deleteRows` does. + stats.deleted++ + return { + key: row.key, + raw: merge(stored.raw, row.data), + updatedAt: row.updatedAt, + deletedAt: later(stored.deletedAt, row.updatedAt), + } + }) + } + return stats +} diff --git a/packages/sync-clickhouse/src/destination.ts b/packages/sync-clickhouse/src/destination.ts new file mode 100644 index 0000000..999e65d --- /dev/null +++ b/packages/sync-clickhouse/src/destination.ts @@ -0,0 +1,299 @@ +/** + * The ClickHouse destination: everything `@workos/sync-engine` needs to know + * about a database, implemented over the official HTTP client. + */ +import type { + Destination, + FreshnessRow, + IdDiff, + Lease, + LeaseResult, + Logger, + RejectedEvent, + Row, + TableDef, + WorkOSEvent, + WriteMode, + WriteStats, +} from '@workos/sync-engine' +import { encodeKey } from '@workos/sync-engine' +import { connect, isoOf, type ChClient } from './db/client.js' +import { upsertRows, deleteRows, relinkRows, unlinkRows, keyExpression, tableRef, KEY_SEPARATOR } from './db/upsert.js' +import { diffIdsInClickHouse } from './db/reconcile.js' +import { acquireLease, releaseLease, getLease, getState, setState, setStateIfHeld } from './db/lease.js' +import { rejectEvents, listRejected, countRejected, clearRejected } from './db/rejected.js' +import { isTransientClickHouseError } from './db/errors.js' +import { migrate, type MigrateResult } from './schema/migrate.js' +import { jsonPathArgs, quoteIdent, safeIdentifier } from './schema/ddl.js' + +const ident = (name: string) => quoteIdent(safeIdentifier(name)) + +export interface ClickHouseDestinationOptions { + /** + * The HTTP interface, `https://user:password@host:8443`. Alternatively + * pass a pre-built `db`. A path in the URL is ignored: every statement + * qualifies its tables with `schema`. + */ + databaseUrl?: string + /** + * PEM-encoded CA certificate(s) the server's certificate must chain to, + * for a server whose certificate a public CA did not issue. Only with an + * `https://` `databaseUrl`; a pre-built `db` brings its own TLS settings. + * `WORKOS_SYNC_CLICKHOUSE_TLS_CA` on the CLI. + */ + sslCa?: string + /** A pre-built client, e.g. `fromClient(client)` to share the app's. Not closed by `close()`. */ + db?: ChClient + /** The database the mirror lives in, created if missing. Defaults to `workos`. */ + schema?: string + /** Identifier when several syncs share one database. Defaults to `default`. */ + syncId?: string + /** Rows per statement. Defaults to 500. */ + batchSize?: number + /** + * On `init()`, replace a `_live` view under a generated name that is not + * the generated shape. Off, `init()` fails and names it instead. `init + * --repair` on the CLI. + */ + repairViews?: boolean + /** + * During `init()`, redefine any projected column whose definition no + * longer matches the catalog. Off by default: drift is then reported at + * error level and the schema fingerprint withheld until it is fixed. Key + * columns, non-generated columns and orphans are never touched either + * way. `init --repair` on the CLI sets this together with `repairViews`. + */ + repairColumns?: boolean +} + +export class ClickHouseDestination implements Destination { + readonly schema: string + private db: ChClient | undefined + private ownsDb = false + private readonly syncId: string + private readonly batchSize: number | undefined + /** + * What the most recent `init()` did, including any column drift it found. + * The engine's `init()` returns nothing, so this is how a caller — the CLI's + * `init` command — gets the drift report back. + */ + lastMigration: MigrateResult | undefined + + constructor(private options: ClickHouseDestinationOptions) { + if (!options.db && !options.databaseUrl) throw new Error('databaseUrl or db is required') + if (options.sslCa !== undefined && (options.db || !options.databaseUrl)) { + throw new Error('sslCa applies to databaseUrl; a pre-built db carries its own TLS settings') + } + this.schema = options.schema ?? 'workos' + if (!/^[A-Za-z_][A-Za-z0-9_]{0,62}$/.test(this.schema)) { + throw new Error('schema must be a plain identifier: letters, digits and underscores, up to 63 characters') + } + this.syncId = options.syncId ?? 'default' + this.batchSize = options.batchSize + this.db = options.db + } + + private get client(): ChClient { + if (!this.db) throw new Error('destination not initialised: call init() first') + return this.db + } + + private get writeOpts() { + return { schema: this.schema, batchSize: this.batchSize } + } + + private get bookkeepingOpts() { + return { schema: this.schema, syncId: this.syncId } + } + + async init(streams: TableDef[], logger?: Logger): Promise { + if (!this.db) { + this.db = await connect(this.options.databaseUrl!, { sslCa: this.options.sslCa, logger }) + this.ownsDb = true + } + this.lastMigration = await migrate(this.db, streams, this.schema, logger, { + repair: this.options.repairColumns, + repairViews: this.options.repairViews, + }) + } + + async close(): Promise { + if (this.ownsDb && this.db) await this.db.close() + } + + upsert(stream: TableDef, env: string, rows: Row[], mode: WriteMode): Promise { + return upsertRows(this.client, this.writeOpts, stream, env, rows, mode) + } + + delete(stream: TableDef, env: string, rows: Row[]): Promise { + return deleteRows(this.client, this.writeOpts, stream, env, rows) + } + + relink(stream: TableDef, env: string, rows: Row[]): Promise { + return relinkRows(this.client, this.writeOpts, stream, env, rows) + } + + unlink(stream: TableDef, env: string, rows: Row[]): Promise { + return unlinkRows(this.client, this.writeOpts, stream, env, rows) + } + + /** `FINAL` for the mirror tables, where a key may have several versions; append tables have one row per key. */ + private final(stream: TableDef): string { + return stream.mode === 'append' ? '' : ' FINAL' + } + + async liveIds(stream: TableDef, env: string): Promise { + const live = stream.mode === 'append' ? '' : ' AND `_deleted_at` IS NULL' + const rows = await this.client.query<{ k: string }>( + `SELECT ${keyExpression(stream)} AS k FROM ${tableRef(this.schema, stream)}${this.final(stream)} + WHERE \`_env\` = {env:String}${live} ORDER BY k`, + { env }, + ) + return rows.map((r) => encodeKey(stream.key.length === 1 ? [r.k] : r.k.split(KEY_SEPARATOR))) + } + + async rawRows(stream: TableDef, env: string): Promise[]> { + const rows = await this.client.query<{ raw: string }>( + `SELECT \`_raw\` AS raw FROM ${tableRef(this.schema, stream)}${this.final(stream)} WHERE \`_env\` = {env:String}`, + { env }, + ) + return rows.map((r) => JSON.parse(r.raw) as Record) + } + + /** Staged into tables of their own and diffed in SQL; see `db/reconcile.ts`. */ + diffIds(stream: TableDef, env: string, remote: AsyncIterable[]>): Promise { + return diffIdsInClickHouse(this.client, this.writeOpts, stream, env, remote) + } + + async freshness(stream: TableDef, env: string, lagWindowHours: number): Promise { + const hasDeleted = stream.mode !== 'append' + const recent = `\`_updated_at\` > now64(3, 'UTC') - toIntervalHour({hours:UInt64})` + const ms = (col: string) => `toUnixTimestamp64Milli(${col})` + const rows = await this.client.query<{ + n: string + deleted: string + last_synced_at: string + staleness: string + lag_s: string | null + lag_n: string + }>( + `SELECT toString(count()) AS n, + ${hasDeleted ? 'toString(countIf(`_deleted_at` IS NOT NULL))' : "'0'"} AS deleted, + max(\`_synced_at\`) AS last_synced_at, + toString((${ms(`now64(3, 'UTC')`)} - ${ms('max(`_updated_at`)')}) / 1000) AS staleness, + toString(avgIf((${ms('`_synced_at`')} - ${ms('`_updated_at`')}) / 1000, ${recent})) AS lag_s, + toString(countIf(${recent})) AS lag_n + FROM ${tableRef(this.schema, stream)}${this.final(stream)} WHERE \`_env\` = {env:String}`, + { env, hours: Math.max(0, Math.floor(lagWindowHours)) }, + ) + const row = rows[0] + const n = Number(row?.n ?? 0) + const lagN = Number(row?.lag_n ?? 0) + // Aggregates over no rows read as epoch and NaN here, not NULL. + return { + stream: stream.name, + rows: n, + deleted: Number(row?.deleted ?? 0), + lastSyncedAt: n > 0 && row ? isoOf(row.last_synced_at) : null, + stalenessSeconds: n > 0 && row ? Number(row.staleness) : null, + lagSeconds: lagN > 0 && row?.lag_s != null && Number.isFinite(Number(row.lag_s)) ? Number(row.lag_s) : null, + lagSampleSize: lagN, + } + } + + async redact(streams: TableDef[], env: string, userId: string): Promise<{ table: string; rows: number }[]> { + const out: { table: string; rows: number }[] = [] + // A delete reports no count, so each table is counted first. + const erase = async ( + label: string, + table: string, + final: string, + where: string, + params: Record, + ) => { + const counted = await this.client.query<{ n: string }>( + `SELECT toString(count()) AS n FROM ${table}${final} WHERE ${where}`, + params, + ) + const n = Number(counted[0]?.n ?? 0) + if (n === 0) return + await this.client.command(`DELETE FROM ${table} WHERE ${where}`, params) + out.push({ table: label, rows: n }) + } + for (const stream of streams) { + const col = stream.columns.find((c) => c.name === 'user_id' || (c.name === 'id' && stream.name === 'users')) + if (!col) continue + await erase( + stream.name, + tableRef(this.schema, stream), + this.final(stream), + `\`_env\` = {env:String} AND ${ident(col.name)} = {id:String}`, + { env, id: userId }, + ) + } + const userIdInData = `JSONExtractString(\`_raw\`, ${jsonPathArgs(['data', 'user_id'])})` + // Archived events embed the payload, so they must be scrubbed too. + await erase( + 'events', + `${quoteIdent(this.schema)}.\`events\``, + '', + `\`_env\` = {env:String} AND (\`object_id\` = {id:String} OR ${userIdInData} = {id:String})`, + { env, id: userId }, + ) + // Quarantined events embed the payload too, and they may be the only copy + // of it: an event rejected before its archive write succeeded is nowhere + // in `events`. + await erase( + '_rejected', + `${quoteIdent(this.schema)}.\`_rejected\``, + ' FINAL', + `sync_id = {syncId:String} AND env = {env:String} AND (object_id = {id:String} OR ${userIdInData} = {id:String})`, + { syncId: this.syncId, env, id: userId }, + ) + return out + } + + reject(env: string, rows: { event: WorkOSEvent; error: string }[]): Promise { + return rejectEvents(this.client, this.bookkeepingOpts, env, rows) + } + + rejected(env: string, opts: { ids?: string[] } = {}): Promise { + return listRejected(this.client, this.bookkeepingOpts, env, opts.ids) + } + + rejectedCount(env: string): Promise { + return countRejected(this.client, this.bookkeepingOpts, env) + } + + clearRejected(env: string, ids: string[]): Promise { + return clearRejected(this.client, this.bookkeepingOpts, env, ids) + } + + isTransientError(err: unknown): boolean { + return isTransientClickHouseError(err) + } + + getState(env: string, stream: string): Promise { + return getState(this.client, this.schema, this.syncId, env, stream) + } + + setState(env: string, stream: string, state: unknown): Promise { + return setState(this.client, this.schema, this.syncId, env, stream, state) + } + + setStateIfHeld(env: string, stream: string, state: unknown, holder: string, expected: unknown): Promise { + return setStateIfHeld(this.client, this.schema, this.syncId, env, stream, state, holder, expected) + } + + acquireLease(env: string, holder: string, ttlMs: number): Promise { + return acquireLease(this.client, this.schema, this.syncId, env, holder, ttlMs) + } + + releaseLease(env: string, holder: string): Promise { + return releaseLease(this.client, this.schema, this.syncId, env, holder) + } + + getLease(env: string): Promise { + return getLease(this.client, this.schema, this.syncId, env) + } +} diff --git a/packages/sync-clickhouse/src/health.ts b/packages/sync-clickhouse/src/health.ts new file mode 100644 index 0000000..4e91bca --- /dev/null +++ b/packages/sync-clickhouse/src/health.ts @@ -0,0 +1,39 @@ +/** + * Health server settings for the `listen` command. + * + * Off unless a port is given, because the CLI also runs as one-shot commands + * and a cron `tick`, where a listening socket would be a surprise. The Docker + * image sets the port for the same reason it sets `NODE_ENV`: a container is + * the deployment where something external wants to ask whether the process is + * alive. Everything the server needs is read here, in one place, so `cli.ts` + * only has to ask for a config and hand it to `startHealthServer`. + */ +import { DEFAULT_MAX_POLL_AGE_MS, type HealthServerOptions } from '@workos/sync-engine' + +type Flags = Record + +/** + * `--health-port` or `WORKOS_SYNC_HEALTH_PORT` turns the server on; + * `WORKOS_SYNC_HEALTH_HOST` and `WORKOS_SYNC_HEALTH_MAX_POLL_AGE_MS` tune it. + * Returns null when it should stay off. + */ +export function healthOptionsFromEnv(flags: Flags, env: NodeJS.ProcessEnv = process.env): HealthServerOptions | null { + const raw = typeof flags['health-port'] === 'string' ? flags['health-port'] : env['WORKOS_SYNC_HEALTH_PORT'] + if (raw === undefined || raw.trim() === '') return null + const port = Number(raw) + if (!Number.isInteger(port) || port < 0 || port > 65535) { + throw new Error(`invalid health port "${raw}": expected an integer between 0 and 65535`) + } + const maxRaw = env['WORKOS_SYNC_HEALTH_MAX_POLL_AGE_MS'] + const maxPollAgeMs = maxRaw === undefined || maxRaw.trim() === '' ? DEFAULT_MAX_POLL_AGE_MS : Number(maxRaw) + if (!Number.isFinite(maxPollAgeMs) || maxPollAgeMs <= 0) { + throw new Error( + `invalid WORKOS_SYNC_HEALTH_MAX_POLL_AGE_MS "${maxRaw}": expected a positive number of milliseconds`, + ) + } + return { + port, + host: env['WORKOS_SYNC_HEALTH_HOST']?.trim() || '0.0.0.0', + maxPollAgeMs, + } +} diff --git a/packages/sync-clickhouse/src/index.ts b/packages/sync-clickhouse/src/index.ts new file mode 100644 index 0000000..5a553de --- /dev/null +++ b/packages/sync-clickhouse/src/index.ts @@ -0,0 +1,53 @@ +/** + * @workos/sync-clickhouse + * + * Mirror WorkOS data into your own ClickHouse: the sign-in and event logs + * that WorkOS keeps for 90 days, as tables you keep for as long as you like, + * and the identity objects beside them for the joins. + * + * The engine itself lives in `@workos/sync-engine` and is re-exported here, so this + * one package is all an application needs to import. This package adds the + * ClickHouse destination, the DDL and write path behind it, and the CLI. + * + * The mirror is a **read model**. It is eventually consistent, typically within + * seconds. Read from it for analytics, dashboards, lists and agents; keep + * writes and authoritative post-signup checks on the WorkOS API. + */ +export * from '@workos/sync-engine' + +export { ClickHouseDestination } from './destination.js' +export type { ClickHouseDestinationOptions } from './destination.js' + +export { migrate, fingerprint, DDL_VERSION } from './schema/migrate.js' +export type { MigrateOptions, MigrateResult } from './schema/migrate.js' +export { detectColumnDrift, repairColumnDrift, unrepairableReason } from './schema/drift.js' +export type { ColumnDrift, DriftKind, RepairResult } from './schema/drift.js' +export { + buildSchemaSql, + createTableSql, + columnExpression, + columnType, + generatedColumnSql, + indexSpecs, + jsonPathArgs, +} from './schema/ddl.js' +export { liveViewSql, liveViewName } from './schema/views.js' + +export { + upsertRows, + deleteRows, + relinkRows, + unlinkRows, + dedupeByKey, + wavesByKey, + keyValues, + parseUpdatedAt, + jsonEqual, +} from './db/upsert.js' +export { diffIdsInClickHouse } from './db/reconcile.js' +export type { ChDiff } from './db/reconcile.js' +export { rejectEvents, listRejected, countRejected, clearRejected } from './db/rejected.js' +export { canonicalJson } from './db/lease.js' +export { isTransientClickHouseError } from './db/errors.js' +export { connect, fromClient, clientConfig, APPLICATION_NAME, DEFAULT_SETTINGS, KEEPER_SETTINGS } from './db/client.js' +export type { ConnectOptions, ChClient } from './db/client.js' diff --git a/packages/sync-clickhouse/src/otel.ts b/packages/sync-clickhouse/src/otel.ts new file mode 100644 index 0000000..78472f2 --- /dev/null +++ b/packages/sync-clickhouse/src/otel.ts @@ -0,0 +1,75 @@ +/** + * OpenTelemetry for the command line. + * + * The engine depends on the OpenTelemetry API only, and the API does nothing + * until a process installs an SDK. The CLI is a process of its own, so it is + * where one can live: `WORKOS_SYNC_OTEL=1` starts the Node SDK, configured by + * the standard `OTEL_*` variables (exporter, endpoint, headers, service + * name), before the engine is built, and shuts it down once the command + * returns. The shutdown is the part that matters for `tick` and the sliced + * commands: they exit seconds after they start, long before the SDK's + * periodic export would have fired, and shutting down flushes. + * + * The SDK is an optional dependency of this package, so a host that embeds + * the library and runs its own SDK can leave it out; the container image has + * it. A missing module is reported as such rather than as a stack trace. + */ +export interface OtelHandle { + /** Flush and stop the SDK. Call it once the command has returned. */ + shutdown(): Promise +} + +/** The part of `@opentelemetry/sdk-node` this module uses, so nothing here types against the package. */ +export interface SdkModule { + NodeSDK: new (configuration?: { serviceName?: string }) => { start(): void; shutdown(): Promise } +} + +/** The `service.name` reported, unless `OTEL_SERVICE_NAME` says otherwise. */ +export const DEFAULT_SERVICE_NAME = 'workos-sync-clickhouse' + +const ON = new Set(['1', 'true', 'yes', 'on']) +const OFF = new Set(['0', 'false', 'no', 'off']) + +/** `WORKOS_SYNC_OTEL` is on, off, or unset, which is off. Anything else is refused up front. */ +export function otelEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + const raw = env['WORKOS_SYNC_OTEL'] + if (raw === undefined || raw.trim() === '') return false + const v = raw.trim().toLowerCase() + if (ON.has(v)) return true + if (OFF.has(v)) return false + throw new Error(`invalid WORKOS_SYNC_OTEL "${raw}": expected 1 or 0`) +} + +/** + * Start the SDK when `WORKOS_SYNC_OTEL` asks for it; null when it does not. + * The SDK reads its own configuration from the process environment; `env` + * only decides whether to start it and what to call the service. `load` is + * a test seam standing in for the import. + */ +export async function startOtel( + env: NodeJS.ProcessEnv = process.env, + load: () => Promise = () => import('@opentelemetry/sdk-node'), +): Promise { + if (!otelEnabled(env)) return null + let sdkNode: SdkModule + try { + sdkNode = await load() + } catch (err) { + if (isModuleNotFound(err)) { + throw new Error( + 'WORKOS_SYNC_OTEL is set but @opentelemetry/sdk-node is not installed: ' + + 'add it to the host (npm install @opentelemetry/sdk-node), or unset WORKOS_SYNC_OTEL', + ) + } + throw err + } + const named = Boolean(env['OTEL_SERVICE_NAME']?.trim()) + const sdk = new sdkNode.NodeSDK(named ? {} : { serviceName: DEFAULT_SERVICE_NAME }) + sdk.start() + return { shutdown: () => sdk.shutdown() } +} + +function isModuleNotFound(err: unknown): boolean { + const code = (err as { code?: unknown } | null)?.code + return code === 'ERR_MODULE_NOT_FOUND' || code === 'MODULE_NOT_FOUND' +} diff --git a/packages/sync-clickhouse/src/schema/ddl.ts b/packages/sync-clickhouse/src/schema/ddl.ts new file mode 100644 index 0000000..ebceb6a --- /dev/null +++ b/packages/sync-clickhouse/src/schema/ddl.ts @@ -0,0 +1,264 @@ +/** + * DDL generation. + * + * `_raw` is the only column the engine ever writes. Every typed column is a + * `DEFAULT` expression over it, which ClickHouse computes when the engine's + * insert leaves the column out, and which `SELECT *` includes (a + * `MATERIALIZED` column would not be). A new field appearing upstream can + * never break ingest, is immediately queryable with `JSONExtract(_raw, ...)`, + * and can be promoted to a typed column later with pure DDL: a column added + * with `ALTER TABLE ... ADD COLUMN` is computed for the rows already there. + * + * Three things are ClickHouse's rather than Postgres's: + * + * - **Versions, not updates.** A mirror table is a `ReplacingMergeTree` + * versioned by `_updated_at`: every write inserts the row's new state, and + * the newest source timestamp wins at merge time and under `FINAL`. That + * is the staleness gate, enforced by the table itself whatever order the + * writes land in. The `_live` views read with `FINAL`. Append-only + * streams are plain `MergeTree`, the shape analytics wants, with + * re-delivery filtered by the write path. + * - **A projection that cannot be cast is NULL, not an error.** `JSONExtract` + * with a `Nullable` return type gives NULL for a missing key, a JSON null + * and a value of the wrong shape alike, and the value stays in `_raw`. + * - **Indexes are data-skipping indexes.** A column marked `index` gets a + * bloom filter (text) or a min-max index (timestamps, integers); the sort + * key is `(_env, )`, which is also what the write path looks + * rows up by. + * + * There are no foreign keys between mirror tables. Events arrive with + * children before parents routinely (a `group.user_added` before the + * `user.created`), and an FK would turn that ordering artifact into a hard + * write failure. + */ +import type { ColumnDef, TableDef } from '@workos/sync-engine' +import { quoteIdent, quoteLiteral, safeIdentifier } from './identifiers.js' +import { liveViewSql } from './views.js' + +// Re-exported so callers keep importing the identifier helpers from here. +export { quoteIdent, quoteLiteral, safeIdentifier } from './identifiers.js' + +export const RAW = '_raw' +export const ENV = '_env' + +/** Keeper path under `keeper_map_path_prefix` for a schema's bookkeeping. */ +export const keeperPath = (schema: string, table: string) => quoteLiteral(`/workos_sync/${schema}/${table}`) + +const TIMESTAMP = `DateTime64(3, 'UTC')` + +/** The JSON path as `JSONExtract` takes it: one quoted step per argument. */ +export function jsonPathArgs(path: string | string[]): string { + const parts = Array.isArray(path) ? path : [path] + return parts.map((p) => quoteLiteral(p)).join(', ') +} + +/** The ClickHouse column type for a projected column. Keys are not nullable: they sort the table. */ +export function columnType(col: ColumnDef, isKey: boolean): string { + switch (col.type) { + case 'text': + return isKey ? 'String' : 'Nullable(String)' + case 'boolean': + return 'Nullable(Bool)' + case 'bigint': + return 'Nullable(Int64)' + case 'numeric': + return 'Nullable(Decimal(38, 10))' + case 'jsonb': + return 'Nullable(String)' + case 'timestamptz': + return `Nullable(${TIMESTAMP})` + } +} + +/** The default expression for a projected field. */ +export function columnExpression(col: ColumnDef, isKey: boolean): string { + const raw = quoteIdent(RAW) + const path = jsonPathArgs(col.path) + switch (col.type) { + case 'text': + // A key reads as '' when missing; the engine never writes a keyless row. + return isKey ? `JSONExtractString(${raw}, ${path})` : `JSONExtract(${raw}, ${path}, 'Nullable(String)')` + case 'boolean': + case 'bigint': + case 'numeric': + return `JSONExtract(${raw}, ${path}, '${columnType(col, isKey)}')` + case 'jsonb': + // The subtree as JSON text; `JSONExtractRaw` alone reads '' for a missing key. + return `if(JSONHas(${raw}, ${path}), JSONExtractRaw(${raw}, ${path}), NULL)` + case 'timestamptz': + // Reads every ISO-8601 shape WorkOS emits, offsets included; NULL for anything else. + return `parseDateTime64BestEffortOrNull(JSONExtractString(${raw}, ${path}), 3, 'UTC')` + } +} + +/** + * One `` `name` type DEFAULT expr `` clause. Every path that creates a + * projected column — table creation, the additive upgrade, drift detection + * and repair — goes through here, so they cannot disagree about what a + * column should look like. The text is also what `_columns` records, and + * what drift is measured against. + */ +export function generatedColumnSql(col: ColumnDef, stream: TableDef): string { + const isKey = stream.key.includes(col.name) + return `${quoteIdent(safeIdentifier(col.name))} ${columnType(col, isKey)} DEFAULT ${columnExpression(col, isKey)}` +} + +export function createSchemaSql(schema: string): string { + return `CREATE DATABASE IF NOT EXISTS ${quoteIdent(schema)}` +} + +/** + * Engine bookkeeping. Cursors and the single-writer lease share a KeeperMap + * row per environment: KeeperMap is a key-value table held in ClickHouse + * Keeper with linearizable writes, and under `keeper_map_strict_mode` an + * `ALTER ... UPDATE` that names the row's current values is a compare-and- + * swap, which is what the lease and the fenced checkpoint need. See + * `db/lease.ts`. The applied-schema fingerprints and the record of generated + * columns live there too: an `INSERT` of an existing key is refused, which + * makes "record once" a single statement. + * + * `_rejected` holds events the tail set aside after they failed + * deterministically on two consecutive polls (see the engine's + * `sync/quarantine.ts`). It keeps the whole event, not a reference into + * `events`, because the archive write may itself have been the failure. A + * rejection written again replaces the earlier one: the table is a + * `ReplacingMergeTree` versioned by `rejected_at`. `created_at` is + * nullable and parsed in Node on the way in, so a quarantine write can + * never fail on the same bad data that caused it. + */ +export function bookkeepingSql(schema: string): [name: string, sql: string][] { + const s = quoteIdent(schema) + return [ + [ + '_sync', + `CREATE TABLE IF NOT EXISTS ${s}.\`_sync\` ( + k String, + holder String, + acquired_at ${TIMESTAMP}, + expires_at ${TIMESTAMP}, + states String +) ENGINE = KeeperMap(${keeperPath(schema, '_sync')}) PRIMARY KEY k`, + ], + [ + '_migrations', + `CREATE TABLE IF NOT EXISTS ${s}.\`_migrations\` ( + fingerprint String, + applied_at ${TIMESTAMP}, + detail String +) ENGINE = KeeperMap(${keeperPath(schema, '_migrations')}) PRIMARY KEY fingerprint`, + ], + [ + '_columns', + `CREATE TABLE IF NOT EXISTS ${s}.\`_columns\` ( + k String, + table_name String, + column_name String, + ddl String +) ENGINE = KeeperMap(${keeperPath(schema, '_columns')}) PRIMARY KEY k`, + ], + [ + '_rejected', + `CREATE TABLE IF NOT EXISTS ${s}.\`_rejected\` ( + sync_id String, + env String, + event_id String, + event_type String, + object_id Nullable(String), + created_at Nullable(${TIMESTAMP}), + rejected_at ${TIMESTAMP} DEFAULT now64(3), + error String, + \`_raw\` String +) ENGINE = ReplacingMergeTree(rejected_at) ORDER BY (sync_id, env, event_id)`, + ], + ] +} + +export interface IndexSpec { + name: string + column: string + type: 'bloom_filter' | 'minmax' + granularity: number +} + +/** The data-skipping indexes a stream's table carries, by name, so `migrate` can tell which are missing. */ +export function indexSpecs(stream: TableDef): IndexSpec[] { + const tableName = safeIdentifier(stream.name) + const out: IndexSpec[] = [] + for (const c of stream.columns) { + if (!c.index) continue + const minmax = c.type === 'timestamptz' || c.type === 'bigint' || c.type === 'numeric' + out.push({ + name: safeIdentifier(`${tableName}_${c.name}_idx`), + column: safeIdentifier(c.name), + type: minmax ? 'minmax' : 'bloom_filter', + granularity: minmax ? 1 : 4, + }) + } + // Freshness and reconciliation queries scan these. + out.push({ name: safeIdentifier(`${tableName}_synced_at_idx`), column: '_synced_at', type: 'minmax', granularity: 1 }) + if (stream.mode !== 'append') { + out.push({ + name: safeIdentifier(`${tableName}_deleted_at_idx`), + column: '_deleted_at', + type: 'minmax', + granularity: 1, + }) + } + return out +} + +const indexClause = (idx: IndexSpec) => + `INDEX ${quoteIdent(idx.name)} ${quoteIdent(idx.column)} TYPE ${idx.type} GRANULARITY ${idx.granularity}` + +export function createTableSql(stream: TableDef, schema: string): string { + const s = quoteIdent(schema) + const table = quoteIdent(safeIdentifier(stream.name)) + const cols: string[] = [ + `${quoteIdent(ENV)} String`, + `${quoteIdent(RAW)} String`, + // Source-derived. The staleness gate compares this, and so does the merge. + `\`_updated_at\` ${TIMESTAMP}`, + // Destination write time. Lag is `_synced_at - _updated_at`. + `\`_synced_at\` DateTime64(6, 'UTC') DEFAULT now64(6)`, + ] + // Append-only streams are immutable logs: no tombstones. + if (stream.mode !== 'append') cols.push(`\`_deleted_at\` Nullable(${TIMESTAMP})`) + for (const c of stream.columns) cols.push(generatedColumnSql(c, stream)) + for (const idx of indexSpecs(stream)) cols.push(indexClause(idx)) + const order = [ENV, ...stream.key].map((k) => quoteIdent(safeIdentifier(k))).join(', ') + const engine = stream.mode === 'append' ? 'MergeTree' : 'ReplacingMergeTree(`_updated_at`)' + return `CREATE TABLE IF NOT EXISTS ${s}.${table} (\n ${cols.join(',\n ')}\n) ENGINE = ${engine} ORDER BY (${order})` +} + +/** Additive DDL for a table that already exists; both are idempotent. */ +export function addColumnSql(stream: TableDef, col: ColumnDef, schema: string): string { + const table = `${quoteIdent(schema)}.${quoteIdent(safeIdentifier(stream.name))}` + return `ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS ${generatedColumnSql(col, stream)}` +} + +/** Write a column added later into the parts that predate it, so reads stop recomputing it. */ +export function materializeColumnSql(stream: TableDef, col: ColumnDef, schema: string): string { + const table = `${quoteIdent(schema)}.${quoteIdent(safeIdentifier(stream.name))}` + return `ALTER TABLE ${table} MATERIALIZE COLUMN ${quoteIdent(safeIdentifier(col.name))}` +} + +export function addIndexSql(stream: TableDef, idx: IndexSpec, schema: string): string { + const table = `${quoteIdent(schema)}.${quoteIdent(safeIdentifier(stream.name))}` + return `ALTER TABLE ${table} ADD ${indexClause(idx).replace('INDEX ', 'INDEX IF NOT EXISTS ')}` +} + +/** + * Full DDL for a set of streams, in dependency-free order: what + * `init --print-sql` prints. Every statement is idempotent, so the script + * can be applied to a database in any state. + */ +export function buildSchemaSql(streams: TableDef[], schema: string): string[] { + const out: string[] = [createSchemaSql(schema), ...bookkeepingSql(schema).map(([, sql]) => sql)] + for (const s of streams) { + out.push(createTableSql(s, schema)) + for (const c of s.columns) out.push(addColumnSql(s, c, schema)) + for (const idx of indexSpecs(s)) out.push(addIndexSql(s, idx, schema)) + out.push(...liveViewSql(s, schema)) + } + return out +} diff --git a/packages/sync-clickhouse/src/schema/drift.ts b/packages/sync-clickhouse/src/schema/drift.ts new file mode 100644 index 0000000..96474d1 --- /dev/null +++ b/packages/sync-clickhouse/src/schema/drift.ts @@ -0,0 +1,194 @@ +/** + * Column drift: a projected column whose definition no longer matches the + * catalog. + * + * Additive DDL cannot change a column that already exists, so once a + * projection is wrong — an `extraColumns` entry whose path or type was + * edited, a catalog change to a column that is already there — it stays + * wrong until something rebuilds it. This finds those columns and, when + * asked, redefines them with `ALTER TABLE ... MODIFY COLUMN`, which + * ClickHouse applies to the rows already there. + * + * The expression is compared against `_columns`, the engine's own record of + * the DDL it emitted for every projected column, not against what + * `system.columns` shows: the server stores a re-formatted expression, so + * text written by the catalog and text read back never compare equal. The + * live type is compared with the catalog's directly; `system.columns` + * spells types canonically, and so does `columnType`. + * + * Never touched, only reported: key columns (they sort the table), columns + * that are not `DEFAULT` expressions (they are yours), and orphans — + * recorded columns the catalog no longer names, which still compute from + * `_raw` with their old expression and are harmless to leave. + */ +import type { Logger, TableDef } from '@workos/sync-engine' +import { KEEPER_SETTINGS, type ChClient } from '../db/client.js' +import { isKeeperConflict } from '../db/errors.js' +import { columnExpression, columnType, generatedColumnSql, quoteIdent, safeIdentifier } from './ddl.js' + +export type DriftKind = 'expression' | 'type' | 'orphan' | 'not_generated' + +export interface ColumnDrift { + stream: string + column: string + kind: DriftKind + /** The clause the catalog would emit now; null for an orphan. */ + expected: string | null + /** What is recorded or, failing that, what `system.columns` shows. */ + actual: string | null + /** Part of the sort key, so never rebuilt. */ + key: boolean +} + +export interface RepairResult { + repaired: ColumnDrift[] + failed: { drift: ColumnDrift; error: string }[] +} + +interface LiveColumn { + type: string + default_kind: string + default_expression: string +} + +async function liveColumns(db: ChClient, schema: string, table: string): Promise> { + const rows = await db.query<{ name: string; type: string; default_kind: string; default_expression: string }>( + `SELECT name, type, default_kind, default_expression FROM system.columns + WHERE database = {schema:String} AND table = {table:String}`, + { schema, table }, + ) + return new Map(rows.map((r) => [r.name, r])) +} + +async function recordedColumns(db: ChClient, schema: string, table: string): Promise> { + const rows = await db.query<{ column_name: string; ddl: string }>( + `SELECT column_name, ddl FROM ${quoteIdent(schema)}.\`_columns\` WHERE table_name = {table:String}`, + { table }, + ) + return new Map(rows.map((r) => [r.column_name, r.ddl])) +} + +/** + * Record (or re-record) the clause a column was created with. Strict mode + * is off, so this overwrites; two processes recording the same column at + * once can still race on the first write, and the loser simply writes again. + */ +export async function recordColumn(db: ChClient, schema: string, table: string, column: string, ddl: string) { + for (let attempt = 0; ; attempt++) { + try { + await db.command( + `INSERT INTO ${quoteIdent(schema)}.\`_columns\` (k, table_name, column_name, ddl) + VALUES ({k:String}, {table:String}, {column:String}, {ddl:String})`, + { k: `${table}\u0001${column}`, table, column, ddl }, + { ...KEEPER_SETTINGS, keeper_map_strict_mode: 0 }, + ) + return + } catch (err) { + if (attempt >= 3 || !isKeeperConflict(err)) throw err + } + } +} + +async function forgetColumn(db: ChClient, schema: string, table: string, column: string) { + await db.command( + `ALTER TABLE ${quoteIdent(schema)}.\`_columns\` DELETE WHERE k = {k:String}`, + { k: `${table}\u0001${column}` }, + { ...KEEPER_SETTINGS, keeper_map_strict_mode: 0 }, + ) +} + +export async function detectColumnDrift(db: ChClient, streams: TableDef[], schema: string): Promise { + const out: ColumnDrift[] = [] + for (const stream of streams) { + const table = safeIdentifier(stream.name) + const live = await liveColumns(db, schema, table) + const recorded = await recordedColumns(db, schema, table) + const catalog = new Set() + for (const col of stream.columns) { + const name = safeIdentifier(col.name) + catalog.add(name) + const actual = live.get(name) + // Missing entirely: `migrate` adds it, or already failed loudly trying. + if (!actual) continue + const key = stream.key.includes(col.name) + const expected = generatedColumnSql(col, stream) + if (actual.default_kind !== 'DEFAULT') { + out.push({ stream: stream.name, column: name, kind: 'not_generated', expected, actual: actual.type, key }) + continue + } + // The live type is checked on its own, before the record is consulted: + // a column retyped by hand leaves `_columns` saying what the engine + // created, and that must not hide what is actually there. + if (actual.type !== columnType(col, key)) { + out.push({ stream: stream.name, column: name, kind: 'type', expected, actual: actual.type, key }) + continue + } + const record = recorded.get(name) + if (record === undefined) { + await recordColumn(db, schema, table, name, expected) + continue + } + if (record !== expected) { + out.push({ stream: stream.name, column: name, kind: 'expression', expected, actual: record, key }) + } + } + for (const [name, ddl] of recorded) { + if (catalog.has(name)) continue + if (!live.has(name)) { + // Dropped by hand: nothing to report, and the record would only mislead. + await forgetColumn(db, schema, table, name) + continue + } + out.push({ stream: stream.name, column: name, kind: 'orphan', expected: null, actual: ddl, key: false }) + } + } + return out +} + +/** Why a drifted column is not rebuilt, or null when `repairColumnDrift` may try. */ +export function unrepairableReason(d: ColumnDrift): string | null { + if (d.kind === 'orphan') return 'the catalog no longer names this column; drop it by hand if it is no longer wanted' + if (d.kind === 'not_generated') + return 'the column is not a DEFAULT expression, so it is yours: rename it or project the field under another name' + if (d.key) return 'key columns are never rebuilt: they sort the table' + return null +} + +/** + * Redefine each repairable drifted column with the catalog's type and + * expression, then recompute it for the rows already there: `MODIFY COLUMN` + * changes what future inserts compute and converts the stored values to the + * new type, but only a mutation re-evaluates the expression over `_raw`. + * Both run under `mutations_sync`, so the call returns once they are done. + */ +export async function repairColumnDrift( + db: ChClient, + drift: ColumnDrift[], + streams: TableDef[], + schema: string, + logger?: Logger, +): Promise { + const repaired: ColumnDrift[] = [] + const failed: RepairResult['failed'] = [] + for (const d of drift) { + if (unrepairableReason(d)) continue + const stream = streams.find((s) => s.name === d.stream) + const col = stream?.columns.find((c) => safeIdentifier(c.name) === d.column) + if (!stream || !col) continue + const table = safeIdentifier(stream.name) + try { + const ref = `${quoteIdent(schema)}.${quoteIdent(table)}` + const expr = columnExpression(col, false) + await db.command( + `ALTER TABLE ${ref} MODIFY COLUMN ${quoteIdent(d.column)} ${columnType(col, false)} DEFAULT ${expr}`, + ) + await db.command(`ALTER TABLE ${ref} UPDATE ${quoteIdent(d.column)} = ${expr} WHERE 1`) + await recordColumn(db, schema, table, d.column, generatedColumnSql(col, stream)) + repaired.push(d) + logger?.info('redefined drifted column from _raw', { stream: d.stream, column: d.column, kind: d.kind }) + } catch (err) { + failed.push({ drift: d, error: err instanceof Error ? err.message : String(err) }) + } + } + return { repaired, failed } +} diff --git a/packages/sync-clickhouse/src/schema/identifiers.ts b/packages/sync-clickhouse/src/schema/identifiers.ts new file mode 100644 index 0000000..b20b775 --- /dev/null +++ b/packages/sync-clickhouse/src/schema/identifiers.ts @@ -0,0 +1,27 @@ +/** + * Identifier and literal helpers shared by the table DDL and the view DDL. + * + * They live apart from `ddl.ts` so that `views.ts` can use them without + * importing the module that imports `views.ts` back for `buildSchemaSql`. + */ + +/** + * ClickHouse has no hard identifier limit, but the stream names become + * Keeper paths and file names on disk; the SQL packages' 63-byte habit is + * kept so one catalog produces one set of names everywhere. + */ +export const MAX_IDENTIFIER_LENGTH = 63 + +export function safeIdentifier(name: string): string { + if (name.length <= MAX_IDENTIFIER_LENGTH) return name + let hash = 5381 + for (let i = 0; i < name.length; i++) hash = ((hash << 5) + hash + name.charCodeAt(i)) >>> 0 + const suffix = `_${hash.toString(16).padStart(8, '0')}` + return name.slice(0, MAX_IDENTIFIER_LENGTH - suffix.length) + suffix +} + +/** Backtick-quoted, with embedded backticks and backslashes escaped. */ +export const quoteIdent = (name: string): string => `\`${name.replace(/\\/g, '\\\\').replace(/`/g, '\\`')}\`` + +/** A single-quoted SQL string literal, with backslashes and quotes escaped the way ClickHouse reads them. */ +export const quoteLiteral = (value: string): string => `'${value.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'` diff --git a/packages/sync-clickhouse/src/schema/migrate.ts b/packages/sync-clickhouse/src/schema/migrate.ts new file mode 100644 index 0000000..cc1f27f --- /dev/null +++ b/packages/sync-clickhouse/src/schema/migrate.ts @@ -0,0 +1,214 @@ +/** + * Schema setup. + * + * The DDL is derived from the stream catalog rather than hand-written per + * field, and a fingerprint gates the per-stream work, so an unchanged + * catalog costs one query. Every statement is idempotent — `CREATE ... IF + * NOT EXISTS`, `ADD COLUMN IF NOT EXISTS`, `ADD INDEX IF NOT EXISTS` — so + * two processes applying the schema at once, or a run that dies halfway, + * leave the same database as one clean run; there is no lock to take. + * + * The DDL is additive, which cannot change a column that already exists. So + * after it runs, and only when the fingerprint is new, the live tables are + * compared against the catalog (see `drift.ts`). The fingerprint is withheld + * while any projection is wrong: recording it would turn the next `init` + * into a silent no-op on a database that needs attention. + */ +import { createHash } from 'node:crypto' +import type { Logger, TableDef } from '@workos/sync-engine' +import { KEEPER_SETTINGS, type ChClient } from '../db/client.js' +import { + addColumnSql, + addIndexSql, + materializeColumnSql, + bookkeepingSql, + createSchemaSql, + createTableSql, + generatedColumnSql, + indexSpecs, + quoteIdent, + safeIdentifier, +} from './ddl.js' +import { applyLiveView, liveViewName, type ApplyViewOptions } from './views.js' +import { detectColumnDrift, recordColumn, repairColumnDrift, unrepairableReason, type ColumnDrift } from './drift.js' + +/** + * Bumped whenever the DDL generated for an unchanged catalog changes: a new + * kind of object, a reworked index, a view. The fingerprint otherwise hashes + * only the catalog, so without this a database whose streams did not change + * would never run the new statements on upgrade. + */ +export const DDL_VERSION = 1 + +export function fingerprint(streams: TableDef[], schema: string): string { + const canonical = JSON.stringify({ + ddl: DDL_VERSION, + schema, + streams: streams + .map((s) => ({ + n: s.name, + k: s.key, + m: s.mode, + c: s.columns.map((c) => [c.name, c.type, c.path, c.index ?? false]), + })) + .sort((a, b) => a.n.localeCompare(b.n)), + }) + return createHash('sha256').update(canonical).digest('hex').slice(0, 16) +} + +export interface MigrateOptions extends ApplyViewOptions { + /** + * Redefine drifted columns instead of only reporting them. Key columns, + * non-generated columns and orphans are still only reported; see + * `repairColumnDrift`. + */ + repair?: boolean +} + +export interface MigrateResult { + /** DDL was run. `false` means the fingerprint was already recorded. */ + applied: boolean + fingerprint: string + statements: number + /** + * Columns that still disagree with the catalog after this run. Any + * `expression` or `type` item here means the fingerprint was *not* + * recorded, so the next run detects and reports again. Orphans are + * reported but do not hold the fingerprint back. + */ + drift: ColumnDrift[] + /** Columns redefined by this run's repair pass. Only with `repair: true`. */ + repaired: ColumnDrift[] +} + +const names = async (db: ChClient, sql: string, params: Record) => + new Set((await db.query<{ name: string }>(sql, params)).map((r) => r.name)) + +export async function migrate( + db: ChClient, + streams: TableDef[], + schema: string, + logger?: Logger, + options: MigrateOptions = {}, +): Promise { + const fp = fingerprint(streams, schema) + const s = quoteIdent(schema) + + // Only when missing: creating needs a privilege the day-to-day user may not have. + const databases = await names(db, `SELECT name FROM system.databases WHERE name = {schema:String}`, { schema }) + if (!databases.has(schema)) await db.command(createSchemaSql(schema)) + // The bookkeeping tables must exist before we can check whether they say + // this fingerprint is already applied. + const tables = await names(db, `SELECT name FROM system.tables WHERE database = {schema:String}`, { schema }) + for (const [name, sql] of bookkeepingSql(schema)) if (!tables.has(name)) await db.command(sql) + + const applied = await db.query(`SELECT 1 FROM ${s}.\`_migrations\` WHERE fingerprint = {fp:String}`, { fp }) + if (applied.length > 0) { + logger?.debug('schema up to date', { fingerprint: fp }) + return { applied: false, fingerprint: fp, statements: 0, drift: [], repaired: [] } + } + + let statements = 0 + for (const stream of streams) { + const table = safeIdentifier(stream.name) + const live = await names( + db, + `SELECT name FROM system.columns WHERE database = {schema:String} AND table = {table:String}`, + { schema, table }, + ) + if (live.size === 0) { + await db.command(createTableSql(stream, schema)) + statements++ + for (const col of stream.columns) { + await recordColumn(db, schema, table, safeIdentifier(col.name), generatedColumnSql(col, stream)) + } + } else { + for (const col of stream.columns) { + if (live.has(safeIdentifier(col.name))) continue + // Computed for the rows already present, and written into their + // parts: left unmaterialised, every read of an old part would + // evaluate the expression again. + await db.command(addColumnSql(stream, col, schema)) + await db.command(materializeColumnSql(stream, col, schema)) + await recordColumn(db, schema, table, safeIdentifier(col.name), generatedColumnSql(col, stream)) + statements++ + } + const indexes = await names( + db, + `SELECT name FROM system.data_skipping_indices WHERE database = {schema:String} AND table = {table:String}`, + { schema, table }, + ) + for (const idx of indexSpecs(stream)) { + if (indexes.has(idx.name)) continue + await db.command(addIndexSql(stream, idx, schema)) + statements++ + } + } + // Last, so the view carries every column, including any added above. A + // view that already does is left alone: `init` runs no DDL it does not need. + if (stream.mode !== 'append' && live.size > 0) { + const viewColumns = await names( + db, + `SELECT name FROM system.columns WHERE database = {schema:String} AND table = {table:String}`, + { schema, table: liveViewName(stream) }, + ) + const tableColumns = await names( + db, + `SELECT name FROM system.columns WHERE database = {schema:String} AND table = {table:String}`, + { schema, table }, + ) + if (viewColumns.size > 0 && [...tableColumns].every((c) => viewColumns.has(c))) continue + } + if (await applyLiveView(db, stream, schema, logger, options)) statements++ + } + + // Detection runs only on this path — once per catalog change, not on every + // restart — because a recorded fingerprint means it already came back clean. + let drift = await detectColumnDrift(db, streams, schema) + let repaired: ColumnDrift[] = [] + const failures = new Map() + if (options.repair && drift.some((d) => unrepairableReason(d) === null)) { + // A recorded fingerprint means "the live columns were checked against + // this catalog". Redefining a column breaks that promise for every + // catalog recorded before now, so every other fingerprint is forgotten, + // before the first statement rather than after the last, and a rollback + // re-runs detection. + await db.command( + `ALTER TABLE ${s}.\`_migrations\` DELETE WHERE fingerprint != {fp:String}`, + { fp }, + { ...KEEPER_SETTINGS, keeper_map_strict_mode: 0 }, + ) + const r = await repairColumnDrift(db, drift, streams, schema, logger) + repaired = r.repaired + for (const f of r.failed) failures.set(f.drift, f.error) + drift = drift.filter((d) => !repaired.includes(d)) + } + + for (const d of drift) { + const fields = { stream: d.stream, column: d.column, expected: d.expected, actual: d.actual, key: d.key } + if (d.kind === 'orphan') { + logger?.warn('orphan projected column', fields) + continue + } + const reason = failures.get(d) ?? unrepairableReason(d) ?? 'run `init --repair`, or apply the DDL by hand' + logger?.error('column drift: projection does not match the catalog', { ...fields, kind: d.kind, reason }) + } + + const unresolved = drift.filter((d) => d.kind !== 'orphan') + if (unresolved.length > 0) { + logger?.error('schema fingerprint not recorded: projected columns have drifted', { + fingerprint: fp, + columns: unresolved.length, + }) + return { applied: true, fingerprint: fp, statements, drift, repaired } + } + + await db.command( + `INSERT INTO ${s}.\`_migrations\` (fingerprint, applied_at, detail) VALUES ({fp:String}, now64(3, 'UTC'), {detail:String})`, + { fp, detail: JSON.stringify({ streams: streams.map((st) => st.name), statements }) }, + { ...KEEPER_SETTINGS, keeper_map_strict_mode: 0 }, + ) + + logger?.info('schema applied', { fingerprint: fp, statements, repaired: repaired.length }) + return { applied: true, fingerprint: fp, statements, drift, repaired } +} diff --git a/packages/sync-clickhouse/src/schema/views.ts b/packages/sync-clickhouse/src/schema/views.ts new file mode 100644 index 0000000..26619af --- /dev/null +++ b/packages/sync-clickhouse/src/schema/views.ts @@ -0,0 +1,91 @@ +/** + * Live views. + * + * Every query against a mirror table has to remember `FINAL` and `WHERE + * _deleted_at IS NULL`, and forgetting either is the most common query + * mistake: `FINAL` left out counts every version a row ever had, the filter + * left out counts users who left. Ours are generated from the catalog, in + * the same pass as the tables, so they can never drift from them. + * + * ClickHouse fixes a view's column list when it is created, so a column + * promoted later only reaches the view once it is re-created; `migrate` + * does that whenever the catalog changes. The engine recognises its own + * views by the query `system.tables` stores for them: `select * from + *
final where _deleted_at is null` is ours and is replaced freely, + * anything else under that name belongs to somebody and is left alone + * unless `repairViews` says to replace it. + */ +import type { Logger, TableDef } from '@workos/sync-engine' +import type { ChClient } from '../db/client.js' +import { quoteIdent, safeIdentifier } from './identifiers.js' + +export const LIVE_VIEW_SUFFIX = '_live' + +/** `_live`, shortened the same way table names are. */ +export function liveViewName(stream: TableDef): string { + return safeIdentifier(`${stream.name}${LIVE_VIEW_SUFFIX}`) +} + +/** + * DDL for one stream's live view. Empty for append-only streams: they have + * no versions to collapse and no `_deleted_at` to filter, so a view would + * only suggest there was. + */ +export function liveViewSql(stream: TableDef, schema: string): string[] { + if (stream.mode === 'append') return [] + const s = quoteIdent(schema) + const table = quoteIdent(safeIdentifier(stream.name)) + const view = quoteIdent(liveViewName(stream)) + return [`CREATE OR REPLACE VIEW ${s}.${view} AS SELECT * FROM ${s}.${table} FINAL WHERE \`_deleted_at\` IS NULL`] +} + +export interface ApplyViewOptions { + /** + * Replace a view under a generated name that is not the generated shape. + * Off, such a view fails `init` and is reported, because replacing it + * would silently change what its users see. + */ + repairViews?: boolean +} + +/** Whether a stored view query is the generated shape over `table`. */ +export function isGeneratedDefinition(query: string, schema: string, table: string): boolean { + const normalized = query.replace(/`/g, '').replace(/\s+/g, ' ').trim().toLowerCase() + return normalized.endsWith(`from ${schema}.${table} final where _deleted_at is null`.toLowerCase()) +} + +/** + * Create or refresh a stream's live view, refusing to overwrite one that is + * not the engine's. Returns whether a statement ran. + */ +export async function applyLiveView( + db: ChClient, + stream: TableDef, + schema: string, + logger: Logger | undefined, + options: ApplyViewOptions = {}, +): Promise { + const [sql] = liveViewSql(stream, schema) + if (!sql) return false + const name = liveViewName(stream) + const existing = await db.query<{ engine: string; as_select: string }>( + `SELECT engine, as_select FROM system.tables WHERE database = {schema:String} AND name = {name:String}`, + { schema, name }, + ) + const found = existing[0] + if (found && found.engine !== 'View') { + throw new Error( + `${schema}.${name} is a table, where the engine generates a view over ${stream.name}: rename or drop it, or exclude the stream`, + ) + } + if (found && !isGeneratedDefinition(found.as_select, schema, safeIdentifier(stream.name))) { + if (!options.repairViews) { + throw new Error( + `${schema}.${name} is a view the engine did not generate; run init --repair to replace it with the generated definition, or rename it`, + ) + } + logger?.warn('replacing a view that is not the generated shape', { view: name, previous: found.as_select }) + } + await db.command(sql) + return true +} diff --git a/packages/sync-clickhouse/test/cli.test.ts b/packages/sync-clickhouse/test/cli.test.ts new file mode 100644 index 0000000..9c8996e --- /dev/null +++ b/packages/sync-clickhouse/test/cli.test.ts @@ -0,0 +1,298 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { SyncEngine, WorkOSClient, selectStreams, silentLogger } from '@workos/sync-engine' +import { + main, + parseHydrate, + parseExcludeStreams, + parseLeaseTtl, + parseMaxPages, + parseSslCa, + printSchemaSql, + repairRequested, +} from '../src/cli.js' +import { fingerprint, migrate } from '../src/schema/migrate.js' +import { createTestDb, describeDb, type TestDb } from './helpers/clickhouse.js' +import { FaultyDestination } from './helpers/faulty.js' +import { TEST_CA_PEM } from './helpers/ca.js' +import { FakeWorkOS, resetIds } from '../../sync-engine/test/helpers/fake-workos.js' + +describe('WORKOS_SYNC_HYDRATE', () => { + it('leaves the engine default in place when unset', () => { + expect(parseHydrate(undefined)).toBeUndefined() + expect(parseHydrate('')).toBeUndefined() + }) + + it('accepts one mode for every stream, or per-stream overrides', () => { + expect(parseHydrate('always')).toBe('always') + expect(parseHydrate('users=off, organizations=off')).toEqual({ users: 'off', organizations: 'off' }) + }) + + it('rejects anything else loudly', () => { + expect(() => parseHydrate('sometimes')).toThrow(/invalid WORKOS_SYNC_HYDRATE/) + }) +}) + +describe('backfill --max-pages', () => { + it('means unbounded when absent and accepts a positive integer', () => { + expect(parseMaxPages(undefined)).toBeUndefined() + expect(parseMaxPages('25')).toBe(25) + }) + + it('refuses a bare flag, zero, a fraction, and anything that is not a number', () => { + for (const bad of [true, '', '0', '-3', '1.5', 'ten']) + expect(() => parseMaxPages(bad)).toThrow(/invalid --max-pages/) + }) +}) + +describe('reconcile --exclude-streams', () => { + it('means nothing excluded when absent and splits a list', () => { + expect(parseExcludeStreams(undefined)).toBeUndefined() + expect(parseExcludeStreams('users, invitations')).toEqual(['users', 'invitations']) + }) + + it('refuses a bare flag: an exclusion of nothing is the mistake it exists to prevent', () => { + expect(() => parseExcludeStreams(true)).toThrow(/invalid --exclude-streams/) + expect(() => parseExcludeStreams(' , ')).toThrow(/invalid --exclude-streams/) + }) +}) + +describe('init --repair', () => { + it('asks for repair on init and nothing else', () => { + expect(repairRequested('init', { repair: true })).toBe(true) + expect(repairRequested('reconcile', { repair: true })).toBe(false) + }) + + it('is in the help text', async () => { + const out: string[] = [] + const spy = vi.spyOn(process.stdout, 'write').mockImplementation(((chunk: string) => { + out.push(String(chunk)) + return true + }) as typeof process.stdout.write) + try { + expect(await main(['help'])).toBe(0) + } finally { + spy.mockRestore() + } + const help = out.join('') + expect(help).toContain('workos-sync-clickhouse') + expect(help).toContain('init [--repair] [--print-sql]') + expect(help).toContain('reconcile [--repair] [--resume] [--max-pages ] [--exclude-streams ]') + expect(help).toContain('WORKOS_SYNC_CLICKHOUSE_TLS_CA') + expect(help).not.toMatch(/MySQL|Postgres/) + }) +}) + +describe('WORKOS_SYNC_LEASE_TTL_MS', () => { + it('accepts a number of milliseconds and refuses a value that would expire on arrival', () => { + expect(parseLeaseTtl(undefined)).toBeUndefined() + expect(parseLeaseTtl('250')).toBe(250) + for (const bad of ['0', '-1', 'abc']) expect(() => parseLeaseTtl(bad)).toThrow(/invalid WORKOS_SYNC_LEASE_TTL_MS/) + }) +}) + +describe('WORKOS_SYNC_CLICKHOUSE_TLS_CA', () => { + it('accepts PEM as is or as base64, and refuses anything that is not a certificate', () => { + expect(parseSslCa(undefined)).toBeUndefined() + expect(parseSslCa(TEST_CA_PEM)).toBe(TEST_CA_PEM.trim()) + expect(parseSslCa(Buffer.from(TEST_CA_PEM).toString('base64'))).toBe(TEST_CA_PEM) + expect(() => parseSslCa('not a cert')).toThrow(/invalid WORKOS_SYNC_CLICKHOUSE_TLS_CA/) + }) +}) + +describe('init --print-sql', () => { + let out: string[] + let spy: ReturnType + + beforeEach(() => { + out = [] + spy = vi.spyOn(process.stdout, 'write').mockImplementation(((chunk: string) => { + out.push(String(chunk)) + return true + }) as typeof process.stdout.write) + }) + afterEach(() => { + spy.mockRestore() + }) + + it('prints the DDL init would run, without a database or an API key', async () => { + const saved = { url: process.env['DATABASE_URL'], key: process.env['WORKOS_API_KEY'] } + delete process.env['DATABASE_URL'] + delete process.env['WORKOS_API_KEY'] + try { + expect(await main(['init', '--print-sql'])).toBe(0) + } finally { + if (saved.url !== undefined) process.env['DATABASE_URL'] = saved.url + if (saved.key !== undefined) process.env['WORKOS_API_KEY'] = saved.key + } + const sql = out.join('') + expect(sql).toBe(printSchemaSql(undefined, 'workos')) + expect(sql).toContain('CREATE DATABASE IF NOT EXISTS `workos`') + expect(sql).toContain('CREATE TABLE IF NOT EXISTS `workos`.`users`') + expect(sql).toContain('CREATE OR REPLACE VIEW `workos`.`users_live`') + expect(sql).toContain(`fingerprint ${fingerprint(selectStreams(), 'workos')}`) + expect(sql).not.toMatch(/INSERT INTO [^\n]*_migrations/) + }) + + it('honors --schema and --streams the way init does', async () => { + expect(await main(['init', '--print-sql', '--schema', 'mirror', '--streams', 'users'])).toBe(0) + const sql = out.join('') + expect(sql).toContain('CREATE TABLE IF NOT EXISTS `mirror`.`users`') + expect(sql).toContain('CREATE TABLE IF NOT EXISTS `mirror`.`events`') + expect(sql).not.toContain('`organizations`') + }) + + it('refuses a stream the catalog does not know, like init', async () => { + await expect(main(['init', '--print-sql', '--streams', 'customers'])).rejects.toThrow(/unknown stream: customers/) + }) + + describeDb('applied by hand', () => { + let db: TestDb + beforeEach(async () => { + db = await createTestDb() + }) + afterEach(async () => { + await db.close() + }) + + it('prints a script a server accepts twice, after which init records the fingerprint', async () => { + expect(await main(['init', '--print-sql', '--schema', db.schema])).toBe(0) + const statements = out + .join('') + .split(/;\n\n/) + .map((s) => + s + .replace(/^--.*$/gm, '') + .trim() + .replace(/;$/, ''), + ) + .filter(Boolean) + for (const sql of statements) await db.command(sql) + for (const sql of statements) await db.command(sql) + const result = await migrate(db, selectStreams(), db.schema) + expect(result.applied).toBe(true) + expect(result.drift).toEqual([]) + const recorded = await db.query<{ fingerprint: string }>( + `SELECT fingerprint FROM \`${db.schema}\`.\`_migrations\``, + ) + expect(recorded).toEqual([{ fingerprint: fingerprint(selectStreams(), db.schema) }]) + }) + }) +}) + +describeDb('commands against a server', () => { + let db: TestDb + let fake: FakeWorkOS + let out: string[] + + beforeEach(async () => { + db = await createTestDb() + resetIds() + fake = new FakeWorkOS() + out = [] + vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => { + out.push(String(chunk)) + return true + }) + }) + afterEach(async () => { + vi.restoreAllMocks() + await db.close() + }) + + const lastJson = () => JSON.parse(out[out.length - 1]!) as Record + const makeEngine = (destination: FaultyDestination, streams = ['users']) => + new SyncEngine( + { + destination, + environments: [{ name: 'prod', apiKey: 'sk_test_1' }], + streams, + hydrate: 'off', + logger: silentLogger, + lagWindowMs: 0, + }, + { + clientFor: () => + new WorkOSClient({ apiKey: 'sk_test_1', apiUrl: 'http://fake.workos.local', fetchImpl: fake.fetchImpl }), + }, + ) + + it('exits 2 while quarantined events exist, and 0 once they replay', async () => { + fake.createUser({ email: 'good@example.com' }) + const bad = fake.createUser({ email: 'bad@example.com' }) + const destination = new FaultyDestination({ db, schema: db.schema }) + destination.poison = bad['id'] as string + const engine = makeEngine(destination) + + expect(await main(['rejected'], { engine })).toBe(0) + await engine.tick() + await engine.tick() + expect(await main(['rejected'], { engine })).toBe(2) + const listed = lastJson() as { prod: { eventId: string; eventType: string; error: string }[] } + expect(listed.prod).toHaveLength(1) + expect(listed.prod[0]!.error).toMatch(/Cannot parse string/) + + expect(await main(['rejected', '--replay'], { engine })).toBe(2) + destination.poison = null + expect(await main(['rejected', '--replay', '--id', listed.prod[0]!.eventId], { engine })).toBe(0) + expect(lastJson()).toEqual({ replayed: 1, stillRejected: [] }) + expect(await main(['rejected'], { engine })).toBe(0) + + const users = await db.query<{ email: string }>(`SELECT email FROM \`${db.schema}\`.users_live ORDER BY email`) + expect(users.map((r) => r.email)).toEqual(['bad@example.com', 'good@example.com']) + }) + + it('runs backfill, status, reconcile and streams end to end', async () => { + fake.createUser({ email: 'ada@example.com' }) + const engine = makeEngine(new FaultyDestination({ db, schema: db.schema }), ['users', 'organizations']) + + expect(await main(['backfill'], { engine })).toBe(0) + const backfill = lastJson() as unknown as { stream: string; complete: boolean }[] + expect(backfill.find((s) => s.stream === 'users')).toMatchObject({ complete: true }) + + expect(await main(['status'], { engine })).toBe(0) + const status = lastJson() as Record + expect(status['prod']!.streams.find((s) => s.stream === 'users')!.rows).toBe(1) + expect(status['prod']!.rejected).toBe(0) + + fake.users.clear() + expect(await main(['reconcile'], { engine })).toBe(2) + expect(await main(['reconcile', '--repair'], { engine })).toBe(0) + expect(await main(['reconcile'], { engine })).toBe(0) + + out = [] + expect(await main(['streams'])).toBe(0) + expect((JSON.parse(out.join('')) as { stream: string }[]).some((s) => s.stream === 'users')).toBe(true) + }) + + it('leaves out --exclude-streams on reconcile, and refuses a name the catalog does not know', async () => { + fake.createUser() + const engine = makeEngine(new FaultyDestination({ db, schema: db.schema }), ['users', 'organizations']) + await engine.backfill() + + expect(await main(['reconcile', '--exclude-streams', 'users'], { engine })).toBe(0) + const names = (lastJson() as { diffs: Record }).diffs['prod']!.map((d) => d.stream) + expect(names).not.toContain('users') + expect(names).toContain('organizations') + + await expect(main(['reconcile', '--exclude-streams', 'customers'], { engine })).rejects.toThrow( + /unknown stream: customers/, + ) + }) + + it('reports a stream reconcile abandoned as oversized, in the diff and in status', async () => { + for (let i = 0; i < 250; i++) fake.createUser() + const engine = makeEngine(new FaultyDestination({ db, schema: db.schema })) + await engine.backfill() + + // Three pages upstream, two budgeted: abandoned on the third, exit 0. + expect(await main(['reconcile', '--max-pages', '2'], { engine })).toBe(0) + const users = ( + lastJson() as { + diffs: Record + } + ).diffs['prod']!.find((d) => d.stream === 'users') + expect(users).toMatchObject({ pages: 3, complete: false, oversized: true }) + expect(await main(['status'], { engine })).toBe(0) + expect(lastJson()).toMatchObject({ prod: { oversized: ['users'] } }) + }) +}) diff --git a/packages/sync-clickhouse/test/clickhouse/config.xml b/packages/sync-clickhouse/test/clickhouse/config.xml new file mode 100644 index 0000000..eafd6dc --- /dev/null +++ b/packages/sync-clickhouse/test/clickhouse/config.xml @@ -0,0 +1,30 @@ + + + + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + 10000 + 30000 + + + 1localhost9234 + + + + localhost9181 + + /keeper_map_tables + disabled + diff --git a/packages/sync-clickhouse/test/clickhouse/users.xml b/packages/sync-clickhouse/test/clickhouse/users.xml new file mode 100644 index 0000000..a394182 --- /dev/null +++ b/packages/sync-clickhouse/test/clickhouse/users.xml @@ -0,0 +1,8 @@ + + + + + 0 + + + diff --git a/packages/sync-clickhouse/test/client.test.ts b/packages/sync-clickhouse/test/client.test.ts new file mode 100644 index 0000000..d030b4f --- /dev/null +++ b/packages/sync-clickhouse/test/client.test.ts @@ -0,0 +1,79 @@ +import { describe, it, expect } from 'vitest' +import { + clientConfig, + fromClient, + connect, + toCh, + fromCh, + isoOf, + APPLICATION_NAME, + DEFAULT_SETTINGS, +} from '../src/db/client.js' +import { describeDb, TEST_CLICKHOUSE_URL } from './helpers/clickhouse.js' + +describe('clientConfig', () => { + it('splits the URL into origin and credentials, and drops the path', () => { + const cfg = clientConfig('https://me:p%40ss@host.example:8443/some_db?x=1') + expect(cfg).toMatchObject({ + url: 'https://host.example:8443', + username: 'me', + password: 'p@ss', + application: APPLICATION_NAME, + }) + expect(cfg['tls']).toBeUndefined() + expect(cfg['clickhouse_settings']).toBe(DEFAULT_SETTINGS) + }) + + it('leaves credentials out when the URL has none', () => { + const cfg = clientConfig('http://127.0.0.1:8123') + expect(cfg['username']).toBeUndefined() + expect(cfg['password']).toBeUndefined() + }) + + it('verifies against a CA over https only', () => { + const cfg = clientConfig('https://host:8443', { sslCa: 'pem' }) + expect((cfg['tls'] as { ca_cert: Buffer }).ca_cert.toString()).toBe('pem') + expect(() => clientConfig('http://host:8123', { sslCa: 'pem' })).toThrow(/https/) + }) + + it('refuses a URL that is not the HTTP interface', () => { + expect(() => clientConfig('clickhouse://host:9000')).toThrow(/http:\/\/ or https:\/\//) + expect(() => clientConfig('not a url')).toThrow() + }) + + it('writes and reads DateTime64 in the one plain form', () => { + expect(toCh(new Date('2026-01-02T03:04:05.678Z'))).toBe('2026-01-02 03:04:05.678') + expect(fromCh('2026-01-02 03:04:05.678').toISOString()).toBe('2026-01-02T03:04:05.678Z') + expect(isoOf('2026-01-02 03:04:05.678123')).toBe('2026-01-02T03:04:05.678Z') + }) + + it('sends no request for an empty insert', async () => { + let inserted = 0 + const client = fromClient({ insert: async () => void inserted++ } as never) + await client.insert('t', []) + expect(inserted).toBe(0) + }) +}) + +describeDb('connect', () => { + it('connects, runs with the engine settings, and tolerates a path in the URL', async () => { + const url = new URL(TEST_CLICKHOUSE_URL!) + url.pathname = '/does_not_exist_' + Date.now() + const db = await connect(url.toString()) + try { + expect(await db.query<{ one: number }>('SELECT 1 AS one')).toEqual([{ one: 1 }]) + const settings = await db.query<{ name: string; value: string }>( + `SELECT name, value FROM system.settings WHERE name IN ('async_insert', 'wait_for_async_insert', 'mutations_sync', 'keeper_map_strict_mode') ORDER BY name`, + ) + expect(settings).toEqual([ + { name: 'async_insert', value: '1' }, + { name: 'keeper_map_strict_mode', value: '1' }, + { name: 'mutations_sync', value: '2' }, + { name: 'wait_for_async_insert', value: '1' }, + ]) + expect(await db.query<{ ua: string }>(`SELECT http_user_agent AS ua FROM system.query_log LIMIT 0`)).toEqual([]) + } finally { + await db.close() + } + }) +}) diff --git a/packages/sync-clickhouse/test/destination.test.ts b/packages/sync-clickhouse/test/destination.test.ts new file mode 100644 index 0000000..9abb901 --- /dev/null +++ b/packages/sync-clickhouse/test/destination.test.ts @@ -0,0 +1,32 @@ +import { describe, it, expect } from 'vitest' +import { ClickHouseDestination } from '../src/destination.js' +import type { ChClient } from '../src/db/client.js' + +const db = {} as ChClient + +describe('ClickHouseDestination options', () => { + it('needs a connection string or a client', () => { + expect(() => new ClickHouseDestination({})).toThrow(/databaseUrl or db is required/) + expect(new ClickHouseDestination({ db }).schema).toBe('workos') + expect(new ClickHouseDestination({ databaseUrl: 'https://u:p@host:8443', schema: 'mirror' }).schema).toBe('mirror') + }) + + it('takes a CA alongside an https connection string', () => { + expect(() => new ClickHouseDestination({ databaseUrl: 'https://host:8443', sslCa: 'pem' })).not.toThrow() + }) + + it('refuses a CA with a pre-built client, which carries its own TLS settings', () => { + expect(() => new ClickHouseDestination({ db, sslCa: 'pem' })).toThrow(/sslCa applies to databaseUrl/) + }) + + it('refuses a schema name that is not a plain identifier', () => { + expect(() => new ClickHouseDestination({ db, schema: '' })).toThrow(/plain identifier/) + expect(() => new ClickHouseDestination({ db, schema: 'with space' })).toThrow(/plain identifier/) + expect(() => new ClickHouseDestination({ db, schema: 'x'.repeat(64) })).toThrow(/plain identifier/) + }) + + it('refuses to be used before init', () => { + const dest = new ClickHouseDestination({ databaseUrl: 'http://localhost:8123' }) + expect(() => dest.getLease('prod')).toThrow(/call init\(\) first/) + }) +}) diff --git a/packages/sync-clickhouse/test/drift.test.ts b/packages/sync-clickhouse/test/drift.test.ts new file mode 100644 index 0000000..b965d92 --- /dev/null +++ b/packages/sync-clickhouse/test/drift.test.ts @@ -0,0 +1,288 @@ +/** + * Projected-column drift, against a real ClickHouse. Detection compares the + * DDL the engine recorded in `_columns` with what the catalog would emit + * now, and the live type with the catalog's; only a real server can show + * that a repair redefines the column for the rows already there. + */ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { createTestDb, describeDb, type TestDb } from './helpers/clickhouse.js' +import { migrate, fingerprint } from '../src/schema/migrate.js' +import { detectColumnDrift, unrepairableReason } from '../src/schema/drift.js' +import { main } from '../src/cli.js' +import { STREAMS, streamByName, type ColumnDef, type Logger, type StreamDef } from '@workos/sync-engine' + +const users = streamByName('users')! + +/** The stream with one column swapped out, the way a catalog release would change it. */ +const withColumn = (stream: StreamDef, col: ColumnDef): StreamDef => ({ + ...stream, + columns: stream.columns.map((c) => (c.name === col.name ? col : c)), +}) + +/** `email` projected from a different key: the classic path change. */ +const usersMovedEmail = withColumn(users, { name: 'email', type: 'text', path: 'primary_email', index: true }) + +const capture = () => { + const lines: { level: string; msg: string; fields?: Record }[] = [] + const at = (level: string) => (msg: string, fields?: Record) => { + lines.push({ level, msg, fields }) + } + const logger: Logger = { debug: at('debug'), info: at('info'), warn: at('warn'), error: at('error') } + return { logger, lines } +} + +describe('unrepairable reasons', () => { + it('names why a column is left alone', () => { + const base = { stream: 'users', column: 'x', expected: 'e', actual: 'a', key: false } + expect(unrepairableReason({ ...base, kind: 'orphan' })).toMatch(/no longer names/) + expect(unrepairableReason({ ...base, kind: 'not_generated' })).toMatch(/not a DEFAULT/) + expect(unrepairableReason({ ...base, kind: 'expression', key: true })).toMatch(/key columns/) + expect(unrepairableReason({ ...base, kind: 'expression' })).toBeNull() + expect(unrepairableReason({ ...base, kind: 'type' })).toBeNull() + }) +}) + +describeDb('column drift detection', () => { + let db: TestDb + + beforeEach(async () => { + db = await createTestDb() + }) + afterEach(async () => { + await db.close() + }) + + const insertUser = (raw: Record) => + db.insert(`\`${db.schema}\`.users`, [ + { _env: 'prod', _raw: JSON.stringify(raw), _updated_at: '2026-01-01 00:00:00.000' }, + ]) + const recorded = async (streams: StreamDef[]) => + ( + await db.query(`SELECT 1 FROM \`${db.schema}\`.\`_migrations\` WHERE fingerprint = {fp:String}`, { + fp: fingerprint(streams, db.schema), + }) + ).length > 0 + const columnExists = async (column: string) => + ( + await db.query( + `SELECT 1 FROM system.columns WHERE database = {s:String} AND table = 'users' AND name = {c:String}`, + { + s: db.schema, + c: column, + }, + ) + ).length > 0 + + it('reports nothing on a fresh schema built from the full catalog', async () => { + await migrate(db, STREAMS, db.schema) + expect(await detectColumnDrift(db, STREAMS, db.schema)).toEqual([]) + }) + + it('reports a changed path as expression drift, with both clauses', async () => { + await migrate(db, [users], db.schema) + const r = await migrate(db, [usersMovedEmail], db.schema) + expect(r.repaired).toEqual([]) + expect(r.drift).toHaveLength(1) + expect(r.drift[0]).toMatchObject({ stream: 'users', column: 'email', kind: 'expression', key: false }) + expect(r.drift[0]!.expected).toContain("'primary_email'") + expect(r.drift[0]!.actual).toContain("'email'") + }) + + it('reports a changed type as type drift', async () => { + await migrate(db, [users], db.schema) + const changed = withColumn(users, { name: 'first_name', type: 'boolean', path: 'first_name' }) + const r = await migrate(db, [changed], db.schema) + expect(r.drift).toHaveLength(1) + expect(r.drift[0]).toMatchObject({ column: 'first_name', kind: 'type', actual: 'Nullable(String)' }) + expect(r.drift[0]!.expected).toContain('`first_name` Nullable(Bool)') + }) + + it('withholds the fingerprint while drift is unrepaired, and records it once repaired', async () => { + await migrate(db, [users], db.schema) + await migrate(db, [usersMovedEmail], db.schema) + expect(await recorded([usersMovedEmail])).toBe(false) + const again = await migrate(db, [usersMovedEmail], db.schema) + expect(again.applied).toBe(true) + expect(again.drift).toHaveLength(1) + const fixed = await migrate(db, [usersMovedEmail], db.schema, undefined, { repair: true }) + expect(fixed.drift).toEqual([]) + expect(fixed.repaired).toHaveLength(1) + expect(await recorded([usersMovedEmail])).toBe(true) + expect((await migrate(db, [usersMovedEmail], db.schema)).applied).toBe(false) + }) + + it('repairs non-key drift in place and recomputes rows already present', async () => { + await migrate(db, [users], db.schema) + await insertUser({ id: 'user_1', email: 'old@example.com', primary_email: 'new@example.com', first_name: true }) + const changed = withColumn(usersMovedEmail, { name: 'first_name', type: 'boolean', path: 'first_name' }) + const r = await migrate(db, [changed], db.schema, undefined, { repair: true }) + expect(r.repaired.map((d) => [d.column, d.kind])).toEqual([ + ['email', 'expression'], + ['first_name', 'type'], + ]) + expect(r.drift).toEqual([]) + const row = await db.query<{ email: string; first_name: boolean }>( + `SELECT email, first_name FROM \`${db.schema}\`.users FINAL`, + ) + expect(row[0]).toEqual({ email: 'new@example.com', first_name: true }) + // The skipping index on the column survives a redefinition. + const idx = await db.query<{ name: string }>( + `SELECT name FROM system.data_skipping_indices WHERE database = {s:String} AND table = 'users'`, + { s: db.schema }, + ) + expect(idx.map((i) => i.name)).toContain('users_email_idx') + expect(await detectColumnDrift(db, [changed], db.schema)).toEqual([]) + expect(await recorded([changed])).toBe(true) + }) + + it('refuses to repair a key column', async () => { + await migrate(db, [users], db.schema) + const rekeyed = withColumn(users, { name: 'id', type: 'text', path: 'uid' }) + const { logger, lines } = capture() + const r = await migrate(db, [rekeyed], db.schema, logger, { repair: true }) + expect(r.repaired).toEqual([]) + expect(r.drift).toHaveLength(1) + expect(r.drift[0]).toMatchObject({ column: 'id', kind: 'expression', key: true }) + expect(await recorded([rekeyed])).toBe(false) + const logged = lines.find((l) => l.level === 'error' && l.msg.startsWith('column drift')) + expect(String(logged?.fields?.['reason'])).toMatch(/key columns are never rebuilt/) + }) + + it('reports an orphan and leaves it alone, even when repairing', async () => { + const extended: StreamDef = { + ...users, + columns: [...users.columns, { name: 'nickname', type: 'text', path: 'nickname' }], + } + await migrate(db, [extended], db.schema) + const { logger, lines } = capture() + const r = await migrate(db, [users], db.schema, logger, { repair: true }) + expect(r.repaired).toEqual([]) + expect(r.drift).toHaveLength(1) + expect(r.drift[0]).toMatchObject({ + stream: 'users', + column: 'nickname', + kind: 'orphan', + expected: null, + key: false, + }) + expect(await columnExists('nickname')).toBe(true) + expect(lines.some((l) => l.level === 'warn' && l.msg === 'orphan projected column')).toBe(true) + expect(await recorded([users])).toBe(true) + }) + + it('forgets a record whose column was dropped by hand', async () => { + const extended: StreamDef = { + ...users, + columns: [...users.columns, { name: 'nickname', type: 'text', path: 'nickname' }], + } + await migrate(db, [extended], db.schema) + await db.command(`ALTER TABLE \`${db.schema}\`.users DROP COLUMN nickname`) + expect(await detectColumnDrift(db, [users], db.schema)).toEqual([]) + const left = await db.query(`SELECT 1 FROM \`${db.schema}\`.\`_columns\` WHERE column_name = 'nickname'`) + expect(left).toEqual([]) + }) + + it('reports a column of yours that is not a DEFAULT expression, and never touches it', async () => { + await migrate(db, [{ ...users, columns: users.columns.filter((c) => c.name !== 'locale') }], db.schema) + await db.command(`ALTER TABLE \`${db.schema}\`.users ADD COLUMN locale String`) + const r = await migrate(db, [users], db.schema, undefined, { repair: true }) + expect(r.repaired).toEqual([]) + expect(r.drift).toHaveLength(1) + expect(r.drift[0]).toMatchObject({ column: 'locale', kind: 'not_generated', actual: 'String' }) + expect(await recorded([users])).toBe(false) + }) + + it('reports a live type changed by hand, whatever the record says', async () => { + await migrate(db, [users], db.schema) + await db.command( + `ALTER TABLE \`${db.schema}\`.users MODIFY COLUMN locale String DEFAULT JSONExtractString(\`_raw\`, 'locale')`, + ) + const drift = await detectColumnDrift(db, [users], db.schema) + expect(drift).toHaveLength(1) + expect(drift[0]).toMatchObject({ column: 'locale', kind: 'type', actual: 'String' }) + await db.command( + `ALTER TABLE \`${db.schema}\`.\`_migrations\` DELETE WHERE 1`, + {}, + { async_insert: 0, keeper_map_strict_mode: 0 }, + ) + const r = await migrate(db, [users], db.schema, undefined, { repair: true }) + expect(r.repaired.map((d) => [d.column, d.kind])).toEqual([['locale', 'type']]) + expect(await detectColumnDrift(db, [users], db.schema)).toEqual([]) + expect(await recorded([users])).toBe(true) + }) + + it('forgets earlier fingerprints before a repair runs any DDL', async () => { + await migrate(db, [users], db.schema) + await migrate(db, [usersMovedEmail], db.schema, undefined, { repair: true }) + expect(await recorded([users])).toBe(false) + expect(await recorded([usersMovedEmail])).toBe(true) + // Rolled back to the old catalog: the live column is still the new one. + const r = await migrate(db, [users], db.schema) + expect(r.applied).toBe(true) + expect(r.drift.map((d) => [d.column, d.kind])).toEqual([['email', 'expression']]) + }) + + it('keeps earlier fingerprints when a change is only additive', async () => { + await migrate(db, [users], db.schema) + const extended: StreamDef = { + ...users, + columns: [...users.columns, { name: 'nickname', type: 'text', path: 'nickname' }], + } + await migrate(db, [extended], db.schema) + expect(await recorded([users])).toBe(true) + expect(await recorded([extended])).toBe(true) + }) +}) + +describeDb('init via the CLI', () => { + let db: TestDb + let out: string[] + + beforeEach(async () => { + db = await createTestDb() + out = [] + vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => { + out.push(String(chunk)) + return true + }) + process.env['WORKOS_API_KEY'] = 'sk_test_1' + process.env['WORKOS_SYNC_SCHEMA'] = db.schema + }) + afterEach(async () => { + vi.restoreAllMocks() + delete process.env['WORKOS_API_KEY'] + delete process.env['WORKOS_SYNC_SCHEMA'] + await db.close() + }) + + it('exits 2 on drift, repairs with --repair, then is clean', async () => { + expect(await main(['init', '--streams', 'users'], { db })).toBe(0) + expect(out.join('')).toContain('schema ready') + // A release moved a column: simulate by rewriting the record. + await db.command( + `ALTER TABLE \`${db.schema}\`.\`_columns\` UPDATE ddl = 'something else' WHERE column_name = 'email'`, + {}, + { async_insert: 0, keeper_map_strict_mode: 0 }, + ) + await db.command( + `ALTER TABLE \`${db.schema}\`.\`_migrations\` DELETE WHERE 1`, + {}, + { async_insert: 0, keeper_map_strict_mode: 0 }, + ) + + out = [] + expect(await main(['init', '--streams', 'users'], { db })).toBe(2) + const report = JSON.parse(out.join('')) as { drift: { column: string; kind: string }[]; repaired: unknown[] } + expect(report.drift.map((d) => [d.column, d.kind])).toEqual([['email', 'expression']]) + + out = [] + expect(await main(['init', '--streams', 'users', '--repair'], { db })).toBe(0) + const fixed = JSON.parse(out[0]!) as { drift: unknown[]; repaired: { column: string }[] } + expect(fixed.repaired.map((d) => d.column)).toEqual(['email']) + expect(out.join('')).toContain('schema ready') + + out = [] + expect(await main(['init', '--streams', 'users'], { db })).toBe(0) + expect(out.join('')).toBe('schema ready\n') + }) +}) diff --git a/packages/sync-clickhouse/test/engine.test.ts b/packages/sync-clickhouse/test/engine.test.ts new file mode 100644 index 0000000..e4d36b5 --- /dev/null +++ b/packages/sync-clickhouse/test/engine.test.ts @@ -0,0 +1,665 @@ +/** + * Whole-engine tests: backfill, tail, drift and repair, driven against the + * fake WorkOS API and a real ClickHouse. The engine's own behaviour is + * covered exhaustively in `@workos/sync-postgres`; what is exercised here is + * every path through this destination, with the same fake, so that a + * ClickHouse-only difference in a write cannot hide behind a unit test. + */ +import { it, expect, beforeEach, afterEach } from 'vitest' +import { SyncEngine, WorkOSClient, LeaseHeldError, silentLogger } from '@workos/sync-engine' +import { createTestDb, describeDb, type TestDb } from './helpers/clickhouse.js' +import { FakeWorkOS, resetIds, id } from '../../sync-engine/test/helpers/fake-workos.js' +import { ClickHouseDestination } from '../src/destination.js' +import { FaultyDestination, captureLogger } from './helpers/faulty.js' + +const TIER1 = [ + 'users', + 'organizations', + 'organization_memberships', + 'organization_domains', + 'invitations', + 'events', + 'auth_attempts', +] + +function makeEngine(db: TestDb, fake: FakeWorkOS, streams = TIER1, extra = {}) { + return new SyncEngine( + { + destination: new ClickHouseDestination({ db, schema: db.schema }), + environments: [{ name: 'prod', apiKey: 'sk_test_1' }], + streams, + logger: silentLogger, + lagWindowMs: 0, + ...extra, + }, + { + clientFor: () => + new WorkOSClient({ + apiKey: 'sk_test_1', + apiUrl: 'http://fake.workos.local', + fetchImpl: fake.fetchImpl, + pageSize: 100, + }), + }, + ) +} + +const count = async (db: TestDb, table: string, final = true) => + Number( + ( + await db.query<{ n: string }>( + `SELECT toString(count()) AS n FROM \`${db.schema}\`.\`${table}\`${final ? ' FINAL' : ''}`, + ) + )[0]!.n, + ) +const emails = async (db: TestDb, table = 'users_live') => + (await db.query<{ email: string }>(`SELECT email FROM \`${db.schema}\`.\`${table}\` ORDER BY email`)).map( + (x) => x.email, + ) + +describeDb('sync engine', () => { + let db: TestDb + let fake: FakeWorkOS + + beforeEach(async () => { + db = await createTestDb() + resetIds() + fake = new FakeWorkOS() + }) + afterEach(async () => { + await db.close() + }) + + it('backfills existing state, including the field events omit', async () => { + const org = fake.createOrganization({ name: 'Acme' }) + const user = fake.createUser({ email: 'ada@acme.com' }) + fake.createMembership(user['id'] as string, org['id'] as string, 'admin') + const engine = makeEngine(db, fake) + await engine.backfill() + + expect(await count(db, 'users')).toBe(1) + expect(await count(db, 'organizations')).toBe(1) + expect(await count(db, 'organization_memberships')).toBe(1) + const u = await db.query<{ email: string; external_id: string; metadata: string }>( + `SELECT email, external_id, metadata FROM \`${db.schema}\`.users_live`, + ) + expect(u[0]!.email).toBe('ada@acme.com') + expect(u[0]!.external_id).toMatch(/^ext_/) + expect(JSON.parse(u[0]!.metadata)).toEqual({ plan: 'pro' }) + const o = await db.query<{ allow: boolean | null }>( + `SELECT allow_profiles_outside_organization AS allow FROM \`${db.schema}\`.organizations_live`, + ) + expect(o[0]!.allow).toBe(false) + const m = await db.query<{ role_slug: string }>( + `SELECT role_slug FROM \`${db.schema}\`.organization_memberships_live`, + ) + expect(m[0]!.role_slug).toBe('admin') + }) + + it('applies live events after backfill', async () => { + const user = fake.createUser({ email: 'before@example.com' }) + const engine = makeEngine(db, fake) + await engine.backfill() + fake.updateUser(user['id'] as string, { email: 'after@example.com' }) + fake.createOrganization({ name: 'New Co' }) + await engine.tick() + expect(await emails(db)).toEqual(['after@example.com']) + expect((await db.query<{ name: string }>(`SELECT name FROM \`${db.schema}\`.organizations_live`))[0]!.name).toBe( + 'New Co', + ) + }) + + it('does not null out backfilled fields when an event arrives', async () => { + const org = fake.createOrganization({ name: 'Acme' }) + const engine = makeEngine(db, fake) + await engine.backfill() + await engine.tick() + fake.updateOrganization(org['id'] as string, { name: 'Acme Corp' }) + await engine.tick() + const row = await db.query<{ name: string; allow: boolean | null }>( + `SELECT name, allow_profiles_outside_organization AS allow FROM \`${db.schema}\`.organizations_live`, + ) + expect(row[0]).toEqual({ name: 'Acme Corp', allow: false }) + }) + + it('is idempotent when the same events are delivered twice', async () => { + fake = new FakeWorkOS({ duplicateEvents: true }) + fake.createUser() + fake.createOrganization() + const engine = makeEngine(db, fake) + await engine.tick() + await engine.tick() + expect(await count(db, 'users')).toBe(1) + expect(await count(db, 'organizations')).toBe(1) + expect(await count(db, 'events', false)).toBe(2) + }) + + it('converges regardless of the order events are applied in', async () => { + const user = fake.createUser({ email: 'v1@example.com' }) + fake.updateUser(user['id'] as string, { email: 'v2@example.com' }) + fake.updateUser(user['id'] as string, { email: 'v3@example.com' }) + const shuffled = new FakeWorkOS({ shuffleEvents: true }) + shuffled.users = fake.users + shuffled.events = fake.events + const engine = makeEngine(db, shuffled) + await engine.tick() + expect(await emails(db)).toEqual(['v3@example.com']) + }) + + it('tombstones a deleted user, drops it from the live view, and keeps the base row', async () => { + const gone = fake.createUser({ email: 'gone@example.com' }) + fake.createUser({ email: 'stays@example.com' }) + const engine = makeEngine(db, fake) + await engine.backfill() + await engine.tick() + expect(await count(db, 'users_live')).toBe(2) + fake.deleteUser(gone['id'] as string) + await engine.tick() + expect(await count(db, 'users')).toBe(2) + expect(await count(db, 'users_live')).toBe(1) + expect(await emails(db)).toEqual(['stays@example.com']) + }) + + it('records sign-in attempts that exist nowhere else in the API', async () => { + const user = fake.createUser() + fake.emitAuthAttempt(user['id'] as string, true) + fake.emitAuthAttempt(user['id'] as string, false, 'password') + const engine = makeEngine(db, fake) + await engine.tick() + expect(await count(db, 'auth_attempts', false)).toBe(2) + const failed = await db.query<{ n: string }>( + `SELECT toString(countIf(status = 'failed')) AS n FROM \`${db.schema}\`.auth_attempts`, + ) + expect(Number(failed[0]!.n)).toBe(1) + }) + + it('resumes from the stored cursor rather than replaying', async () => { + fake.createUser() + const engine = makeEngine(db, fake) + await engine.tick() + const first = await engine.status() + fake.createUser() + await engine.tick() + expect((await engine.status())['prod']!.cursor).not.toBe(first['prod']!.cursor) + expect(await count(db, 'users')).toBe(2) + expect((await makeEngine(db, fake).tick()).processed).toBe(0) + }) + + it('explodes organization domains out of the organization payload', async () => { + const org = fake.createOrganization({ name: 'Acme' }) + fake.createDomain(org['id'] as string, 'acme.com') + const engine = makeEngine(db, fake) + await engine.backfill() + const d = await db.query<{ domain: string; organization_id: string }>( + `SELECT domain, organization_id FROM \`${db.schema}\`.organization_domains_live`, + ) + expect(d).toEqual([{ domain: 'acme.com', organization_id: org['id'] }]) + }) + + it('never mirrors invitation acceptance tokens, however the row arrives', async () => { + const org = fake.createOrganization() + fake.createInvitation(org['id'] as string, { email: 'new@example.com' }) + const engine = makeEngine(db, fake) + await engine.backfill() + await engine.tick() + const rows = await db.query<{ raw: string }>(`SELECT \`_raw\` AS raw FROM \`${db.schema}\`.invitations_live`) + expect(rows).toHaveLength(1) + expect(JSON.parse(rows[0]!.raw)).not.toHaveProperty('token') + const archived = await db.query<{ raw: string }>( + `SELECT \`_raw\` AS raw FROM \`${db.schema}\`.events WHERE event LIKE 'invitation.%'`, + ) + for (const e of archived) expect(JSON.parse(e.raw)['data']).not.toHaveProperty('token') + }) + + it('keeps two environments apart in one database', async () => { + fake.createUser({ email: 'prod@example.com' }) + const other = new FakeWorkOS() + other.createUser({ email: 'staging@example.com' }) + const engine = new SyncEngine( + { + destination: new ClickHouseDestination({ db, schema: db.schema }), + environments: [ + { name: 'prod', apiKey: 'sk_prod' }, + { name: 'staging', apiKey: 'sk_staging' }, + ], + streams: ['users'], + logger: silentLogger, + lagWindowMs: 0, + }, + { + clientFor: (env) => + new WorkOSClient({ + apiKey: env.apiKey, + apiUrl: 'http://fake.workos.local', + fetchImpl: env.apiKey === 'sk_prod' ? fake.fetchImpl : other.fetchImpl, + }), + }, + ) + await engine.backfill() + const rows = await db.query<{ _env: string; email: string }>( + `SELECT \`_env\`, email FROM \`${db.schema}\`.users_live ORDER BY \`_env\``, + ) + expect(rows).toEqual([ + { _env: 'prod', email: 'prod@example.com' }, + { _env: 'staging', email: 'staging@example.com' }, + ]) + expect(Object.keys(await engine.status()).sort()).toEqual(['prod', 'staging']) + }) + + it('projects extra columns out of _raw without touching the catalog', async () => { + fake.createUser({ email: 'ada@example.com', metadata: { plan: 'pro', seats: 12 } }) + const engine = makeEngine(db, fake, TIER1, { + extraColumns: { + users: [ + { name: 'plan', type: 'text', path: ['metadata', 'plan'], index: true }, + { name: 'seats', type: 'bigint', path: ['metadata', 'seats'] }, + ], + }, + }) + await engine.backfill() + const r = await db.query<{ plan: string; seats: string }>(`SELECT plan, seats FROM \`${db.schema}\`.users_live`) + expect(r[0]!.plan).toBe('pro') + expect(Number(r[0]!.seats)).toBe(12) + const idx = await db.query<{ name: string }>( + `SELECT name FROM system.data_skipping_indices WHERE database = {s:String} AND table = 'users'`, + { s: db.schema }, + ) + expect(idx.map((i) => i.name)).toContain('users_plan_idx') + }) + + it('lands a payload a typed column cannot cast, with the column NULL and the value in _raw', async () => { + fake.createUser({ email: 'good@example.com', metadata: { beta: true } }) + fake.createUser({ email: 'bad@example.com', metadata: { beta: 'maybe' } }) + const engine = makeEngine(db, fake, TIER1, { + extraColumns: { users: [{ name: 'beta', type: 'boolean', path: ['metadata', 'beta'] }] }, + }) + expect(await engine.tick()).toMatchObject({ processed: 2, rejected: 0 }) + const rows = await db.query<{ email: string; beta: boolean | null; raw: string }>( + `SELECT email, beta, \`_raw\` AS raw FROM \`${db.schema}\`.users_live ORDER BY email`, + ) + expect(rows.map((r) => [r.email, r.beta])).toEqual([ + ['bad@example.com', null], + ['good@example.com', true], + ]) + expect(JSON.parse(rows[0]!.raw)['metadata']['beta']).toBe('maybe') + }) + + it('joins mirrored identity data with the sign-in log', async () => { + const org = fake.createOrganization({ name: 'Acme' }) + const user = fake.createUser({ email: 'ada@acme.com' }) + fake.createMembership(user['id'] as string, org['id'] as string, 'admin') + fake.emitAuthAttempt(user['id'] as string, false) + fake.emitAuthAttempt(user['id'] as string, true) + const engine = makeEngine(db, fake) + await engine.backfill() + await engine.tick() + const r = await db.query<{ email: string; organization: string; failures: string }>( + `SELECT u.email AS email, o.name AS organization, toString(countIf(a.status = 'failed')) AS failures + FROM \`${db.schema}\`.auth_attempts a + JOIN \`${db.schema}\`.users_live u ON u.id = a.user_id AND u.\`_env\` = a.\`_env\` + JOIN \`${db.schema}\`.organization_memberships_live m ON m.user_id = u.id AND m.\`_env\` = u.\`_env\` + JOIN \`${db.schema}\`.organizations_live o ON o.id = m.organization_id AND o.\`_env\` = m.\`_env\` + GROUP BY email, organization`, + ) + expect(r).toEqual([{ email: 'ada@acme.com', organization: 'Acme', failures: '1' }]) + }) +}) + +describeDb('reconciliation', () => { + let db: TestDb + let fake: FakeWorkOS + + beforeEach(async () => { + db = await createTestDb() + resetIds() + fake = new FakeWorkOS() + }) + afterEach(async () => { + await db.close() + }) + + it('reports a clean mirror as matching with no drift', async () => { + fake.createUser() + fake.createOrganization() + const engine = makeEngine(db, fake) + await engine.backfill() + const { diffs } = await engine.reconcile() + expect(diffs['prod']!.find((d) => d.stream === 'users')).toMatchObject({ + matched: 1, + localOnly: [], + remoteOnly: [], + }) + }) + + it('detects a missed delete and a missed create, and repairs both', async () => { + const stale = fake.createUser({ email: 'gone@example.com' }) + const engine = makeEngine(db, fake) + await engine.backfill() + fake.users.delete(stale['id'] as string) + const added = fake.createUser({ email: 'added@example.com' }) + fake.events = [] + const first = await engine.reconcile() + const users = first.diffs['prod']!.find((d) => d.stream === 'users')! + expect(users.localOnly).toEqual([stale['id']]) + expect(users.remoteOnly).toEqual([added['id']]) + await engine.reconcile({ repair: true }) + expect(await emails(db)).toEqual(['added@example.com']) + const tombstoned = await db.query<{ email: string }>( + `SELECT email FROM \`${db.schema}\`.users FINAL WHERE \`_deleted_at\` IS NOT NULL`, + ) + expect(tombstoned[0]!.email).toBe('gone@example.com') + expect((await engine.reconcile()).diffs['prod']!.find((d) => d.stream === 'users')).toMatchObject({ + localOnly: [], + remoteOnly: [], + }) + }) + + it('keeps an oversized verdict across runs, and forgets it once the stream completes', async () => { + for (let i = 0; i < 250; i++) fake.createUser() + const engine = makeEngine(db, fake, ['users', 'invitations']) + await engine.backfill() + + // Three pages upstream and two budgeted: abandoned on the third page, + // with the verdict written to the stream's state in `_sync`. + const first = await engine.reconcile({ maxPages: 2 }) + expect(first.diffs['prod']!.find((d) => d.stream === 'users')).toMatchObject({ + pages: 3, + complete: false, + oversized: true, + }) + expect(await engine.status()).toMatchObject({ prod: { oversized: ['users'] } }) + + // A resumed slice under the same budget reads the verdict back and skips the stream. + const second = await engine.reconcile({ resume: true, maxPages: 2 }) + expect(second.diffs['prod']!.find((d) => d.stream === 'users')).toMatchObject({ pages: 0, oversized: true }) + expect(second.diffs['prod']!.find((d) => d.stream === 'invitations')).toMatchObject({ complete: true }) + + // A larger budget diffs it in full and clears the verdict. + const third = await engine.reconcile({ resume: true, maxPages: 3 }) + expect(third.diffs['prod']!.find((d) => d.stream === 'users')).toMatchObject({ complete: true, matched: 250 }) + expect(await engine.status()).toMatchObject({ prod: { oversized: [] } }) + }) + + it('reports freshness and lag per stream', async () => { + fake.createUser() + const engine = makeEngine(db, fake) + await engine.backfill() + const status = await engine.status() + const users = status['prod']!.streams.find((s) => s.stream === 'users')! + expect(users.rows).toBe(1) + expect(users.lastSyncedAt).toBeTruthy() + expect(users.lagSeconds).not.toBeNull() + }) + + it('refreshes fields that event payloads omit', async () => { + fake.createOrganization() + const engine = makeEngine(db, fake) + await engine.tick() + const read = async () => + ( + await db.query<{ allow: boolean | null }>( + `SELECT allow_profiles_outside_organization AS allow FROM \`${db.schema}\`.organizations_live`, + ) + )[0]!.allow + expect(await read()).toBeNull() + await engine.refresh() + expect(await read()).toBe(false) + }) +}) + +describeDb('single-writer lease', () => { + let db: TestDb + let fake: FakeWorkOS + + beforeEach(async () => { + db = await createTestDb() + resetIds() + fake = new FakeWorkOS() + }) + afterEach(async () => { + await db.close() + }) + + it('tick() refuses to run while another process holds the lease', async () => { + fake.createUser() + const holder = makeEngine(db, fake) + await holder.init() + const dest = new ClickHouseDestination({ db, schema: db.schema }) + await dest.init([]) + expect((await dest.acquireLease('prod', 'someone-else', 60_000)).acquired).toBe(true) + await expect(holder.tick()).rejects.toBeInstanceOf(LeaseHeldError) + await dest.releaseLease('prod', 'someone-else') + expect((await holder.tick()).processed).toBe(1) + expect(await dest.getLease('prod')).toBeNull() + }) + + it('listen() waits for the lease, takes over when it is released, and gives it back on stop', async () => { + fake.createUser() + const dest = new ClickHouseDestination({ db, schema: db.schema }) + await dest.init([]) + await dest.acquireLease('prod', 'old-process', 60_000) + const engine = makeEngine(db, fake, TIER1, { pollIntervalMs: 20 }) + await engine.init() + const listening = engine.listen() + await new Promise((r) => setTimeout(r, 200)) + expect(await count(db, 'users')).toBe(0) + await dest.releaseLease('prod', 'old-process') + const deadline = Date.now() + 10_000 + while (Date.now() < deadline && (await count(db, 'users')) < 1) await new Promise((r) => setTimeout(r, 50)) + expect(await count(db, 'users')).toBe(1) + expect((await dest.getLease('prod'))?.holder).not.toBe('old-process') + engine.stop() + await listening + expect(await dest.getLease('prod')).toBeNull() + }) +}) + +describeDb('redaction', () => { + let db: TestDb + let fake: FakeWorkOS + + beforeEach(async () => { + db = await createTestDb() + resetIds() + fake = new FakeWorkOS() + }) + afterEach(async () => { + await db.close() + }) + + it('erases a subject from the mirror, the sign-in log and the event archive', async () => { + const user = fake.createUser({ email: 'forget@example.com' }) + fake.emitAuthAttempt(user['id'] as string, true) + const engine = makeEngine(db, fake) + await engine.backfill() + await engine.tick() + const removed = await engine.redact(user['id'] as string) + expect(removed).toContainEqual({ table: 'users', rows: 1 }) + expect(removed).toContainEqual({ table: 'auth_attempts', rows: 1 }) + expect(removed.find((r) => r.table === 'events')!.rows).toBeGreaterThan(0) + expect(await count(db, 'users', false)).toBe(0) + const events = await db.query<{ n: string }>( + `SELECT toString(count()) AS n FROM \`${db.schema}\`.events WHERE \`_raw\` LIKE {p:String}`, + { p: `%${user['id']}%` }, + ) + expect(Number(events[0]!.n)).toBe(0) + }) +}) + +describeDb('poison events', () => { + let db: TestDb + let fake: FakeWorkOS + + beforeEach(async () => { + db = await createTestDb() + resetIds() + fake = new FakeWorkOS() + }) + afterEach(async () => { + await db.close() + }) + + const eventFor = (objectId: string) => fake.events.find((e) => e.data['id'] === objectId)! + + function makeFaulty(extra: Record = {}) { + const destination = new FaultyDestination({ db, schema: db.schema }) + const { logger, lines } = captureLogger() + const engine = new SyncEngine( + { + destination, + environments: [{ name: 'prod', apiKey: 'sk_test_1' }], + streams: TIER1, + hydrate: 'off', + logger, + lagWindowMs: 0, + ...extra, + }, + { + clientFor: () => + new WorkOSClient({ + apiKey: 'sk_test_1', + apiUrl: 'http://fake.workos.local', + fetchImpl: fake.fetchImpl, + maxRetries: 0, + }), + }, + ) + return { engine, destination, lines } + } + + it('quarantines one event after two sightings and keeps the rest of the environment moving', async () => { + fake.createUser({ email: 'good1@example.com' }) + const bad = fake.createUser({ email: 'bad@example.com' }) + fake.createUser({ email: 'good2@example.com' }) + const badEvent = eventFor(bad['id'] as string) + const { engine, destination, lines } = makeFaulty() + destination.poison = bad['id'] as string + + expect(await engine.tick()).toMatchObject({ processed: 0, rejected: 0 }) + expect(await count(db, '_rejected')).toBe(0) + expect(lines.find((l) => l.level === 'warn' && l.fields['eventId'] === badEvent.id)?.msg).toMatch( + /quarantined if it fails again/, + ) + + expect(await engine.tick()).toMatchObject({ processed: 3, rejected: 1 }) + const status = await engine.status() + expect(status['prod']!.cursor).toBe(fake.events[fake.events.length - 1]!.id) + expect(status['prod']!.rejected).toBe(1) + expect(await emails(db)).toEqual(['good1@example.com', 'good2@example.com']) + const rejected = (await engine.rejected())['prod']! + expect(rejected).toHaveLength(1) + expect(rejected[0]).toMatchObject({ + eventId: badEvent.id, + eventType: 'user.created', + objectId: bad['id'], + raw: badEvent, + }) + expect(rejected[0]!.error).toMatch(/Cannot parse string/) + expect((await engine.tick()).processed).toBe(0) + expect(await count(db, 'events', false)).toBe(3) + }) + + it('replays a quarantined event through the normal path once the cause is fixed', async () => { + fake.createUser({ email: 'good@example.com' }) + const bad = fake.createUser({ email: 'bad@example.com' }) + const { engine, destination } = makeFaulty() + destination.poison = bad['id'] as string + await engine.tick() + await engine.tick() + expect((await engine.status())['prod']!.rejected).toBe(1) + destination.fault = () => Object.assign(new Error('still broken'), { code: '36', type: 'BAD_ARGUMENTS' }) + const again = await engine.replayRejected() + expect(again.replayed).toBe(0) + expect(again.stillRejected).toEqual([{ eventId: eventFor(bad['id'] as string).id, error: 'Error: still broken' }]) + destination.poison = null + expect(await engine.replayRejected()).toEqual({ replayed: 1, stillRejected: [] }) + expect(await count(db, '_rejected')).toBe(0) + expect(await emails(db)).toEqual(['bad@example.com', 'good@example.com']) + }) + + it('never quarantines on an error the destination classifies as transient', async () => { + fake.createUser({ email: 'a@example.com' }) + const { engine, destination } = makeFaulty() + destination.failAll = true + destination.fault = () => + Object.assign(new Error('Keeper session expired'), { code: '999', type: 'KEEPER_EXCEPTION' }) + for (let i = 0; i < 3; i++) expect((await engine.tick()).processed).toBe(0) + expect(await count(db, '_rejected')).toBe(0) + destination.failAll = false + expect((await engine.tick()).processed).toBe(1) + }) + + it('does not resurrect a membership removed after a quarantined link', async () => { + const at = (ms: number) => new Date(Date.now() - 50_000 + ms).toISOString() + const pair = { group_id: 'group_01', organization_membership_id: 'om_01' } + const push = (event: string, created_at: string) => { + const e = { id: id('event_'), event, data: pair, created_at } + fake.events.push(e) + return e + } + push('group.member_added', at(0)) + push('group.member_removed', at(1_000)) + const readd = push('group.member_added', at(2_000)) + push('group.member_removed', at(3_000)) + const { engine, destination } = makeFaulty({ streams: ['group_memberships', 'events'] }) + destination.poison = readd.id + await engine.tick() + expect(await engine.tick()).toMatchObject({ processed: 4, rejected: 1 }) + expect(await count(db, 'group_memberships')).toBe(1) + expect(await count(db, 'group_memberships_live')).toBe(0) + destination.poison = null + expect(await engine.replayRejected()).toEqual({ replayed: 1, stillRejected: [] }) + expect(await count(db, 'group_memberships_live')).toBe(0) + }) + + it('does not hide a membership re-added after a quarantined removal', async () => { + const at = (ms: number) => new Date(Date.now() - 50_000 + ms).toISOString() + const pair = { group_id: 'group_01', organization_membership_id: 'om_01' } + const push = (event: string, created_at: string) => { + const e = { id: id('event_'), event, data: pair, created_at } + fake.events.push(e) + return e + } + push('group.member_added', at(0)) + const removal = push('group.member_removed', at(1_000)) + push('group.member_added', at(2_000)) + const { engine, destination } = makeFaulty({ streams: ['group_memberships', 'events'] }) + destination.poison = removal.id + await engine.tick() + expect(await engine.tick()).toMatchObject({ processed: 3, rejected: 1 }) + expect(await count(db, 'group_memberships_live')).toBe(1) + destination.poison = null + expect(await engine.replayRejected()).toEqual({ replayed: 1, stillRejected: [] }) + expect(await count(db, 'group_memberships_live')).toBe(1) + push('group.member_removed', at(3_000)) + expect(await engine.tick()).toMatchObject({ processed: 1, rejected: 0 }) + expect(await count(db, 'group_memberships_live')).toBe(0) + }) + + it('quarantines an event whose own timestamp cannot be stored, with no fault injected', async () => { + const bad = fake.createUser({ email: 'bad@example.com' }) + const badEvent = eventFor(bad['id'] as string) + badEvent.created_at = '2026-13-01T00:00:00Z' + fake.createUser({ email: 'good@example.com' }) + const { engine } = makeFaulty() + expect((await engine.tick()).processed).toBe(0) + expect(await engine.tick()).toMatchObject({ processed: 2, rejected: 1 }) + const rejected = (await engine.rejected())['prod']! + expect(rejected[0]).toMatchObject({ eventId: badEvent.id, createdAt: '2026-13-01T00:00:00Z' }) + expect(rejected[0]!.error).toMatch(/invalid updatedAt timestamp/) + expect(await emails(db)).toEqual(['good@example.com']) + }) + + it('is scrubbed by redact like the event archive', async () => { + fake.createUser({ email: 'keep@example.com' }) + const subject = fake.createUser({ email: 'forget@example.com' }) + const { engine, destination } = makeFaulty() + destination.poison = subject['id'] as string + await engine.tick() + await engine.tick() + expect(await count(db, '_rejected')).toBe(1) + const removed = await engine.redact(subject['id'] as string) + expect(removed).toContainEqual({ table: '_rejected', rows: 1 }) + expect(await count(db, '_rejected')).toBe(0) + }) +}) diff --git a/packages/sync-clickhouse/test/errors.test.ts b/packages/sync-clickhouse/test/errors.test.ts new file mode 100644 index 0000000..e9938be --- /dev/null +++ b/packages/sync-clickhouse/test/errors.test.ts @@ -0,0 +1,49 @@ +import { describe, it, expect } from 'vitest' +import { isTransientClickHouseError, isKeeperConflict } from '../src/db/errors.js' + +const err = (fields: Record, message = 'x') => Object.assign(new Error(message), fields) + +describe('isTransientClickHouseError', () => { + it('calls timeouts, lost connections, resource pressure and Keeper races transient', () => { + for (const code of ['159', '209', '210', '241', '252', '319', '394', '999', '242']) { + expect(isTransientClickHouseError(err({ code, type: 'SOMETHING' }))).toBe(true) + } + }) + + it('calls verdicts on the statement deterministic: bad values, missing objects, syntax, access', () => { + const cases: [string, string][] = [ + ['6', 'CANNOT_PARSE_TEXT'], + ['27', 'CANNOT_PARSE_INPUT_ASSERTION_FAILED'], + ['36', 'BAD_ARGUMENTS'], + ['47', 'UNKNOWN_IDENTIFIER'], + ['60', 'UNKNOWN_TABLE'], + ['62', 'SYNTAX_ERROR'], + ['497', 'ACCESS_DENIED'], + ] + for (const [code, type] of cases) expect(isTransientClickHouseError(err({ code, type }))).toBe(false) + }) + + it('recognises socket errors and the client-side failures that carry no code', () => { + for (const code of ['ECONNRESET', 'ECONNREFUSED', 'ETIMEDOUT', 'EPIPE']) { + expect(isTransientClickHouseError(err({ code }))).toBe(true) + } + expect(isTransientClickHouseError(new Error('socket hang up'))).toBe(true) + expect(isTransientClickHouseError(new Error('Timeout error.'))).toBe(true) + }) + + it('presumes anything it cannot classify is deterministic', () => { + expect(isTransientClickHouseError(new Error('plain'))).toBe(false) + expect(isTransientClickHouseError(err({ code: 'INVALID_TIMESTAMP' }))).toBe(false) + expect(isTransientClickHouseError(null)).toBe(false) + expect(isTransientClickHouseError('ECONNRESET')).toBe(false) + }) +}) + +describe('isKeeperConflict', () => { + it('names the two strict-mode refusals and nothing else', () => { + expect(isKeeperConflict(err({ code: '999' }, 'Transaction failed (Node exists): Op #0'))).toBe(true) + expect(isKeeperConflict(err({ code: '999' }, 'Transaction failed (Bad version): Op #0'))).toBe(true) + expect(isKeeperConflict(err({ code: '999' }, 'Session expired'))).toBe(false) + expect(isKeeperConflict(err({ code: '6' }, 'Node exists'))).toBe(false) + }) +}) diff --git a/packages/sync-clickhouse/test/helpers/ca.ts b/packages/sync-clickhouse/test/helpers/ca.ts new file mode 100644 index 0000000..0c48621 --- /dev/null +++ b/packages/sync-clickhouse/test/helpers/ca.ts @@ -0,0 +1,17 @@ +/** + * A self-signed certificate for tests that need a value `X509Certificate` + * accepts. Generated once with `openssl req -x509 -newkey ec ...` and its + * private key discarded on the spot, so it can sign nothing and is not a + * secret; it names no real host and is trusted by nothing. + */ +export const TEST_CA_PEM = `-----BEGIN CERTIFICATE----- +MIIBdTCCARygAwIBAgIUYA/j8vmXLoOxBg17MVsxr0WptZgwCgYIKoZIzj0EAwIw +IDEeMBwGA1UEAwwVc3luYy1wb3N0Z3JlcyB0ZXN0IENBMCAXDTI2MTAyOTE2NDE1 +OVoYDzIxMjYxMDA1MTY0MTU5WjAgMR4wHAYDVQQDDBVzeW5jLXBvc3RncmVzIHRl +c3QgQ0EwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAR/tIk0cVsFVh49bOYbZ2UB +blKkXKfCfgj3+6A5B7t+dz+phLfY0PJe/RfKCPnbvQI4ZDO8jkLctAjItiiNjbBm +ozIwMDAdBgNVHQ4EFgQUdosu7nttrBB5We4MUhtGRm7Q/gEwDwYDVR0TAQH/BAUw +AwEB/zAKBggqhkjOPQQDAgNHADBEAiBWaRIW/Ef4zN9nmK01dsPVor+p06ihoxO8 +G/oBUsR6zAIgOJSTKCW/ymLt0xjT42M7xm2AoJximsh0cy3Wbk+mohU= +-----END CERTIFICATE----- +` diff --git a/packages/sync-clickhouse/test/helpers/clickhouse.ts b/packages/sync-clickhouse/test/helpers/clickhouse.ts new file mode 100644 index 0000000..a6c6664 --- /dev/null +++ b/packages/sync-clickhouse/test/helpers/clickhouse.ts @@ -0,0 +1,49 @@ +/** + * A real ClickHouse for the tests, one throwaway database per test. + * + * There is no embedded ClickHouse with a Keeper in it, and KeeperMap needs + * one, so the suite needs a server: `TEST_CLICKHOUSE_URL=http://default@127.0.0.1:8123` + * against a server started with `test/clickhouse/config.xml`. Without it the + * database-backed tests skip themselves (`describeDb`), and + * `pnpm test:database` refuses to run. + * + * Each test gets a database named after a random suffix and drops it on + * `close()`, so tests can run in parallel workers against one server. The + * KeeperMap tables keep their Keeper paths under the database's name, so + * two tests never share a lease. + */ +import { randomBytes } from 'node:crypto' +import { describe } from 'vitest' +import { connect, type ChClient } from '../../src/db/client.js' + +export const TEST_CLICKHOUSE_URL = process.env['TEST_CLICKHOUSE_URL'] + +/** `describe` when a server is configured, `describe.skip` otherwise. */ +export const describeDb = (name: string, fn: () => void): void => { + if (TEST_CLICKHOUSE_URL) describe(name, fn) + else describe.skip(name, fn) +} + +export interface TestDb extends ChClient { + /** The throwaway database this test owns; pass it as the destination's `schema`. */ + schema: string +} + +export async function createTestDb(): Promise { + if (!TEST_CLICKHOUSE_URL) throw new Error('TEST_CLICKHOUSE_URL is not set') + const schema = `workos_test_${randomBytes(6).toString('hex')}` + const client = await connect(TEST_CLICKHOUSE_URL) + return { + schema, + query: client.query.bind(client), + command: client.command.bind(client), + insert: client.insert.bind(client), + async close() { + try { + await client.command(`DROP DATABASE IF EXISTS \`${schema}\` SYNC`) + } finally { + await client.close() + } + }, + } +} diff --git a/packages/sync-clickhouse/test/helpers/faulty.ts b/packages/sync-clickhouse/test/helpers/faulty.ts new file mode 100644 index 0000000..3f1b87c --- /dev/null +++ b/packages/sync-clickhouse/test/helpers/faulty.ts @@ -0,0 +1,65 @@ +/** + * Test doubles for the poison-event quarantine. + * + * `FaultyDestination` is the real ClickHouse destination with one write + * fault injected: it stands in for a write a payload cannot survive, or a + * router bug on one shape, without having to construct the offending data + * for every scenario. The error it throws is configurable so the same + * double can imitate a deterministic failure, a dropped connection, or a + * socket error. + */ +import type { Logger, Row, TableDef, WriteMode, WriteStats } from '@workos/sync-engine' +import { ClickHouseDestination } from '../../src/destination.js' + +export class FaultyDestination extends ClickHouseDestination { + /** Object id whose mirror write fails. `null` means no fault. */ + poison: string | null = null + /** Fail every write, as an outage would. */ + failAll = false + /** + * Fail only multi-row writes: the batch attempt fails, every event succeeds + * alone. That is what an interaction between rows, or a transient error the + * classifier does not know, looks like to the quarantine. + */ + failBatches = false + /** What the fault throws; defaults to the shape of a ClickHouse parse error. */ + fault: () => Error = () => + Object.assign(new Error("Cannot parse string 'maybe' as DateTime64: syntax error"), { + code: '6', + type: 'CANNOT_PARSE_TEXT', + }) + /** How many times the fault fired, so a test can prove isolation happened. */ + faults = 0 + + override async upsert(stream: TableDef, env: string, rows: Row[], mode: WriteMode): Promise { + const hit = + this.failAll || + (this.failBatches && rows.length > 1) || + (this.poison !== null && rows.some((r) => r.data['id'] === this.poison)) + if (hit) { + this.faults++ + throw this.fault() + } + return super.upsert(stream, env, rows, mode) + } +} + +export interface LogLine { + level: 'debug' | 'info' | 'warn' | 'error' + msg: string + fields: Record +} + +/** A logger that keeps every line, so a test can assert on what was said. */ +export function captureLogger(): { logger: Logger; lines: LogLine[] } { + const lines: LogLine[] = [] + const at = + (level: LogLine['level']) => + (msg: string, fields: Record = {}) => { + lines.push({ level, msg, fields }) + } + return { + lines, + logger: { debug: at('debug'), info: at('info'), warn: at('warn'), error: at('error') }, + } +} diff --git a/packages/sync-clickhouse/test/lease.test.ts b/packages/sync-clickhouse/test/lease.test.ts new file mode 100644 index 0000000..0b2f725 --- /dev/null +++ b/packages/sync-clickhouse/test/lease.test.ts @@ -0,0 +1,203 @@ +/** + * Lease and cursor semantics against a real ClickHouse Keeper, through + * KeeperMap: the compare-and-swap the engine's fenced checkpoint relies on, + * exercised with real races. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { streamByName } from '@workos/sync-engine' +import { createTestDb, describeDb, type TestDb } from './helpers/clickhouse.js' +import { ClickHouseDestination } from '../src/destination.js' +import { canonicalJson } from '../src/db/lease.js' + +describe('canonical json', () => { + it('serialises equal values equally whatever the key order', () => { + expect(canonicalJson({ b: 1, a: { d: [1, { z: 1, y: 2 }], c: null } })).toBe( + canonicalJson({ a: { c: null, d: [1, { y: 2, z: 1 }] }, b: 1 }), + ) + expect(canonicalJson({ a: 1, b: undefined })).toBe('{"a":1}') + expect(canonicalJson([3, 1])).toBe('[3,1]') + }) +}) + +describeDb('events tail lease', () => { + let db: TestDb + let dest: ClickHouseDestination + + beforeEach(async () => { + db = await createTestDb() + dest = new ClickHouseDestination({ db, schema: db.schema }) + await dest.init([streamByName('users')!]) + }) + afterEach(async () => { + await db.close() + }) + + const expire = () => + db.command( + `ALTER TABLE \`${db.schema}\`.\`_sync\` UPDATE expires_at = now64(3, 'UTC') - toIntervalSecond(1) WHERE holder != ''`, + {}, + { async_insert: 0, keeper_map_strict_mode: 0 }, + ) + + it('is taken by the first process to ask', async () => { + const r = await dest.acquireLease('prod', 'a', 60_000) + expect(r.acquired).toBe(true) + expect(r.holder).toBe('a') + expect(Date.parse(r.expiresAt!)).toBeGreaterThan(Date.now() + 30_000) + expect(r.expiresAt).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/) + expect(await dest.getLease('prod')).toEqual({ holder: 'a', expiresAt: r.expiresAt }) + }) + + it('refuses a second process and tells it who holds the lease', async () => { + const first = await dest.acquireLease('prod', 'a', 60_000) + const second = await dest.acquireLease('prod', 'b', 60_000) + expect(second).toEqual({ acquired: false, holder: 'a', expiresAt: first.expiresAt }) + expect(await dest.getLease('prod')).toEqual({ holder: 'a', expiresAt: first.expiresAt }) + }) + + it('renews for the holder without changing hands', async () => { + const first = await dest.acquireLease('prod', 'a', 1_000) + const renewed = await dest.acquireLease('prod', 'a', 60_000) + expect(renewed.acquired).toBe(true) + expect(renewed.holder).toBe('a') + expect(Date.parse(renewed.expiresAt!)).toBeGreaterThan(Date.parse(first.expiresAt!)) + // acquired_at marks the original takeover, not the latest heartbeat. + const row = await db.query<{ same: number }>( + `SELECT acquired_at < expires_at - toIntervalSecond(30) AS same FROM \`${db.schema}\`.\`_sync\``, + ) + expect(Number(row[0]!.same)).toBe(1) + }) + + it('passes to a new process once it has expired', async () => { + const stale = await dest.acquireLease('prod', 'a', 0) + expect(stale.acquired).toBe(true) + await new Promise((r) => setTimeout(r, 20)) + expect(await dest.getLease('prod')).toBeNull() + const takeover = await dest.acquireLease('prod', 'b', 60_000) + expect(takeover.acquired).toBe(true) + expect(takeover.holder).toBe('b') + expect((await dest.acquireLease('prod', 'a', 60_000)).holder).toBe('b') + }) + + it('judges expiry by the database clock', async () => { + await dest.acquireLease('prod', 'a', 60_000) + await expire() + expect(await dest.getLease('prod')).toBeNull() + expect((await dest.acquireLease('prod', 'b', 60_000)).acquired).toBe(true) + }) + + it('is freed by release, and the cursors survive it', async () => { + await dest.acquireLease('prod', 'a', 60_000) + await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c1' }, 'a', null) + await dest.releaseLease('prod', 'a') + expect(await dest.getLease('prod')).toBeNull() + expect(await dest.getState('prod', '__tail__')).toEqual({ cursor: 'c1' }) + expect((await dest.acquireLease('prod', 'b', 60_000)).acquired).toBe(true) + }) + + it('ignores a release from anyone but the holder', async () => { + const r = await dest.acquireLease('prod', 'a', 60_000) + await dest.releaseLease('prod', 'b') + await dest.releaseLease('prod', 'a-but-older') + expect(await dest.getLease('prod')).toEqual({ holder: 'a', expiresAt: r.expiresAt }) + }) + + it('is scoped to the sync id and the environment', async () => { + await dest.acquireLease('prod', 'a', 60_000) + const other = new ClickHouseDestination({ db, schema: db.schema, syncId: 'reporting' }) + expect((await other.acquireLease('prod', 'b', 60_000)).acquired).toBe(true) + expect((await dest.acquireLease('staging', 'c', 60_000)).acquired).toBe(true) + expect(await dest.getLease('prod')).toMatchObject({ holder: 'a' }) + }) + + it('gives exactly one of many simultaneous contenders the lease', async () => { + const holders = Array.from({ length: 8 }, (_, i) => `p${i}`) + const results = await Promise.all(holders.map((h) => dest.acquireLease('prod', h, 60_000))) + const winners = results.filter((r) => r.acquired) + expect(winners).toHaveLength(1) + const winner = winners[0]!.holder + for (const r of results) if (!r.acquired) expect(r.holder).toBe(winner) + expect(await dest.getLease('prod')).toMatchObject({ holder: winner }) + }) + + it('gives exactly one of many simultaneous contenders an expired lease', async () => { + await dest.acquireLease('prod', 'old', 60_000) + await expire() + const results = await Promise.all(Array.from({ length: 8 }, (_, i) => dest.acquireLease('prod', `p${i}`, 60_000))) + expect(results.filter((r) => r.acquired)).toHaveLength(1) + }) + + describe('fences the state write', () => { + const stored = () => dest.getState>('prod', '__tail__') + + it('writes for the holder, and refuses anyone else', async () => { + await dest.acquireLease('prod', 'a', 60_000) + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c1' }, 'b', null)).toBe(false) + expect(await stored()).toBeNull() + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c1' }, 'a', null)).toBe(true) + expect(await stored()).toEqual({ cursor: 'c1' }) + }) + + it('refuses a holder whose lease has expired or been taken over', async () => { + await dest.acquireLease('prod', 'a', 60_000) + await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c1' }, 'a', null) + await expire() + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c2' }, 'a', { cursor: 'c1' })).toBe(false) + await dest.acquireLease('prod', 'b', 60_000) + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c2' }, 'a', { cursor: 'c1' })).toBe(false) + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c2' }, 'b', { cursor: 'c1' })).toBe(true) + expect(await stored()).toEqual({ cursor: 'c2' }) + }) + + it('refuses a write whose expected state is no longer the stored one', async () => { + await dest.acquireLease('prod', 'a', 60_000) + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c1' }, 'a', { cursor: 'c0' })).toBe(false) + expect(await stored()).toBeNull() + await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c1' }, 'a', null) + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c2' }, 'a', null)).toBe(false) + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c2' }, 'a', { cursor: 'c0' })).toBe(false) + expect(await stored()).toEqual({ cursor: 'c1' }) + // Equality is semantic, not textual: key order does not matter. + await dest.setState('prod', '__tail__', { cursor: 'c1', lastEventAt: null }) + expect( + await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c2' }, 'a', { lastEventAt: null, cursor: 'c1' }), + ).toBe(true) + expect(await stored()).toEqual({ cursor: 'c2' }) + }) + + it('refuses a checkpoint that races a cursor moved by someone else', async () => { + await dest.acquireLease('prod', 'a', 60_000) + await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c1' }, 'a', null) + // Two writers with the same expectation: only one can be right. + const [x, y] = await Promise.all([ + dest.setStateIfHeld('prod', '__tail__', { cursor: 'x' }, 'a', { cursor: 'c1' }), + dest.setStateIfHeld('prod', '__tail__', { cursor: 'y' }, 'a', { cursor: 'c1' }), + ]) + expect([x, y].filter(Boolean)).toHaveLength(1) + expect(await stored()).toEqual({ cursor: x ? 'x' : 'y' }) + }) + + it('keeps other streams’ cursors when writing one', async () => { + await dest.acquireLease('prod', 'a', 60_000) + await dest.setState('prod', 'users', { kind: 'backfill', status: 'complete' }) + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c1' }, 'a', null)).toBe(true) + expect(await dest.getState('prod', 'users')).toEqual({ kind: 'backfill', status: 'complete' }) + expect(await stored()).toEqual({ cursor: 'c1' }) + }) + }) + + describe('plain state', () => { + it('round-trips JSON and overwrites in place', async () => { + expect(await dest.getState('prod', 'users')).toBeNull() + await dest.setState('prod', 'users', { kind: 'backfill', status: 'running', after: null }) + await dest.setState('prod', 'users', { kind: 'backfill', status: 'complete', count: 3 }) + expect(await dest.getState('prod', 'users')).toEqual({ kind: 'backfill', status: 'complete', count: 3 }) + expect(await dest.getState('staging', 'users')).toBeNull() + }) + + it('survives concurrent writers to different streams', async () => { + await Promise.all(Array.from({ length: 6 }, (_, i) => dest.setState('prod', `stream_${i}`, { i }))) + for (let i = 0; i < 6; i++) expect(await dest.getState('prod', `stream_${i}`)).toEqual({ i }) + }) + }) +}) diff --git a/packages/sync-clickhouse/test/reconcile.test.ts b/packages/sync-clickhouse/test/reconcile.test.ts new file mode 100644 index 0000000..73ceb7e --- /dev/null +++ b/packages/sync-clickhouse/test/reconcile.test.ts @@ -0,0 +1,232 @@ +/** + * The reconcile set difference, computed in ClickHouse rather than in Node. + * + * These exercise the database half on its own, with the remote side handed + * in as pages. The engine half — pagination, repair — is covered by the + * drift and repair cases in engine.test.ts. + */ +import { it, expect, beforeEach, afterEach } from 'vitest' +import { diffIdsInMemory, diffStream, streamByName, WorkOSClient } from '@workos/sync-engine' +import { createTestDb, describeDb, type TestDb } from './helpers/clickhouse.js' +import { migrate } from '../src/schema/migrate.js' +import { upsertRows, deleteRows } from '../src/db/upsert.js' +import { diffIdsInClickHouse } from '../src/db/reconcile.js' +import { ClickHouseDestination } from '../src/destination.js' + +const users = streamByName('users')! +const STREAMS = ['users', 'organizations', 'organization_roles'].map((n) => streamByName(n)!) + +// Zero-padded so lexical order (what the SQL sorts by) is numeric order. +const user = (n: number, over: Record = {}) => ({ + object: 'user', + id: `user_${String(n).padStart(5, '0')}`, + email: `u${n}@example.com`, + created_at: '2026-01-01T00:00:00.000Z', + updated_at: '2026-01-01T00:00:00.000Z', + ...over, +}) + +const range = (from: number, to: number) => Array.from({ length: to - from + 1 }, (_, i) => user(from + i)) + +async function* pages(rows: Record[], size = 100) { + for (let i = 0; i < rows.length; i += size) yield rows.slice(i, i + size) +} + +describeDb('diff in sql', () => { + let db: TestDb + let opts: { schema: string } + + beforeEach(async () => { + db = await createTestDb() + opts = { schema: db.schema } + await migrate(db, STREAMS, db.schema) + }) + afterEach(async () => { + await db.close() + }) + + const seed = (rows: Record[], env = 'prod') => + upsertRows( + db, + opts, + users, + env, + rows.map((data) => ({ data, updatedAt: data['updated_at'] as string })), + 'replace', + ) + + it('reports matched, localOnly and remoteOnly, with the payload intact', async () => { + await seed([user(1), user(2), user(3)]) + const four = user(4, { email: 'four@example.com', metadata: { plan: 'pro' } }) + const diff = await diffIdsInClickHouse(db, opts, users, 'prod', pages([user(2), user(3), four])) + expect(diff.matched).toBe(2) + expect(diff.localOnly).toEqual(['user_00001']) + expect(diff.remoteOnly).toEqual([four]) + expect(diff.skipped).toBe(0) + }) + + it('compares complete composite keys in SQL and in memory', async () => { + const destination = new ClickHouseDestination({ db, schema: db.schema }) + const roles = streamByName('organization_roles')! + const removed = { organization_id: 'org_1', slug: 'admin' } + const kept = { organization_id: 'org_1', slug: 'auditor' } + const elsewhere = { organization_id: 'org_2', slug: 'admin' } + const added = { organization_id: 'org_1', slug: 'member', permissions: ['read'] } + await destination.upsert( + roles, + 'prod', + [removed, kept, elsewhere].map((data) => ({ data, updatedAt: '2026-01-01T00:00:00.000Z' })), + 'replace', + ) + const remote = [ + kept, + elsewhere, + added, + { ...added, permissions: [] }, + { organization_id: 'org_1' }, + { slug: 'admin' }, + { organization_id: 'org_1', slug: '' }, + ] + const diff = await diffIdsInClickHouse(db, opts, roles, 'prod', pages(remote, 1)) + expect(diff).toEqual({ matched: 2, localOnly: ['["org_1","admin"]'], remoteOnly: [added], skipped: 3 }) + expect(await destination.liveIds(roles, 'prod')).toEqual([ + '["org_1","admin"]', + '["org_1","auditor"]', + '["org_2","admin"]', + ]) + expect(await diffIdsInMemory(destination, roles, 'prod', pages(remote, 1))).toEqual({ + matched: diff.matched, + localOnly: diff.localOnly, + remoteOnly: diff.remoteOnly, + }) + }) + + it('repairs a missing role and a missed deletion without touching sibling roles', async () => { + const destination = new ClickHouseDestination({ db, schema: db.schema }) + const roles = streamByName('organization_roles')! + const removed = { organization_id: 'org_1', slug: 'admin' } + const kept = { organization_id: 'org_1', slug: 'auditor' } + const added = { organization_id: 'org_1', slug: 'member' } + const updatedAt = '2026-01-01T00:00:00.000Z' + await destination.upsert(streamByName('organizations')!, 'prod', [{ data: { id: 'org_1' }, updatedAt }], 'replace') + await destination.upsert( + roles, + 'prod', + [removed, kept].map((data) => ({ data, updatedAt })), + 'replace', + ) + const api = new WorkOSClient({ + apiKey: 'sk_test_1', + fetchImpl: async (input) => { + expect(new URL(String(input)).pathname).toBe('/authorization/organizations/org_1/roles') + return Response.json({ data: [kept, added] }) + }, + }) + const result = await diffStream(destination, api, 'prod', roles, { repair: true }) + expect(result).toMatchObject({ + matched: 1, + localOnly: ['["org_1","admin"]'], + remoteOnly: ['["org_1","member"]'], + repaired: { deleted: 1, inserted: 1 }, + }) + const rows = await db.query<{ slug: string; deleted: number }>( + `SELECT slug, \`_deleted_at\` IS NOT NULL AS deleted FROM \`${db.schema}\`.organization_roles FINAL ORDER BY slug`, + ) + // Comparisons come back as UInt8, not Bool. + expect(rows).toEqual([ + { slug: 'admin', deleted: 1 }, + { slug: 'auditor', deleted: 0 }, + { slug: 'member', deleted: 0 }, + ]) + expect(await diffStream(destination, api, 'prod', roles)).toMatchObject({ + matched: 2, + localOnly: [], + remoteOnly: [], + }) + }) + + it('stages in batches of 500 across pages, keeping the first copy of a repeated id', async () => { + await seed(range(1, 2000)) + const remote = [...range(11, 2010), user(11, { email: 'again@example.com' })] + const diff = await diffIdsInClickHouse(db, opts, users, 'prod', pages(remote, 1200)) + expect(diff.matched).toBe(1990) + expect(diff.localOnly).toEqual(range(1, 10).map((u) => u.id)) + expect(diff.remoteOnly.map((r) => r['id'])).toEqual(range(2001, 2010).map((u) => u.id)) + expect(diff.skipped).toBe(0) + }) + + it('keeps the first copy of a remote-only id that arrives twice', async () => { + const diff = await diffIdsInClickHouse( + db, + opts, + users, + 'prod', + pages([user(1, { email: 'first@example.com' }), user(1, { email: 'second@example.com' })], 1), + ) + expect(diff.remoteOnly).toEqual([user(1, { email: 'first@example.com' })]) + }) + + it('reports a tombstoned local row that still exists upstream as remoteOnly', async () => { + await seed([user(1), user(2)]) + await deleteRows(db, opts, users, 'prod', [{ data: { id: user(2).id }, updatedAt: '2026-01-02T00:00:00.000Z' }]) + const diff = await diffIdsInClickHouse(db, opts, users, 'prod', pages([user(1), user(2)])) + expect(diff.matched).toBe(1) + expect(diff.localOnly).toEqual([]) + expect(diff.remoteOnly.map((r) => r['id'])).toEqual([user(2).id]) + }) + + it('is not fooled by a row the tail writes while the list is in progress', async () => { + await seed([user(1), user(2)]) + async function* remote() { + yield [user(1)] + await seed([user(3)]) + yield [user(2), user(4)] + } + const diff = await diffIdsInClickHouse(db, opts, users, 'prod', remote()) + expect(diff.matched).toBe(2) + expect(diff.localOnly).toEqual([]) + expect(diff.remoteOnly.map((r) => r['id'])).toEqual([user(4).id]) + }) + + it('skips remote rows without a key instead of throwing', async () => { + await seed([user(1)]) + const remote = [user(1), { object: 'user', email: 'no-id@example.com' }, user(2, { id: '' })] + const diff = await diffIdsInClickHouse(db, opts, users, 'prod', pages(remote)) + expect(diff).toMatchObject({ matched: 1, localOnly: [], remoteOnly: [], skipped: 2 }) + }) + + it('only looks at the requested environment', async () => { + await seed([user(1)], 'prod') + await seed([user(2)], 'staging') + const diff = await diffIdsInClickHouse(db, opts, users, 'prod', pages([user(1), user(2)])) + expect(diff.matched).toBe(1) + expect(diff.localOnly).toEqual([]) + expect(diff.remoteOnly.map((r) => r['id'])).toEqual([user(2).id]) + }) + + it('leaves no staging table behind, even when the listing throws', async () => { + const destination = new ClickHouseDestination({ db, schema: db.schema }) + await destination.init([users]) + await seed([user(1)]) + const staging = async () => + ( + await db.query<{ name: string }>( + `SELECT name FROM system.tables WHERE database = {schema:String} AND name LIKE '_reconcile_%'`, + { schema: db.schema }, + ) + ).map((r) => r.name) + const first = await destination.diffIds(users, 'prod', pages([user(1), user(2)])) + expect(first.remoteOnly.map((r) => r['id'])).toEqual([user(2).id]) + expect(await staging()).toEqual([]) + async function* failing() { + yield [user(1)] + throw new Error('upstream went away') + } + await expect(destination.diffIds(users, 'prod', failing())).rejects.toThrow('upstream went away') + expect(await staging()).toEqual([]) + const again = await destination.diffIds(users, 'prod', pages([user(3)])) + expect(again.matched).toBe(0) + expect(again.localOnly).toEqual([user(1).id]) + expect(again.remoteOnly.map((r) => r['id'])).toEqual([user(3).id]) + }) +}) diff --git a/packages/sync-clickhouse/test/rejected.test.ts b/packages/sync-clickhouse/test/rejected.test.ts new file mode 100644 index 0000000..8fe7a11 --- /dev/null +++ b/packages/sync-clickhouse/test/rejected.test.ts @@ -0,0 +1,100 @@ +/** + * The quarantine store on its own, against a real ClickHouse. How the engine + * fills it is covered by the poison-event cases in engine.test.ts. + */ +import { it, expect, beforeEach, afterEach } from 'vitest' +import { streamByName, type WorkOSEvent } from '@workos/sync-engine' +import { createTestDb, describeDb, type TestDb } from './helpers/clickhouse.js' +import { ClickHouseDestination } from '../src/destination.js' + +const event = (n: number, over: Partial = {}): WorkOSEvent => ({ + id: `event_${String(n).padStart(3, '0')}`, + event: 'user.created', + data: { id: `user_${n}`, email: `u${n}@example.com`, nested: { 'odd key': [1, null, 'x'] } }, + created_at: '2026-01-01T00:00:00.000Z', + ...over, +}) + +describeDb('quarantine store', () => { + let db: TestDb + let dest: ClickHouseDestination + + beforeEach(async () => { + db = await createTestDb() + dest = new ClickHouseDestination({ db, schema: db.schema }) + await dest.init([streamByName('users')!, streamByName('events')!]) + }) + afterEach(async () => { + await db.close() + }) + + it('stores whole events, lists them oldest first by id, and counts without reading them', async () => { + await dest.reject('prod', [ + { event: event(2), error: 'second' }, + { event: event(1), error: 'first' }, + ]) + expect(await dest.rejectedCount('prod')).toBe(2) + expect(await dest.rejectedCount('staging')).toBe(0) + const rows = await dest.rejected('prod') + expect(rows.map((r) => r.eventId)).toEqual(['event_001', 'event_002']) + expect(rows[0]).toMatchObject({ + eventType: 'user.created', + objectId: 'user_1', + createdAt: '2026-01-01T00:00:00.000Z', + error: 'first', + raw: event(1), + }) + expect(rows[0]!.rejectedAt).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/) + expect(Date.parse(rows[0]!.rejectedAt)).toBeGreaterThan(Date.now() - 60_000) + }) + + it('replaces the error and timestamp when an id is rejected again', async () => { + await dest.reject('prod', [{ event: event(1), error: 'first try' }]) + const before = (await dest.rejected('prod'))[0]! + await new Promise((r) => setTimeout(r, 5)) + await dest.reject('prod', [{ event: event(1, { data: { id: 'user_1', retried: true } }), error: 'second try' }]) + const rows = await dest.rejected('prod') + expect(rows).toHaveLength(1) + expect(rows[0]!.error).toBe('second try') + expect(rows[0]!.raw.data).toEqual({ id: 'user_1', retried: true }) + expect(Date.parse(rows[0]!.rejectedAt)).toBeGreaterThanOrEqual(Date.parse(before.rejectedAt)) + expect(await dest.rejectedCount('prod')).toBe(1) + }) + + it('filters by id and forgets what was replayed', async () => { + await dest.reject( + 'prod', + [1, 2, 3].map((n) => ({ event: event(n), error: 'e' })), + ) + expect((await dest.rejected('prod', { ids: ['event_002'] })).map((r) => r.eventId)).toEqual(['event_002']) + expect(await dest.rejected('prod', { ids: [] })).toEqual([]) + expect(await dest.clearRejected('prod', ['event_001', 'event_003', 'event_nope'])).toBe(2) + expect(await dest.clearRejected('prod', [])).toBe(0) + expect((await dest.rejected('prod')).map((r) => r.eventId)).toEqual(['event_002']) + expect(await dest.rejectedCount('prod')).toBe(1) + }) + + it('never fails on the event timestamp it is quarantining', async () => { + await dest.reject('prod', [{ event: event(1, { created_at: '2026-13-01T00:00:00Z' }), error: 'bad month' }]) + const rows = await dest.rejected('prod') + expect(rows[0]!.createdAt).toBe('2026-13-01T00:00:00Z') + const stored = await db.query<{ created_at: string | null }>( + `SELECT created_at FROM \`${db.schema}\`.\`_rejected\` FINAL`, + ) + expect(stored[0]!.created_at).toBeNull() + }) + + it('keeps membership events, which carry no object id', async () => { + const link = event(1, { event: 'group.member_added', data: { group_id: 'g', organization_membership_id: 'om' } }) + await dest.reject('prod', [{ event: link, error: 'e' }]) + expect((await dest.rejected('prod'))[0]!.objectId).toBeNull() + }) + + it('keeps syncs sharing a database apart', async () => { + const other = new ClickHouseDestination({ db, schema: db.schema, syncId: 'reporting' }) + await dest.reject('prod', [{ event: event(1), error: 'e' }]) + await other.reject('prod', [{ event: event(2), error: 'e' }]) + expect((await dest.rejected('prod')).map((r) => r.eventId)).toEqual(['event_001']) + expect((await other.rejected('prod')).map((r) => r.eventId)).toEqual(['event_002']) + }) +}) diff --git a/packages/sync-clickhouse/test/schema.test.ts b/packages/sync-clickhouse/test/schema.test.ts new file mode 100644 index 0000000..75dbe67 --- /dev/null +++ b/packages/sync-clickhouse/test/schema.test.ts @@ -0,0 +1,344 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { createTestDb, describeDb, type TestDb } from './helpers/clickhouse.js' +import { migrate, fingerprint, DDL_VERSION } from '../src/schema/migrate.js' +import { + buildSchemaSql, + safeIdentifier, + columnExpression, + columnType, + createTableSql, + indexSpecs, + generatedColumnSql, + jsonPathArgs, +} from '../src/schema/ddl.js' +import { liveViewName, liveViewSql, isGeneratedDefinition } from '../src/schema/views.js' +import { STREAMS, streamByName, type Logger } from '@workos/sync-engine' + +const users = streamByName('users')! + +describe('DDL generation', () => { + it('projects every typed column out of _raw as a DEFAULT, never writing them directly', () => { + const sql = createTableSql(users, 'workos') + expect(sql).toContain('`_raw` String') + for (const col of users.columns) expect(sql).toMatch(new RegExp(`\`${col.name}\` [^\\n]* DEFAULT `)) + expect(sql).not.toContain('MATERIALIZED') + }) + + it('versions mirror tables by the source timestamp and keeps append logs plain', () => { + expect(createTableSql(users, 'workos')).toContain( + 'ENGINE = ReplacingMergeTree(`_updated_at`) ORDER BY (`_env`, `id`)', + ) + expect(createTableSql(users, 'workos')).toContain('`_deleted_at`') + const events = createTableSql(streamByName('events')!, 'workos') + expect(events).toContain('ENGINE = MergeTree ORDER BY (`_env`, `id`)') + expect(events).not.toContain('`_deleted_at`') + const memberships = createTableSql(streamByName('directory_group_memberships')!, 'workos') + expect(memberships).toContain('ORDER BY (`_env`, `directory_group_id`, `directory_user_id`)') + }) + + it('keeps keys non-nullable and everything else nullable', () => { + expect(generatedColumnSql({ name: 'id', type: 'text', path: 'id' }, users)).toBe( + "`id` String DEFAULT JSONExtractString(`_raw`, 'id')", + ) + expect(generatedColumnSql({ name: 'email', type: 'text', path: 'email' }, users)).toBe( + "`email` Nullable(String) DEFAULT JSONExtract(`_raw`, 'email', 'Nullable(String)')", + ) + expect(columnType({ name: 'x', type: 'timestamptz', path: 'x' }, false)).toBe("Nullable(DateTime64(3, 'UTC'))") + expect(columnType({ name: 'x', type: 'numeric', path: 'x' }, false)).toBe('Nullable(Decimal(38, 10))') + }) + + it('walks nested paths one argument per step, quoting them', () => { + expect(jsonPathArgs(['role', 'slug'])).toBe("'role', 'slug'") + expect(jsonPathArgs(`it's`)).toBe(`'it\\'s'`) + expect(columnExpression({ name: 'x', type: 'jsonb', path: ['a', 'b'] }, false)).toBe( + "if(JSONHas(`_raw`, 'a', 'b'), JSONExtractRaw(`_raw`, 'a', 'b'), NULL)", + ) + expect(columnExpression({ name: 'x', type: 'timestamptz', path: 'created_at' }, false)).toBe( + "parseDateTime64BestEffortOrNull(JSONExtractString(`_raw`, 'created_at'), 3, 'UTC')", + ) + }) + + it('turns indexed columns into skipping indexes of the right kind', () => { + expect(indexSpecs(users)).toContainEqual({ + name: 'users_email_idx', + column: 'email', + type: 'bloom_filter', + granularity: 4, + }) + expect(indexSpecs(streamByName('events')!)).toContainEqual({ + name: 'events_created_at_idx', + column: 'created_at', + type: 'minmax', + granularity: 1, + }) + expect(indexSpecs(users).map((i) => i.column)).toContain('_deleted_at') + expect(indexSpecs(streamByName('events')!).map((i) => i.column)).not.toContain('_deleted_at') + }) + + it('declares no foreign keys between mirror tables', () => { + const all = buildSchemaSql(STREAMS, 'workos').join('\n') + expect(all.toLowerCase()).not.toContain('foreign key') + for (const sql of buildSchemaSql(STREAMS, 'workos')) expect(sql).toMatch(/IF NOT EXISTS|OR REPLACE/) + }) + + it('shortens identifiers longer than the shared limit', () => { + const long = 'a'.repeat(80) + expect(safeIdentifier(long)).toHaveLength(63) + expect(safeIdentifier(long)).not.toBe(safeIdentifier(long + 'b')) + expect(safeIdentifier('users')).toBe('users') + }) + + it('changes the fingerprint when the catalog changes', () => { + const a = fingerprint([users], 'workos') + const b = fingerprint([{ ...users, columns: [...users.columns, { name: 'x', type: 'text', path: 'x' }] }], 'workos') + expect(a).not.toBe(b) + expect(fingerprint([users], 'other')).not.toBe(a) + expect(typeof DDL_VERSION).toBe('number') + }) + + it('generates a live view for every stream that has tombstones, and no other', () => { + for (const s of STREAMS) { + const sql = liveViewSql(s, 'workos') + if (s.mode === 'append') expect(sql).toEqual([]) + else { + expect(sql).toEqual([ + `CREATE OR REPLACE VIEW \`workos\`.\`${s.name}_live\` AS SELECT * FROM \`workos\`.\`${s.name}\` FINAL WHERE \`_deleted_at\` IS NULL`, + ]) + } + } + expect(liveViewName({ ...users, name: 'x'.repeat(70) })).toHaveLength(63) + }) + + it('recognises its own view definition by shape', () => { + const ours = 'SELECT * FROM workos.users FINAL WHERE _deleted_at IS NULL' + expect(isGeneratedDefinition(ours, 'workos', 'users')).toBe(true) + expect( + isGeneratedDefinition('SELECT * FROM `workos`.`users` FINAL WHERE `_deleted_at` IS NULL', 'workos', 'users'), + ).toBe(true) + expect(isGeneratedDefinition(ours.replace('IS NULL', 'IS NOT NULL'), 'workos', 'users')).toBe(false) + expect(isGeneratedDefinition(ours, 'workos', 'organizations')).toBe(false) + }) +}) + +describeDb('migrations against real ClickHouse', () => { + let db: TestDb + + beforeEach(async () => { + db = await createTestDb() + }) + afterEach(async () => { + await db.close() + }) + + const insertUser = (raw: Record) => + db.insert(`\`${db.schema}\`.users`, [ + { _env: 'prod', _raw: JSON.stringify(raw), _updated_at: '2026-01-01 00:00:00.000' }, + ]) + const tables = async () => + ( + await db.query<{ name: string }>(`SELECT name FROM system.tables WHERE database = {s:String} ORDER BY name`, { + s: db.schema, + }) + ).map((r) => r.name) + + it('creates every stream table with working projected columns, and records them', async () => { + await migrate(db, STREAMS, db.schema) + const names = await tables() + for (const s of STREAMS) expect(names).toContain(s.name) + for (const t of ['_sync', '_migrations', '_rejected', '_columns']) expect(names).toContain(t) + const recorded = await db.query<{ n: string }>(`SELECT toString(count()) AS n FROM \`${db.schema}\`.\`_columns\``) + expect(Number(recorded[0]!.n)).toBe(STREAMS.reduce((n, s) => n + s.columns.length, 0)) + }) + + it('is a no-op on the second run', async () => { + const first = await migrate(db, STREAMS, db.schema) + const second = await migrate(db, STREAMS, db.schema) + expect(first.applied).toBe(true) + expect(second.applied).toBe(false) + expect(second.statements).toBe(0) + }) + + it('creates the database when it is missing and leaves it alone when it is not', async () => { + await db.command(`CREATE DATABASE \`${db.schema}\``) + await db.command(`CREATE TABLE \`${db.schema}\`.keep (id Int32) ENGINE = MergeTree ORDER BY id`) + await migrate(db, [users], db.schema) + expect(await tables()).toContain('keep') + }) + + it('adds a newly projected column, computed for the rows already there, and the view picks it up', async () => { + await migrate(db, [users], db.schema) + await insertUser({ id: 'user_1', email: 'a@b.com', nickname: 'ada' }) + const extended = { + ...users, + columns: [...users.columns, { name: 'nickname', type: 'text' as const, path: 'nickname' }], + } + const r = await migrate(db, [extended], db.schema) + expect(r.applied).toBe(true) + expect(r.drift).toEqual([]) + const row = await db.query<{ nickname: string }>(`SELECT nickname FROM \`${db.schema}\`.users_live`) + expect(row[0]?.nickname).toBe('ada') + }) + + it('adds a missing index on a table that already exists', async () => { + await migrate(db, [users], db.schema) + await db.command(`ALTER TABLE \`${db.schema}\`.users DROP INDEX users_email_idx`) + await db.command( + `ALTER TABLE \`${db.schema}\`.\`_migrations\` DELETE WHERE 1`, + {}, + { async_insert: 0, keeper_map_strict_mode: 0 }, + ) + const r = await migrate(db, [users], db.schema) + expect(r.applied).toBe(true) + const idx = await db.query<{ name: string }>( + `SELECT name FROM system.data_skipping_indices WHERE database = {s:String} AND table = 'users'`, + { s: db.schema }, + ) + expect(idx.map((i) => i.name)).toContain('users_email_idx') + }) + + it('parses the ISO-8601 formats WorkOS emits, and nulls anything else', async () => { + await migrate(db, [users], db.schema) + const cases: [string | null, string | null][] = [ + ['2026-01-01T00:00:00.000Z', '2026-01-01 00:00:00.000'], + ['2026-08-06T17:44:03.481Z', '2026-08-06 17:44:03.481'], + ['2026-01-01T00:00:00Z', '2026-01-01 00:00:00.000'], + ['2026-01-01T00:00:00.123456789Z', '2026-01-01 00:00:00.123'], + ['2026-01-01T05:00:00+02:00', '2026-01-01 03:00:00.000'], + ['2024-02-29T00:00:00Z', '2024-02-29 00:00:00.000'], + ['not-a-timestamp', null], + [null, null], + ['2026-02-30T12:00:00Z', null], + ['2026-13-01T00:00:00Z', null], + // Best-effort parsing reads 24:00 as the next midnight, where Postgres refuses it. + ['2026-01-01T24:00:00Z', '2026-01-02 00:00:00.000'], + ] + for (const [input, expected] of cases) { + const id = `user_${String(input)}` + await insertUser({ id, created_at: input }) + const r = await db.query<{ created_at: string | null }>( + `SELECT created_at FROM \`${db.schema}\`.users FINAL WHERE id = {id:String}`, + { id }, + ) + expect([input, r[0]?.created_at ?? null]).toEqual([input, expected]) + } + }) + + it('projects booleans, integers, decimals and JSON the way the catalog types say', async () => { + const stream = { + ...users, + columns: [ + ...users.columns, + { name: 'n', type: 'bigint' as const, path: 'n' }, + { name: 'd', type: 'numeric' as const, path: 'd' }, + { name: 'flag', type: 'boolean' as const, path: ['meta', 'flag'] }, + ], + } + await migrate(db, [stream], db.schema) + await insertUser({ + id: 'u1', + n: 9007199254740993n.toString(), + d: '12.50', + meta: { flag: false, k: [1, 'two'] }, + metadata: { a: 1 }, + }) + await insertUser({ id: 'u2', n: 'twelve', d: 'cheap', meta: { flag: 'maybe' } }) + const rows = await db.query>( + `SELECT id, toString(n) AS n, d, flag, metadata FROM \`${db.schema}\`.users FINAL ORDER BY id`, + ) + expect(String(rows[0]!['n'])).toBe('9007199254740993') + expect(Number(rows[0]!['d'])).toBe(12.5) + expect(rows[0]!['flag']).toBe(false) + expect(JSON.parse(rows[0]!['metadata'] as string)).toEqual({ a: 1 }) + expect(rows[1]).toMatchObject({ n: null, d: null, flag: null, metadata: null }) + }) + + it('serialises nothing: two concurrent migrations both finish with one schema', async () => { + const results = await Promise.all([migrate(db, STREAMS, db.schema), migrate(db, STREAMS, db.schema)]) + expect(results.map((r) => r.applied)).toContain(true) + expect((await migrate(db, STREAMS, db.schema)).applied).toBe(false) + }) +}) + +describeDb('live views against real ClickHouse', () => { + let db: TestDb + + beforeEach(async () => { + db = await createTestDb() + }) + afterEach(async () => { + await db.close() + }) + + const views = async () => + ( + await db.query<{ name: string }>( + `SELECT name FROM system.tables WHERE database = {s:String} AND engine = 'View' ORDER BY name`, + { s: db.schema }, + ) + ).map((r) => r.name) + const forget = () => + db.command( + `ALTER TABLE \`${db.schema}\`.\`_migrations\` DELETE WHERE 1`, + {}, + { async_insert: 0, keeper_map_strict_mode: 0 }, + ) + + it('creates a view for every mirror stream and none for the append-only logs', async () => { + await migrate(db, STREAMS, db.schema) + const names = await views() + for (const s of STREAMS) { + if (s.mode === 'append') expect(names).not.toContain(`${s.name}_live`) + else expect(names).toContain(`${s.name}_live`) + } + }) + + it('collapses versions and excludes tombstoned rows, and nothing else', async () => { + await migrate(db, [users], db.schema) + await db.insert(`\`${db.schema}\`.users`, [ + { _env: 'prod', _raw: JSON.stringify({ id: 'live', email: 'old@x' }), _updated_at: '2026-01-01 00:00:00.000' }, + { _env: 'prod', _raw: JSON.stringify({ id: 'live', email: 'new@x' }), _updated_at: '2026-01-02 00:00:00.000' }, + { + _env: 'prod', + _raw: JSON.stringify({ id: 'gone' }), + _updated_at: '2026-01-01 00:00:00.000', + _deleted_at: '2026-01-03 00:00:00.000', + }, + ]) + const live = await db.query<{ id: string; email: string }>(`SELECT id, email FROM \`${db.schema}\`.users_live`) + expect(live).toEqual([{ id: 'live', email: 'new@x' }]) + }) + + it('re-creates a view that was dropped once the schema is applied again', async () => { + await migrate(db, [users], db.schema) + await db.command(`DROP VIEW \`${db.schema}\`.users_live`) + await forget() + await migrate(db, [users], db.schema) + expect(await views()).toContain('users_live') + }) + + it('refuses to replace a view under its name that it did not generate, until repairViews', async () => { + await migrate(db, [users], db.schema) + await db.command( + `CREATE OR REPLACE VIEW \`${db.schema}\`.users_live AS SELECT id FROM \`${db.schema}\`.users WHERE email LIKE '%@acme.com'`, + ) + await forget() + await expect(migrate(db, [users], db.schema)).rejects.toThrow(/did not generate/) + const lines: string[] = [] + const logger: Logger = { debug() {}, info() {}, warn: (m) => lines.push(m), error() {} } + await migrate(db, [users], db.schema, logger, { repairViews: true }) + expect(lines).toContain('replacing a view that is not the generated shape') + const r = await db.query<{ as_select: string }>( + `SELECT as_select FROM system.tables WHERE database = {s:String} AND name = 'users_live'`, + { + s: db.schema, + }, + ) + expect(isGeneratedDefinition(r[0]!.as_select, db.schema, 'users')).toBe(true) + }) + + it('refuses a table standing where its view would go', async () => { + await db.command(`CREATE DATABASE \`${db.schema}\``) + await db.command(`CREATE TABLE \`${db.schema}\`.users_live (id Int32) ENGINE = MergeTree ORDER BY id`) + await expect(migrate(db, [users], db.schema)).rejects.toThrow(/is a table/) + }) +}) diff --git a/packages/sync-clickhouse/test/upsert.test.ts b/packages/sync-clickhouse/test/upsert.test.ts new file mode 100644 index 0000000..9688e7c --- /dev/null +++ b/packages/sync-clickhouse/test/upsert.test.ts @@ -0,0 +1,596 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { createTestDb, describeDb, type TestDb } from './helpers/clickhouse.js' +import { migrate } from '../src/schema/migrate.js' +import { + upsertRows, + deleteRows, + relinkRows, + unlinkRows, + dedupeByKey, + wavesByKey, + keyValues, + parseUpdatedAt, + jsonEqual, + keyExpression, +} from '../src/db/upsert.js' +import { isTransientClickHouseError } from '../src/db/errors.js' +import { streamByName } from '@workos/sync-engine' + +const users = streamByName('users')! + +const userPayload = (over: Record = {}) => ({ + object: 'user', + id: 'user_1', + email: 'ada@example.com', + first_name: 'Ada', + created_at: '2026-01-01T00:00:00.000Z', + updated_at: '2026-01-01T00:00:00.000Z', + ...over, +}) + +describe('key extraction', () => { + it('reads slug-keyed streams that have no id in their event payload', () => { + const roles = streamByName('roles')! + expect(keyValues(roles, { slug: 'admin', permissions: [] })).toEqual(['admin']) + expect(keyValues(roles, { permissions: [] })).toBeUndefined() + }) + + it('reads composite keys, and looks them up as one joined expression', () => { + const orgRoles = streamByName('organization_roles')! + expect(keyValues(orgRoles, { organization_id: 'org_1', slug: 'admin' })).toEqual(['org_1', 'admin']) + expect(keyExpression(orgRoles)).toBe('concat(`organization_id`, char(1), `slug`)') + expect(keyExpression(users)).toBe('`id`') + }) + + it('collapses duplicate keys within a batch, keeping the newest', () => { + const { rows, dropped } = dedupeByKey(users, [ + { data: userPayload({ email: 'old@example.com' }), updatedAt: '2026-01-01T00:00:00.000Z' }, + { data: userPayload({ email: 'new@example.com' }), updatedAt: '2026-01-02T00:00:00.000Z' }, + { data: { email: 'no-id@example.com' }, updatedAt: '2026-01-03T00:00:00.000Z' }, + ]) + expect(rows).toHaveLength(1) + expect(rows[0]!.data['email']).toBe('new@example.com') + expect(dropped).toBe(1) + }) + + it('splits repeated keys into ordered waves and leaves distinct keys in one', () => { + const first = { data: userPayload({ external_id: 'ext_1' }), updatedAt: '2026-01-01T00:00:00.000Z' } + const other = { data: userPayload({ id: 'user_2' }), updatedAt: '2026-01-01T00:00:00.000Z' } + const second = { data: { id: 'user_1', first_name: 'Grace' }, updatedAt: '2026-01-02T00:00:00.000Z' } + const { waves, dropped } = wavesByKey(users, [ + first, + other, + second, + { data: { email: 'no-id@example.com' }, updatedAt: '2026-01-03T00:00:00.000Z' }, + ]) + expect(waves).toEqual([[first, other], [second]]) + expect(dropped).toBe(1) + }) +}) + +describe('timestamps and equality', () => { + it('parses an ISO-8601 updatedAt and refuses anything else as a deterministic error', () => { + expect(parseUpdatedAt('2026-01-01T00:00:00.000Z').toISOString()).toBe('2026-01-01T00:00:00.000Z') + let err: unknown + try { + parseUpdatedAt('maybe') + } catch (e) { + err = e + } + expect((err as { code: string }).code).toBe('INVALID_TIMESTAMP') + expect(isTransientClickHouseError(err)).toBe(false) + }) + + it('compares JSON structurally, ignoring key order', () => { + expect(jsonEqual({ a: 1, b: [1, { c: null }] }, { b: [1, { c: null }], a: 1 })).toBe(true) + expect(jsonEqual({ a: 1 }, { a: 1, b: undefined })).toBe(false) + expect(jsonEqual({ a: [1, 2] }, { a: [2, 1] })).toBe(false) + }) +}) + +const SOME_STREAMS = ['users', 'roles', 'group_memberships', 'events'].map((n) => streamByName(n)!) + +describeDb('write path', () => { + let db: TestDb + let opts: { schema: string; batchSize?: number } + + beforeEach(async () => { + db = await createTestDb() + opts = { schema: db.schema } + await migrate(db, SOME_STREAMS, db.schema) + }) + afterEach(async () => { + await db.close() + }) + + const readUser = async (id = 'user_1') => + ( + await db.query>( + `SELECT * FROM \`${db.schema}\`.users FINAL WHERE \`_env\` = 'prod' AND id = {id:String}`, + { id }, + ) + )[0] + const versions = async (id = 'user_1') => + Number( + ( + await db.query<{ n: string }>( + `SELECT toString(count()) AS n FROM \`${db.schema}\`.users WHERE \`_env\` = 'prod' AND id = {id:String}`, + { id }, + ) + )[0]!.n, + ) + + it('inserts and reports created vs updated, keeping every version until a merge', async () => { + const first = await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload(), updatedAt: '2026-01-01T00:00:00.000Z' }], + 'replace', + ) + expect(first).toMatchObject({ created: 1, updated: 0 }) + const second = await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ email: 'changed@example.com' }), updatedAt: '2026-01-02T00:00:00.000Z' }], + 'replace', + ) + expect(second).toMatchObject({ created: 0, updated: 1 }) + expect((await readUser())!['email']).toBe('changed@example.com') + // The base table keeps both versions; FINAL and the view collapse them. + expect(await versions()).toBe(2) + }) + + it('rejects a write that is older than the stored row', async () => { + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ email: 'current@example.com' }), updatedAt: '2026-01-05T00:00:00.000Z' }], + 'replace', + ) + const stale = await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ email: 'stale@example.com' }), updatedAt: '2026-01-01T00:00:00.000Z' }], + 'replace', + ) + expect(stale).toMatchObject({ created: 0, updated: 0, skipped: 1 }) + expect((await readUser())!['email']).toBe('current@example.com') + expect(await versions()).toBe(1) + }) + + it('accepts a write with an identical timestamp, and the table keeps the later one', async () => { + const ts = '2026-01-05T00:00:00.000Z' + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ email: 'first@example.com' }), updatedAt: ts }], + 'replace', + ) + const same = await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ email: 'second@example.com' }), updatedAt: ts }], + 'replace', + ) + expect(same.updated).toBe(1) + // Equal versions: ReplacingMergeTree keeps the last inserted, which is the gate's `>=`. + expect((await readUser())!['email']).toBe('second@example.com') + }) + + it('skips a write with the same payload and timestamp, writing no version', async () => { + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload(), updatedAt: '2026-01-01T00:00:00.000Z' }], + 'replace', + ) + const again = await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload(), updatedAt: '2026-01-01T00:00:00.000Z' }], + 'replace', + ) + expect(again).toMatchObject({ created: 0, updated: 0, skipped: 1 }) + expect(await versions()).toBe(1) + }) + + it('keeps the fields of a full snapshot under a later subset in one merge batch', async () => { + const stats = await upsertRows( + db, + opts, + users, + 'prod', + [ + { + data: userPayload({ external_id: 'ext_1', metadata: { plan: 'pro' } }), + updatedAt: '2026-01-01T00:00:00.000Z', + }, + { + data: { object: 'user', id: 'user_1', email: 'ada@example.com', first_name: 'Grace' }, + updatedAt: '2026-01-02T00:00:00.000Z', + }, + ], + 'merge', + ) + expect(stats).toMatchObject({ created: 1, updated: 1, skipped: 0 }) + const row = (await readUser())! + expect(row['first_name']).toBe('Grace') + expect(row['external_id']).toBe('ext_1') + expect(JSON.parse(row['metadata'] as string)).toEqual({ plan: 'pro' }) + }) + + it('gates every row of a merge batch against the stored row, not only against the batch', async () => { + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ metadata: { plan: 'current' } }), updatedAt: '2026-01-02T00:00:00.000Z' }], + 'replace', + ) + const stats = await upsertRows( + db, + opts, + users, + 'prod', + [ + { data: { object: 'user', id: 'user_1', metadata: { plan: 'old' } }, updatedAt: '2026-01-01T00:00:00.000Z' }, + { data: { object: 'user', id: 'user_1', first_name: 'Grace' }, updatedAt: '2026-01-03T00:00:00.000Z' }, + ], + 'merge', + ) + expect(stats).toMatchObject({ created: 0, updated: 1, skipped: 1 }) + const row = (await readUser())! + expect(row['first_name']).toBe('Grace') + expect(JSON.parse(row['metadata'] as string)).toEqual({ plan: 'current' }) + expect(row['_updated_at']).toBe('2026-01-03 00:00:00.000') + }) + + it('merges shallowly: a nested object in the event replaces the stored one', async () => { + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ metadata: { plan: 'pro', seats: 5 } }), updatedAt: '2026-01-01T00:00:00.000Z' }], + 'replace', + ) + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: { object: 'user', id: 'user_1', metadata: { seats: 6 } }, updatedAt: '2026-01-02T00:00:00.000Z' }], + 'merge', + ) + expect(JSON.parse((await readUser())!['metadata'] as string)).toEqual({ seats: 6 }) + }) + + it('merges event payloads instead of nulling the fields events omit', async () => { + await upsertRows( + db, + opts, + users, + 'prod', + [ + { + data: userPayload({ external_id: 'ext_42', metadata: { plan: 'enterprise' }, locale: 'fr' }), + updatedAt: '2026-01-01T00:00:00.000Z', + }, + ], + 'replace', + ) + await upsertRows( + db, + opts, + users, + 'prod', + [ + { + data: { object: 'user', id: 'user_1', email: 'renamed@example.com', first_name: 'Ada' }, + updatedAt: '2026-01-09T00:00:00.000Z', + }, + ], + 'merge', + ) + const row = (await readUser())! + expect(row['email']).toBe('renamed@example.com') + expect(row['external_id']).toBe('ext_42') + expect(row['locale']).toBe('fr') + }) + + it('replaces the payload when the row came from the REST API', async () => { + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ locale: 'fr' }), updatedAt: '2026-01-01T00:00:00.000Z' }], + 'replace', + ) + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ updated_at: '2026-02-01T00:00:00.000Z' }), updatedAt: '2026-02-01T00:00:00.000Z' }], + 'replace', + ) + expect((await readUser())!['locale']).toBeNull() + }) + + it('projects a JSON null as NULL, a boolean as a boolean, and a bad cast as NULL with the value kept in _raw', async () => { + await upsertRows( + db, + opts, + users, + 'prod', + [ + { + data: userPayload({ id: 'user_t', first_name: null, email_verified: true }), + updatedAt: '2026-01-01T00:00:00.000Z', + }, + { data: userPayload({ id: 'user_m', email_verified: 'maybe' }), updatedAt: '2026-01-01T00:00:00.000Z' }, + ], + 'replace', + ) + const t = (await readUser('user_t'))! + expect(t['first_name']).toBeNull() + expect(t['email_verified']).toBe(true) + const m = (await readUser('user_m'))! + expect(m['email_verified']).toBeNull() + expect(JSON.parse(m['_raw'] as string)['email_verified']).toBe('maybe') + }) + + it('tombstones on delete and keeps the last known payload', async () => { + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload(), updatedAt: '2026-01-01T00:00:00.000Z' }], + 'replace', + ) + const stats = await deleteRows(db, opts, users, 'prod', [ + { data: { id: 'user_1' }, updatedAt: '2026-01-10T00:00:00.000Z' }, + ]) + expect(stats.deleted).toBe(1) + const row = (await readUser())! + expect(row['_deleted_at']).toBe('2026-01-10 00:00:00.000') + expect(row['email']).toBe('ada@example.com') + // A delete is not a staleness-gated write: the row's own time stays, and + // the tombstone wins the version tie by being the later insert. + expect(row['_updated_at']).toBe('2026-01-01 00:00:00.000') + }) + + it('records a delete for an object it never saw created', async () => { + const stats = await deleteRows(db, opts, users, 'prod', [ + { data: { id: 'user_ghost', email: 'ghost@example.com' }, updatedAt: '2026-01-10T00:00:00.000Z' }, + ]) + expect(stats.deleted).toBe(1) + expect((await readUser('user_ghost'))!['_deleted_at']).not.toBeNull() + }) + + it('does not resurrect a deleted row when a late update arrives', async () => { + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload(), updatedAt: '2026-01-01T00:00:00.000Z' }], + 'replace', + ) + await deleteRows(db, opts, users, 'prod', [{ data: { id: 'user_1' }, updatedAt: '2026-01-10T00:00:00.000Z' }]) + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ email: 'zombie@example.com' }), updatedAt: '2026-01-20T00:00:00.000Z' }], + 'merge', + ) + const row = (await readUser())! + expect(row['_deleted_at']).not.toBeNull() + expect(row['email']).toBe('zombie@example.com') + }) + + it('keeps the latest removal when a row is deleted twice', async () => { + await deleteRows(db, opts, users, 'prod', [{ data: { id: 'user_1' }, updatedAt: '2026-01-10T00:00:00.000Z' }]) + await deleteRows(db, opts, users, 'prod', [{ data: { id: 'user_1' }, updatedAt: '2026-01-05T00:00:00.000Z' }]) + expect((await readUser())!['_deleted_at']).toBe('2026-01-10 00:00:00.000') + await deleteRows(db, opts, users, 'prod', [{ data: { id: 'user_1' }, updatedAt: '2026-01-12T00:00:00.000Z' }]) + expect((await readUser())!['_deleted_at']).toBe('2026-01-12 00:00:00.000') + }) + + it('keeps environments isolated under one database', async () => { + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ email: 'prod@example.com' }), updatedAt: '2026-01-01T00:00:00.000Z' }], + 'replace', + ) + await upsertRows( + db, + opts, + users, + 'staging', + [{ data: userPayload({ email: 'staging@example.com' }), updatedAt: '2026-01-01T00:00:00.000Z' }], + 'replace', + ) + const r = await db.query<{ _env: string; email: string }>( + `SELECT \`_env\`, email FROM \`${db.schema}\`.users FINAL ORDER BY \`_env\``, + ) + expect(r).toEqual([ + { _env: 'prod', email: 'prod@example.com' }, + { _env: 'staging', email: 'staging@example.com' }, + ]) + }) + + it('treats a re-added membership as live again', async () => { + const memberships = streamByName('group_memberships')! + const pair = { group_id: 'group_1', organization_membership_id: 'om_1' } + await relinkRows(db, opts, memberships, 'prod', [{ data: pair, updatedAt: '2026-01-01T00:00:00.000Z' }]) + await unlinkRows(db, opts, memberships, 'prod', [{ data: pair, updatedAt: '2026-01-02T00:00:00.000Z' }]) + const back = await relinkRows(db, opts, memberships, 'prod', [ + { data: pair, updatedAt: '2026-01-03T00:00:00.000Z' }, + ]) + expect(back).toMatchObject({ created: 0, updated: 1, skipped: 0 }) + const r = await db.query<{ deleted_at: string | null }>( + `SELECT \`_deleted_at\` AS deleted_at FROM \`${db.schema}\`.group_memberships FINAL WHERE group_id = 'group_1'`, + ) + expect(r[0]!.deleted_at).toBeNull() + }) + + it('ignores a re-add older than the removal, even when the row itself is older', async () => { + const memberships = streamByName('group_memberships')! + const pair = { group_id: 'group_1', organization_membership_id: 'om_1' } + await relinkRows(db, opts, memberships, 'prod', [{ data: pair, updatedAt: '2026-01-01T00:00:00.000Z' }]) + await deleteRows(db, opts, memberships, 'prod', [{ data: pair, updatedAt: '2026-01-05T00:00:00.000Z' }]) + const stale = await relinkRows(db, opts, memberships, 'prod', [ + { data: pair, updatedAt: '2026-01-03T00:00:00.000Z' }, + ]) + expect(stale).toMatchObject({ created: 0, updated: 0, skipped: 1 }) + const r = await db.query<{ deleted_at: string | null }>( + `SELECT \`_deleted_at\` AS deleted_at FROM \`${db.schema}\`.group_memberships FINAL WHERE group_id = 'group_1'`, + ) + expect(r[0]!.deleted_at).not.toBeNull() + }) + + it('ignores a membership removal older than the row, unlike an object delete', async () => { + const memberships = streamByName('group_memberships')! + const pair = { group_id: 'group_1', organization_membership_id: 'om_1' } + const deletedAt = async () => + ( + await db.query<{ deleted_at: string | null }>( + `SELECT \`_deleted_at\` AS deleted_at FROM \`${db.schema}\`.group_memberships FINAL WHERE group_id = 'group_1'`, + ) + )[0]!.deleted_at + await relinkRows(db, opts, memberships, 'prod', [{ data: pair, updatedAt: '2026-01-03T00:00:00.000Z' }]) + const stale = await unlinkRows(db, opts, memberships, 'prod', [ + { data: pair, updatedAt: '2026-01-02T00:00:00.000Z' }, + ]) + expect(stale).toMatchObject({ deleted: 0, skipped: 1 }) + expect(await deletedAt()).toBeNull() + const fresh = await unlinkRows(db, opts, memberships, 'prod', [ + { data: pair, updatedAt: '2026-01-04T00:00:00.000Z' }, + ]) + expect(fresh).toMatchObject({ deleted: 1, skipped: 0 }) + expect(await deletedAt()).not.toBeNull() + }) + + it('never writes an append-only row twice', async () => { + const events = streamByName('events')! + const e = { id: 'event_1', event: 'user.created', data: { id: 'user_1' }, created_at: '2026-01-01T00:00:00.000Z' } + const first = await upsertRows(db, opts, events, 'prod', [{ data: e, updatedAt: e.created_at }], 'replace') + expect(first).toMatchObject({ created: 1, skipped: 0 }) + const dup = await upsertRows( + db, + opts, + events, + 'prod', + [{ data: { ...e, event: 'tampered' }, updatedAt: e.created_at }], + 'replace', + ) + expect(dup).toMatchObject({ created: 0, skipped: 1 }) + const r = await db.query<{ event: string }>(`SELECT event FROM \`${db.schema}\`.events`) + expect(r).toEqual([{ event: 'user.created' }]) + }) + + it('writes large batches in chunks', async () => { + const rows = Array.from({ length: 1200 }, (_, i) => ({ + data: userPayload({ id: `user_${i}`, email: `u${i}@example.com` }), + updatedAt: '2026-01-01T00:00:00.000Z', + })) + const stats = await upsertRows(db, { schema: db.schema, batchSize: 250 }, users, 'prod', rows, 'replace') + expect(stats.created).toBe(1200) + const count = await db.query<{ n: string }>(`SELECT toString(count()) AS n FROM \`${db.schema}\`.users FINAL`) + expect(Number(count[0]!.n)).toBe(1200) + }) + + it('stores a payload whose string values look like SQL, JSON paths or parameters verbatim', async () => { + const data = userPayload({ first_name: 'O\'Brien `$.x` \\ "quoted" {id:String}', metadata: { '$.weird key': 'v' } }) + await upsertRows(db, opts, users, 'prod', [{ data, updatedAt: '2026-01-01T00:00:00.000Z' }], 'replace') + const row = (await readUser())! + expect(row['first_name']).toBe('O\'Brien `$.x` \\ "quoted" {id:String}') + expect(JSON.parse(row['metadata'] as string)).toEqual({ '$.weird key': 'v' }) + }) +}) + +describeDb('freshness reporting', () => { + let db: TestDb + + beforeEach(async () => { + db = await createTestDb() + await migrate(db, SOME_STREAMS, db.schema) + }) + afterEach(async () => { + await db.close() + }) + + it('does not report the age of old objects as sync lag', async () => { + const { ClickHouseDestination } = await import('../src/destination.js') + const destination = new ClickHouseDestination({ db, schema: db.schema }) + const opts = { schema: db.schema } + + expect(await destination.freshness(users, 'prod', 24)).toEqual({ + stream: 'users', + rows: 0, + deleted: 0, + lastSyncedAt: null, + stalenessSeconds: null, + lagSeconds: null, + lagSampleSize: 0, + }) + + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ id: 'user_old' }), updatedAt: '2024-01-01T00:00:00.000Z' }], + 'replace', + ) + const stale = await destination.freshness(users, 'prod', 24) + expect(stale.rows).toBe(1) + expect(stale.lagSampleSize).toBe(0) + expect(stale.lagSeconds).toBeNull() + expect(stale.stalenessSeconds!).toBeGreaterThan(365 * 86400) + expect(Date.parse(stale.lastSyncedAt!)).toBeGreaterThan(Date.now() - 60_000) + + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ id: 'user_fresh' }), updatedAt: new Date(Date.now() - 2000).toISOString() }], + 'replace', + ) + await deleteRows(db, opts, users, 'prod', [{ data: { id: 'user_old' }, updatedAt: new Date().toISOString() }]) + + const fresh = await destination.freshness(users, 'prod', 24) + expect(fresh.rows).toBe(2) + expect(fresh.deleted).toBe(1) + expect(fresh.lagSampleSize).toBe(1) + expect(fresh.lagSeconds).not.toBeNull() + expect(fresh.lagSeconds!).toBeLessThan(60) + expect(fresh.lagSeconds!).toBeGreaterThan(0) + }) +}) diff --git a/packages/sync-clickhouse/tsconfig.build.json b/packages/sync-clickhouse/tsconfig.build.json new file mode 100644 index 0000000..f44fac8 --- /dev/null +++ b/packages/sync-clickhouse/tsconfig.build.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "outDir": "dist", + "rootDir": "src", + "paths": {} + }, + "include": ["src/**/*"], + "exclude": ["test/**/*"] +} diff --git a/packages/sync-clickhouse/tsconfig.json b/packages/sync-clickhouse/tsconfig.json new file mode 100644 index 0000000..09ebd6a --- /dev/null +++ b/packages/sync-clickhouse/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2023"], + "strict": true, + "noUncheckedIndexedAccess": true, + "exactOptionalPropertyTypes": false, + "esModuleInterop": true, + "skipLibCheck": true, + "declaration": true, + "sourceMap": true, + "noEmit": true, + "types": ["node"], + // Typecheck and editors see @workos/sync-engine source directly, so a change + // there never needs a rebuild to show up here. The build (tsconfig.build) + // clears this and resolves the real package. + "paths": { "@workos/sync-engine": ["../sync-engine/src/index.ts"] } + }, + "include": ["src/**/*", "test/**/*"] +} diff --git a/packages/sync-clickhouse/vitest.config.ts b/packages/sync-clickhouse/vitest.config.ts new file mode 100644 index 0000000..a27ae83 --- /dev/null +++ b/packages/sync-clickhouse/vitest.config.ts @@ -0,0 +1,14 @@ +import { fileURLToPath } from 'node:url' +import { defineConfig } from 'vitest/config' +export default defineConfig({ + resolve: { + // Tests run against @workos/sync-engine source, not its last build. + alias: { '@workos/sync-engine': fileURLToPath(new URL('../sync-engine/src/index.ts', import.meta.url)) }, + }, + test: { + include: ['test/**/*.test.ts'], + testTimeout: 60_000, + hookTimeout: 60_000, + pool: 'forks', + }, +}) diff --git a/packages/sync-engine/README.md b/packages/sync-engine/README.md index 1012fa1..5d711ff 100644 --- a/packages/sync-engine/README.md +++ b/packages/sync-engine/README.md @@ -8,10 +8,11 @@ particular database. Writes go through a small `Destination` interface. **You probably want a destination package instead.** Each one depends on this package and re-exports it, so it is the only thing to install: -| Destination | Package | -| ----------- | ------------------------------------------------------------------------------ | -| Postgres | [`@workos/sync-postgres`](https://www.npmjs.com/package/@workos/sync-postgres) | -| MySQL | [`@workos/sync-mysql`](https://www.npmjs.com/package/@workos/sync-mysql) | +| Destination | Package | +| ----------- | ---------------------------------------------------------------------------------- | +| Postgres | [`@workos/sync-postgres`](https://www.npmjs.com/package/@workos/sync-postgres) | +| MySQL | [`@workos/sync-mysql`](https://www.npmjs.com/package/@workos/sync-mysql) | +| ClickHouse | [`@workos/sync-clickhouse`](https://www.npmjs.com/package/@workos/sync-clickhouse) | Install this package directly only to write a new destination. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1cf3cb3..d24a00d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -229,6 +229,35 @@ importers: specifier: ^7.0.2 version: 7.0.2 + packages/sync-clickhouse: + dependencies: + '@clickhouse/client': + specifier: ^1.11.0 + version: 1.25.0 + '@workos/sync-engine': + specifier: workspace:* + version: link:../sync-engine + devDependencies: + '@opentelemetry/api': + specifier: ^1.10.0 + version: 1.10.0 + '@opentelemetry/sdk-metrics': + specifier: ^3.0.0 + version: 3.0.0(@opentelemetry/api@1.10.0) + '@types/node': + specifier: ^26.6.5 + version: 26.6.5 + typescript: + specifier: ^7.0.2 + version: 7.0.2 + vitest: + specifier: ^5.0.5 + version: 5.0.5(@opentelemetry/api@1.10.0)(@types/node@26.6.5)(vite@8.3.4(@types/node@26.6.5)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1)) + optionalDependencies: + '@opentelemetry/sdk-node': + specifier: ^0.300.0 + version: 0.300.0(@opentelemetry/api@1.10.0) + packages/sync-engine: dependencies: '@opentelemetry/api': @@ -331,6 +360,10 @@ packages: resolution: {integrity: sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==} engines: {node: '>=6.9.0'} + '@clickhouse/client@1.25.0': + resolution: {integrity: sha512-sxoCDlA8iOEGpGrJzl9SeNvG7II6e5kSeo0JI4Mru2QhwhjydWuDEE55ryZg3vIJ0CSJ1bhn9+Xtnuvf1BZT/w==} + engines: {node: '>=20'} + '@commitlint/cli@21.2.4': resolution: {integrity: sha512-g3M/wrQdElqGH5vROIeL2QiYIxrzqUkhCgsDyDlWfKEyQdCt10OyMB28W2cJtpZ+VNGMvmaN923infr1hOfyrg==} engines: {node: '>=22.12.0'} @@ -1961,6 +1994,8 @@ snapshots: '@babel/helper-validator-identifier@7.29.7': {} + '@clickhouse/client@1.25.0': {} + '@commitlint/cli@21.2.4(conventional-commits-parser@7.1.4)(typescript@7.0.2)': dependencies: '@commitlint/config-conventional': 21.2.4 diff --git a/release-please-config.json b/release-please-config.json index d7edd0c..dd4936f 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -19,6 +19,10 @@ "packages/sync-mysql": { "release-type": "node", "component": "sync-mysql" + }, + "packages/sync-clickhouse": { + "release-type": "node", + "component": "sync-clickhouse" } } } From 484eec6cd28e35bca4e53763a09df4d338e08e93 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 5 Nov 2026 16:21:04 -0500 Subject: [PATCH 02/10] fix(clickhouse): lock writes per stream, verify live views, retry checkpoint races Every mirror write is read-decide-write, and ClickHouse has no row lock to serialize it, so two writers on one key (the tail beside a backfill, or a poll outliving its lease) judged against the state before the other's insert landed: a merge could drop fields, a tombstone could lose a version tie, an append-only event could land twice. Each batch now holds a per-stream lock in a `_locks` KeeperMap table: strict-mode INSERT to take it, DELETE to release, conditional UPDATE to take over one a crashed process left behind once it expires. A lock wait that runs out is a transient error, so the engine retries rather than quarantines. `migrate` left a view alone whenever it had every column; a foreign definition with every column now goes through `applyLiveView` and is refused, or replaced under `--repair`. A checkpoint that Keeper refused over a version race (the holder's own renewal is enough) re-reads and retries instead of reporting a moved cursor and replaying the poll. Also: release-please calls the publish workflow for this package, and the test server config loses the `--` XML forbids inside a comment, which kept CI's ClickHouse from starting. --- .github/workflows/release-please.yml | 16 +- packages/sync-clickhouse/README.md | 8 +- packages/sync-clickhouse/src/db/errors.ts | 3 + packages/sync-clickhouse/src/db/lease.ts | 70 ++++++--- packages/sync-clickhouse/src/db/lock.ts | 140 ++++++++++++++++++ packages/sync-clickhouse/src/db/upsert.ts | 105 ++++++++----- packages/sync-clickhouse/src/schema/ddl.ts | 11 +- .../sync-clickhouse/src/schema/migrate.ts | 34 +++-- .../test/clickhouse/config.xml | 8 +- packages/sync-clickhouse/test/lease.test.ts | 43 ++++++ packages/sync-clickhouse/test/lock.test.ts | 108 ++++++++++++++ packages/sync-clickhouse/test/schema.test.ts | 17 ++- 12 files changed, 478 insertions(+), 85 deletions(-) create mode 100644 packages/sync-clickhouse/src/db/lock.ts create mode 100644 packages/sync-clickhouse/test/lock.test.ts diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 5f8ece8..f09b988 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -1,6 +1,6 @@ name: Release Please -# Both packages have npm trusted publishers that name THIS file, not +# Every package has an npm trusted publisher that names THIS file, not # release.yml: when a publish runs inside a workflow_call, npm validates the # calling workflow's filename. Provenance is attached automatically once the # repo is public; nothing here changes for that. @@ -35,6 +35,8 @@ jobs: sync-engine--tag_name: ${{ steps.release.outputs['packages/sync-engine--tag_name'] }} sync-postgres--release_created: ${{ steps.release.outputs['packages/sync-postgres--release_created'] }} sync-postgres--tag_name: ${{ steps.release.outputs['packages/sync-postgres--tag_name'] }} + sync-clickhouse--release_created: ${{ steps.release.outputs['packages/sync-clickhouse--release_created'] }} + sync-clickhouse--tag_name: ${{ steps.release.outputs['packages/sync-clickhouse--tag_name'] }} steps: # Release PRs and tags come from the SDK Automation Bot, as in workos-node. # Events raised with GITHUB_TOKEN start no workflow runs, so a release PR @@ -80,3 +82,15 @@ jobs: tag_name: ${{ needs.release-please.outputs.sync-postgres--tag_name }} secrets: PUBLIC_SOCKET_FIREWALL_TOKEN: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }} + + publish-sync-clickhouse: + needs: [release-please, publish-sync-engine] + if: >- + ${{ !cancelled() + && needs.publish-sync-engine.result != 'failure' + && needs.release-please.outputs.sync-clickhouse--release_created == 'true' }} + uses: ./.github/workflows/release.yml + with: + tag_name: ${{ needs.release-please.outputs.sync-clickhouse--tag_name }} + secrets: + PUBLIC_SOCKET_FIREWALL_TOKEN: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }} diff --git a/packages/sync-clickhouse/README.md b/packages/sync-clickhouse/README.md index 50efccb..efc0e5e 100644 --- a/packages/sync-clickhouse/README.md +++ b/packages/sync-clickhouse/README.md @@ -31,9 +31,11 @@ the mirror streams when you want to join against them. - ClickHouse 23.3 or later, self-hosted or ClickHouse Cloud. Tested against 26.x in CI and against ClickHouse Cloud. - A ClickHouse Keeper (or ZooKeeper) reachable by the server, with - `keeper_map_path_prefix` set. The single-writer lease and the cursors live - in a [KeeperMap](https://clickhouse.com/docs/engines/table-engines/special/keeper-map) - table: it is the one place ClickHouse offers an atomic compare-and-swap. + `keeper_map_path_prefix` set. The single-writer lease, the cursors, and + the per-stream write lock that keeps two processes from writing one row + at once live in + [KeeperMap](https://clickhouse.com/docs/engines/table-engines/special/keeper-map) + tables: it is the one place ClickHouse offers an atomic compare-and-swap. Cloud and any replicated cluster have this already. A single self-hosted node needs the embedded Keeper enabled; the few lines are in [`test/clickhouse/config.xml`](test/clickhouse/config.xml). diff --git a/packages/sync-clickhouse/src/db/errors.ts b/packages/sync-clickhouse/src/db/errors.ts index 8db1284..f7bdcb5 100644 --- a/packages/sync-clickhouse/src/db/errors.ts +++ b/packages/sync-clickhouse/src/db/errors.ts @@ -55,6 +55,9 @@ const TRANSIENT_MESSAGES = ['socket hang up', 'Timeout error', 'ECONNRESET', 'fe export function isTransientClickHouseError(err: unknown): boolean { if (typeof err !== 'object' || err === null) return false const { code, message } = err as { code?: unknown; message?: unknown } + // Raised here, not by the server: a write lock (`lock.ts`) that stayed + // taken for the whole wait. Whoever holds it finishes or expires. + if (code === 'LOCK_TIMEOUT') return true if (typeof code === 'string' && (TRANSIENT_CODES.has(code) || TRANSIENT_SOCKET_CODES.has(code))) return true if (typeof message === 'string' && TRANSIENT_MESSAGES.some((m) => message.includes(m))) return true return false diff --git a/packages/sync-clickhouse/src/db/lease.ts b/packages/sync-clickhouse/src/db/lease.ts index ad963d1..79c7902 100644 --- a/packages/sync-clickhouse/src/db/lease.ts +++ b/packages/sync-clickhouse/src/db/lease.ts @@ -186,7 +186,13 @@ export async function getState( return stored === undefined ? null : (JSON.parse(stored) as T) } -/** Replace `states` with `next` only if it still reads `previous`. Whether it did. */ +/** + * Replace `states` with `next` only if it still reads `previous`. `swapped` + * when it did; `unchanged` when the `WHERE` matched nothing; `conflict` + * when Keeper refused the write because the row's version moved between the + * mutation's read and its write — which any write to the row does, a lease + * renewal included, whether or not it touched `states`. + */ async function swapStates( db: ChClient, schema: string, @@ -196,7 +202,7 @@ async function swapStates( next: string, guard = '', params: Record = {}, -): Promise { +): Promise<'swapped' | 'unchanged' | 'conflict'> { try { await db.command( `ALTER TABLE ${table(schema)} UPDATE states = {next:String} @@ -206,10 +212,10 @@ async function swapStates( ) } catch (err) { if (!isKeeperConflict(err)) throw err - return false + return 'conflict' } const after = await readSync(db, schema, syncId, env) - return after !== null && after.states === next + return after !== null && after.states === next ? 'swapped' : 'unchanged' } export async function setState( @@ -230,7 +236,7 @@ export async function setState( const states = parseStates(row) if (states[stream] === value) return const next = JSON.stringify({ ...states, [stream]: value }) - if (await swapStates(db, schema, syncId, env, row.states, next)) return + if ((await swapStates(db, schema, syncId, env, row.states, next)) === 'swapped') return await backoff() } throw new Error(`could not save the ${stream} state for ${env}: the row kept changing under ${ATTEMPTS} attempts`) @@ -240,6 +246,15 @@ export async function setState( * One read, one conditional write: the holder, the unexpired lease and the * cursor as it was expected to be are all judged on the row as read, and * the write names that exact row, so it lands only if nothing changed. + * + * A write Keeper refused over the row's version, or a `WHERE` that matched + * nothing, is not yet a refusal: the holder's own renewal moves the version + * without touching a cursor, and a backfill checkpoint for another stream + * changes `states` without touching this one. So the row is read again and + * judged again, and only a lease that is no longer held or a cursor that + * is no longer where it was expected refuses the checkpoint. The engine + * treats a refusal as "someone else moved the cursor" and replays the + * poll, which must not happen over a race nobody lost. */ export async function setStateIfHeld( db: ChClient, @@ -251,22 +266,31 @@ export async function setStateIfHeld( holder: string, expected: unknown, ): Promise { - const row = await readSync(db, schema, syncId, env) - if (!isHeld(row, holder)) return false - const states = parseStates(row) - const stored = states[stream] - if (expected == null ? stored !== undefined : stored !== canonicalJson(expected)) return false - const next = JSON.stringify({ ...states, [stream]: canonicalJson(state) }) - // Expiry is judged again by the server, in the write itself: the read - // above may have waited, and a lease that ran out meanwhile must refuse. - return swapStates( - db, - schema, - syncId, - env, - row!.states, - next, - ` AND holder = {holder:String} AND expires_at > now64(3, 'UTC')`, - { holder }, - ) + const value = canonicalJson(state) + const wanted = expected == null ? undefined : canonicalJson(expected) + for (let attempt = 0; attempt < ATTEMPTS; attempt++) { + if (attempt > 0) await backoff() + const row = await readSync(db, schema, syncId, env) + if (!isHeld(row, holder)) return false + const states = parseStates(row) + const stored = states[stream] + // A write of ours on an earlier attempt may have landed while its + // verdict was lost to a race; the row then already says what we meant. + if (stored !== wanted) return attempt > 0 && stored === value + const next = JSON.stringify({ ...states, [stream]: value }) + // Expiry is judged again by the server, in the write itself: the read + // above may have waited, and a lease that ran out meanwhile must refuse. + const outcome = await swapStates( + db, + schema, + syncId, + env, + row!.states, + next, + ` AND holder = {holder:String} AND expires_at > now64(3, 'UTC')`, + { holder }, + ) + if (outcome === 'swapped') return true + } + throw new Error(`could not checkpoint ${stream} for ${env}: the row kept changing under ${ATTEMPTS} attempts`) } diff --git a/packages/sync-clickhouse/src/db/lock.ts b/packages/sync-clickhouse/src/db/lock.ts new file mode 100644 index 0000000..dc3b365 --- /dev/null +++ b/packages/sync-clickhouse/src/db/lock.ts @@ -0,0 +1,140 @@ +/** + * The per-stream write lock. + * + * Every mirror write is read-decide-write: a batch reads the current state + * of its rows, judges each in Node — the staleness gate, the shallow merge, + * which tombstone to carry — and inserts the next versions. Postgres and + * MySQL serialise that with row locks inside a transaction. ClickHouse has + * neither, and two writers can meet on one key: the tail and a backfill run + * side by side, and a poll can finish its writes after its lease has passed + * to another process. Each would judge against the state before the other's + * insert landed: a merge could drop the fields the other wrote, a tombstone + * could lose a version tie to a live row it never saw, an append-only event + * could be inserted twice. + * + * So each batch holds a lock for its table and environment while it reads + * and writes. The lock is a KeeperMap row: a strict-mode `INSERT` takes it + * (refused when the key is there), a `DELETE` of the holder's own row + * releases it, and a row a crashed process left behind is taken over, once + * it has expired on the server's clock, with the conditional `UPDATE` the + * lease uses — so a live holder is never displaced. Every acquisition has + * its own token, so two batches in one process contend like two processes + * do. A writer that cannot get the lock within `waitMs` fails with + * `LOCK_TIMEOUT`, which the error classifier reports as transient: the + * engine retries the poll rather than quarantining the event. + */ +import { randomUUID } from 'node:crypto' +import { KEEPER_SETTINGS, type ChClient } from './client.js' +import { isKeeperConflict } from './errors.js' +import { quoteIdent } from '../schema/ddl.js' + +/** How long a lock outlives the process that took it. A batch holds it for well under a second. */ +export const LOCK_TTL_MS = 30_000 +/** How long a writer waits for a lock before reporting a transient failure. */ +export const LOCK_WAIT_MS = 60_000 + +const SEP = '\u0001' +/** Strict mode named outright: a client built without it would overwrite the row instead of being refused. */ +const SETTINGS = { ...KEEPER_SETTINGS, keeper_map_strict_mode: 1 } as const + +export interface LockOptions { + schema: string + /** Prefix of every token this writer takes; identifies the process in `_locks`. */ + holder: string + ttlMs?: number + waitMs?: number +} + +const table = (schema: string) => `${quoteIdent(schema)}.\`_locks\`` + +/** The lock for one table in one environment. */ +export const lockKey = (tableName: string, env: string) => `${tableName}${SEP}${env}` + +const backoff = () => new Promise((r) => setTimeout(r, 10 + Math.random() * 40)) + +/** Take the lock, waiting for whoever holds it. Resolves to the token that releases it. */ +export async function acquireStreamLock(db: ChClient, opts: LockOptions, key: string): Promise { + const token = `${opts.holder}:${randomUUID()}` + const ttl = Math.max(1, Math.floor(opts.ttlMs ?? LOCK_TTL_MS)) + const waitMs = opts.waitMs ?? LOCK_WAIT_MS + const deadline = Date.now() + waitMs + const params = { k: key, holder: token, ttl } + for (let attempt = 0; ; attempt++) { + if (attempt > 0) { + if (Date.now() >= deadline) { + throw Object.assign( + new Error(`could not take the write lock for ${key.replace(SEP, ' in ')} within ${waitMs} ms`), + { code: 'LOCK_TIMEOUT' }, + ) + } + await backoff() + } + try { + await db.command( + `INSERT INTO ${table(opts.schema)} (k, holder, expires_at) + VALUES ({k:String}, {holder:String}, now64(3, 'UTC') + toIntervalMillisecond({ttl:UInt64}))`, + params, + SETTINGS, + ) + return token + } catch (err) { + if (!isKeeperConflict(err)) throw err + } + // Taken. Ours only if whoever holds it is gone: the takeover names an + // expired row, and a row that is not expired is left exactly as it was. + try { + await db.command( + `ALTER TABLE ${table(opts.schema)} UPDATE + holder = {holder:String}, + expires_at = now64(3, 'UTC') + toIntervalMillisecond({ttl:UInt64}) + WHERE k = {k:String} AND expires_at <= now64(3, 'UTC')`, + params, + SETTINGS, + ) + } catch (err) { + if (!isKeeperConflict(err)) throw err + continue + } + const rows = await db.query<{ holder: string }>(`SELECT holder FROM ${table(opts.schema)} WHERE k = {k:String}`, { + k: key, + }) + if (rows[0]?.holder === token) return token + } +} + +/** Give the lock back. A lock that was taken over meanwhile is not ours to release and is left alone. */ +export async function releaseStreamLock(db: ChClient, opts: LockOptions, key: string, token: string): Promise { + for (let attempt = 0; attempt < 8; attempt++) { + try { + await db.command( + `ALTER TABLE ${table(opts.schema)} DELETE WHERE k = {k:String} AND holder = {holder:String}`, + { k: key, holder: token }, + SETTINGS, + ) + return + } catch (err) { + if (!isKeeperConflict(err)) throw err + } + const rows = await db.query<{ holder: string }>(`SELECT holder FROM ${table(opts.schema)} WHERE k = {k:String}`, { + k: key, + }) + if (rows[0]?.holder !== token) return + await backoff() + } + // Still ours after every attempt: it expires on its own, and says so in the log the caller keeps. +} + +/** Run `fn` holding the lock for `key`, releasing it however `fn` ends. */ +export async function withStreamLock( + db: ChClient, + opts: LockOptions, + key: string, + fn: () => Promise, +): Promise { + const token = await acquireStreamLock(db, opts, key) + try { + return await fn() + } finally { + await releaseStreamLock(db, opts, key, token) + } +} diff --git a/packages/sync-clickhouse/src/db/upsert.ts b/packages/sync-clickhouse/src/db/upsert.ts index 3c77de2..6983867 100644 --- a/packages/sync-clickhouse/src/db/upsert.ts +++ b/packages/sync-clickhouse/src/db/upsert.ts @@ -13,14 +13,19 @@ * — the gate, the shallow merge, whether anything changed, which tombstone * to carry forward — and inserts only the rows that passed. The table then * keeps, for each key, the version with the newest `_updated_at`, and the - * last inserted among equals; so even two writers racing resolve the way - * the gate would have, and a refused row, never inserted, changes nothing. - * Append-only streams are plain `MergeTree`: the batch reads which ids are - * already there and inserts the rest, which is what makes re-delivery a - * no-op. + * last inserted among equals; a refused row, never inserted, changes + * nothing. Reading and writing are two statements, so each batch holds the + * stream's write lock (`lock.ts`) across both: two writers on one key — the + * tail and a backfill, or a poll outliving its lease — then see each + * other's rows instead of each judging against the state before the other + * wrote. Append-only streams are plain `MergeTree`: under the same lock the + * batch reads which ids are already there and inserts the rest, which is + * what makes re-delivery a no-op. */ import { emptyStats, type Row, type TableDef, type WriteMode, type WriteStats } from '@workos/sync-engine' +import { hostname } from 'node:os' import { fromCh, toCh, type ChClient } from './client.js' +import { lockKey, withStreamLock, type LockOptions } from './lock.js' import { quoteIdent, safeIdentifier, ENV, RAW } from '../schema/ddl.js' /** Joins the components of a composite key, on both sides of every lookup. */ @@ -145,9 +150,25 @@ const later = (a: Date | null, b: Date): Date => (a !== null && a.getTime() > b. export interface WriteOptions { schema: string batchSize?: number + /** Names this writer in `_locks`. One per process by default. */ + lockHolder?: string + lockTtlMs?: number + lockWaitMs?: number } const DEFAULT_BATCH_SIZE = 500 +const PROCESS_ID = `${hostname()}:${process.pid}` + +const lockOptions = (opts: WriteOptions): LockOptions => ({ + schema: opts.schema, + holder: opts.lockHolder ?? PROCESS_ID, + ttlMs: opts.lockTtlMs, + waitMs: opts.lockWaitMs, +}) + +/** Run `fn` holding the write lock for `stream` in `env`. */ +const locked = (db: ChClient, opts: WriteOptions, stream: TableDef, env: string, fn: () => Promise) => + withStreamLock(db, lockOptions(opts), lockKey(safeIdentifier(stream.name), env), fn) export const tableRef = (schema: string, stream: TableDef) => `${quoteIdent(schema)}.${quoteIdent(safeIdentifier(stream.name))}` @@ -244,9 +265,9 @@ const keyed = (stream: TableDef, rows: Row[]): Keyed[] => rows.map((r) => ({ key: keyValues(stream, r.data)!, data: r.data, updatedAt: parseUpdatedAt(r.updatedAt) })) /** - * Read, decide, write: the shape of every mirror write. `decide` is handed - * each row with what is stored for it and returns what to write, or null to - * leave the row alone. + * Read, decide, write: the shape of every mirror write, under the stream's + * lock. `decide` is handed each row with what is stored for it and returns + * what to write, or null to leave the row alone. */ async function applyBatch( db: ChClient, @@ -256,19 +277,21 @@ async function applyBatch( batch: Keyed[], decide: (row: Keyed, stored: StoredRow | undefined) => Pending | null, ): Promise { - const stored = await readRows( - db, - opts, - stream, - env, - batch.map((r) => r.key), - ) - const pending: Pending[] = [] - for (const row of batch) { - const out = decide(row, stored.get(row.key.join(KEY_SEPARATOR))) - if (out) pending.push(out) - } - await writeRows(db, opts, stream, env, pending) + await locked(db, opts, stream, env, async () => { + const stored = await readRows( + db, + opts, + stream, + env, + batch.map((r) => r.key), + ) + const pending: Pending[] = [] + for (const row of batch) { + const out = decide(row, stored.get(row.key.join(KEY_SEPARATOR))) + if (out) pending.push(out) + } + await writeRows(db, opts, stream, env, pending) + }) } export async function upsertRows( @@ -288,24 +311,28 @@ export async function upsertRows( for (const wave of waves) { for (const batch of chunk(keyed(stream, wave), opts.batchSize ?? DEFAULT_BATCH_SIZE)) { if (stream.mode === 'append') { - // Immutable log. Re-delivery is a no-op, which is the whole idempotency story. - const present = await existingKeys( - db, - opts, - stream, - env, - batch.map((r) => r.key), - ) - const fresh = batch.filter((r) => !present.has(r.key.join(KEY_SEPARATOR))) - await writeRows( - db, - opts, - stream, - env, - fresh.map((r) => ({ key: r.key, raw: r.data, updatedAt: r.updatedAt, deletedAt: null })), - ) - stats.created += fresh.length - stats.skipped += batch.length - fresh.length + // Immutable log. Re-delivery is a no-op, which is the whole + // idempotency story; the lock is what makes it hold for two polls + // delivering the same page at once. + await locked(db, opts, stream, env, async () => { + const present = await existingKeys( + db, + opts, + stream, + env, + batch.map((r) => r.key), + ) + const fresh = batch.filter((r) => !present.has(r.key.join(KEY_SEPARATOR))) + await writeRows( + db, + opts, + stream, + env, + fresh.map((r) => ({ key: r.key, raw: r.data, updatedAt: r.updatedAt, deletedAt: null })), + ) + stats.created += fresh.length + stats.skipped += batch.length - fresh.length + }) continue } await applyBatch(db, opts, stream, env, batch, (row, stored) => { diff --git a/packages/sync-clickhouse/src/schema/ddl.ts b/packages/sync-clickhouse/src/schema/ddl.ts index ebceb6a..87c871c 100644 --- a/packages/sync-clickhouse/src/schema/ddl.ts +++ b/packages/sync-clickhouse/src/schema/ddl.ts @@ -115,7 +115,8 @@ export function createSchemaSql(schema: string): string { * swap, which is what the lease and the fenced checkpoint need. See * `db/lease.ts`. The applied-schema fingerprints and the record of generated * columns live there too: an `INSERT` of an existing key is refused, which - * makes "record once" a single statement. + * makes "record once" a single statement — and "take the lock" one as well, + * for the per-stream write lock in `_locks` (see `db/lock.ts`). * * `_rejected` holds events the tail set aside after they failed * deterministically on two consecutive polls (see the engine's @@ -138,6 +139,14 @@ export function bookkeepingSql(schema: string): [name: string, sql: string][] { expires_at ${TIMESTAMP}, states String ) ENGINE = KeeperMap(${keeperPath(schema, '_sync')}) PRIMARY KEY k`, + ], + [ + '_locks', + `CREATE TABLE IF NOT EXISTS ${s}.\`_locks\` ( + k String, + holder String, + expires_at ${TIMESTAMP} +) ENGINE = KeeperMap(${keeperPath(schema, '_locks')}) PRIMARY KEY k`, ], [ '_migrations', diff --git a/packages/sync-clickhouse/src/schema/migrate.ts b/packages/sync-clickhouse/src/schema/migrate.ts index cc1f27f..dc77c22 100644 --- a/packages/sync-clickhouse/src/schema/migrate.ts +++ b/packages/sync-clickhouse/src/schema/migrate.ts @@ -29,7 +29,7 @@ import { quoteIdent, safeIdentifier, } from './ddl.js' -import { applyLiveView, liveViewName, type ApplyViewOptions } from './views.js' +import { applyLiveView, isGeneratedDefinition, liveViewName, type ApplyViewOptions } from './views.js' import { detectColumnDrift, recordColumn, repairColumnDrift, unrepairableReason, type ColumnDrift } from './drift.js' /** @@ -145,19 +145,29 @@ export async function migrate( } } // Last, so the view carries every column, including any added above. A - // view that already does is left alone: `init` runs no DDL it does not need. + // view that is the generated shape and already does is left alone: + // `init` runs no DDL it does not need. Anything else under the name — + // a column short, or a definition that is not ours, however many + // columns it has — goes through `applyLiveView`, which refuses a + // foreign definition unless `repairViews` says to replace it. if (stream.mode !== 'append' && live.size > 0) { - const viewColumns = await names( - db, - `SELECT name FROM system.columns WHERE database = {schema:String} AND table = {table:String}`, - { schema, table: liveViewName(stream) }, + const [view] = await db.query<{ engine: string; as_select: string }>( + `SELECT engine, as_select FROM system.tables WHERE database = {schema:String} AND name = {name:String}`, + { schema, name: liveViewName(stream) }, ) - const tableColumns = await names( - db, - `SELECT name FROM system.columns WHERE database = {schema:String} AND table = {table:String}`, - { schema, table }, - ) - if (viewColumns.size > 0 && [...tableColumns].every((c) => viewColumns.has(c))) continue + if (view?.engine === 'View' && isGeneratedDefinition(view.as_select, schema, table)) { + const viewColumns = await names( + db, + `SELECT name FROM system.columns WHERE database = {schema:String} AND table = {table:String}`, + { schema, table: liveViewName(stream) }, + ) + const tableColumns = await names( + db, + `SELECT name FROM system.columns WHERE database = {schema:String} AND table = {table:String}`, + { schema, table }, + ) + if ([...tableColumns].every((c) => viewColumns.has(c))) continue + } } if (await applyLiveView(db, stream, schema, logger, options)) statements++ } diff --git a/packages/sync-clickhouse/test/clickhouse/config.xml b/packages/sync-clickhouse/test/clickhouse/config.xml index eafd6dc..0e27bdc 100644 --- a/packages/sync-clickhouse/test/clickhouse/config.xml +++ b/packages/sync-clickhouse/test/clickhouse/config.xml @@ -2,11 +2,9 @@ Server configuration for the test container: an embedded ClickHouse Keeper (KeeperMap needs one), the `keeper_map_path_prefix` that enables the engine, and seccomp off for Docker Desktop's VM, which refuses the filter. - - docker run -d --name clickhouse -p 127.0.0.1:8123:8123 \ - -v "$PWD/packages/sync-clickhouse/test/clickhouse/config.xml:/etc/clickhouse-server/config.d/sync.xml:ro" \ - -v "$PWD/packages/sync-clickhouse/test/clickhouse/users.xml:/etc/clickhouse-server/users.d/sync.xml:ro" \ - -e CLICKHOUSE_SKIP_USER_SETUP=1 clickhouse/clickhouse-server:latest + The README's Testing section has the docker run command that mounts this + file and users.xml. (No double hyphens in here: XML forbids them in a + comment, and the server refuses the whole file.) --> diff --git a/packages/sync-clickhouse/test/lease.test.ts b/packages/sync-clickhouse/test/lease.test.ts index 0b2f725..555e330 100644 --- a/packages/sync-clickhouse/test/lease.test.ts +++ b/packages/sync-clickhouse/test/lease.test.ts @@ -39,6 +39,49 @@ describeDb('events tail lease', () => { { async_insert: 0, keeper_map_strict_mode: 0 }, ) + it('retries a checkpoint that lost a Keeper version race, and still refuses a cursor that moved', async () => { + await dest.acquireLease('prod', 'a', 60_000) + await dest.setState('prod', '__tail__', { cursor: 1 }) + // Refuses the first checkpoint write the way Keeper does when the row's + // version moved under it — the holder's own renewal is enough — then + // passes everything through. + let refused = 0 + const racing: TestDb = { + ...db, + async command(sql, params, settings) { + if (refused === 0 && /UPDATE states =/.test(sql)) { + refused++ + throw Object.assign(new Error('Code: 999. DB::Exception: Bad version. (KEEPER_EXCEPTION)'), { code: '999' }) + } + return db.command(sql, params, settings) + }, + } + const viaRace = new ClickHouseDestination({ db: racing, schema: db.schema }) + expect(await viaRace.setStateIfHeld('prod', '__tail__', { cursor: 2 }, 'a', { cursor: 1 })).toBe(true) + expect(refused).toBe(1) + expect(await dest.getState('prod', '__tail__')).toEqual({ cursor: 2 }) + + refused = 0 + await dest.setState('prod', '__tail__', { cursor: 5 }) + expect(await viaRace.setStateIfHeld('prod', '__tail__', { cursor: 3 }, 'a', { cursor: 2 })).toBe(false) + expect(await dest.getState('prod', '__tail__')).toEqual({ cursor: 5 }) + }) + + it('checkpoints while the holder renews, without a refusal', async () => { + await dest.acquireLease('prod', 'a', 60_000) + let cursor = 0 + await dest.setState('prod', '__tail__', { cursor }) + for (let i = 0; i < 10; i++) { + const [, ok] = await Promise.all([ + dest.acquireLease('prod', 'a', 60_000), + dest.setStateIfHeld('prod', '__tail__', { cursor: cursor + 1 }, 'a', { cursor }), + ]) + expect(ok).toBe(true) + cursor++ + } + expect(await dest.getState('prod', '__tail__')).toEqual({ cursor: 10 }) + }) + it('is taken by the first process to ask', async () => { const r = await dest.acquireLease('prod', 'a', 60_000) expect(r.acquired).toBe(true) diff --git a/packages/sync-clickhouse/test/lock.test.ts b/packages/sync-clickhouse/test/lock.test.ts new file mode 100644 index 0000000..750d7f9 --- /dev/null +++ b/packages/sync-clickhouse/test/lock.test.ts @@ -0,0 +1,108 @@ +import { it, expect, beforeEach, afterEach } from 'vitest' +import { streamByName } from '@workos/sync-engine' +import { createTestDb, describeDb, type TestDb } from './helpers/clickhouse.js' +import { migrate } from '../src/schema/migrate.js' +import { acquireStreamLock, releaseStreamLock, withStreamLock, lockKey } from '../src/db/lock.js' +import { upsertRows, deleteRows } from '../src/db/upsert.js' +import { isTransientClickHouseError } from '../src/db/errors.js' + +const users = streamByName('users')! +const events = streamByName('events')! + +describeDb('per-stream write lock', () => { + let db: TestDb + const key = lockKey('users', 'prod') + + beforeEach(async () => { + db = await createTestDb() + await migrate(db, [users, events], db.schema) + }) + afterEach(async () => { + await db.close() + }) + + it('admits one holder at a time, and the next once it is released', async () => { + const a = { schema: db.schema, holder: 'a' } + const b = { schema: db.schema, holder: 'b', waitMs: 300 } + const token = await acquireStreamLock(db, a, key) + const err = await withStreamLock(db, b, key, async () => 'ran').catch((e: unknown) => e) + expect(err).toMatchObject({ code: 'LOCK_TIMEOUT' }) + expect(isTransientClickHouseError(err)).toBe(true) + await releaseStreamLock(db, a, key, token) + expect(await withStreamLock(db, b, key, async () => 'ran')).toBe('ran') + }) + + it("is taken over once a crashed holder's lock expires, and not before", async () => { + await acquireStreamLock(db, { schema: db.schema, holder: 'crashed', ttlMs: 400 }, key) + const started = Date.now() + expect(await withStreamLock(db, { schema: db.schema, holder: 'next', waitMs: 5_000 }, key, async () => 'ran')).toBe( + 'ran', + ) + expect(Date.now() - started).toBeGreaterThanOrEqual(300) + }) + + it('is per table and per environment', async () => { + const a = { schema: db.schema, holder: 'a', waitMs: 300 } + await acquireStreamLock(db, a, key) + expect(await withStreamLock(db, a, lockKey('users', 'staging'), async () => 'ran')).toBe('ran') + expect(await withStreamLock(db, a, lockKey('events', 'prod'), async () => 'ran')).toBe('ran') + }) + + it('never lets two critical sections overlap, even within one process', async () => { + let active = 0 + let overlaps = 0 + let runs = 0 + await Promise.all( + Array.from({ length: 8 }, () => + withStreamLock(db, { schema: db.schema, holder: 'same' }, key, async () => { + if (active++ > 0) overlaps++ + await new Promise((r) => setTimeout(r, 20)) + active-- + runs++ + }), + ), + ) + expect(runs).toBe(8) + expect(overlaps).toBe(0) + }) + + it('serialises concurrent writers so an append-only event lands once', async () => { + const rows = Array.from({ length: 20 }, (_, i) => ({ + data: { id: `event_${i}`, event: 'user.created', created_at: '2026-01-01T00:00:00.000Z', data: {} }, + updatedAt: '2026-01-01T00:00:00.000Z', + })) + // Six polls delivering the same page at once: a lease takeover's worst case. + await Promise.all( + Array.from({ length: 6 }, (_, p) => + upsertRows(db, { schema: db.schema, lockHolder: `poll${p}` }, events, 'prod', rows, 'merge'), + ), + ) + const n = await db.query<{ n: string }>( + `SELECT toString(count()) AS n FROM \`${db.schema}\`.events WHERE \`_env\` = 'prod'`, + ) + expect(Number(n[0]!.n)).toBe(20) + }) + + it('keeps a tombstone that a concurrent snapshot write would otherwise outrank', async () => { + const payload = { object: 'user', id: 'user_1', email: 'ada@example.com', updated_at: '2026-01-01T00:00:00.000Z' } + const at = '2026-01-01T00:00:00.000Z' + await upsertRows(db, { schema: db.schema }, users, 'prod', [{ data: payload, updatedAt: at }], 'replace') + // A refresh re-listing the same snapshot with a field events omit, while + // the tail deletes the user. Both versions carry the same `_updated_at`, + // so unserialised the tie would go to whichever insert landed last. + await Promise.all([ + deleteRows(db, { schema: db.schema, lockHolder: 'tail' }, users, 'prod', [ + { data: { id: 'user_1' }, updatedAt: '2026-01-01T00:00:00.500Z' }, + ]), + upsertRows( + db, + { schema: db.schema, lockHolder: 'refresh' }, + users, + 'prod', + [{ data: { ...payload, last_sign_in_at: at }, updatedAt: at }], + 'replace', + ), + ]) + expect(await db.query(`SELECT id FROM \`${db.schema}\`.users_live`)).toEqual([]) + }) +}) diff --git a/packages/sync-clickhouse/test/schema.test.ts b/packages/sync-clickhouse/test/schema.test.ts index 75dbe67..fdaea90 100644 --- a/packages/sync-clickhouse/test/schema.test.ts +++ b/packages/sync-clickhouse/test/schema.test.ts @@ -145,7 +145,7 @@ describeDb('migrations against real ClickHouse', () => { await migrate(db, STREAMS, db.schema) const names = await tables() for (const s of STREAMS) expect(names).toContain(s.name) - for (const t of ['_sync', '_migrations', '_rejected', '_columns']) expect(names).toContain(t) + for (const t of ['_sync', '_locks', '_migrations', '_rejected', '_columns']) expect(names).toContain(t) const recorded = await db.query<{ n: string }>(`SELECT toString(count()) AS n FROM \`${db.schema}\`.\`_columns\``) expect(Number(recorded[0]!.n)).toBe(STREAMS.reduce((n, s) => n + s.columns.length, 0)) }) @@ -336,6 +336,21 @@ describeDb('live views against real ClickHouse', () => { expect(isGeneratedDefinition(r[0]!.as_select, db.schema, 'users')).toBe(true) }) + it('refuses a foreign view that happens to have every column, rather than mistaking it for its own', async () => { + await migrate(db, [users], db.schema) + await db.command( + `CREATE OR REPLACE VIEW \`${db.schema}\`.users_live AS SELECT * FROM \`${db.schema}\`.users WHERE email LIKE '%@acme.com'`, + ) + await forget() + await expect(migrate(db, [users], db.schema)).rejects.toThrow(/did not generate/) + await migrate(db, [users], db.schema, undefined, { repairViews: true }) + const r = await db.query<{ as_select: string }>( + `SELECT as_select FROM system.tables WHERE database = {s:String} AND name = 'users_live'`, + { s: db.schema }, + ) + expect(isGeneratedDefinition(r[0]!.as_select, db.schema, 'users')).toBe(true) + }) + it('refuses a table standing where its view would go', async () => { await db.command(`CREATE DATABASE \`${db.schema}\``) await db.command(`CREATE TABLE \`${db.schema}\`.users_live (id Int32) ENGINE = MergeTree ORDER BY id`) From 729e0b415decba1f38f8ab214336a02582189941 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 5 Nov 2026 16:33:24 -0500 Subject: [PATCH 03/10] docs(clickhouse): say why the tombstone's tie-break is deterministic --- packages/sync-clickhouse/src/db/upsert.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/sync-clickhouse/src/db/upsert.ts b/packages/sync-clickhouse/src/db/upsert.ts index 6983867..7d6b33c 100644 --- a/packages/sync-clickhouse/src/db/upsert.ts +++ b/packages/sync-clickhouse/src/db/upsert.ts @@ -379,6 +379,9 @@ export async function deleteRows( // re-added and removed again must carry the last removal, because that // is what `relinkRows` compares a re-add against. `_updated_at` stays, // so the version is a tie and the tombstone, inserted later, wins it. + // "Later" is certain: the batch holds the stream's write lock from its + // read to its write, so no other writer's row — a refresh re-listing + // the same snapshot, say — can land between the two and take the tie. return { key: row.key, raw: merge(stored.raw, row.data), From df50485ebd01d6dbe045a13a1f24c99d572a2777 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 5 Nov 2026 16:42:02 -0500 Subject: [PATCH 04/10] fix(clickhouse): refuse a write once the lock is lost, version tombstones at deletion time A batch that stalled past the lock's TTL could still insert rows judged against state another writer has since changed. It now checks it still holds the lock right before writing and fails with LOCK_LOST, a transient error the engine retries from a fresh read. An object delete's tombstone takes the deletion time as its version instead of the row's last update time, so it outranks any older snapshot on the version itself rather than on which insert landed last. The deletion came after anything a listing could still show of the row, so nothing legitimate is outranked. --- packages/sync-clickhouse/README.md | 4 +-- packages/sync-clickhouse/src/db/errors.ts | 7 +++-- packages/sync-clickhouse/src/db/lock.ts | 30 +++++++++++++++--- packages/sync-clickhouse/src/db/upsert.ts | 32 ++++++++++++-------- packages/sync-clickhouse/test/lock.test.ts | 25 +++++++++++++++ packages/sync-clickhouse/test/upsert.test.ts | 10 +++--- 6 files changed, 82 insertions(+), 26 deletions(-) diff --git a/packages/sync-clickhouse/README.md b/packages/sync-clickhouse/README.md index efc0e5e..f75a604 100644 --- a/packages/sync-clickhouse/README.md +++ b/packages/sync-clickhouse/README.md @@ -171,7 +171,7 @@ The engine writes one column, `_raw`, and computes the rest from it with CREATE TABLE workos.auth_attempts ( _env String, _raw String, -- the event payload, the only column the engine writes - _updated_at DateTime64(3, 'UTC'), -- the event's own time + _updated_at DateTime64(3, 'UTC'), -- the event's own time; a tombstone's is its deletion time _synced_at DateTime64(6, 'UTC') DEFAULT now64(6), event_id String DEFAULT JSONExtractString(_raw, 'event_id'), status Nullable(String) DEFAULT JSONExtract(_raw, 'status', 'Nullable(String)'), @@ -188,7 +188,7 @@ per event, never updated, the shape every ClickHouse query expects. Mirror streams are `ReplacingMergeTree` versioned by `_updated_at`. Every write inserts the row's next state, and the newest source timestamp wins at merge time and under `FINAL`; that is the staleness gate, enforced by the -table whatever order writes land in. Each mirror table has a generated +table whatever order writes land in. A deletion's tombstone carries the deletion time as its version, so it outranks every earlier version of the row. Each mirror table has a generated `
_live` view — `workos.users_live`, `workos.organizations_live`, and so on — that reads with `FINAL` and hides tombstoned rows. **Query the views.** The base table shows every version a row ever had, plus its tombstones diff --git a/packages/sync-clickhouse/src/db/errors.ts b/packages/sync-clickhouse/src/db/errors.ts index f7bdcb5..9c6e11c 100644 --- a/packages/sync-clickhouse/src/db/errors.ts +++ b/packages/sync-clickhouse/src/db/errors.ts @@ -55,9 +55,10 @@ const TRANSIENT_MESSAGES = ['socket hang up', 'Timeout error', 'ECONNRESET', 'fe export function isTransientClickHouseError(err: unknown): boolean { if (typeof err !== 'object' || err === null) return false const { code, message } = err as { code?: unknown; message?: unknown } - // Raised here, not by the server: a write lock (`lock.ts`) that stayed - // taken for the whole wait. Whoever holds it finishes or expires. - if (code === 'LOCK_TIMEOUT') return true + // Raised here, not by the server (`lock.ts`): a write lock that stayed + // taken for the whole wait, or one lost to a takeover while the batch + // stalled. Whoever holds it finishes or expires, and a retry reads afresh. + if (code === 'LOCK_TIMEOUT' || code === 'LOCK_LOST') return true if (typeof code === 'string' && (TRANSIENT_CODES.has(code) || TRANSIENT_SOCKET_CODES.has(code))) return true if (typeof message === 'string' && TRANSIENT_MESSAGES.some((m) => message.includes(m))) return true return false diff --git a/packages/sync-clickhouse/src/db/lock.ts b/packages/sync-clickhouse/src/db/lock.ts index dc3b365..e4831dc 100644 --- a/packages/sync-clickhouse/src/db/lock.ts +++ b/packages/sync-clickhouse/src/db/lock.ts @@ -20,8 +20,10 @@ * lease uses — so a live holder is never displaced. Every acquisition has * its own token, so two batches in one process contend like two processes * do. A writer that cannot get the lock within `waitMs` fails with - * `LOCK_TIMEOUT`, which the error classifier reports as transient: the - * engine retries the poll rather than quarantining the event. + * `LOCK_TIMEOUT`, and a batch that stalled past the lock's TTL, lost it, + * and asks `held()` before writing fails with `LOCK_LOST`; the error + * classifier reports both as transient, so the engine retries the poll + * rather than quarantining the event, and the retry reads afresh. */ import { randomUUID } from 'node:crypto' import { KEEPER_SETTINGS, type ChClient } from './client.js' @@ -124,16 +126,34 @@ export async function releaseStreamLock(db: ChClient, opts: LockOptions, key: st // Still ours after every attempt: it expires on its own, and says so in the log the caller keeps. } -/** Run `fn` holding the lock for `key`, releasing it however `fn` ends. */ +/** Throws `LOCK_LOST` unless the lock is still the caller's. */ +export type HeldCheck = () => Promise + +/** + * Run `fn` holding the lock for `key`, releasing it however `fn` ends. `fn` + * is handed a check to call right before it writes: a batch that stalled + * past the TTL has lost the lock to a writer whose rows it never read. + */ export async function withStreamLock( db: ChClient, opts: LockOptions, key: string, - fn: () => Promise, + fn: (held: HeldCheck) => Promise, ): Promise { const token = await acquireStreamLock(db, opts, key) + const held: HeldCheck = async () => { + const rows = await db.query<{ holder: string }>(`SELECT holder FROM ${table(opts.schema)} WHERE k = {k:String}`, { + k: key, + }) + if (rows[0]?.holder !== token) { + throw Object.assign( + new Error(`the write lock for ${key.replace(SEP, ' in ')} was taken over: this batch stalled past its TTL`), + { code: 'LOCK_LOST' }, + ) + } + } try { - return await fn() + return await fn(held) } finally { await releaseStreamLock(db, opts, key, token) } diff --git a/packages/sync-clickhouse/src/db/upsert.ts b/packages/sync-clickhouse/src/db/upsert.ts index 7d6b33c..48506c9 100644 --- a/packages/sync-clickhouse/src/db/upsert.ts +++ b/packages/sync-clickhouse/src/db/upsert.ts @@ -25,7 +25,7 @@ import { emptyStats, type Row, type TableDef, type WriteMode, type WriteStats } from '@workos/sync-engine' import { hostname } from 'node:os' import { fromCh, toCh, type ChClient } from './client.js' -import { lockKey, withStreamLock, type LockOptions } from './lock.js' +import { lockKey, withStreamLock, type HeldCheck, type LockOptions } from './lock.js' import { quoteIdent, safeIdentifier, ENV, RAW } from '../schema/ddl.js' /** Joins the components of a composite key, on both sides of every lookup. */ @@ -167,9 +167,13 @@ const lockOptions = (opts: WriteOptions): LockOptions => ({ }) /** Run `fn` holding the write lock for `stream` in `env`. */ -const locked = (db: ChClient, opts: WriteOptions, stream: TableDef, env: string, fn: () => Promise) => - withStreamLock(db, lockOptions(opts), lockKey(safeIdentifier(stream.name), env), fn) - +const locked = ( + db: ChClient, + opts: WriteOptions, + stream: TableDef, + env: string, + fn: (held: HeldCheck) => Promise, +) => withStreamLock(db, lockOptions(opts), lockKey(safeIdentifier(stream.name), env), fn) export const tableRef = (schema: string, stream: TableDef) => `${quoteIdent(schema)}.${quoteIdent(safeIdentifier(stream.name))}` @@ -277,7 +281,7 @@ async function applyBatch( batch: Keyed[], decide: (row: Keyed, stored: StoredRow | undefined) => Pending | null, ): Promise { - await locked(db, opts, stream, env, async () => { + await locked(db, opts, stream, env, async (held) => { const stored = await readRows( db, opts, @@ -290,6 +294,8 @@ async function applyBatch( const out = decide(row, stored.get(row.key.join(KEY_SEPARATOR))) if (out) pending.push(out) } + // Decided against rows read under the lock; written only if it is still ours. + if (pending.length > 0) await held() await writeRows(db, opts, stream, env, pending) }) } @@ -314,7 +320,7 @@ export async function upsertRows( // Immutable log. Re-delivery is a no-op, which is the whole // idempotency story; the lock is what makes it hold for two polls // delivering the same page at once. - await locked(db, opts, stream, env, async () => { + await locked(db, opts, stream, env, async (held) => { const present = await existingKeys( db, opts, @@ -323,6 +329,7 @@ export async function upsertRows( batch.map((r) => r.key), ) const fresh = batch.filter((r) => !present.has(r.key.join(KEY_SEPARATOR))) + if (fresh.length > 0) await held() await writeRows( db, opts, @@ -377,15 +384,16 @@ export async function deleteRows( if (!stored) return { key: row.key, raw: row.data, updatedAt: row.updatedAt, deletedAt: row.updatedAt } // The *latest* removal is kept, not the first: a membership removed, // re-added and removed again must carry the last removal, because that - // is what `relinkRows` compares a re-add against. `_updated_at` stays, - // so the version is a tie and the tombstone, inserted later, wins it. - // "Later" is certain: the batch holds the stream's write lock from its - // read to its write, so no other writer's row — a refresh re-listing - // the same snapshot, say — can land between the two and take the tie. + // is what `relinkRows` compares a re-add against. The version moves to + // the deletion time, where Postgres leaves `_updated_at` alone: here + // the tombstone has to be the newest version of the row, and the + // deletion came after anything a listing could still show of it, so + // it outranks any older snapshot on the version itself — not on which + // insert happened to land last. return { key: row.key, raw: merge(stored.raw, row.data), - updatedAt: stored.updatedAt, + updatedAt: later(stored.updatedAt, row.updatedAt), deletedAt: later(stored.deletedAt, row.updatedAt), } }) diff --git a/packages/sync-clickhouse/test/lock.test.ts b/packages/sync-clickhouse/test/lock.test.ts index 750d7f9..2ad54ee 100644 --- a/packages/sync-clickhouse/test/lock.test.ts +++ b/packages/sync-clickhouse/test/lock.test.ts @@ -41,6 +41,17 @@ describeDb('per-stream write lock', () => { expect(Date.now() - started).toBeGreaterThanOrEqual(300) }) + it('refuses to write once a stalled batch has lost its lock', async () => { + const err = await withStreamLock(db, { schema: db.schema, holder: 'stalled', ttlMs: 300 }, key, async (held) => { + await new Promise((r) => setTimeout(r, 400)) + // Someone else took the expired lock and finished; the stalled batch must not write. + await withStreamLock(db, { schema: db.schema, holder: 'next' }, key, async () => {}) + await held() + }).catch((e: unknown) => e) + expect(err).toMatchObject({ code: 'LOCK_LOST' }) + expect(isTransientClickHouseError(err)).toBe(true) + }) + it('is per table and per environment', async () => { const a = { schema: db.schema, holder: 'a', waitMs: 300 } await acquireStreamLock(db, a, key) @@ -105,4 +116,18 @@ describeDb('per-stream write lock', () => { ]) expect(await db.query(`SELECT id FROM \`${db.schema}\`.users_live`)).toEqual([]) }) + it('versions a tombstone at the deletion time, so an older snapshot cannot outrank it however late it lands', async () => { + const at = '2026-01-01T00:00:00.000Z' + const payload = { object: 'user', id: 'user_1', email: 'ada@example.com', updated_at: at } + await upsertRows(db, { schema: db.schema }, users, 'prod', [{ data: payload, updatedAt: at }], 'replace') + await deleteRows(db, { schema: db.schema }, users, 'prod', [ + { data: { id: 'user_1' }, updatedAt: '2026-01-01T00:00:01.000Z' }, + ]) + // The row as it was before the deletion, landing after the tombstone: + // what a writer that got past the lock would insert. + await db.insert(`\`${db.schema}\`.users`, [ + { _env: 'prod', _raw: JSON.stringify(payload), _updated_at: '2026-01-01 00:00:00.000', _deleted_at: null }, + ]) + expect(await db.query(`SELECT id FROM \`${db.schema}\`.users_live`)).toEqual([]) + }) }) diff --git a/packages/sync-clickhouse/test/upsert.test.ts b/packages/sync-clickhouse/test/upsert.test.ts index 9688e7c..0e18223 100644 --- a/packages/sync-clickhouse/test/upsert.test.ts +++ b/packages/sync-clickhouse/test/upsert.test.ts @@ -374,9 +374,9 @@ describeDb('write path', () => { const row = (await readUser())! expect(row['_deleted_at']).toBe('2026-01-10 00:00:00.000') expect(row['email']).toBe('ada@example.com') - // A delete is not a staleness-gated write: the row's own time stays, and - // the tombstone wins the version tie by being the later insert. - expect(row['_updated_at']).toBe('2026-01-01 00:00:00.000') + // A delete is not a staleness-gated write, but the tombstone takes the + // deletion time as its version: the newest version of the row must be it. + expect(row['_updated_at']).toBe('2026-01-10 00:00:00.000') }) it('records a delete for an object it never saw created', async () => { @@ -588,7 +588,9 @@ describeDb('freshness reporting', () => { const fresh = await destination.freshness(users, 'prod', 24) expect(fresh.rows).toBe(2) expect(fresh.deleted).toBe(1) - expect(fresh.lagSampleSize).toBe(1) + // The deletion just now is a change inside the window too: a tombstone + // is versioned at its deletion time, so it joins the sample. + expect(fresh.lagSampleSize).toBe(2) expect(fresh.lagSeconds).not.toBeNull() expect(fresh.lagSeconds!).toBeLessThan(60) expect(fresh.lagSeconds!).toBeGreaterThan(0) From b3b61b59024c09c8edafe3e285db87a886f323c0 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 5 Nov 2026 18:30:20 -0500 Subject: [PATCH 05/10] fix(clickhouse): run the installed command, dedupe by instant, refuse out-of-range offsets The same three fixes Postgres (#49) and MySQL (#50) got. npm installs the bin as a symlink named after the command, which the old argv check did not recognise, so `npx workos-sync-clickhouse` imported the CLI and did nothing. Duplicate keys in a batch are now judged by the instant each timestamp names rather than by its text, so `01:00:00+01:00` no longer outranks `00:30:00Z`. And the timestamp projection range-checks the clock and the offset ahead of the best-effort parser, which read `24:00` as the next midnight and `+00:99` as a zone where the other destinations project NULL. --- packages/sync-clickhouse/README.md | 3 +- packages/sync-clickhouse/src/cli.ts | 22 +++++++++-- packages/sync-clickhouse/src/db/upsert.ts | 32 +++++++++++++-- packages/sync-clickhouse/src/schema/ddl.ts | 22 +++++++++-- packages/sync-clickhouse/test/cli.test.ts | 41 ++++++++++++++++++++ packages/sync-clickhouse/test/schema.test.ts | 18 +++++++-- packages/sync-clickhouse/test/upsert.test.ts | 21 ++++++++++ 7 files changed, 145 insertions(+), 14 deletions(-) diff --git a/packages/sync-clickhouse/README.md b/packages/sync-clickhouse/README.md index f75a604..1a2c9db 100644 --- a/packages/sync-clickhouse/README.md +++ b/packages/sync-clickhouse/README.md @@ -176,7 +176,8 @@ CREATE TABLE workos.auth_attempts ( event_id String DEFAULT JSONExtractString(_raw, 'event_id'), status Nullable(String) DEFAULT JSONExtract(_raw, 'status', 'Nullable(String)'), user_id Nullable(String) DEFAULT JSONExtract(_raw, 'user_id', 'Nullable(String)'), - created_at Nullable(DateTime64(3, 'UTC')) DEFAULT parseDateTime64BestEffortOrNull(JSONExtractString(_raw, 'created_at'), 3, 'UTC'), + created_at Nullable(DateTime64(3, 'UTC')) DEFAULT if(match(JSONExtractString(_raw, 'created_at'), ''), + parseDateTime64BestEffortOrNull(upper(JSONExtractString(_raw, 'created_at')), 3, 'UTC'), NULL), INDEX auth_attempts_user_id_idx user_id TYPE bloom_filter GRANULARITY 4, INDEX auth_attempts_created_at_idx created_at TYPE minmax GRANULARITY 1 ) ENGINE = MergeTree ORDER BY (_env, event_id); diff --git a/packages/sync-clickhouse/src/cli.ts b/packages/sync-clickhouse/src/cli.ts index d918dc0..62945e4 100644 --- a/packages/sync-clickhouse/src/cli.ts +++ b/packages/sync-clickhouse/src/cli.ts @@ -32,6 +32,8 @@ import { type SyncEngineConfig, } from '@workos/sync-engine' import { X509Certificate } from 'node:crypto' +import { realpathSync } from 'node:fs' +import { fileURLToPath } from 'node:url' import { ClickHouseDestination } from './destination.js' import { buildSchemaSql } from './schema/ddl.js' import { fingerprint } from './schema/migrate.js' @@ -538,9 +540,23 @@ async function runCommand(command: string, flags: Record process.exit(code), (err) => { diff --git a/packages/sync-clickhouse/src/db/upsert.ts b/packages/sync-clickhouse/src/db/upsert.ts index 48506c9..0d11aee 100644 --- a/packages/sync-clickhouse/src/db/upsert.ts +++ b/packages/sync-clickhouse/src/db/upsert.ts @@ -59,7 +59,7 @@ export function keyValues(stream: TableDef, data: Record): stri * `wavesByKey`. */ export function dedupeByKey(stream: TableDef, rows: Row[]): { rows: Row[]; dropped: number } { - const byKey = new Map() + const byKey = new Map() let dropped = 0 for (const row of rows) { const key = keyValues(stream, row.data) @@ -68,10 +68,36 @@ export function dedupeByKey(stream: TableDef, rows: Row[]): { rows: Row[]; dropp continue } const k = key.join(KEY_SEPARATOR) + const at = micros(row.updatedAt) const existing = byKey.get(k) - if (!existing || row.updatedAt >= existing.updatedAt) byKey.set(k, row) + if (!existing || !isBefore(row.updatedAt, at, existing.row.updatedAt, existing.at)) byKey.set(k, { row, at }) } - return { rows: [...byKey.values()], dropped } + return { rows: [...byKey.values()].map((e) => e.row), dropped } +} + +/** + * The instant `iso` names, in microseconds since the epoch, or NaN. + * `Date.parse` reads three fractional digits, and so does the column + * (`DateTime64(3)`), so two rows within a millisecond tie at the staleness + * gate and the last one written wins; reading the rest of the fraction + * here is what makes that last one the newer of the two. + */ +function micros(iso: string): number { + const frac = /\.(\d+)/.exec(iso)?.[1] ?? '' + const ms = Date.parse(frac.length > 3 ? iso.replace(`.${frac}`, `.${frac.slice(0, 3)}`) : iso) + if (Number.isNaN(ms)) return ms + return ms * 1000 + Number(frac.slice(3, 6).padEnd(3, '0')) +} + +/** + * Whether one timestamp is strictly earlier than another, by the instant + * each names: `01:00:00+01:00` sorts after `00:30:00Z` as text and is half an + * hour earlier as a time. A value that is not a date is ordered as text; + * `parseUpdatedAt` refuses it on the write either way. + */ +function isBefore(a: string, atA: number, b: string, atB: number): boolean { + if (Number.isNaN(atA) || Number.isNaN(atB)) return a < b + return atA < atB } /** diff --git a/packages/sync-clickhouse/src/schema/ddl.ts b/packages/sync-clickhouse/src/schema/ddl.ts index 87c871c..a2b6e2d 100644 --- a/packages/sync-clickhouse/src/schema/ddl.ts +++ b/packages/sync-clickhouse/src/schema/ddl.ts @@ -85,12 +85,28 @@ export function columnExpression(col: ColumnDef, isKey: boolean): string { case 'jsonb': // The subtree as JSON text; `JSONExtractRaw` alone reads '' for a missing key. return `if(JSONHas(${raw}, ${path}), JSONExtractRaw(${raw}, ${path}), NULL)` - case 'timestamptz': - // Reads every ISO-8601 shape WorkOS emits, offsets included; NULL for anything else. - return `parseDateTime64BestEffortOrNull(JSONExtractString(${raw}, ${path}), 3, 'UTC')` + case 'timestamptz': { + // Every ISO-8601 shape WorkOS emits, offsets included, with the clock + // and the offset range-checked ahead of the parser; NULL for anything + // else. Upper-cased for the parser, which reads `T` and `Z` only. + const s = `JSONExtractString(${raw}, ${path})` + return `if(match(${s}, ${sqlString(ISO_8601)}), parseDateTime64BestEffortOrNull(upper(${s}), 3, 'UTC'), NULL)` + } } } +/** + * The timestamps the projection accepts: the Postgres destination's grammar, + * every field range-checked in the pattern. The best-effort parser alone + * reads `24:00` as the next midnight, `23:60` as the next hour and `+00:99` + * as a zone, where the other destinations project NULL. The day of the + * month is left to the parser, which refuses `02-30` and `04-31` itself. + */ +const ISO_8601 = String.raw`^(?:[1-9]\d{3}|0[1-9]\d{2}|00[1-9]\d|000[1-9])-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12]\d|3[01])[Tt ](?:[01]\d|2[0-3]):[0-5]\d:(?:[0-5]\d|60)(?:\.\d+)?\s*(?:[Zz]|[+-](?:[01]\d|2[0-3]):?[0-5]\d)?$` + +/** A single-quoted ClickHouse string literal; backslashes are escapes there, so each is doubled. */ +const sqlString = (text: string) => `'${text.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'` + /** * One `` `name` type DEFAULT expr `` clause. Every path that creates a * projected column — table creation, the additive upgrade, drift detection diff --git a/packages/sync-clickhouse/test/cli.test.ts b/packages/sync-clickhouse/test/cli.test.ts index 9c8996e..083d1cd 100644 --- a/packages/sync-clickhouse/test/cli.test.ts +++ b/packages/sync-clickhouse/test/cli.test.ts @@ -1,6 +1,11 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { pathToFileURL } from 'node:url' import { SyncEngine, WorkOSClient, selectStreams, silentLogger } from '@workos/sync-engine' import { + invokedDirectly, main, parseHydrate, parseExcludeStreams, @@ -16,6 +21,42 @@ import { FaultyDestination } from './helpers/faulty.js' import { TEST_CA_PEM } from './helpers/ca.js' import { FakeWorkOS, resetIds } from '../../sync-engine/test/helpers/fake-workos.js' +describe('entrypoint', () => { + let dir: string + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'workos-sync-clickhouse-bin-')) + }) + afterEach(() => { + rmSync(dir, { recursive: true, force: true }) + }) + + it('recognises the script itself and the symlink npm installs as the command', () => { + const script = join(dir, 'cli.js') + writeFileSync(script, '') + const url = pathToFileURL(script).href + expect(invokedDirectly(script, url)).toBe(true) + + // `npm install` links `node_modules/.bin/workos-sync-clickhouse` at dist/cli.js, + // so argv[1] is the link, not the file. + const link = join(dir, 'workos-sync-clickhouse') + symlinkSync(script, link) + expect(invokedDirectly(link, url)).toBe(true) + }) + + it('is false for an import, a missing path, or another script', () => { + const script = join(dir, 'cli.js') + writeFileSync(script, '') + const url = pathToFileURL(script).href + expect(invokedDirectly(undefined, url)).toBe(false) + expect(invokedDirectly(join(dir, 'missing'), url)).toBe(false) + const other = join(dir, 'vitest.mjs') + writeFileSync(other, '') + expect(invokedDirectly(other, url)).toBe(false) + // This test file was started by the test runner, not by the CLI. + expect(invokedDirectly(process.argv[1], url)).toBe(false) + }) +}) + describe('WORKOS_SYNC_HYDRATE', () => { it('leaves the engine default in place when unset', () => { expect(parseHydrate(undefined)).toBeUndefined() diff --git a/packages/sync-clickhouse/test/schema.test.ts b/packages/sync-clickhouse/test/schema.test.ts index fdaea90..8caaf9c 100644 --- a/packages/sync-clickhouse/test/schema.test.ts +++ b/packages/sync-clickhouse/test/schema.test.ts @@ -53,8 +53,8 @@ describe('DDL generation', () => { expect(columnExpression({ name: 'x', type: 'jsonb', path: ['a', 'b'] }, false)).toBe( "if(JSONHas(`_raw`, 'a', 'b'), JSONExtractRaw(`_raw`, 'a', 'b'), NULL)", ) - expect(columnExpression({ name: 'x', type: 'timestamptz', path: 'created_at' }, false)).toBe( - "parseDateTime64BestEffortOrNull(JSONExtractString(`_raw`, 'created_at'), 3, 'UTC')", + expect(columnExpression({ name: 'x', type: 'timestamptz', path: 'created_at' }, false)).toMatch( + /^if\(match\(JSONExtractString\(`_raw`, 'created_at'\), '\^.*\$'\), parseDateTime64BestEffortOrNull\(upper\(JSONExtractString\(`_raw`, 'created_at'\)\), 3, 'UTC'\), NULL\)$/, ) }) @@ -209,8 +209,18 @@ describeDb('migrations against real ClickHouse', () => { [null, null], ['2026-02-30T12:00:00Z', null], ['2026-13-01T00:00:00Z', null], - // Best-effort parsing reads 24:00 as the next midnight, where Postgres refuses it. - ['2026-01-01T24:00:00Z', '2026-01-02 00:00:00.000'], + // The clock and the offset are range-checked ahead of the parser, which + // would otherwise read 24:00 as the next midnight and +00:99 as a zone. + ['2026-01-01T24:00:00Z', null], + ['2026-01-01T23:60:00Z', null], + ['0000-01-01T00:00:00Z', null], + ['2026-01-01T00:00:00+00:99', null], + ['2026-01-01T00:00:00-24:00', null], + ['2026-01-01T00:00:00+23:59', '2025-12-31 00:01:00.000'], + ['2026-01-01T00:00:00+0530', '2025-12-31 18:30:00.000'], + ['2026-01-01 00:00:00', '2026-01-01 00:00:00.000'], + ['2026-01-01t00:00:00z', '2026-01-01 00:00:00.000'], + ['2026-01-01', null], ] for (const [input, expected] of cases) { const id = `user_${String(input)}` diff --git a/packages/sync-clickhouse/test/upsert.test.ts b/packages/sync-clickhouse/test/upsert.test.ts index 0e18223..b1ed6e5 100644 --- a/packages/sync-clickhouse/test/upsert.test.ts +++ b/packages/sync-clickhouse/test/upsert.test.ts @@ -53,6 +53,27 @@ describe('key extraction', () => { expect(dropped).toBe(1) }) + it('judges the newest row by its instant, not by the text of its timestamp', () => { + // "01:00:00+01:00" sorts after "00:30:00Z" as text and is half an hour earlier as a time. + const { rows } = dedupeByKey(users, [ + { data: userPayload({ email: 'older@example.com' }), updatedAt: '2026-01-01T01:00:00+01:00' }, + { data: userPayload({ email: 'newer@example.com' }), updatedAt: '2026-01-01T00:30:00Z' }, + ]) + expect(rows.map((r) => r.data['email'])).toEqual(['newer@example.com']) + // Two rows within a millisecond tie at the gate, so the one kept here must be the newer. + const fine = dedupeByKey(users, [ + { data: userPayload({ email: 'newer@example.com' }), updatedAt: '2026-01-01T00:00:00.000900Z' }, + { data: userPayload({ email: 'older@example.com' }), updatedAt: '2026-01-01T00:00:00.000100Z' }, + ]) + expect(fine.rows.map((r) => r.data['email'])).toEqual(['newer@example.com']) + // What is not a date is ordered as text, and refused on the write. + const junk = dedupeByKey(users, [ + { data: userPayload({ email: 'a@example.com' }), updatedAt: 'later' }, + { data: userPayload({ email: 'b@example.com' }), updatedAt: 'yesterday' }, + ]) + expect(junk.rows.map((r) => r.data['email'])).toEqual(['b@example.com']) + }) + it('splits repeated keys into ordered waves and leaves distinct keys in one', () => { const first = { data: userPayload({ external_id: 'ext_1' }), updatedAt: '2026-01-01T00:00:00.000Z' } const other = { data: userPayload({ id: 'user_2' }), updatedAt: '2026-01-01T00:00:00.000Z' } From 66ea27af3d16371460d94456050ddd81a4c5a4d5 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 5 Nov 2026 18:30:23 -0500 Subject: [PATCH 06/10] fix(clickhouse): re-create live views after a repair, collapse duplicate append rows `init --repair` redefines a column, and its `_live` view keeps the type it declared when it was created: a query through the view then returns the old type or fails to convert. The views of the streams a repair touched are now re-created after it, and the check that leaves an up-to-date view alone compares column types as well as names. Append-only tables are a ReplacingMergeTree keyed by the event id, so a second copy of an event collapses at the next merge, and under FINAL before then. The write path's existence check under the stream lock still keeps copies out in the first place; the table is the backstop for a poll that outlived its lease. --- packages/sync-clickhouse/README.md | 9 +++-- packages/sync-clickhouse/src/db/upsert.ts | 7 ++-- packages/sync-clickhouse/src/destination.ts | 6 ++-- packages/sync-clickhouse/src/schema/ddl.ts | 9 +++-- .../sync-clickhouse/src/schema/migrate.ts | 33 ++++++++++++------- packages/sync-clickhouse/test/drift.test.ts | 18 ++++++++++ packages/sync-clickhouse/test/schema.test.ts | 16 ++++++++- 7 files changed, 74 insertions(+), 24 deletions(-) diff --git a/packages/sync-clickhouse/README.md b/packages/sync-clickhouse/README.md index 1a2c9db..7cac68c 100644 --- a/packages/sync-clickhouse/README.md +++ b/packages/sync-clickhouse/README.md @@ -180,11 +180,14 @@ CREATE TABLE workos.auth_attempts ( parseDateTime64BestEffortOrNull(upper(JSONExtractString(_raw, 'created_at')), 3, 'UTC'), NULL), INDEX auth_attempts_user_id_idx user_id TYPE bloom_filter GRANULARITY 4, INDEX auth_attempts_created_at_idx created_at TYPE minmax GRANULARITY 1 -) ENGINE = MergeTree ORDER BY (_env, event_id); +) ENGINE = ReplacingMergeTree ORDER BY (_env, event_id); ``` -Append-only streams (`auth_attempts`, `events`) are plain `MergeTree`: one row -per event, never updated, the shape every ClickHouse query expects. +Append-only streams (`auth_attempts`, `events`) are a `ReplacingMergeTree` +keyed by the event id, with no version column: one row per event, never +updated. A second copy of an event, which only a poll that outlived its lease +could write, collapses at the next merge; add `FINAL` where an exact count +matters before then. Mirror streams are `ReplacingMergeTree` versioned by `_updated_at`. Every write inserts the row's next state, and the newest source timestamp wins at diff --git a/packages/sync-clickhouse/src/db/upsert.ts b/packages/sync-clickhouse/src/db/upsert.ts index 0d11aee..abd0617 100644 --- a/packages/sync-clickhouse/src/db/upsert.ts +++ b/packages/sync-clickhouse/src/db/upsert.ts @@ -18,9 +18,10 @@ * stream's write lock (`lock.ts`) across both: two writers on one key — the * tail and a backfill, or a poll outliving its lease — then see each * other's rows instead of each judging against the state before the other - * wrote. Append-only streams are plain `MergeTree`: under the same lock the - * batch reads which ids are already there and inserts the rest, which is - * what makes re-delivery a no-op. + * wrote. For an append-only stream, under the same lock, the batch reads + * which ids are already there and inserts the rest, which is what makes + * re-delivery a no-op; the table, a `ReplacingMergeTree` keyed by the id, + * collapses a copy that got past that anyway. */ import { emptyStats, type Row, type TableDef, type WriteMode, type WriteStats } from '@workos/sync-engine' import { hostname } from 'node:os' diff --git a/packages/sync-clickhouse/src/destination.ts b/packages/sync-clickhouse/src/destination.ts index 999e65d..5186180 100644 --- a/packages/sync-clickhouse/src/destination.ts +++ b/packages/sync-clickhouse/src/destination.ts @@ -137,9 +137,9 @@ export class ClickHouseDestination implements Destination { return unlinkRows(this.client, this.writeOpts, stream, env, rows) } - /** `FINAL` for the mirror tables, where a key may have several versions; append tables have one row per key. */ - private final(stream: TableDef): string { - return stream.mode === 'append' ? '' : ' FINAL' + /** `FINAL`: a key may have several versions in a mirror table, and a second copy in an append table. */ + private final(_stream: TableDef): string { + return ' FINAL' } async liveIds(stream: TableDef, env: string): Promise { diff --git a/packages/sync-clickhouse/src/schema/ddl.ts b/packages/sync-clickhouse/src/schema/ddl.ts index a2b6e2d..51e3712 100644 --- a/packages/sync-clickhouse/src/schema/ddl.ts +++ b/packages/sync-clickhouse/src/schema/ddl.ts @@ -16,8 +16,11 @@ * the newest source timestamp wins at merge time and under `FINAL`. That * is the staleness gate, enforced by the table itself whatever order the * writes land in. The `_live` views read with `FINAL`. Append-only - * streams are plain `MergeTree`, the shape analytics wants, with - * re-delivery filtered by the write path. + * streams are a `ReplacingMergeTree` keyed by the event id with no + * version column: one row per event, never updated, and a second copy + * of one — which only a poll that outlived its lease can write, past + * the write path's own check — collapses at the next merge, or under + * `FINAL` before then. * - **A projection that cannot be cast is NULL, not an error.** `JSONExtract` * with a `Nullable` return type gives NULL for a missing key, a JSON null * and a value of the wrong shape alike, and the value stays in `_raw`. @@ -251,7 +254,7 @@ export function createTableSql(stream: TableDef, schema: string): string { for (const c of stream.columns) cols.push(generatedColumnSql(c, stream)) for (const idx of indexSpecs(stream)) cols.push(indexClause(idx)) const order = [ENV, ...stream.key].map((k) => quoteIdent(safeIdentifier(k))).join(', ') - const engine = stream.mode === 'append' ? 'MergeTree' : 'ReplacingMergeTree(`_updated_at`)' + const engine = stream.mode === 'append' ? 'ReplacingMergeTree' : 'ReplacingMergeTree(`_updated_at`)' return `CREATE TABLE IF NOT EXISTS ${s}.${table} (\n ${cols.join(',\n ')}\n) ENGINE = ${engine} ORDER BY (${order})` } diff --git a/packages/sync-clickhouse/src/schema/migrate.ts b/packages/sync-clickhouse/src/schema/migrate.ts index dc77c22..ab790f0 100644 --- a/packages/sync-clickhouse/src/schema/migrate.ts +++ b/packages/sync-clickhouse/src/schema/migrate.ts @@ -84,6 +84,16 @@ export interface MigrateResult { const names = async (db: ChClient, sql: string, params: Record) => new Set((await db.query<{ name: string }>(sql, params)).map((r) => r.name)) +const typedColumns = async (db: ChClient, schema: string, table: string) => + new Map( + ( + await db.query<{ name: string; type: string }>( + `SELECT name, type FROM system.columns WHERE database = {schema:String} AND table = {table:String}`, + { schema, table }, + ) + ).map((r) => [r.name, r.type]), + ) + export async function migrate( db: ChClient, streams: TableDef[], @@ -156,17 +166,11 @@ export async function migrate( { schema, name: liveViewName(stream) }, ) if (view?.engine === 'View' && isGeneratedDefinition(view.as_select, schema, table)) { - const viewColumns = await names( - db, - `SELECT name FROM system.columns WHERE database = {schema:String} AND table = {table:String}`, - { schema, table: liveViewName(stream) }, - ) - const tableColumns = await names( - db, - `SELECT name FROM system.columns WHERE database = {schema:String} AND table = {table:String}`, - { schema, table }, - ) - if ([...tableColumns].every((c) => viewColumns.has(c))) continue + // Name and type both: a view fixes its columns' types when it is + // created, so a column redefined since declares the old one there. + const viewColumns = await typedColumns(db, schema, liveViewName(stream)) + const tableColumns = await typedColumns(db, schema, table) + if ([...tableColumns].every(([c, t]) => viewColumns.get(c) === t)) continue } } if (await applyLiveView(db, stream, schema, logger, options)) statements++ @@ -192,6 +196,13 @@ export async function migrate( repaired = r.repaired for (const f of r.failed) failures.set(f.drift, f.error) drift = drift.filter((d) => !repaired.includes(d)) + // A redefined column changes the type its view declares for it, and the + // view fixed that when it was created: the views of the streams touched + // are re-created now, after the repair rather than before it. + for (const name of new Set(repaired.map((d) => d.stream))) { + const stream = streams.find((st) => st.name === name) + if (stream && stream.mode !== 'append' && (await applyLiveView(db, stream, schema, logger, options))) statements++ + } } for (const d of drift) { diff --git a/packages/sync-clickhouse/test/drift.test.ts b/packages/sync-clickhouse/test/drift.test.ts index b965d92..dd27117 100644 --- a/packages/sync-clickhouse/test/drift.test.ts +++ b/packages/sync-clickhouse/test/drift.test.ts @@ -6,6 +6,7 @@ */ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' import { createTestDb, describeDb, type TestDb } from './helpers/clickhouse.js' +import { columnType } from '../src/schema/ddl.js' import { migrate, fingerprint } from '../src/schema/migrate.js' import { detectColumnDrift, unrepairableReason } from '../src/schema/drift.js' import { main } from '../src/cli.js' @@ -135,6 +136,23 @@ describeDb('column drift detection', () => { expect(await recorded([changed])).toBe(true) }) + it('re-creates the live view after a repair changes a column type, so the view declares the new one', async () => { + await migrate(db, [users], db.schema) + const typeOf = async (table: string) => + ( + await db.query<{ type: string }>( + `SELECT type FROM system.columns WHERE database = {s:String} AND table = {t:String} AND name = 'first_name'`, + { s: db.schema, t: table }, + ) + )[0]?.type + expect(await typeOf('users_live')).toBe(await typeOf('users')) + const column = { name: 'first_name', type: 'boolean' as const, path: 'first_name' } + const r = await migrate(db, [withColumn(users, column)], db.schema, undefined, { repair: true }) + expect(r.repaired.map((d) => d.column)).toEqual(['first_name']) + expect(await typeOf('users')).toBe(columnType(column, false)) + expect(await typeOf('users_live')).toBe(await typeOf('users')) + }) + it('refuses to repair a key column', async () => { await migrate(db, [users], db.schema) const rekeyed = withColumn(users, { name: 'id', type: 'text', path: 'uid' }) diff --git a/packages/sync-clickhouse/test/schema.test.ts b/packages/sync-clickhouse/test/schema.test.ts index 8caaf9c..d8926f1 100644 --- a/packages/sync-clickhouse/test/schema.test.ts +++ b/packages/sync-clickhouse/test/schema.test.ts @@ -30,7 +30,7 @@ describe('DDL generation', () => { ) expect(createTableSql(users, 'workos')).toContain('`_deleted_at`') const events = createTableSql(streamByName('events')!, 'workos') - expect(events).toContain('ENGINE = MergeTree ORDER BY (`_env`, `id`)') + expect(events).toContain('ENGINE = ReplacingMergeTree ORDER BY (`_env`, `id`)') expect(events).not.toContain('`_deleted_at`') const memberships = createTableSql(streamByName('directory_group_memberships')!, 'workos') expect(memberships).toContain('ORDER BY (`_env`, `directory_group_id`, `directory_user_id`)') @@ -361,6 +361,20 @@ describeDb('live views against real ClickHouse', () => { expect(isGeneratedDefinition(r[0]!.as_select, db.schema, 'users')).toBe(true) }) + it('collapses a second copy of an append-only row under FINAL', async () => { + const events = streamByName('events')! + await migrate(db, [events], db.schema) + const row = { + _env: 'prod', + _raw: JSON.stringify({ id: 'event_1', event: 'user.created' }), + _updated_at: '2026-01-01 00:00:00.000', + } + await db.insert(`\`${db.schema}\`.events`, [row]) + await db.insert(`\`${db.schema}\`.events`, [row]) + const n = await db.query<{ n: string }>(`SELECT toString(count()) AS n FROM \`${db.schema}\`.events FINAL`) + expect(Number(n[0]!.n)).toBe(1) + }) + it('refuses a table standing where its view would go', async () => { await db.command(`CREATE DATABASE \`${db.schema}\``) await db.command(`CREATE TABLE \`${db.schema}\`.users_live (id Int32) ENGINE = MergeTree ORDER BY id`) From 154dc7a5cb676ad6267366d58e18cd91092f9350 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Fri, 6 Nov 2026 08:16:00 -0500 Subject: [PATCH 07/10] fix(clickhouse): survive losing a KeeperMap create race while recording the schema Two processes applying the same catalog at once both record the same column clauses and the same fingerprint. A KeeperMap insert decides create-or-set per key before its transaction, so with strict mode off the second writer can still be refused with "Node exists" when the key appeared in between, and `init` failed on it. Both records now read what landed: the same value counts as recorded, a different column clause is overwritten in place. --- packages/sync-clickhouse/src/schema/drift.ts | 36 +++++++++++++++---- .../sync-clickhouse/src/schema/migrate.ts | 34 +++++++++++++++--- packages/sync-clickhouse/test/schema.test.ts | 28 +++++++++++++++ 3 files changed, 86 insertions(+), 12 deletions(-) diff --git a/packages/sync-clickhouse/src/schema/drift.ts b/packages/sync-clickhouse/src/schema/drift.ts index 96474d1..9ecaf66 100644 --- a/packages/sync-clickhouse/src/schema/drift.ts +++ b/packages/sync-clickhouse/src/schema/drift.ts @@ -68,25 +68,47 @@ async function recordedColumns(db: ChClient, schema: string, table: string): Pro return new Map(rows.map((r) => [r.column_name, r.ddl])) } +const backoff = () => new Promise((r) => setTimeout(r, 5 + Math.random() * 20)) + /** * Record (or re-record) the clause a column was created with. Strict mode - * is off, so this overwrites; two processes recording the same column at - * once can still race on the first write, and the loser simply writes again. + * is off, so an existing record is overwritten. Even so, a KeeperMap + * insert decides create-or-set per key before its transaction, so two + * processes recording the same column at once can both decide "create" + * and the second is refused with "Node exists". The loser then reads what + * landed: the same clause means the record is already right, another + * clause is overwritten in place. */ export async function recordColumn(db: ChClient, schema: string, table: string, column: string, ddl: string) { - for (let attempt = 0; ; attempt++) { + const ref = `${quoteIdent(schema)}.\`_columns\`` + const k = `${table}\u0001${column}` + for (let attempt = 0; attempt < 8; attempt++) { + if (attempt > 0) await backoff() + try { + await db.command( + `INSERT INTO ${ref} (k, table_name, column_name, ddl) VALUES ({k:String}, {table:String}, {column:String}, {ddl:String})`, + { k, table, column, ddl }, + { ...KEEPER_SETTINGS, keeper_map_strict_mode: 0 }, + ) + return + } catch (err) { + if (!isKeeperConflict(err)) throw err + } + const [stored] = await db.query<{ ddl: string }>(`SELECT ddl FROM ${ref} WHERE k = {k:String}`, { k }) + if (!stored) continue + if (stored.ddl === ddl) return try { await db.command( - `INSERT INTO ${quoteIdent(schema)}.\`_columns\` (k, table_name, column_name, ddl) - VALUES ({k:String}, {table:String}, {column:String}, {ddl:String})`, - { k: `${table}\u0001${column}`, table, column, ddl }, + `ALTER TABLE ${ref} UPDATE table_name = {table:String}, column_name = {column:String}, ddl = {ddl:String} WHERE k = {k:String}`, + { k, table, column, ddl }, { ...KEEPER_SETTINGS, keeper_map_strict_mode: 0 }, ) return } catch (err) { - if (attempt >= 3 || !isKeeperConflict(err)) throw err + if (!isKeeperConflict(err)) throw err } } + throw new Error(`could not record the column ${table}.${column}: the row kept changing under 8 attempts`) } async function forgetColumn(db: ChClient, schema: string, table: string, column: string) { diff --git a/packages/sync-clickhouse/src/schema/migrate.ts b/packages/sync-clickhouse/src/schema/migrate.ts index ab790f0..6665582 100644 --- a/packages/sync-clickhouse/src/schema/migrate.ts +++ b/packages/sync-clickhouse/src/schema/migrate.ts @@ -17,6 +17,7 @@ import { createHash } from 'node:crypto' import type { Logger, TableDef } from '@workos/sync-engine' import { KEEPER_SETTINGS, type ChClient } from '../db/client.js' +import { isKeeperConflict } from '../db/errors.js' import { addColumnSql, addIndexSql, @@ -81,6 +82,33 @@ export interface MigrateResult { repaired: ColumnDrift[] } +const backoff = () => new Promise((r) => setTimeout(r, 5 + Math.random() * 20)) + +/** + * Record the fingerprint. Two processes applying the same catalog at once + * both reach this with the same value, and a KeeperMap insert that loses + * the create race is refused with "Node exists" even with strict mode off: + * the row the other process wrote is this one's, so that counts as done. + */ +async function recordFingerprint(db: ChClient, s: string, fp: string, detail: string): Promise { + for (let attempt = 0; attempt < 8; attempt++) { + if (attempt > 0) await backoff() + try { + await db.command( + `INSERT INTO ${s}.\`_migrations\` (fingerprint, applied_at, detail) VALUES ({fp:String}, now64(3, 'UTC'), {detail:String})`, + { fp, detail }, + { ...KEEPER_SETTINGS, keeper_map_strict_mode: 0 }, + ) + return + } catch (err) { + if (!isKeeperConflict(err)) throw err + } + const rows = await db.query(`SELECT 1 FROM ${s}.\`_migrations\` WHERE fingerprint = {fp:String}`, { fp }) + if (rows.length > 0) return + } + throw new Error(`could not record the schema fingerprint ${fp}: the row kept changing under 8 attempts`) +} + const names = async (db: ChClient, sql: string, params: Record) => new Set((await db.query<{ name: string }>(sql, params)).map((r) => r.name)) @@ -224,11 +252,7 @@ export async function migrate( return { applied: true, fingerprint: fp, statements, drift, repaired } } - await db.command( - `INSERT INTO ${s}.\`_migrations\` (fingerprint, applied_at, detail) VALUES ({fp:String}, now64(3, 'UTC'), {detail:String})`, - { fp, detail: JSON.stringify({ streams: streams.map((st) => st.name), statements }) }, - { ...KEEPER_SETTINGS, keeper_map_strict_mode: 0 }, - ) + await recordFingerprint(db, s, fp, JSON.stringify({ streams: streams.map((st) => st.name), statements })) logger?.info('schema applied', { fingerprint: fp, statements, repaired: repaired.length }) return { applied: true, fingerprint: fp, statements, drift, repaired } diff --git a/packages/sync-clickhouse/test/schema.test.ts b/packages/sync-clickhouse/test/schema.test.ts index d8926f1..43dda1c 100644 --- a/packages/sync-clickhouse/test/schema.test.ts +++ b/packages/sync-clickhouse/test/schema.test.ts @@ -262,6 +262,34 @@ describeDb('migrations against real ClickHouse', () => { expect(rows[1]).toMatchObject({ n: null, d: null, flag: null, metadata: null }) }) + it('finishes when a concurrent run recorded the same fingerprint and columns a moment earlier', async () => { + // A KeeperMap insert refused with "Node exists" after the other process + // created the key between this insert's existence check and its write: + // the first `_columns` and `_migrations` inserts are refused that way, + // and what the other process wrote is found in place. + const refused: string[] = [] + const racing: TestDb = { + ...db, + async command(sql, params, settings) { + const target = /INSERT INTO [^\n]*`(_columns|_migrations)`/.exec(sql)?.[1] + if (target && !refused.includes(target)) { + refused.push(target) + await db.command(sql, params, settings) + throw Object.assign(new Error('Code: 999. DB::Exception: Transaction failed (Node exists): Op #0'), { + code: '999', + }) + } + return db.command(sql, params, settings) + }, + } + const r = await migrate(racing, [users], db.schema) + expect(r.applied).toBe(true) + expect(refused).toEqual(['_columns', '_migrations']) + expect((await migrate(db, [users], db.schema)).applied).toBe(false) + const recorded = await db.query<{ n: string }>(`SELECT toString(count()) AS n FROM \`${db.schema}\`.\`_columns\``) + expect(Number(recorded[0]!.n)).toBe(users.columns.length) + }) + it('serialises nothing: two concurrent migrations both finish with one schema', async () => { const results = await Promise.all([migrate(db, STREAMS, db.schema), migrate(db, STREAMS, db.schema)]) expect(results.map((r) => r.applied)).toContain(true) From 6e07feca10c494961c6ce0b055782b7e67af26cb Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Fri, 6 Nov 2026 08:28:52 -0500 Subject: [PATCH 08/10] fix(clickhouse): redact invitations that name the user, verify unrecorded columns `redact --user` erased rows by `user_id` only. An invitation names a user as its inviter and as the user who accepted it, so those rows, and the archived and quarantined events carrying those payloads, kept the subject's id and email after the command reported success. Every column and payload field that names a user is matched now. A column with no `_columns` record had its expected clause recorded without a look at the table, so after a schema applied by hand from `init --print-sql` a changed path was recorded as installed, with the column still reading the old path. The catalog's clause now goes through the server's own formatter, by way of a throwaway Memory table, and is compared with the column's stored expression; a match is recorded and anything else is drift that `--repair` rewrites. --- packages/sync-clickhouse/src/destination.ts | 26 +++++++--- packages/sync-clickhouse/src/schema/drift.ts | 51 ++++++++++++++++++-- packages/sync-clickhouse/test/drift.test.ts | 41 ++++++++++++++++ packages/sync-clickhouse/test/engine.test.ts | 8 +++ 4 files changed, 115 insertions(+), 11 deletions(-) diff --git a/packages/sync-clickhouse/src/destination.ts b/packages/sync-clickhouse/src/destination.ts index 5186180..78b2a3e 100644 --- a/packages/sync-clickhouse/src/destination.ts +++ b/packages/sync-clickhouse/src/destination.ts @@ -28,6 +28,14 @@ import { jsonPathArgs, quoteIdent, safeIdentifier } from './schema/ddl.js' const ident = (name: string) => quoteIdent(safeIdentifier(name)) +/** + * The columns that name a user: a membership's or sign-in's `user_id`, and + * an invitation's inviter and the user who accepted it. `redact` erases a + * row that names the subject in any of them, and an archived or quarantined + * event whose payload does. + */ +const USER_REFERENCES = ['user_id', 'inviter_user_id', 'accepted_user_id'] + export interface ClickHouseDestinationOptions { /** * The HTTP interface, `https://user:password@host:8443`. Alternatively @@ -221,23 +229,27 @@ export class ClickHouseDestination implements Destination { out.push({ table: label, rows: n }) } for (const stream of streams) { - const col = stream.columns.find((c) => c.name === 'user_id' || (c.name === 'id' && stream.name === 'users')) - if (!col) continue + const refs = stream.columns.filter( + (c) => USER_REFERENCES.includes(c.name) || (c.name === 'id' && stream.name === 'users'), + ) + if (refs.length === 0) continue await erase( stream.name, tableRef(this.schema, stream), this.final(stream), - `\`_env\` = {env:String} AND ${ident(col.name)} = {id:String}`, + `\`_env\` = {env:String} AND (${refs.map((c) => `${ident(c.name)} = {id:String}`).join(' OR ')})`, { env, id: userId }, ) } - const userIdInData = `JSONExtractString(\`_raw\`, ${jsonPathArgs(['data', 'user_id'])})` + const namedInData = USER_REFERENCES.map( + (field) => `JSONExtractString(\`_raw\`, ${jsonPathArgs(['data', field])}) = {id:String}`, + ).join(' OR ') // Archived events embed the payload, so they must be scrubbed too. await erase( 'events', `${quoteIdent(this.schema)}.\`events\``, - '', - `\`_env\` = {env:String} AND (\`object_id\` = {id:String} OR ${userIdInData} = {id:String})`, + ' FINAL', + `\`_env\` = {env:String} AND (\`object_id\` = {id:String} OR ${namedInData})`, { env, id: userId }, ) // Quarantined events embed the payload too, and they may be the only copy @@ -247,7 +259,7 @@ export class ClickHouseDestination implements Destination { '_rejected', `${quoteIdent(this.schema)}.\`_rejected\``, ' FINAL', - `sync_id = {syncId:String} AND env = {env:String} AND (object_id = {id:String} OR ${userIdInData} = {id:String})`, + `sync_id = {syncId:String} AND env = {env:String} AND (object_id = {id:String} OR ${namedInData})`, { syncId: this.syncId, env, id: userId }, ) return out diff --git a/packages/sync-clickhouse/src/schema/drift.ts b/packages/sync-clickhouse/src/schema/drift.ts index 9ecaf66..012d2a9 100644 --- a/packages/sync-clickhouse/src/schema/drift.ts +++ b/packages/sync-clickhouse/src/schema/drift.ts @@ -12,15 +12,20 @@ * The expression is compared against `_columns`, the engine's own record of * the DDL it emitted for every projected column, not against what * `system.columns` shows: the server stores a re-formatted expression, so - * text written by the catalog and text read back never compare equal. The - * live type is compared with the catalog's directly; `system.columns` - * spells types canonically, and so does `columnType`. + * text written by the catalog and text read back never compare equal. A + * column with no record — a schema applied by hand from `init --print-sql` + * — has only the table as a witness, so the catalog's clause is put through + * the same formatter, by way of a throwaway table, before the two are + * compared; only a match is recorded, anything else is drift. The live type + * is compared with the catalog's directly; `system.columns` spells types + * canonically, and so does `columnType`. * * Never touched, only reported: key columns (they sort the table), columns * that are not `DEFAULT` expressions (they are yours), and orphans — * recorded columns the catalog no longer names, which still compute from * `_raw` with their old expression and are harmless to leave. */ +import { randomBytes } from 'node:crypto' import type { Logger, TableDef } from '@workos/sync-engine' import { KEEPER_SETTINGS, type ChClient } from '../db/client.js' import { isKeeperConflict } from '../db/errors.js' @@ -119,12 +124,36 @@ async function forgetColumn(db: ChClient, schema: string, table: string, column: ) } +/** + * The server's own spelling of each of a stream's generated expressions, + * read back from a throwaway `Memory` table built from the catalog's + * clauses. It is what `system.columns.default_expression` would show for a + * column created from those clauses, so an unrecorded column can be judged + * against the table it is in. + */ +async function canonicalExpressions(db: ChClient, schema: string, stream: TableDef): Promise> { + const probe = `_probe_${randomBytes(4).toString('hex')}` + const ref = `${quoteIdent(schema)}.${quoteIdent(probe)}` + const clauses = stream.columns.map((c) => generatedColumnSql(c, stream)) + await db.command(`CREATE TABLE ${ref} (\`_raw\` String, ${clauses.join(', ')}) ENGINE = Memory`) + try { + const rows = await db.query<{ name: string; default_expression: string }>( + `SELECT name, default_expression FROM system.columns WHERE database = {schema:String} AND table = {table:String}`, + { schema, table: probe }, + ) + return new Map(rows.map((r) => [r.name, r.default_expression])) + } finally { + await db.command(`DROP TABLE IF EXISTS ${ref}`) + } +} + export async function detectColumnDrift(db: ChClient, streams: TableDef[], schema: string): Promise { const out: ColumnDrift[] = [] for (const stream of streams) { const table = safeIdentifier(stream.name) const live = await liveColumns(db, schema, table) const recorded = await recordedColumns(db, schema, table) + let canonical: Map | undefined const catalog = new Set() for (const col of stream.columns) { const name = safeIdentifier(col.name) @@ -147,7 +176,21 @@ export async function detectColumnDrift(db: ChClient, streams: TableDef[], schem } const record = recorded.get(name) if (record === undefined) { - await recordColumn(db, schema, table, name, expected) + // Nothing says what this column was created with, so the table is + // asked, in its own spelling of the clause the catalog would write. + canonical ??= await canonicalExpressions(db, schema, stream) + if (canonical.get(name) === actual.default_expression) { + await recordColumn(db, schema, table, name, expected) + continue + } + out.push({ + stream: stream.name, + column: name, + kind: 'expression', + expected, + actual: actual.default_expression, + key, + }) continue } if (record !== expected) { diff --git a/packages/sync-clickhouse/test/drift.test.ts b/packages/sync-clickhouse/test/drift.test.ts index dd27117..1f696e7 100644 --- a/packages/sync-clickhouse/test/drift.test.ts +++ b/packages/sync-clickhouse/test/drift.test.ts @@ -153,6 +153,47 @@ describeDb('column drift detection', () => { expect(await typeOf('users_live')).toBe(await typeOf('users')) }) + it('judges an unrecorded column by the table itself, as after a schema applied by hand', async () => { + await migrate(db, [users], db.schema) + const forgetAll = async () => { + for (const t of ['_columns', '_migrations']) { + await db.command( + `ALTER TABLE \`${db.schema}\`.\`${t}\` DELETE WHERE 1`, + {}, + { + async_insert: 0, + keeper_map_strict_mode: 0, + }, + ) + } + } + // The same catalog again: every column checks out and is recorded. + await forgetAll() + const same = await migrate(db, [users], db.schema) + expect(same.drift).toEqual([]) + expect(await recorded([users])).toBe(true) + + // A changed path with no record to compare against. `ADD COLUMN IF NOT + // EXISTS` leaves the column as it was; the table says so, and that is + // drift to report, not a clause to record as installed. + await forgetAll() + await insertUser({ id: 'user_1', email: 'old@example.com', primary_email: 'new@example.com' }) + const moved = await migrate(db, [usersMovedEmail], db.schema) + expect(moved.drift.map((d) => [d.column, d.kind])).toEqual([['email', 'expression']]) + expect(await recorded([usersMovedEmail])).toBe(false) + const fixed = await migrate(db, [usersMovedEmail], db.schema, undefined, { repair: true }) + expect(fixed.drift).toEqual([]) + const row = await db.query<{ email: string }>(`SELECT email FROM \`${db.schema}\`.users FINAL`) + expect(row[0]!.email).toBe('new@example.com') + expect(await recorded([usersMovedEmail])).toBe(true) + // The probe table is gone again. + const probes = await db.query( + `SELECT name FROM system.tables WHERE database = {s:String} AND name LIKE '\\_probe\\_%'`, + { s: db.schema }, + ) + expect(probes).toEqual([]) + }) + it('refuses to repair a key column', async () => { await migrate(db, [users], db.schema) const rekeyed = withColumn(users, { name: 'id', type: 'text', path: 'uid' }) diff --git a/packages/sync-clickhouse/test/engine.test.ts b/packages/sync-clickhouse/test/engine.test.ts index e4d36b5..4accbe1 100644 --- a/packages/sync-clickhouse/test/engine.test.ts +++ b/packages/sync-clickhouse/test/engine.test.ts @@ -470,12 +470,20 @@ describeDb('redaction', () => { it('erases a subject from the mirror, the sign-in log and the event archive', async () => { const user = fake.createUser({ email: 'forget@example.com' }) fake.emitAuthAttempt(user['id'] as string, true) + // Invitations name a user as the one who accepted and as the inviter. + const org = fake.createOrganization() + const accepted = fake.createInvitation(org['id'] as string) + fake.acceptInvitation(accepted['id'] as string, user['id'] as string) + fake.createInvitation(org['id'] as string, { inviter_user_id: user['id'] }) + fake.createInvitation(org['id'] as string) const engine = makeEngine(db, fake) await engine.backfill() await engine.tick() const removed = await engine.redact(user['id'] as string) expect(removed).toContainEqual({ table: 'users', rows: 1 }) expect(removed).toContainEqual({ table: 'auth_attempts', rows: 1 }) + expect(removed).toContainEqual({ table: 'invitations', rows: 2 }) + expect(await count(db, 'invitations')).toBe(1) expect(removed.find((r) => r.table === 'events')!.rows).toBeGreaterThan(0) expect(await count(db, 'users', false)).toBe(0) const events = await db.query<{ n: string }>( From 1218afe038ee49d5e14d1fdeaa0d6c4eb8fe2d48 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Fri, 6 Nov 2026 08:40:59 -0500 Subject: [PATCH 09/10] fix(clickhouse): format expressions with formatQuerySingleLine, not a probe table Checking an unrecorded column built a throwaway table, which `init` had no business needing a `DROP TABLE` grant for. The server's formatter is a function too: `formatQuerySingleLine` spells a clause exactly as `system.columns.default_expression` does, so the comparison is one query with no DDL. The README's grants gain `DROP TABLE` for the staging tables `reconcile` has always built and dropped, and the version floor moves to 23.11, where the function arrived. --- packages/sync-clickhouse/README.md | 5 ++- packages/sync-clickhouse/src/schema/drift.ts | 40 +++++++++----------- packages/sync-clickhouse/test/drift.test.ts | 6 --- 3 files changed, 21 insertions(+), 30 deletions(-) diff --git a/packages/sync-clickhouse/README.md b/packages/sync-clickhouse/README.md index 7cac68c..2fee0d5 100644 --- a/packages/sync-clickhouse/README.md +++ b/packages/sync-clickhouse/README.md @@ -28,7 +28,7 @@ the mirror streams when you want to join against them. ## Requirements -- ClickHouse 23.3 or later, self-hosted or ClickHouse Cloud. Tested against +- ClickHouse 23.11 or later, self-hosted or ClickHouse Cloud. Tested against 26.x in CI and against ClickHouse Cloud. - A ClickHouse Keeper (or ZooKeeper) reachable by the server, with `keeper_map_path_prefix` set. The single-writer lease, the cursors, and @@ -121,7 +121,8 @@ the same setting, the flag wins. The user the sync connects as needs `CREATE DATABASE` the first time, or an existing database it may create tables in, plus `CREATE TABLE`, `CREATE VIEW`, -`ALTER`, `INSERT`, `SELECT` and `ALTER DELETE` on that database. +`DROP TABLE` (for the staging tables `reconcile` builds and drops), `ALTER`, +`INSERT`, `SELECT` and `ALTER DELETE` on that database. ## Embed it diff --git a/packages/sync-clickhouse/src/schema/drift.ts b/packages/sync-clickhouse/src/schema/drift.ts index 012d2a9..1cca4db 100644 --- a/packages/sync-clickhouse/src/schema/drift.ts +++ b/packages/sync-clickhouse/src/schema/drift.ts @@ -15,7 +15,7 @@ * text written by the catalog and text read back never compare equal. A * column with no record — a schema applied by hand from `init --print-sql` * — has only the table as a witness, so the catalog's clause is put through - * the same formatter, by way of a throwaway table, before the two are + * the server's own formatter (`formatQuerySingleLine`) before the two are * compared; only a match is recorded, anything else is drift. The live type * is compared with the catalog's directly; `system.columns` spells types * canonically, and so does `columnType`. @@ -25,7 +25,6 @@ * recorded columns the catalog no longer names, which still compute from * `_raw` with their old expression and are harmless to leave. */ -import { randomBytes } from 'node:crypto' import type { Logger, TableDef } from '@workos/sync-engine' import { KEEPER_SETTINGS, type ChClient } from '../db/client.js' import { isKeeperConflict } from '../db/errors.js' @@ -125,26 +124,23 @@ async function forgetColumn(db: ChClient, schema: string, table: string, column: } /** - * The server's own spelling of each of a stream's generated expressions, - * read back from a throwaway `Memory` table built from the catalog's - * clauses. It is what `system.columns.default_expression` would show for a - * column created from those clauses, so an unrecorded column can be judged - * against the table it is in. + * The server's own spelling of each of a stream's generated expressions: + * `system.columns.default_expression` is the clause re-formatted by the + * server, and `formatQuerySingleLine` applies the same formatter to the + * clause the catalog would write, so the two compare equal when the column + * was created from it. A function call, not a table: `init` needs no grant + * beyond the schema's own. */ -async function canonicalExpressions(db: ChClient, schema: string, stream: TableDef): Promise> { - const probe = `_probe_${randomBytes(4).toString('hex')}` - const ref = `${quoteIdent(schema)}.${quoteIdent(probe)}` - const clauses = stream.columns.map((c) => generatedColumnSql(c, stream)) - await db.command(`CREATE TABLE ${ref} (\`_raw\` String, ${clauses.join(', ')}) ENGINE = Memory`) - try { - const rows = await db.query<{ name: string; default_expression: string }>( - `SELECT name, default_expression FROM system.columns WHERE database = {schema:String} AND table = {table:String}`, - { schema, table: probe }, - ) - return new Map(rows.map((r) => [r.name, r.default_expression])) - } finally { - await db.command(`DROP TABLE IF EXISTS ${ref}`) - } +async function canonicalExpressions(db: ChClient, stream: TableDef): Promise> { + const names = stream.columns.map((c) => safeIdentifier(c.name)) + const exprs = stream.columns.map((c) => columnExpression(c, stream.key.includes(c.name))) + const rows = await db.query<{ name: string; expr: string }>( + `SELECT tupleElement(pair, 1) AS name, + substring(formatQuerySingleLine(concat('SELECT ', tupleElement(pair, 2))), 8) AS expr + FROM (SELECT arrayJoin(arrayZip({names:Array(String)}, {exprs:Array(String)})) AS pair)`, + { names, exprs }, + ) + return new Map(rows.map((r) => [r.name, r.expr])) } export async function detectColumnDrift(db: ChClient, streams: TableDef[], schema: string): Promise { @@ -178,7 +174,7 @@ export async function detectColumnDrift(db: ChClient, streams: TableDef[], schem if (record === undefined) { // Nothing says what this column was created with, so the table is // asked, in its own spelling of the clause the catalog would write. - canonical ??= await canonicalExpressions(db, schema, stream) + canonical ??= await canonicalExpressions(db, stream) if (canonical.get(name) === actual.default_expression) { await recordColumn(db, schema, table, name, expected) continue diff --git a/packages/sync-clickhouse/test/drift.test.ts b/packages/sync-clickhouse/test/drift.test.ts index 1f696e7..3f9e9db 100644 --- a/packages/sync-clickhouse/test/drift.test.ts +++ b/packages/sync-clickhouse/test/drift.test.ts @@ -186,12 +186,6 @@ describeDb('column drift detection', () => { const row = await db.query<{ email: string }>(`SELECT email FROM \`${db.schema}\`.users FINAL`) expect(row[0]!.email).toBe('new@example.com') expect(await recorded([usersMovedEmail])).toBe(true) - // The probe table is gone again. - const probes = await db.query( - `SELECT name FROM system.tables WHERE database = {s:String} AND name LIKE '\\_probe\\_%'`, - { s: db.schema }, - ) - expect(probes).toEqual([]) }) it('refuses to repair a key column', async () => { From 9d54eac03dfd2f6355c1f48c1003cae58a9f55dd Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Fri, 6 Nov 2026 08:54:21 -0500 Subject: [PATCH 10/10] chore(clickhouse): start sync-clickhouse at 0.0.1 The same seed as the other packages (#51): the first version is published by hand so the npm trusted publisher can be configured, and release-please reads 0.0.0 as never released and would propose 1.0.0, where from 0.0.1 the next release computes to 0.1.0. --- .release-please-manifest.json | 2 +- packages/sync-clickhouse/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index cfc00dc..3fc3809 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -3,5 +3,5 @@ "packages/sync-postgres": "0.1.0", "packages/sync-mongodb": "0.0.0", "packages/sync-mysql": "0.0.0", - "packages/sync-clickhouse": "0.0.0" + "packages/sync-clickhouse": "0.0.1" } diff --git a/packages/sync-clickhouse/package.json b/packages/sync-clickhouse/package.json index 53238a4..999cc05 100644 --- a/packages/sync-clickhouse/package.json +++ b/packages/sync-clickhouse/package.json @@ -1,6 +1,6 @@ { "name": "@workos/sync-clickhouse", - "version": "0.0.0", + "version": "0.0.1", "description": "Mirror your WorkOS data into ClickHouse. Backfill, tail the Events API, and reconcile drift.", "type": "module", "license": "MIT",