diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d17ce2c..b743503 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -186,3 +186,53 @@ jobs: - name: Build run: pnpm --filter "@workos/sync-mysql..." build + + clickhouse: + name: ClickHouse Node ${{ matrix.node }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + node: [22, 24] + env: + # There is no embedded ClickHouse, so the whole suite runs against the + # container; without this variable the database-backed tests skip. + TEST_CLICKHOUSE_URL: http://default@127.0.0.1:8123 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # Not a service container: those start before the checkout, and the + # server needs the config in test/clickhouse/ — an embedded Keeper for + # the KeeperMap bookkeeping tables and `keeper_map_path_prefix`. + - name: Start ClickHouse + run: | + docker run -d --name clickhouse -p 127.0.0.1:8123:8123 \ + -v "$PWD/packages/sync-clickhouse/test/clickhouse/config.xml:/etc/clickhouse-server/config.d/sync.xml:ro" \ + -v "$PWD/packages/sync-clickhouse/test/clickhouse/users.xml:/etc/clickhouse-server/users.d/sync.xml:ro" \ + -e CLICKHOUSE_SKIP_USER_SETUP=1 \ + --ulimit nofile=262144:262144 \ + clickhouse/clickhouse-server:26.10 + for _ in $(seq 1 60); do + if curl -fsS http://127.0.0.1:8123/ping >/dev/null 2>&1; then exit 0; fi + sleep 1 + done + echo "ClickHouse did not become ready" >&2 + docker logs clickhouse >&2 + exit 1 + + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: ${{ matrix.node }} + + - name: Install + run: pnpm install --frozen-lockfile + + - name: Test + run: pnpm --filter @workos/sync-clickhouse test:database + + - name: Build + run: pnpm --filter "@workos/sync-clickhouse..." build diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a793afb..4208c66 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -24,6 +24,7 @@ jobs: - postgres - mongodb - mysql + - clickhouse runner: [ubuntu-latest, ubuntu-24.04-arm] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 7bd2c70..25edf52 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -39,6 +39,8 @@ jobs: sync-mysql--tag_name: ${{ steps.release.outputs['packages/sync-mysql--tag_name'] }} sync-mongodb--release_created: ${{ steps.release.outputs['packages/sync-mongodb--release_created'] }} sync-mongodb--tag_name: ${{ steps.release.outputs['packages/sync-mongodb--tag_name'] }} + sync-clickhouse--release_created: ${{ steps.release.outputs['packages/sync-clickhouse--release_created'] }} + sync-clickhouse--tag_name: ${{ steps.release.outputs['packages/sync-clickhouse--tag_name'] }} steps: # Release PRs and tags come from the SDK Automation Bot, as in workos-node. # Events raised with GITHUB_TOKEN start no workflow runs, so a release PR @@ -108,3 +110,15 @@ jobs: tag_name: ${{ needs.release-please.outputs.sync-mongodb--tag_name }} secrets: PUBLIC_SOCKET_FIREWALL_TOKEN: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }} + + publish-sync-clickhouse: + needs: [release-please, publish-sync-engine] + if: >- + ${{ !cancelled() + && needs.publish-sync-engine.result != 'failure' + && needs.release-please.outputs.sync-clickhouse--release_created == 'true' }} + uses: ./.github/workflows/release.yml + with: + tag_name: ${{ needs.release-please.outputs.sync-clickhouse--tag_name }} + secrets: + PUBLIC_SOCKET_FIREWALL_TOKEN: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }} diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 88f6ef2..3fc3809 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -2,5 +2,6 @@ "packages/sync-engine": "0.1.0", "packages/sync-postgres": "0.1.0", "packages/sync-mongodb": "0.0.0", - "packages/sync-mysql": "0.0.0" + "packages/sync-mysql": "0.0.0", + "packages/sync-clickhouse": "0.0.1" } diff --git a/README.md b/README.md index b58f26c..6da5dbe 100644 --- a/README.md +++ b/README.md @@ -31,15 +31,16 @@ base tables keep them as tombstones for when you want to ask who left. One engine, one package per database. Each ships a CLI with the same commands and a container image under the same name: -| Destination | Package | Mirror | -| ----------- | ------------------------------------------------- | ----------------------------------------------------------------------- | -| Postgres | [`@workos/sync-postgres`](packages/sync-postgres) | A `workos` schema of tables with generated columns and `_live` views | -| MongoDB | [`@workos/sync-mongodb`](packages/sync-mongodb) | `workos_*` collections in your database, with indexes and `_live` views | -| MySQL | [`@workos/sync-mysql`](packages/sync-mysql) | A `workos` database of tables with generated columns and `_live` views | +| Destination | Package | Mirror | +| ----------- | ----------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Postgres | [`@workos/sync-postgres`](packages/sync-postgres) | A `workos` schema of tables with generated columns and `_live` views | +| MongoDB | [`@workos/sync-mongodb`](packages/sync-mongodb) | `workos_*` collections in your database, with indexes and `_live` views | +| MySQL | [`@workos/sync-mysql`](packages/sync-mysql) | A `workos` database of tables with generated columns and `_live` views | +| ClickHouse | [`@workos/sync-clickhouse`](packages/sync-clickhouse) | A `workos` database of `ReplacingMergeTree` tables with `_live` views, plus append-only tables for the sign-in log and events; for history that outlives the 90-day retention | The rest of this page shows Postgres; every command and option below has a -MongoDB and a MySQL counterpart, and each package's README is the reference -for it. +MongoDB, a MySQL and a ClickHouse counterpart, and each package's README is +the reference for it. ## Try it in 30 seconds diff --git a/packages/sync-clickhouse/LICENSE.txt b/packages/sync-clickhouse/LICENSE.txt new file mode 100644 index 0000000..c0cf745 --- /dev/null +++ b/packages/sync-clickhouse/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 WorkOS + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/sync-clickhouse/README.md b/packages/sync-clickhouse/README.md new file mode 100644 index 0000000..2fee0d5 --- /dev/null +++ b/packages/sync-clickhouse/README.md @@ -0,0 +1,253 @@ +# @workos/sync-clickhouse + +Keep your WorkOS sign-in and event history in ClickHouse for as long as you +like, with the users and organizations beside it for the joins. + +WorkOS keeps events for 90 days, and the `authentication.*` events — every +sign-in, success or failure — have no REST endpoint at all. This package tails +the Events API from a durable cursor into append-only ClickHouse tables, so +"failed sign-ins per organization per hour, last 18 months" is a `GROUP BY` +over data you own: + +```sql +SELECT o.name AS organization, toStartOfHour(a.created_at) AS hour, count() AS failures +FROM workos.auth_attempts a +JOIN workos.users_live u ON u.id = a.user_id AND u._env = a._env +JOIN workos.organization_memberships_live m ON m.user_id = u.id AND m._env = u._env +JOIN workos.organizations_live o ON o.id = m.organization_id AND o._env = m._env +WHERE a.status = 'failed' AND a.created_at > now() - INTERVAL 7 DAY +GROUP BY organization, hour +ORDER BY failures DESC; +``` + +It also mirrors the identity objects (users, organizations, memberships, +directories, connections, roles, …) the way `@workos/sync-postgres` does, as +versioned tables with `_live` views, so the join above works in one database. +Start with `--streams auth_attempts` if the logs are what you came for; add +the mirror streams when you want to join against them. + +## Requirements + +- ClickHouse 23.11 or later, self-hosted or ClickHouse Cloud. Tested against + 26.x in CI and against ClickHouse Cloud. +- A ClickHouse Keeper (or ZooKeeper) reachable by the server, with + `keeper_map_path_prefix` set. The single-writer lease, the cursors, and + the per-stream write lock that keeps two processes from writing one row + at once live in + [KeeperMap](https://clickhouse.com/docs/engines/table-engines/special/keeper-map) + tables: it is the one place ClickHouse offers an atomic compare-and-swap. + Cloud and any replicated cluster have this already. A single self-hosted + node needs the embedded Keeper enabled; the few lines are in + [`test/clickhouse/config.xml`](test/clickhouse/config.xml). +- The HTTP interface (port 8123, or 8443 with TLS). The package uses the + official `@clickhouse/client`. + +## Install + +```bash +npm install @workos/sync-clickhouse +``` + +This package ships the ClickHouse destination, the `workos-sync-clickhouse` +command, and re-exports the engine from +[`@workos/sync-engine`](https://www.npmjs.com/package/@workos/sync-engine), +so it is the only package you need for ClickHouse. + +## Command line + +```bash +export DATABASE_URL=https://default:password@abc123.us-east-1.aws.clickhouse.cloud:8443 +export WORKOS_API_KEY=sk_live_... + +npx workos-sync-clickhouse init # create the database, tables and views +npx workos-sync-clickhouse backfill --streams auth_attempts # nothing to list for logs; sets the cursor +npx workos-sync-clickhouse listen --streams auth_attempts # tail the Events API, forever +``` + +Auth history begins the day you turn the tail on: there is no endpoint to +list past sign-ins, so `backfill` cannot recover them. That is the argument +for starting the tail early. + +| Command | What it does | +| ----------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `init [--repair] [--print-sql]` | Create the database, tables, `_live` views and bookkeeping. Idempotent; safe on every deploy. Exits 2 if a projected column has drifted from the catalog, and fails if a view under a generated `_live` name is not the generated shape; `--repair` redefines columns and replaces such views. `--print-sql` prints the DDL instead of running it. | +| `backfill [--resume] [--max-pages ]` | Load current state from the REST list endpoints. `--resume` continues each stream from its checkpoint and skips streams already complete. `--max-pages` stops after `n` pages across all streams, for a host with a time limit; run again with `--resume` until every stream reports `complete`. | +| `listen [--health-port ]` | Poll the Events API continuously as the sole writer; waits if another listener holds the lease. Finishes the current batch on `SIGINT` or `SIGTERM`. With a port, serves `GET /healthz` and `/metrics` on it. | +| `tick` | One poll, then exit. For a cron job instead of a long-running process. Exits 3 if a listener holds the lease. | +| `reconcile [--repair] [--resume] [--max-pages ] [--exclude-streams ]` | Diff ids against the API. Exits 2 on drift; `--repair` tombstones missed deletes and inserts missed creates. `--max-pages` stops starting streams once `n` pages have been fetched, for a host with a time limit; a stream that starts runs to its last page, since a diff needs the whole listing, unless it alone needs more than `n`: it is then abandoned, reported `oversized`, and skipped by `--resume` until a run with a larger budget or none. Run again with `--resume` until every stream reports `complete`. `--exclude-streams` leaves streams out, for one that has to run somewhere without a time limit. | +| `refresh [--resume] [--max-pages ]` | Re-list top-level streams from the REST API, replacing each row; `--max-pages` and `--resume` slice it like `backfill`, with which it shares checkpoints. Re-run `backfill` for per-parent streams. | +| `status` | Cursor, time since the last successful poll, streams `reconcile` has abandoned as oversized, and per-stream row counts, tombstones and lag. | +| `redact --user ` | Erase one subject from the mirror, the sign-in log and the event archive, with a lightweight `DELETE`. | +| `rejected [--replay] [--id ]` | Show events the tail quarantined; exits 2 if there are any. `--replay` re-applies them (one, with `--id`) once the cause is fixed. | +| `streams` | List what this build can mirror, with group, mode, and whether it is on by default. | + +Flags every command takes: + +| Flag | Overrides | What it does | +| ---------------------- | -------------------- | ---------------------------------------------------------- | +| `--database-url ` | `DATABASE_URL` | The HTTP interface, `https://user:password@host:8443`. | +| `--schema ` | `WORKOS_SYNC_SCHEMA` | The database the mirror lives in. | +| `--streams ` | | Restrict to specific streams. `events` is always included. | +| `--help` | | Print usage. | + +Rows are labelled `_env = 'production'` by default; set +`WORKOS_ENVIRONMENT_NAME` if the key is for something else. The label is +written to every row and must never change once you have data. To mirror +several environments into one database, run one process per key with a +different label each; cursors are kept per label, so they never collide. + +The command reads its settings from the environment; where a flag exists for +the same setting, the flag wins. + +| Variable | Default | What it does | +| ------------------------------------ | ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `DATABASE_URL` | required | The HTTP interface with credentials. A path in it is ignored; see `WORKOS_SYNC_SCHEMA`. | +| `WORKOS_API_KEY` | required | API key for the environment to mirror. One key, one environment, one process; dedicate it to the sync (see Operations). | +| `WORKOS_ENVIRONMENT_NAME` | `production` | Label written to `_env` on every row. Must not change once you have data. Run one process per environment to mirror several. | +| `WORKOS_API_URL` | `https://api.workos.com` | API base URL. | +| `WORKOS_SYNC_SCHEMA` | `workos` | The database the mirror lives in, created if missing. Every statement qualifies its tables with it. | +| `WORKOS_SYNC_ID` | `default` | Identifier when several syncs share one database. Each keeps its own cursors. | +| `WORKOS_SYNC_HYDRATE` | `off` | Re-fetch objects when their events arrive: `off`, `on_create`, `always`, or per stream as `users=always,organizations=off`. | +| `WORKOS_SYNC_POLL_MS` | `5000` | Idle wait between polls, in ms, when the last poll returned nothing. | +| `WORKOS_SYNC_LAG_MS` | `5000` | Hold back events younger than this, in ms, before advancing the cursor. | +| `WORKOS_SYNC_MAX_EVENTS_PER_TICK` | one page (100) | Events to apply per poll, fetched across as many pages as needed. Raise it: each poll is a few inserts, and ClickHouse prefers fewer, larger ones. | +| `WORKOS_SYNC_HEALTH_PORT` | off (`8080` in the container image) | `listen` only: serve `GET /healthz` and `GET /metrics` on this port, from in-memory state, with no database query. | +| `WORKOS_SYNC_HEALTH_HOST` | `0.0.0.0` | Interface the health server binds. | +| `WORKOS_SYNC_HEALTH_MAX_POLL_AGE_MS` | `60000` | `/healthz` returns 503 once any environment's last successful poll is older than this. | +| `WORKOS_SYNC_OTEL` | off | `1` starts the OpenTelemetry Node SDK for the command, configured by the standard `OTEL_*` variables, and flushes it on exit. Every command, `tick` included. | +| `WORKOS_SYNC_LEASE_TTL_MS` | `60000` | How long the tail lease outlives a crashed listener, in ms; at least 1. The most lag a crash can add before a replacement takes over. | +| `WORKOS_SYNC_CLICKHOUSE_TLS_CA` | system trust store | CA certificate the server's certificate must chain to, as PEM or base64 of it. Only with an `https://` URL. | +| `LOG_LEVEL` | `info` | `debug`, `info`, `warn`, `error` or `silent`. Logs are JSON lines on stdout. | + +The user the sync connects as needs `CREATE DATABASE` the first time, or an +existing database it may create tables in, plus `CREATE TABLE`, `CREATE VIEW`, +`DROP TABLE` (for the staging tables `reconcile` builds and drops), `ALTER`, +`INSERT`, `SELECT` and `ALTER DELETE` on that database. + +## Embed it + +```ts +import { SyncEngine, ClickHouseDestination } from "@workos/sync-clickhouse"; + +const engine = new SyncEngine({ + destination: new ClickHouseDestination({ + databaseUrl: process.env.DATABASE_URL, + }), + environments: [{ name: "production", apiKey: process.env.WORKOS_API_KEY! }], + streams: ["auth_attempts", "users", "organizations"], +}); + +await engine.backfill(); +await engine.listen(); // resolves when you call engine.stop() +``` + +To reuse an existing `@clickhouse/client` instance instead of a connection +string, wrap it with `fromClient` and pass it as `db`. Give it the settings +in `DEFAULT_SETTINGS`: the engine relies on waited async inserts, synchronous +mutations and sequentially consistent reads. + +`ClickHouseDestination` options: + +| Option | Default | | +| --------------- | --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `databaseUrl` | — | The HTTP interface with credentials. Either this or `db`. | +| `sslCa` | — | PEM CA certificate the server's certificate must chain to. With an `https://` `databaseUrl` only. | +| `db` | — | A pre-built client, e.g. `fromClient(client)`. Not closed by the engine. | +| `schema` | `workos` | The database the mirror lives in. Created if missing. | +| `syncId` | `default` | Identifier when several syncs share one database. | +| `batchSize` | `500` | Rows per statement. | +| `repairViews` | `false` | On `init()`, replace a view under a generated `_live` name that is not the generated shape. Off, `init()` fails and names it. | +| `repairColumns` | `false` | On `init()`, redefine projected columns that no longer match the catalog and recompute them. Off, drift is reported and `init` withholds the schema fingerprint. | + +Engine options (`environments`, `streams`, `extraColumns`, `hydrate`, +`pollIntervalMs`, `lagWindowMs`, `maxEventsPerTick`, …) are documented in +[`@workos/sync-engine`](https://www.npmjs.com/package/@workos/sync-engine). + +## The schema + +The engine writes one column, `_raw`, and computes the rest from it with +`DEFAULT` expressions: + +```sql +CREATE TABLE workos.auth_attempts ( + _env String, + _raw String, -- the event payload, the only column the engine writes + _updated_at DateTime64(3, 'UTC'), -- the event's own time; a tombstone's is its deletion time + _synced_at DateTime64(6, 'UTC') DEFAULT now64(6), + event_id String DEFAULT JSONExtractString(_raw, 'event_id'), + status Nullable(String) DEFAULT JSONExtract(_raw, 'status', 'Nullable(String)'), + user_id Nullable(String) DEFAULT JSONExtract(_raw, 'user_id', 'Nullable(String)'), + created_at Nullable(DateTime64(3, 'UTC')) DEFAULT if(match(JSONExtractString(_raw, 'created_at'), ''), + parseDateTime64BestEffortOrNull(upper(JSONExtractString(_raw, 'created_at')), 3, 'UTC'), NULL), + INDEX auth_attempts_user_id_idx user_id TYPE bloom_filter GRANULARITY 4, + INDEX auth_attempts_created_at_idx created_at TYPE minmax GRANULARITY 1 +) ENGINE = ReplacingMergeTree ORDER BY (_env, event_id); +``` + +Append-only streams (`auth_attempts`, `events`) are a `ReplacingMergeTree` +keyed by the event id, with no version column: one row per event, never +updated. A second copy of an event, which only a poll that outlived its lease +could write, collapses at the next merge; add `FINAL` where an exact count +matters before then. + +Mirror streams are `ReplacingMergeTree` versioned by `_updated_at`. Every +write inserts the row's next state, and the newest source timestamp wins at +merge time and under `FINAL`; that is the staleness gate, enforced by the +table whatever order writes land in. A deletion's tombstone carries the deletion time as its version, so it outranks every earlier version of the row. Each mirror table has a generated +`_live` view — `workos.users_live`, `workos.organizations_live`, and so +on — that reads with `FINAL` and hides tombstoned rows. **Query the views.** +The base table shows every version a row ever had, plus its tombstones +(`_deleted_at`), for when you want to ask who left. + +A field added upstream can never break ingest — it is queryable as +`JSONExtractString(_raw, 'field')` immediately, and promoting it to a typed +column is pure DDL, computed for the rows already there. A value a typed +column cannot cast projects as NULL, with the value still in `_raw`. Columns +marked for an index get a data-skipping index: a bloom filter for text, a +min-max index for timestamps and numbers. + +## What it mirrors + +Users, organizations, memberships, domains, invitations and sessions; +directories, directory users and groups; SSO connections; AuthKit groups, +roles, permissions and feature flags. Every event is archived verbatim to +`workos.events`, including event types this version has never heard of; the +`authentication.*` events are projected into `auth_attempts`. + +## What it is not + +The mirror is a **read model**: eventually consistent, usually within seconds. +Read from it for analytics, dashboards, lists and agents. Keep writes, +authorization decisions and post-signup checks on the WorkOS API. The sync is +one-directional; there is no write-back. ClickHouse is not where the mirror +tables are at their best: for an application that reads single rows by key, +use `@workos/sync-postgres` or `@workos/sync-mysql`, and if you want both, run +both; the cursors are kept per package. + +## Testing + +`pnpm test` runs the pure tests without a server. The database-backed suite +needs a ClickHouse with a Keeper, started from the config in +`test/clickhouse/`: + +```sh +docker run -d --name clickhouse -p 127.0.0.1:8123:8123 \ + -v "$PWD/packages/sync-clickhouse/test/clickhouse/config.xml:/etc/clickhouse-server/config.d/sync.xml:ro" \ + -v "$PWD/packages/sync-clickhouse/test/clickhouse/users.xml:/etc/clickhouse-server/users.d/sync.xml:ro" \ + -e CLICKHOUSE_SKIP_USER_SETUP=1 clickhouse/clickhouse-server:latest +TEST_CLICKHOUSE_URL=http://default@127.0.0.1:8123 pnpm test +``` + +Each test creates a database of its own and drops it on completion. +`pnpm test:database` fails when the URL is missing instead of skipping. + +## Documentation + +- [Getting started](https://github.com/workos/sync-engine/blob/main/docs/getting-started.md) +- [How it works](https://github.com/workos/sync-engine/blob/main/docs/how-it-works.md) — backfill, tail, reconcile, and why all three +- [Operations](https://github.com/workos/sync-engine/blob/main/docs/operations.md) — deployment, alerting, the 90-day retention cliff +- [Consistency](https://github.com/workos/sync-engine/blob/main/docs/consistency.md) — what the mirror does and does not guarantee + +## License + +MIT diff --git a/packages/sync-clickhouse/package.json b/packages/sync-clickhouse/package.json new file mode 100644 index 0000000..999cc05 --- /dev/null +++ b/packages/sync-clickhouse/package.json @@ -0,0 +1,50 @@ +{ + "name": "@workos/sync-clickhouse", + "version": "0.0.1", + "description": "Mirror your WorkOS data into ClickHouse. Backfill, tail the Events API, and reconcile drift.", + "type": "module", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/workos/sync-engine.git", + "directory": "packages/sync-clickhouse" + }, + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "bin": { + "workos-sync-clickhouse": "./dist/cli.js" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "files": [ + "dist" + ], + "scripts": { + "build": "tsc -p tsconfig.build.json", + "test": "vitest run", + "test:database": "node -e \"if (!process.env.TEST_CLICKHOUSE_URL) { console.error('TEST_CLICKHOUSE_URL is required'); process.exit(1) }\" && vitest run", + "test:watch": "vitest", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@workos/sync-engine": "workspace:*", + "@clickhouse/client": "^1.11.0" + }, + "optionalDependencies": { + "@opentelemetry/sdk-node": "^0.300.0" + }, + "devDependencies": { + "@opentelemetry/api": "^1.10.0", + "@opentelemetry/sdk-metrics": "^3.0.0", + "@types/node": "^26.6.5", + "typescript": "^7.0.2", + "vitest": "^5.0.5" + }, + "engines": { + "node": ">=22" + } +} diff --git a/packages/sync-clickhouse/src/cli.ts b/packages/sync-clickhouse/src/cli.ts new file mode 100644 index 0000000..62945e4 --- /dev/null +++ b/packages/sync-clickhouse/src/cli.ts @@ -0,0 +1,567 @@ +#!/usr/bin/env node +/** + * workos-sync-clickhouse — command line interface. + * + * workos-sync-clickhouse init [--repair] [--print-sql] + * create the schema; --repair rebuilds drifted columns and replaces + * foreign _live views; --print-sql prints the DDL instead of running it + * workos-sync-clickhouse backfill [--resume] [--max-pages ] + * page the list endpoints, optionally n pages at a time + * workos-sync-clickhouse listen tail the Events API + * workos-sync-clickhouse tick one poll, then exit (for cron) + * workos-sync-clickhouse reconcile [--repair] [--exclude-streams ] detect (and optionally fix) drift + * workos-sync-clickhouse refresh re-list to refresh partial event fields + * workos-sync-clickhouse status cursor age and per-stream freshness + * workos-sync-clickhouse redact --user erase one subject + * workos-sync-clickhouse rejected [--replay] quarantined events; replay them once fixed + * workos-sync-clickhouse streams list what this build can mirror + */ +import { + SyncEngine, + LeaseHeldError, + STREAMS, + DEFAULT_HYDRATE, + defaultStreams, + selectStreams, + createLogger, + startHealthServer, + type EnvironmentConfig, + type HydrateMode, + type Logger, + type LogLevel, + type SyncEngineConfig, +} from '@workos/sync-engine' +import { X509Certificate } from 'node:crypto' +import { realpathSync } from 'node:fs' +import { fileURLToPath } from 'node:url' +import { ClickHouseDestination } from './destination.js' +import { buildSchemaSql } from './schema/ddl.js' +import { fingerprint } from './schema/migrate.js' +import { healthOptionsFromEnv } from './health.js' +import { startOtel } from './otel.js' +import type { ChClient } from './db/client.js' + +interface Args { + command: string + flags: Record +} + +function parseArgs(argv: string[]): Args { + const [command = 'help', ...rest] = argv + const flags: Record = {} + for (let i = 0; i < rest.length; i++) { + const arg = rest[i]! + if (!arg.startsWith('--')) continue + const key = arg.slice(2) + const next = rest[i + 1] + if (next && !next.startsWith('--')) { + flags[key] = next + i++ + } else { + flags[key] = true + } + } + return { command, flags } +} + +/** + * One key, one environment, one process. `WORKOS_ENVIRONMENT_NAME` is the label + * written to every row as `_env`. It is yours to choose because no API call + * reports which environment a key belongs to, and it must stay stable across + * key rotation. To mirror several environments into one database, run one + * process per key with a different label each; cursors are kept per label. + */ +function environmentsFromEnv(): EnvironmentConfig[] { + const apiKey = process.env['WORKOS_API_KEY'] + if (!apiKey) throw new Error('set WORKOS_API_KEY') + return [ + { + name: process.env['WORKOS_ENVIRONMENT_NAME'] ?? 'production', + apiKey, + apiUrl: process.env['WORKOS_API_URL'], + }, + ] +} + +const HYDRATE_MODES: HydrateMode[] = ['off', 'on_create', 'always'] + +/** + * `WORKOS_SYNC_HYDRATE` is either one mode for every stream (`off`, + * `on_create`, `always`) or a per-stream list (`users=always,organizations=off`). + * Unset means the engine default: off. + */ +export function parseHydrate(value: string | undefined): SyncEngineConfig['hydrate'] { + if (!value || !value.trim()) return undefined + const v = value.trim() + if ((HYDRATE_MODES as string[]).includes(v)) return v as HydrateMode + const out: Record = { ...DEFAULT_HYDRATE } + for (const pair of v.split(',')) { + const idx = pair.indexOf('=') + const stream = pair.slice(0, idx).trim() + const mode = pair.slice(idx + 1).trim() + if (idx < 1 || !(HYDRATE_MODES as string[]).includes(mode)) { + throw new Error( + `invalid WORKOS_SYNC_HYDRATE entry "${pair}": expected off | on_create | always, or stream=mode,...`, + ) + } + out[stream] = mode as HydrateMode + } + return out +} + +/** + * `--repair` on `init` means rebuilding drifted columns from `_raw` and + * replacing foreign views under generated `_live` names; on `reconcile` it + * means repairing rows. Sharing the flag name is deliberate — it is "fix what + * you found" in both — but each command reads only its own meaning. + */ +export function repairRequested(command: string, flags: Record): boolean { + return command === 'init' && Boolean(flags['repair']) +} + +const PEM_CERTIFICATE = /-----BEGIN CERTIFICATE-----[\s\S]*?-----END CERTIFICATE-----/g + +/** True when `text` holds at least one PEM certificate and every one of them parses. */ +function isCertificateBundle(text: string): boolean { + const blocks = text.match(PEM_CERTIFICATE) + if (!blocks) return false + try { + for (const block of blocks) new X509Certificate(block) + return true + } catch { + return false + } +} + +/** + * `WORKOS_SYNC_CLICKHOUSE_TLS_CA` is the CA certificate the database's certificate + * must chain to: PEM, or the PEM encoded as base64 for a secret store that + * cannot hold a multi-line value. A PEM whose newlines arrived as the two + * characters `\n` is straightened out, since that is how some hosts hand + * back a pasted certificate. Unset or empty means the driver's own defaults. + * Every certificate in the value is parsed, so a truncated or damaged one + * is refused here rather than handed to the driver, where it would surface + * as a handshake failure against the server and read as the server's fault + * rather than the configuration's. + */ +export function parseSslCa(value: string | undefined): string | undefined { + if (!value || !value.trim()) return undefined + const v = value.trim().replace(/\\n/g, '\n') + if (isCertificateBundle(v)) return v + const decoded = Buffer.from(v, 'base64').toString('utf8') + if (isCertificateBundle(decoded)) return decoded + throw new Error( + 'invalid WORKOS_SYNC_CLICKHOUSE_TLS_CA: expected one or more PEM certificates, or them encoded as base64', + ) +} + +function schemaFromFlags(flags: Record): string { + return (flags['schema'] as string) ?? process.env['WORKOS_SYNC_SCHEMA'] ?? 'workos' +} + +function streamsFromFlags(flags: Record): string[] | undefined { + return typeof flags['streams'] === 'string' ? flags['streams'].split(',') : undefined +} + +/** + * `init --print-sql`: the DDL `init` would run, as one script for + * clickhouse-client or a migration file, instead of running it. + * Nothing is touched, so no database or API key is needed; the stream + * selection is the engine's own. + * + * The header names the catalog fingerprint but the script deliberately does + * not record it. A recorded fingerprint means "the live columns were checked + * against this catalog" (see `migrate`), and only `init` can check. So after + * the script is applied, `init` still has to run once: it finds the DDL in + * place, runs that check, and records the fingerprint, after which later + * runs are no-ops. Every statement in it is idempotent, so it can be applied + * to a database in any state. The schema name is JSON-quoted in the header + * so that a line break in it cannot end the comment early; the DDL itself + * quotes it as an identifier. + */ +export function printSchemaSql(streamNames: string[] | undefined, schema: string): string { + const streams = selectStreams(streamNames) + const header = [ + '-- Generated by workos-sync-clickhouse init --print-sql.', + `-- Schema ${JSON.stringify(schema)}, catalog fingerprint ${fingerprint(streams, schema)}.`, + '-- Apply this, then run `init` once: it records the fingerprint after', + '-- checking the live columns against the catalog, which this script cannot do.', + ].join('\n') + return `${header}\n\n${buildSchemaSql(streams, schema) + .map((s) => `${s};`) + .join('\n\n')}\n` +} + +/** + * Built apart from the rest of the config because `init` reads the drift + * report back off it after the engine has run; the engine's own `init()` + * returns nothing. `db` is a test seam: a pre-built client in place + * of a connection string, so the CLI can be exercised end to end. + */ +function destinationFromEnv( + flags: Record, + repair: boolean, + db?: ChClient, +): ClickHouseDestination { + const databaseUrl = (flags['database-url'] as string) ?? process.env['DATABASE_URL'] + if (!db && !databaseUrl) throw new Error('set DATABASE_URL (or pass --database-url)') + return new ClickHouseDestination({ + databaseUrl, + sslCa: parseSslCa(process.env['WORKOS_SYNC_CLICKHOUSE_TLS_CA']), + db, + schema: schemaFromFlags(flags), + syncId: process.env['WORKOS_SYNC_ID'] ?? 'default', + repairColumns: repair, + repairViews: repair, + }) +} + +/** + * `WORKOS_SYNC_LEASE_TTL_MS` is a number of milliseconds, at least 1. Unset + * or empty means the engine default. Anything else is refused here rather + * than further down: an empty value would otherwise become zero, and a lease + * that expires on arrival excludes nobody. + */ +export function parseLeaseTtl(value: string | undefined): number | undefined { + if (!value || !value.trim()) return undefined + const n = Number(value) + if (!Number.isFinite(n) || Math.floor(n) < 1) { + throw new Error(`invalid WORKOS_SYNC_LEASE_TTL_MS "${value}": expected a number of milliseconds of at least 1`) + } + return n +} + +/** + * `--max-pages ` is a positive integer. Absent means unbounded. A bare + * `--max-pages` with no value, zero, or a fraction is refused here, before the + * engine connects: a budget of nothing would stop before the first page and + * look like a backfill that found nothing. + */ +export function parseMaxPages(value: string | boolean | undefined): number | undefined { + if (value === undefined || value === false) return undefined + const n = typeof value === 'string' ? Number(value.trim()) : NaN + if (typeof value !== 'string' || !value.trim() || !Number.isInteger(n) || n < 1) { + throw new Error(`invalid --max-pages "${String(value)}": expected a positive integer`) + } + return n +} + +/** + * `--exclude-streams ` names streams `reconcile` leaves out, for one + * whose listing has to run somewhere without a time limit. A bare flag is + * refused here: excluding nothing by accident is the mistake the flag exists + * to prevent. The names themselves are checked by the engine. + */ +export function parseExcludeStreams(value: string | boolean | undefined): string[] | undefined { + if (value === undefined || value === false) return undefined + const names = + typeof value === 'string' + ? value + .split(',') + .map((s) => s.trim()) + .filter(Boolean) + : [] + if (names.length === 0) { + throw new Error(`invalid --exclude-streams "${String(value)}": expected a comma-separated list of stream names`) + } + return names +} + +function configFromEnv( + flags: Record, + destination: ClickHouseDestination, +): SyncEngineConfig & { logger: Logger } { + const streams = streamsFromFlags(flags) + return { + destination, + environments: environmentsFromEnv(), + streams, + hydrate: parseHydrate(process.env['WORKOS_SYNC_HYDRATE']), + pollIntervalMs: Number(process.env['WORKOS_SYNC_POLL_MS'] ?? 5000), + lagWindowMs: Number(process.env['WORKOS_SYNC_LAG_MS'] ?? 5000), + maxEventsPerTick: process.env['WORKOS_SYNC_MAX_EVENTS_PER_TICK'] + ? Number(process.env['WORKOS_SYNC_MAX_EVENTS_PER_TICK']) + : undefined, + leaseTtlMs: parseLeaseTtl(process.env['WORKOS_SYNC_LEASE_TTL_MS']), + logger: createLogger((process.env['LOG_LEVEL'] as LogLevel) ?? 'info'), + } +} + +const HELP = `workos-sync-clickhouse — mirror WorkOS data into ClickHouse + +Usage + workos-sync-clickhouse [flags] + +Commands + init [--repair] [--print-sql] + Create the database and bookkeeping tables. Exits 2 if a generated + column has drifted from the catalog, and fails if a view under a + generated _live name is not the generated shape; --repair rebuilds + columns from _raw and replaces such views. --print-sql prints the + DDL instead of running it, with no database + needed; apply it, then run init once so it records the fingerprint + backfill [--resume] [--max-pages ] + Load current state from the REST list endpoints. + --resume continues each stream from its checkpoint and + skips streams already complete; --max-pages stops after + n pages across all streams, for hosts with a time limit + (run again with --resume until every stream is complete) + listen [--health-port ] + Tail the Events API and apply changes continuously + (waits while another listener holds the lease); + with a port, serve GET /healthz and /metrics on it + tick One poll of the Events API, then exit (for a cron job); + exits 3 if a listener holds the lease + reconcile [--repair] [--resume] [--max-pages ] [--exclude-streams ] + Compare the mirror against the API; --repair fixes drift. + --max-pages stops starting streams once n pages have been + fetched (a stream that starts runs to its last page, since + a diff needs the whole listing, unless it alone needs more + than n: it is then abandoned and reported oversized until + a run with a larger budget or none); run again with + --resume until every stream reports complete. + --exclude-streams leaves streams out, for one that has to + run somewhere without a time limit + refresh [--resume] [--max-pages ] + Re-list top-level streams, replacing each row; + --max-pages and --resume slice it like backfill, with + which it shares checkpoints + status Cursor position, poll age, and per-stream freshness + redact --user Delete one subject from the mirror and event archive + rejected Show events quarantined by the tail; exits 2 if there are any + --replay Re-apply them all (or one with --id ) once the cause is fixed + streams Show the streams this build can mirror + +Flags + --database-url ClickHouse HTTP URL, https://user:password@host:8443 (or DATABASE_URL) + --schema Target database (or WORKOS_SYNC_SCHEMA; default: workos) + --streams Restrict to specific streams (events is always included) + +Environment + DATABASE_URL ClickHouse HTTP URL + WORKOS_SYNC_CLICKHOUSE_TLS_CA CA certificate the database's certificate must chain to, as PEM + or base64 of it; TLS is then required and verified against it + (default: the system trust store) + WORKOS_API_KEY API key for the environment to mirror + WORKOS_ENVIRONMENT_NAME Label written to _env on every row (default: production) + WORKOS_API_URL API base URL (default: https://api.workos.com) + WORKOS_SYNC_SCHEMA Target database (default: workos) + WORKOS_SYNC_ID Sync identifier when several syncs share a database (default: default) + WORKOS_SYNC_HYDRATE off | on_create | always, or users=always,organizations=off + (default: off) + WORKOS_SYNC_POLL_MS Idle wait between polls (default: 5000) + WORKOS_SYNC_LAG_MS Hold back events younger than this (default: 5000) + WORKOS_SYNC_MAX_EVENTS_PER_TICK Events to apply per poll (default: one page) + WORKOS_SYNC_LEASE_TTL_MS How long the tail lease outlives a crashed listener, at least 1 (default: 60000) + WORKOS_SYNC_HEALTH_PORT listen only: serve /healthz and /metrics on this port (default: off) + WORKOS_SYNC_HEALTH_HOST Interface the health server binds (default: 0.0.0.0) + WORKOS_SYNC_HEALTH_MAX_POLL_AGE_MS + /healthz returns 503 once any environment's last successful + poll is older than this (default: 60000) + WORKOS_SYNC_OTEL 1 starts the OpenTelemetry Node SDK for the command, configured by + the OTEL_* variables, and flushes it on exit (default: off) + LOG_LEVEL debug | info | warn | error | silent +` + +export interface MainDeps { + /** A pre-built client in place of `DATABASE_URL`, so tests can run the CLI against a test server. */ + db?: ChClient + /** A whole engine in place of one built from the environment, for tests of commands that need faults injected. */ + engine?: SyncEngine +} + +export async function main(argv = process.argv.slice(2), deps: MainDeps = {}): Promise { + const { command, flags } = parseArgs(argv) + + if (command === 'help' || flags['help']) { + process.stdout.write(HELP) + return 0 + } + + if (command === 'streams') { + const defaults = new Set(defaultStreams()) + const rows = STREAMS.map((s) => ({ + stream: s.name, + group: s.group, + mode: s.mode, + default: defaults.has(s.name) ? 'yes' : 'no (opt-in)', + backfill: s.list ? 'yes' : s.name === 'organization_domains' ? 'from organizations' : 'no', + events: (s.events?.upsert?.length ?? 0) + (s.events?.delete?.length ?? 0), + })) + process.stdout.write(JSON.stringify(rows, null, 2) + '\n') + return 0 + } + + // The DDL alone, before anything that would need a database or a key. + if (command === 'init' && flags['print-sql']) { + process.stdout.write(printSchemaSql(streamsFromFlags(flags), schemaFromFlags(flags))) + return 0 + } + + // The SDK, when asked for, goes up before the engine is built, so that the + // instruments the engine makes at construction are real ones, and comes + // down after the command returns, which is what exports a one-shot + // command's metrics at all. + const otel = await startOtel() + try { + return await runCommand(command, flags, deps) + } finally { + await otel?.shutdown() + } +} + +async function runCommand(command: string, flags: Record, deps: MainDeps): Promise { + // With an injected engine there is no destination of ours to read the + // drift report from, and nothing to build from the environment. + const destination = deps.engine ? undefined : destinationFromEnv(flags, repairRequested(command, flags), deps.db) + const config = destination ? configFromEnv(flags, destination) : undefined + const engine = deps.engine ?? new SyncEngine(config!) + const logger = config?.logger ?? createLogger((process.env['LOG_LEVEL'] as LogLevel) ?? 'info') + try { + switch (command) { + case 'init': { + await engine.init() + const drift = destination?.lastMigration?.drift ?? [] + const repaired = destination?.lastMigration?.repaired ?? [] + if (drift.length > 0 || repaired.length > 0) { + process.stdout.write(JSON.stringify({ repaired, drift }, null, 2) + '\n') + } + // Orphans are advisory. Anything else means a projection is wrong and + // the fingerprint was withheld, so exit 2 the way `reconcile` does. + const unresolved = drift.some((d) => d.kind !== 'orphan') + if (!unresolved) process.stdout.write('schema ready\n') + return unresolved ? 2 : 0 + } + case 'backfill': { + const results = await engine.backfill({ + resume: Boolean(flags['resume']), + maxPages: parseMaxPages(flags['max-pages']), + }) + process.stdout.write(JSON.stringify(results, null, 2) + '\n') + // A budget that stops short is the outcome asked for, not an error: + // the JSON says which streams are complete, and the next run resumes. + return 0 + } + case 'listen': { + // Up before the engine connects, so a probe during a slow schema init + // sees "starting" rather than a refused connection. + const healthOptions = healthOptionsFromEnv(flags) + const health = healthOptions ? await startHealthServer(engine, healthOptions) : null + if (healthOptions && health) { + logger.info('health server listening', { + host: healthOptions.host, + port: health.port, + maxPollAgeMs: healthOptions.maxPollAgeMs, + }) + } + const shutdown = () => { + engine.stop() + } + process.on('SIGINT', shutdown) + process.on('SIGTERM', shutdown) + try { + await engine.listen() + } finally { + await health?.close() + } + return 0 + } + case 'tick': { + try { + const r = await engine.tick() + process.stdout.write(JSON.stringify(r, null, 2) + '\n') + return 0 + } catch (err) { + // Distinct from a failure (1) and from drift (2): the sync is fine, + // something else is tailing it. Usually a `listen` left running. + if (err instanceof LeaseHeldError) { + process.stderr.write(`${err.message}; nothing to do\n`) + return 3 + } + throw err + } + } + case 'reconcile': { + const r = await engine.reconcile({ + repair: Boolean(flags['repair']), + resume: Boolean(flags['resume']), + maxPages: parseMaxPages(flags['max-pages']), + excludeStreams: parseExcludeStreams(flags['exclude-streams']), + }) + process.stdout.write(JSON.stringify(r, null, 2) + '\n') + // A non-zero exit makes this usable as a scheduled check. + const drift = Object.values(r.diffs) + .flat() + .some((d) => d.localOnly.length > 0 || d.remoteOnly.length > 0) + return drift && !flags['repair'] ? 2 : 0 + } + case 'refresh': { + const r = await engine.refresh({ + resume: Boolean(flags['resume']), + maxPages: parseMaxPages(flags['max-pages']), + }) + process.stdout.write(JSON.stringify(r, null, 2) + '\n') + return 0 + } + case 'status': { + const r = await engine.status() + process.stdout.write(JSON.stringify(r, null, 2) + '\n') + return 0 + } + case 'redact': { + const user = flags['user'] + if (typeof user !== 'string') throw new Error('redact requires --user ') + const r = await engine.redact(user) + process.stdout.write(JSON.stringify(r, null, 2) + '\n') + return 0 + } + case 'rejected': { + const id = flags['id'] + if (id !== undefined && typeof id !== 'string') throw new Error('--id requires an event id') + const only = typeof id === 'string' ? { ids: [id] } : {} + if (flags['replay']) { + const r = await engine.replayRejected(only) + process.stdout.write(JSON.stringify(r, null, 2) + '\n') + // Anything still rejected after a replay is still a problem. + return r.stillRejected.length > 0 ? 2 : 0 + } + const r = await engine.rejected(only) + process.stdout.write(JSON.stringify(r, null, 2) + '\n') + // Like `reconcile`: a non-zero exit makes this a scheduled check. + const any = Object.values(r).some((rows) => rows.length > 0) + return any ? 2 : 0 + } + default: + process.stderr.write(`unknown command: ${command}\n\n${HELP}`) + return 1 + } + } finally { + await engine.close() + } +} + +/** + * Whether this module is the script Node was started with, as opposed to an + * import by a test. npm installs a `bin` as a symlink named after the + * command, so `process.argv[1]` ends in `workos-sync-clickhouse` rather than + * `cli.js`; both sides are resolved to real paths so the installed command + * counts. + */ +export function invokedDirectly(argv1: string | undefined, moduleUrl: string): boolean { + if (!argv1) return false + try { + return realpathSync(argv1) === realpathSync(fileURLToPath(moduleUrl)) + } catch { + return false + } +} + +if (invokedDirectly(process.argv[1], import.meta.url)) { + main().then( + (code) => process.exit(code), + (err) => { + process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`) + process.exit(1) + }, + ) +} diff --git a/packages/sync-clickhouse/src/db/client.ts b/packages/sync-clickhouse/src/db/client.ts new file mode 100644 index 0000000..47ca3c5 --- /dev/null +++ b/packages/sync-clickhouse/src/db/client.ts @@ -0,0 +1,144 @@ +/** + * A minimal ClickHouse interface over the official HTTP client, so the rest + * of the package reads like its SQL siblings: `query`, `command`, `insert`, + * `close`. Values travel as query parameters (`{name:Type}`), never spliced + * into SQL; rows travel as JSONEachRow. + * + * Every connection carries a few settings the engine's arithmetic depends on: + * + * - `async_insert = 1` with `wait_for_async_insert = 1`. The tail writes a + * small batch per stream every few seconds, and a MergeTree part per batch + * would outrun the merges on a busy mirror. Async inserts let the server + * batch them, and waiting for the flush keeps the engine's promise that a + * write is durable when the call returns. KeeperMap statements turn it off + * again: a batched insert collapses duplicate keys silently where a strict + * one must refuse them. + * - `mutations_sync = 2`. Lightweight deletes and KeeperMap updates return + * once every replica has them, so a redaction or a cursor is visible to + * the next query. + * - `select_sequential_consistency = 1`. On a replicated deployment a read + * may otherwise land on a replica the last insert has not reached, and the + * write path reads before it writes. + * - `date_time_input_format = 'best_effort'`. Timestamps are still sent in + * the plain `YYYY-MM-DD hh:mm:ss.fff` form, so this only widens what the + * server accepts; it never changes how a value is read. + */ +import type { Logger } from '@workos/sync-engine' +import { createClient, type ClickHouseClient, type ClickHouseSettings } from '@clickhouse/client' + +/** + * How the engine's connections introduce themselves: the `http_user_agent` + * the server records in `system.query_log`, the ClickHouse counterpart of + * Postgres's `application_name`. Kept a constant rather than carrying a + * version so that filters written against it survive a release. + */ +export const APPLICATION_NAME = 'workos-sync-engine' + +/** Settings every statement runs with unless it says otherwise. */ +export const DEFAULT_SETTINGS: ClickHouseSettings = { + async_insert: 1, + wait_for_async_insert: 1, + // 64-bit settings travel as strings in the client's types. + mutations_sync: '2', + select_sequential_consistency: '1', + date_time_input_format: 'best_effort', + keeper_map_strict_mode: 1, +} + +/** What a KeeperMap statement adds: strict mode is already on, async inserts must be off. */ +export const KEEPER_SETTINGS: ClickHouseSettings = { async_insert: 0 } + +export type Params = Record + +export interface ChClient { + /** The rows of a `SELECT`, parsed from JSONEachRow. */ + query>(sql: string, params?: Params, settings?: ClickHouseSettings): Promise + /** A statement with no result: DDL, `ALTER`, `DELETE`. */ + command(sql: string, params?: Params, settings?: ClickHouseSettings): Promise + /** Rows into `table` (fully qualified) as JSONEachRow. */ + insert(table: string, rows: Record[], settings?: ClickHouseSettings): Promise + close(): Promise +} + +/** Wrap a pre-built `@clickhouse/client` instance. */ +export function fromClient(client: ClickHouseClient): ChClient { + return { + async query(sql: string, params?: Params, settings?: ClickHouseSettings) { + const rs = await client.query({ + query: sql, + query_params: params, + format: 'JSONEachRow', + clickhouse_settings: settings, + }) + return rs.json() + }, + async command(sql: string, params?: Params, settings?: ClickHouseSettings) { + await client.command({ query: sql, query_params: params, clickhouse_settings: settings }) + }, + async insert(table: string, rows: Record[], settings?: ClickHouseSettings) { + if (rows.length === 0) return + await client.insert({ table, values: rows, format: 'JSONEachRow', clickhouse_settings: settings }) + }, + async close() { + await client.close() + }, + } +} + +export interface ConnectOptions { + /** + * PEM-encoded CA certificate(s) the server's certificate must chain to. + * TLS is then required and the certificate verified against this CA and + * the host name. + */ + sslCa?: string + /** Reserved for connection notices; the HTTP client has no idle sockets to lose. */ + logger?: Logger +} + +/** + * What `createClient(...)` is given for a connection string. The URL's user + * info becomes the credentials and its path is dropped: every statement the + * engine runs qualifies its tables with the configured schema, and a path + * would name a default database that `init` may not have created yet. + */ +export function clientConfig(databaseUrl: string, options: ConnectOptions = {}): Record { + const url = new URL(databaseUrl) + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + throw new Error(`unsupported ClickHouse URL scheme ${url.protocol}: use http:// or https:// (the HTTP interface)`) + } + const config: Record = { + url: `${url.protocol}//${url.host}`, + application: APPLICATION_NAME, + clickhouse_settings: DEFAULT_SETTINGS, + request_timeout: 120_000, + } + if (url.username) config['username'] = decodeURIComponent(url.username) + if (url.password) config['password'] = decodeURIComponent(url.password) + if (options.sslCa !== undefined) { + if (url.protocol !== 'https:') { + throw new Error('a CA certificate was given but the connection string is http://, which has no TLS: use https://') + } + config['tls'] = { ca_cert: Buffer.from(options.sslCa) } + } + return config +} + +export async function connect(databaseUrl: string, options: ConnectOptions = {}): Promise { + return fromClient(createClient(clientConfig(databaseUrl, options) as never)) +} + +/** `YYYY-MM-DD hh:mm:ss.fff` in UTC, the form every DateTime64(3) value is sent and received in. */ +export function toCh(d: Date): string { + return d.toISOString().replace('T', ' ').replace('Z', '') +} + +/** A DateTime64 as the server returns it in JSONEachRow, read as UTC. */ +export function fromCh(value: string): Date { + return new Date(value.replace(' ', 'T') + 'Z') +} + +/** ISO-8601 with millisecond precision, from a value the server returned. */ +export function isoOf(value: string): string { + return fromCh(value).toISOString() +} diff --git a/packages/sync-clickhouse/src/db/errors.ts b/packages/sync-clickhouse/src/db/errors.ts new file mode 100644 index 0000000..9c6e11c --- /dev/null +++ b/packages/sync-clickhouse/src/db/errors.ts @@ -0,0 +1,76 @@ +/** + * Classifying ClickHouse errors for the quarantine. + * + * The engine only ever sets an event aside for an error that will not go + * away on retry. The server reports every error with a numeric code and a + * symbolic type (`999 KEEPER_EXCEPTION`), which the client surfaces as + * `code` (a string) and `type`. The list below names the codes that + * describe the server's state rather than the row's: timeouts, lost + * connections, resource pressure, a replica mid-failover, a Keeper + * transaction that lost a race. Not in it, on purpose: a value the server + * cannot parse, a missing table or column, a syntax error, a Keeper node + * that already exists. Those are verdicts on the statement and come back + * identically however many times it is retried. + */ + +/** Server error codes about timing, connections or resources. */ +const TRANSIENT_CODES = new Set([ + '159', // TIMEOUT_EXCEEDED + '160', // TOO_SLOW + '202', // TOO_MANY_SIMULTANEOUS_QUERIES + '203', // NO_FREE_CONNECTION + '209', // SOCKET_TIMEOUT + '210', // NETWORK_ERROR + '236', // ABORTED + '241', // MEMORY_LIMIT_EXCEEDED + '242', // TABLE_IS_READ_ONLY + '243', // NOT_ENOUGH_SPACE + '252', // TOO_MANY_PARTS + '290', // LIMIT_EXCEEDED + '319', // UNKNOWN_STATUS_OF_INSERT + '394', // QUERY_WAS_CANCELLED + '999', // KEEPER_EXCEPTION: a lost race or a lost session, both retried + '1002', // UNKNOWN_EXCEPTION: the server could not say, which is not a verdict on the row +]) + +/** Node socket errors the client surfaces as-is, with the errno name in `code`. */ +const TRANSIENT_SOCKET_CODES = new Set([ + 'ECONNRESET', + 'ECONNREFUSED', + 'ETIMEDOUT', + 'EPIPE', + 'EHOSTUNREACH', + 'ENETUNREACH', + 'EAI_AGAIN', +]) + +/** Message fragments of client-side failures that carry no code. */ +const TRANSIENT_MESSAGES = ['socket hang up', 'Timeout error', 'ECONNRESET', 'fetch failed', 'network error'] + +/** + * Whether `err` is a ClickHouse or socket error that a retry could fix. + * Anything without a recognisable code is reported as deterministic, which + * is the conservative answer: the engine's two-sightings rule covers the gap. + */ +export function isTransientClickHouseError(err: unknown): boolean { + if (typeof err !== 'object' || err === null) return false + const { code, message } = err as { code?: unknown; message?: unknown } + // Raised here, not by the server (`lock.ts`): a write lock that stayed + // taken for the whole wait, or one lost to a takeover while the batch + // stalled. Whoever holds it finishes or expires, and a retry reads afresh. + if (code === 'LOCK_TIMEOUT' || code === 'LOCK_LOST') return true + if (typeof code === 'string' && (TRANSIENT_CODES.has(code) || TRANSIENT_SOCKET_CODES.has(code))) return true + if (typeof message === 'string' && TRANSIENT_MESSAGES.some((m) => message.includes(m))) return true + return false +} + +/** Whether `err` is the server's refusal of a KeeperMap write that lost a race or found its key taken. */ +export function isKeeperConflict(err: unknown): boolean { + if (typeof err !== 'object' || err === null) return false + const { code, message } = err as { code?: unknown; message?: unknown } + return ( + code === '999' && + typeof message === 'string' && + (message.includes('Node exists') || message.includes('Bad version')) + ) +} diff --git a/packages/sync-clickhouse/src/db/lease.ts b/packages/sync-clickhouse/src/db/lease.ts new file mode 100644 index 0000000..79c7902 --- /dev/null +++ b/packages/sync-clickhouse/src/db/lease.ts @@ -0,0 +1,296 @@ +/** + * The single-writer lease and the cursors, as one KeeperMap row. + * + * KeeperMap is a key-value table held in ClickHouse Keeper: linearizable + * writes, and under `keeper_map_strict_mode` an `INSERT` of an existing key + * is refused and an `ALTER ... UPDATE` only lands if the row did not change + * between its read and its write. That is a compare-and-swap, and it is the + * only one ClickHouse offers; a MergeTree row has no such thing. So the + * lease and every cursor for an environment share one row in `_sync`, keyed + * by `(sync_id, env)`: + * + * k = sync_id ‹SEP› env + * holder, acquired_at, expires_at the lease; holder '' when free + * states = {"__tail__": "", "users": "...", ...} + * + * Taking the lease is an `UPDATE` whose `WHERE` names the row as it must be + * for the takeover to be allowed — ours, free, or expired by the server's + * clock (`now64`, never the client's) — so either it lands or nothing + * happens. A checkpoint is an `UPDATE` whose `WHERE` names the holder, the + * unexpired lease and the whole `states` value as it was read: if anyone + * moved a cursor or took the lease in between, the value differs and the + * write is refused by the database. Each write is followed by a read, since + * a mutation reports no row count; strict mode's own race verdict ("Bad + * version") is retried, because the re-read then judges the new state. + * + * Cursor state is stored as canonical JSON — keys sorted, recursively — so + * that equal cursors compare equal as text, as they do as jsonb. + */ +import type { Lease, LeaseResult } from '@workos/sync-engine' +import { isoOf, KEEPER_SETTINGS, type ChClient } from './client.js' +import { isKeeperConflict } from './errors.js' +import { quoteIdent } from '../schema/ddl.js' + +const SEP = '\u0001' +/** + * How often a read-then-swap is retried when the row changed in between. + * Contention is rare in use — one tail per environment, one backfill at a + * time — but each retry is cheap, and a lost race must never be reported as + * a failure to a caller who did nothing wrong. + */ +const ATTEMPTS = 16 + +/** A short, jittered pause between attempts, so contenders do not retry in lockstep. */ +const backoff = () => new Promise((r) => setTimeout(r, 5 + Math.random() * 20)) + +export interface SyncRow { + holder: string + acquired_at: string + expires_at: string + states: string + /** The server's clock at the moment of the read, for judging expiry. */ + now: string +} + +/** `value` as JSON with every object's keys sorted, so equal values serialise equally. */ +export function canonicalJson(value: unknown): string { + return JSON.stringify(sortKeys(value)) +} + +function sortKeys(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortKeys) + if (value !== null && typeof value === 'object') { + const out: Record = {} + for (const key of Object.keys(value as Record).sort()) { + const v = (value as Record)[key] + if (v !== undefined) out[key] = sortKeys(v) + } + return out + } + return value +} + +export const syncKey = (syncId: string, env: string) => `${syncId}${SEP}${env}` + +const table = (schema: string) => `${quoteIdent(schema)}.\`_sync\`` + +export async function readSync(db: ChClient, schema: string, syncId: string, env: string): Promise { + const rows = await db.query( + `SELECT holder, acquired_at, expires_at, states, now64(3, 'UTC') AS now FROM ${table(schema)} WHERE k = {k:String}`, + { k: syncKey(syncId, env) }, + ) + return rows[0] ?? null +} + +/** Insert the row for `(syncId, env)` if it does not exist. False when it already did. */ +async function insertSync(db: ChClient, schema: string, syncId: string, env: string, states: string): Promise { + try { + await db.command( + `INSERT INTO ${table(schema)} (k, holder, acquired_at, expires_at, states) + VALUES ({k:String}, '', toDateTime64(0, 3, 'UTC'), toDateTime64(0, 3, 'UTC'), {states:String})`, + { k: syncKey(syncId, env), states }, + KEEPER_SETTINGS, + ) + return true + } catch (err) { + if (isKeeperConflict(err)) return false + throw err + } +} + +const parseStates = (row: SyncRow | null): Record => + row ? (JSON.parse(row.states) as Record) : {} + +const isHeld = (row: SyncRow | null, holder: string) => + row !== null && row.holder === holder && row.expires_at > row.now + +export async function acquireLease( + db: ChClient, + schema: string, + syncId: string, + env: string, + holder: string, + ttlMs: number, +): Promise { + const k = syncKey(syncId, env) + const ttl = Math.max(0, Math.floor(ttlMs)) + for (let attempt = 0; attempt < ATTEMPTS; attempt++) { + const row = await readSync(db, schema, syncId, env) + if (!row) { + await insertSync(db, schema, syncId, env, '{}') + continue + } + if (attempt > 0) await backoff() + try { + // Free when it is ours, unheld, or expired by the server's clock; + // anything else leaves the row exactly as it was. + await db.command( + `ALTER TABLE ${table(schema)} UPDATE + acquired_at = if(holder = {holder:String}, acquired_at, now64(3, 'UTC')), + expires_at = now64(3, 'UTC') + toIntervalMillisecond({ttl:UInt64}), + holder = {holder:String} + WHERE k = {k:String} AND (holder = {holder:String} OR holder = '' OR expires_at <= now64(3, 'UTC'))`, + { k, holder, ttl }, + KEEPER_SETTINGS, + ) + } catch (err) { + // Somebody else wrote the row between the read and the write: look again. + if (!isKeeperConflict(err)) throw err + continue + } + const after = await readSync(db, schema, syncId, env) + if (after && after.holder === holder) return { acquired: true, holder, expiresAt: isoOf(after.expires_at) } + const current = after && after.holder !== '' && after.expires_at > after.now ? after : null + return { acquired: false, holder: current?.holder ?? null, expiresAt: current ? isoOf(current.expires_at) : null } + } + throw new Error(`could not take the lease for ${env}: the row kept changing under ${ATTEMPTS} attempts`) +} + +export async function releaseLease( + db: ChClient, + schema: string, + syncId: string, + env: string, + holder: string, +): Promise { + // Clearing the holder rather than expiring it keeps `status` honest: a + // released lease reads as free, not as a stale holder with a past date. + // The cursors share the row, so the row itself stays. + try { + await db.command( + `ALTER TABLE ${table(schema)} UPDATE holder = '', expires_at = toDateTime64(0, 3, 'UTC') + WHERE k = {k:String} AND holder = {holder:String}`, + { k: syncKey(syncId, env), holder }, + KEEPER_SETTINGS, + ) + } catch (err) { + // A race with a takeover: the lease is no longer ours to release. + if (!isKeeperConflict(err)) throw err + } +} + +export async function getLease(db: ChClient, schema: string, syncId: string, env: string): Promise { + const row = await readSync(db, schema, syncId, env) + if (!row || row.holder === '' || row.expires_at <= row.now) return null + return { holder: row.holder, expiresAt: isoOf(row.expires_at) } +} + +export async function getState( + db: ChClient, + schema: string, + syncId: string, + env: string, + stream: string, +): Promise { + const stored = parseStates(await readSync(db, schema, syncId, env))[stream] + return stored === undefined ? null : (JSON.parse(stored) as T) +} + +/** + * Replace `states` with `next` only if it still reads `previous`. `swapped` + * when it did; `unchanged` when the `WHERE` matched nothing; `conflict` + * when Keeper refused the write because the row's version moved between the + * mutation's read and its write — which any write to the row does, a lease + * renewal included, whether or not it touched `states`. + */ +async function swapStates( + db: ChClient, + schema: string, + syncId: string, + env: string, + previous: string, + next: string, + guard = '', + params: Record = {}, +): Promise<'swapped' | 'unchanged' | 'conflict'> { + try { + await db.command( + `ALTER TABLE ${table(schema)} UPDATE states = {next:String} + WHERE k = {k:String} AND states = {previous:String}${guard}`, + { ...params, k: syncKey(syncId, env), previous, next }, + KEEPER_SETTINGS, + ) + } catch (err) { + if (!isKeeperConflict(err)) throw err + return 'conflict' + } + const after = await readSync(db, schema, syncId, env) + return after !== null && after.states === next ? 'swapped' : 'unchanged' +} + +export async function setState( + db: ChClient, + schema: string, + syncId: string, + env: string, + stream: string, + state: unknown, +): Promise { + const value = canonicalJson(state) + for (let attempt = 0; attempt < ATTEMPTS; attempt++) { + const row = await readSync(db, schema, syncId, env) + if (!row) { + if (await insertSync(db, schema, syncId, env, JSON.stringify({ [stream]: value }))) return + continue + } + const states = parseStates(row) + if (states[stream] === value) return + const next = JSON.stringify({ ...states, [stream]: value }) + if ((await swapStates(db, schema, syncId, env, row.states, next)) === 'swapped') return + await backoff() + } + throw new Error(`could not save the ${stream} state for ${env}: the row kept changing under ${ATTEMPTS} attempts`) +} + +/** + * One read, one conditional write: the holder, the unexpired lease and the + * cursor as it was expected to be are all judged on the row as read, and + * the write names that exact row, so it lands only if nothing changed. + * + * A write Keeper refused over the row's version, or a `WHERE` that matched + * nothing, is not yet a refusal: the holder's own renewal moves the version + * without touching a cursor, and a backfill checkpoint for another stream + * changes `states` without touching this one. So the row is read again and + * judged again, and only a lease that is no longer held or a cursor that + * is no longer where it was expected refuses the checkpoint. The engine + * treats a refusal as "someone else moved the cursor" and replays the + * poll, which must not happen over a race nobody lost. + */ +export async function setStateIfHeld( + db: ChClient, + schema: string, + syncId: string, + env: string, + stream: string, + state: unknown, + holder: string, + expected: unknown, +): Promise { + const value = canonicalJson(state) + const wanted = expected == null ? undefined : canonicalJson(expected) + for (let attempt = 0; attempt < ATTEMPTS; attempt++) { + if (attempt > 0) await backoff() + const row = await readSync(db, schema, syncId, env) + if (!isHeld(row, holder)) return false + const states = parseStates(row) + const stored = states[stream] + // A write of ours on an earlier attempt may have landed while its + // verdict was lost to a race; the row then already says what we meant. + if (stored !== wanted) return attempt > 0 && stored === value + const next = JSON.stringify({ ...states, [stream]: value }) + // Expiry is judged again by the server, in the write itself: the read + // above may have waited, and a lease that ran out meanwhile must refuse. + const outcome = await swapStates( + db, + schema, + syncId, + env, + row!.states, + next, + ` AND holder = {holder:String} AND expires_at > now64(3, 'UTC')`, + { holder }, + ) + if (outcome === 'swapped') return true + } + throw new Error(`could not checkpoint ${stream} for ${env}: the row kept changing under ${ATTEMPTS} attempts`) +} diff --git a/packages/sync-clickhouse/src/db/lock.ts b/packages/sync-clickhouse/src/db/lock.ts new file mode 100644 index 0000000..e4831dc --- /dev/null +++ b/packages/sync-clickhouse/src/db/lock.ts @@ -0,0 +1,160 @@ +/** + * The per-stream write lock. + * + * Every mirror write is read-decide-write: a batch reads the current state + * of its rows, judges each in Node — the staleness gate, the shallow merge, + * which tombstone to carry — and inserts the next versions. Postgres and + * MySQL serialise that with row locks inside a transaction. ClickHouse has + * neither, and two writers can meet on one key: the tail and a backfill run + * side by side, and a poll can finish its writes after its lease has passed + * to another process. Each would judge against the state before the other's + * insert landed: a merge could drop the fields the other wrote, a tombstone + * could lose a version tie to a live row it never saw, an append-only event + * could be inserted twice. + * + * So each batch holds a lock for its table and environment while it reads + * and writes. The lock is a KeeperMap row: a strict-mode `INSERT` takes it + * (refused when the key is there), a `DELETE` of the holder's own row + * releases it, and a row a crashed process left behind is taken over, once + * it has expired on the server's clock, with the conditional `UPDATE` the + * lease uses — so a live holder is never displaced. Every acquisition has + * its own token, so two batches in one process contend like two processes + * do. A writer that cannot get the lock within `waitMs` fails with + * `LOCK_TIMEOUT`, and a batch that stalled past the lock's TTL, lost it, + * and asks `held()` before writing fails with `LOCK_LOST`; the error + * classifier reports both as transient, so the engine retries the poll + * rather than quarantining the event, and the retry reads afresh. + */ +import { randomUUID } from 'node:crypto' +import { KEEPER_SETTINGS, type ChClient } from './client.js' +import { isKeeperConflict } from './errors.js' +import { quoteIdent } from '../schema/ddl.js' + +/** How long a lock outlives the process that took it. A batch holds it for well under a second. */ +export const LOCK_TTL_MS = 30_000 +/** How long a writer waits for a lock before reporting a transient failure. */ +export const LOCK_WAIT_MS = 60_000 + +const SEP = '\u0001' +/** Strict mode named outright: a client built without it would overwrite the row instead of being refused. */ +const SETTINGS = { ...KEEPER_SETTINGS, keeper_map_strict_mode: 1 } as const + +export interface LockOptions { + schema: string + /** Prefix of every token this writer takes; identifies the process in `_locks`. */ + holder: string + ttlMs?: number + waitMs?: number +} + +const table = (schema: string) => `${quoteIdent(schema)}.\`_locks\`` + +/** The lock for one table in one environment. */ +export const lockKey = (tableName: string, env: string) => `${tableName}${SEP}${env}` + +const backoff = () => new Promise((r) => setTimeout(r, 10 + Math.random() * 40)) + +/** Take the lock, waiting for whoever holds it. Resolves to the token that releases it. */ +export async function acquireStreamLock(db: ChClient, opts: LockOptions, key: string): Promise { + const token = `${opts.holder}:${randomUUID()}` + const ttl = Math.max(1, Math.floor(opts.ttlMs ?? LOCK_TTL_MS)) + const waitMs = opts.waitMs ?? LOCK_WAIT_MS + const deadline = Date.now() + waitMs + const params = { k: key, holder: token, ttl } + for (let attempt = 0; ; attempt++) { + if (attempt > 0) { + if (Date.now() >= deadline) { + throw Object.assign( + new Error(`could not take the write lock for ${key.replace(SEP, ' in ')} within ${waitMs} ms`), + { code: 'LOCK_TIMEOUT' }, + ) + } + await backoff() + } + try { + await db.command( + `INSERT INTO ${table(opts.schema)} (k, holder, expires_at) + VALUES ({k:String}, {holder:String}, now64(3, 'UTC') + toIntervalMillisecond({ttl:UInt64}))`, + params, + SETTINGS, + ) + return token + } catch (err) { + if (!isKeeperConflict(err)) throw err + } + // Taken. Ours only if whoever holds it is gone: the takeover names an + // expired row, and a row that is not expired is left exactly as it was. + try { + await db.command( + `ALTER TABLE ${table(opts.schema)} UPDATE + holder = {holder:String}, + expires_at = now64(3, 'UTC') + toIntervalMillisecond({ttl:UInt64}) + WHERE k = {k:String} AND expires_at <= now64(3, 'UTC')`, + params, + SETTINGS, + ) + } catch (err) { + if (!isKeeperConflict(err)) throw err + continue + } + const rows = await db.query<{ holder: string }>(`SELECT holder FROM ${table(opts.schema)} WHERE k = {k:String}`, { + k: key, + }) + if (rows[0]?.holder === token) return token + } +} + +/** Give the lock back. A lock that was taken over meanwhile is not ours to release and is left alone. */ +export async function releaseStreamLock(db: ChClient, opts: LockOptions, key: string, token: string): Promise { + for (let attempt = 0; attempt < 8; attempt++) { + try { + await db.command( + `ALTER TABLE ${table(opts.schema)} DELETE WHERE k = {k:String} AND holder = {holder:String}`, + { k: key, holder: token }, + SETTINGS, + ) + return + } catch (err) { + if (!isKeeperConflict(err)) throw err + } + const rows = await db.query<{ holder: string }>(`SELECT holder FROM ${table(opts.schema)} WHERE k = {k:String}`, { + k: key, + }) + if (rows[0]?.holder !== token) return + await backoff() + } + // Still ours after every attempt: it expires on its own, and says so in the log the caller keeps. +} + +/** Throws `LOCK_LOST` unless the lock is still the caller's. */ +export type HeldCheck = () => Promise + +/** + * Run `fn` holding the lock for `key`, releasing it however `fn` ends. `fn` + * is handed a check to call right before it writes: a batch that stalled + * past the TTL has lost the lock to a writer whose rows it never read. + */ +export async function withStreamLock( + db: ChClient, + opts: LockOptions, + key: string, + fn: (held: HeldCheck) => Promise, +): Promise { + const token = await acquireStreamLock(db, opts, key) + const held: HeldCheck = async () => { + const rows = await db.query<{ holder: string }>(`SELECT holder FROM ${table(opts.schema)} WHERE k = {k:String}`, { + k: key, + }) + if (rows[0]?.holder !== token) { + throw Object.assign( + new Error(`the write lock for ${key.replace(SEP, ' in ')} was taken over: this batch stalled past its TTL`), + { code: 'LOCK_LOST' }, + ) + } + } + try { + return await fn(held) + } finally { + await releaseStreamLock(db, opts, key, token) + } +} diff --git a/packages/sync-clickhouse/src/db/reconcile.ts b/packages/sync-clickhouse/src/db/reconcile.ts new file mode 100644 index 0000000..df7c9df --- /dev/null +++ b/packages/sync-clickhouse/src/db/reconcile.ts @@ -0,0 +1,99 @@ +/** + * The reconcile set difference, computed in ClickHouse. + * + * The live keys are copied into one staging table by the server, in a + * single `INSERT ... SELECT`, before the first page is requested; the + * remote pages are written into another as they arrive; then three queries + * produce the answer: a count for `matched`, an anti-join from the local + * side for `localOnly`, an anti-join from the remote side for `remoteOnly`. + * Node holds one page of remote rows at a time, plus the drift; the + * Postgres destination explains why that bound matters. + * + * The staging tables are ordinary `MergeTree` tables with a random suffix, + * not temporary ones: a temporary table lives in one HTTP session on one + * replica, and on a replicated deployment the next request may land + * elsewhere. They are dropped when the diff ends, however it ends. + */ +import { encodeKey, type IdDiff, type TableDef } from '@workos/sync-engine' +import { randomBytes } from 'node:crypto' +import type { ChClient } from './client.js' +import { chunk, keyExpression, keyValues, tableRef, KEY_SEPARATOR, type WriteOptions } from './upsert.js' +import { quoteIdent, ENV } from '../schema/ddl.js' + +export interface ChDiff extends IdDiff { + /** Remote rows with no value at the key path. Counted rather than thrown. */ + skipped: number +} + +const DEFAULT_BATCH_SIZE = 500 + +export async function diffIdsInClickHouse( + db: ChClient, + opts: WriteOptions, + stream: TableDef, + env: string, + remote: AsyncIterable[]>, +): Promise { + const token = randomBytes(6).toString('hex') + const s = quoteIdent(opts.schema) + const LOCAL = `${s}.\`_reconcile_local_${token}\`` + const STAGING = `${s}.\`_reconcile_remote_${token}\`` + const decode = (k: string) => encodeKey(stream.key.length === 1 ? [k] : k.split(KEY_SEPARATOR)) + + const drop = async () => { + await db.command(`DROP TABLE IF EXISTS ${LOCAL}`) + await db.command(`DROP TABLE IF EXISTS ${STAGING}`) + } + try { + await db.command(`CREATE TABLE ${LOCAL} (k String) ENGINE = MergeTree ORDER BY k`) + await db.command(`CREATE TABLE ${STAGING} (k String, seq UInt64, raw String) ENGINE = MergeTree ORDER BY (k, seq)`) + + // The live keys are frozen before the first page is requested: a row the + // tail inserts after the listing has passed that object's position is + // simply not a candidate, so `--repair` can never tombstone an object + // that exists upstream. One statement, one snapshot. + const live = stream.mode === 'append' ? '' : ` AND \`_deleted_at\` IS NULL` + const final = stream.mode === 'append' ? '' : ' FINAL' + await db.command( + `INSERT INTO ${LOCAL} (k) SELECT ${keyExpression(stream)} FROM ${tableRef(opts.schema, stream)}${final} + WHERE ${quoteIdent(ENV)} = {env:String}${live}`, + { env }, + ) + + let skipped = 0 + let seq = 0 + for await (const page of remote) { + const keyed: { k: string; seq: number; raw: string }[] = [] + for (const data of page) { + const id = keyValues(stream, data) + if (!id) { + skipped++ + continue + } + keyed.push({ k: id.join(KEY_SEPARATOR), seq: seq++, raw: JSON.stringify(data) }) + } + for (const batch of chunk(keyed, opts.batchSize ?? DEFAULT_BATCH_SIZE)) await db.insert(STAGING, batch) + } + + // An id the API returns twice (a page boundary moving under a concurrent + // write) keeps its first copy, by `seq`. Nothing is lost that the next + // reconcile would not find again. + const [matched, localOnly, remoteOnly] = await Promise.all([ + db.query<{ n: string }>( + `SELECT toString(count(DISTINCT k)) AS n FROM ${STAGING} WHERE k IN (SELECT k FROM ${LOCAL})`, + ), + db.query<{ k: string }>(`SELECT k FROM ${LOCAL} WHERE k NOT IN (SELECT k FROM ${STAGING}) ORDER BY k`), + db.query<{ raw: string }>( + `SELECT argMin(raw, seq) AS raw FROM ${STAGING} WHERE k NOT IN (SELECT k FROM ${LOCAL}) GROUP BY k ORDER BY k`, + ), + ]) + return { + matched: Number(matched[0]?.n ?? 0), + localOnly: localOnly.map((r) => decode(r.k)), + remoteOnly: remoteOnly.map((r) => JSON.parse(r.raw) as Record), + skipped, + } + } finally { + await drop() + } +} diff --git a/packages/sync-clickhouse/src/db/rejected.ts b/packages/sync-clickhouse/src/db/rejected.ts new file mode 100644 index 0000000..258944b --- /dev/null +++ b/packages/sync-clickhouse/src/db/rejected.ts @@ -0,0 +1,116 @@ +/** + * The quarantine store: `_rejected`, one row per event the tail set aside. + * + * Keyed by `(sync_id, env, event_id)` like the rest of the bookkeeping, so + * several syncs sharing a database never see each other's rejects. The whole + * event is stored verbatim in `_raw`, which is what makes a replay possible + * and is also why `redact` has to reach in here as well as into `events`. + * + * The table is a `ReplacingMergeTree` versioned by `rejected_at`, so a + * rejection written again is a new version that wins, and reads use + * `FINAL`. Forgetting an event is a lightweight `DELETE`, which the + * client's `mutations_sync` makes visible before the call returns. + */ +import type { RejectedEvent, WorkOSEvent } from '@workos/sync-engine' +import { isoOf, toCh, type ChClient } from './client.js' +import { quoteIdent } from '../schema/ddl.js' + +export interface RejectedOptions { + schema: string + syncId: string +} + +const table = (schema: string) => `${quoteIdent(schema)}.\`_rejected\`` + +/** The event's `created_at` as the server stores it, or null: a quarantine write must never fail on the data it is quarantining. */ +function createdAt(value: unknown): string | null { + if (typeof value !== 'string') return null + const d = new Date(value) + return Number.isNaN(d.getTime()) ? null : toCh(d) +} + +export async function rejectEvents( + db: ChClient, + opts: RejectedOptions, + env: string, + rows: { event: WorkOSEvent; error: string }[], +): Promise { + if (rows.length === 0) return + // A replay that fails again lands here a second time, as a newer version: + // `rejected` then reports what is wrong now, not what was. + await db.insert( + table(opts.schema), + rows.map(({ event, error }) => { + const objectId = (event.data as Record | undefined)?.['id'] + return { + sync_id: opts.syncId, + env, + event_id: event.id, + event_type: event.event, + object_id: typeof objectId === 'string' ? objectId : null, + created_at: createdAt(event.created_at), + error, + _raw: JSON.stringify(event), + } + }), + ) +} + +export async function listRejected( + db: ChClient, + opts: RejectedOptions, + env: string, + ids?: string[], +): Promise { + if (ids && ids.length === 0) return [] + const rows = await db.query<{ + event_id: string + event_type: string + object_id: string | null + rejected_at: string + error: string + raw: string + }>( + `SELECT event_id, event_type, object_id, rejected_at, error, \`_raw\` AS raw + FROM ${table(opts.schema)} FINAL + WHERE sync_id = {syncId:String} AND env = {env:String}${ids ? ' AND event_id IN ({ids:Array(String)})' : ''} + ORDER BY event_id`, + { syncId: opts.syncId, env, ids: ids ?? [] }, + ) + return rows.map((row) => { + const raw = JSON.parse(row.raw) as WorkOSEvent + return { + eventId: row.event_id, + eventType: row.event_type, + objectId: row.object_id, + // The event's own timestamp, as it arrived; the column is only for SQL. + createdAt: raw.created_at, + rejectedAt: isoOf(row.rejected_at), + error: row.error, + raw, + } + }) +} + +/** How many events are quarantined, without reading their payloads. */ +export async function countRejected(db: ChClient, opts: RejectedOptions, env: string): Promise { + const rows = await db.query<{ n: string }>( + `SELECT toString(count()) AS n FROM ${table(opts.schema)} FINAL WHERE sync_id = {syncId:String} AND env = {env:String}`, + { syncId: opts.syncId, env }, + ) + return Number(rows[0]?.n ?? 0) +} + +export async function clearRejected(db: ChClient, opts: RejectedOptions, env: string, ids: string[]): Promise { + if (ids.length === 0) return 0 + const params = { syncId: opts.syncId, env, ids } + const where = `sync_id = {syncId:String} AND env = {env:String} AND event_id IN ({ids:Array(String)})` + // A delete reports no count, so count first; the quarantine has one writer. + const rows = await db.query<{ n: string }>( + `SELECT toString(count(DISTINCT event_id)) AS n FROM ${table(opts.schema)} FINAL WHERE ${where}`, + params, + ) + const n = Number(rows[0]?.n ?? 0) + if (n > 0) await db.command(`DELETE FROM ${table(opts.schema)} WHERE ${where}`, params) + return n +} diff --git a/packages/sync-clickhouse/src/db/upsert.ts b/packages/sync-clickhouse/src/db/upsert.ts new file mode 100644 index 0000000..abd0617 --- /dev/null +++ b/packages/sync-clickhouse/src/db/upsert.ts @@ -0,0 +1,510 @@ +/** + * Write path. + * + * The rules are the Postgres destination's, and for the same reasons (see + * `db/upsert.ts` there): the staleness gate is `>=` and never `>`, event + * writes merge where REST writes replace, deletion is terminal except for + * memberships, which relink and unlink only when the write is at least as + * new as the row. + * + * ClickHouse has no row update, so a write is an insert of the row's next + * state into a `ReplacingMergeTree` versioned by `_updated_at`. Each batch + * reads the current state of its rows (`FINAL`), decides every row in Node + * — the gate, the shallow merge, whether anything changed, which tombstone + * to carry forward — and inserts only the rows that passed. The table then + * keeps, for each key, the version with the newest `_updated_at`, and the + * last inserted among equals; a refused row, never inserted, changes + * nothing. Reading and writing are two statements, so each batch holds the + * stream's write lock (`lock.ts`) across both: two writers on one key — the + * tail and a backfill, or a poll outliving its lease — then see each + * other's rows instead of each judging against the state before the other + * wrote. For an append-only stream, under the same lock, the batch reads + * which ids are already there and inserts the rest, which is what makes + * re-delivery a no-op; the table, a `ReplacingMergeTree` keyed by the id, + * collapses a copy that got past that anyway. + */ +import { emptyStats, type Row, type TableDef, type WriteMode, type WriteStats } from '@workos/sync-engine' +import { hostname } from 'node:os' +import { fromCh, toCh, type ChClient } from './client.js' +import { lockKey, withStreamLock, type HeldCheck, type LockOptions } from './lock.js' +import { quoteIdent, safeIdentifier, ENV, RAW } from '../schema/ddl.js' + +/** Joins the components of a composite key, on both sides of every lookup. */ +export const KEY_SEPARATOR = '\u0001' + +export function readPath(data: Record, path: string | string[]): unknown { + const parts = Array.isArray(path) ? path : [path] + let cur: unknown = data + for (const p of parts) { + if (cur == null || typeof cur !== 'object') return undefined + cur = (cur as Record)[p] + } + return cur +} + +/** Key values for a row, in `stream.key` order. Undefined if any part is missing. */ +export function keyValues(stream: TableDef, data: Record): string[] | undefined { + const out: string[] = [] + for (const k of stream.key) { + const col = stream.columns.find((c) => c.name === k) + const v = readPath(data, col ? col.path : k) + if (v == null || v === '') return undefined + out.push(String(v)) + } + return out +} + +/** + * Collapse rows that share a primary key, keeping the newest. It serves the + * delete, relink and unlink paths, whose rows are whole; upserts use + * `wavesByKey`. + */ +export function dedupeByKey(stream: TableDef, rows: Row[]): { rows: Row[]; dropped: number } { + const byKey = new Map() + let dropped = 0 + for (const row of rows) { + const key = keyValues(stream, row.data) + if (!key) { + dropped++ + continue + } + const k = key.join(KEY_SEPARATOR) + const at = micros(row.updatedAt) + const existing = byKey.get(k) + if (!existing || !isBefore(row.updatedAt, at, existing.row.updatedAt, existing.at)) byKey.set(k, { row, at }) + } + return { rows: [...byKey.values()].map((e) => e.row), dropped } +} + +/** + * The instant `iso` names, in microseconds since the epoch, or NaN. + * `Date.parse` reads three fractional digits, and so does the column + * (`DateTime64(3)`), so two rows within a millisecond tie at the staleness + * gate and the last one written wins; reading the rest of the fraction + * here is what makes that last one the newer of the two. + */ +function micros(iso: string): number { + const frac = /\.(\d+)/.exec(iso)?.[1] ?? '' + const ms = Date.parse(frac.length > 3 ? iso.replace(`.${frac}`, `.${frac.slice(0, 3)}`) : iso) + if (Number.isNaN(ms)) return ms + return ms * 1000 + Number(frac.slice(3, 6).padEnd(3, '0')) +} + +/** + * Whether one timestamp is strictly earlier than another, by the instant + * each names: `01:00:00+01:00` sorts after `00:30:00Z` as text and is half an + * hour earlier as a time. A value that is not a date is ordered as text; + * `parseUpdatedAt` refuses it on the write either way. + */ +function isBefore(a: string, atA: number, b: string, atB: number): boolean { + if (Number.isNaN(atA) || Number.isNaN(atB)) return a < b + return atA < atB +} + +/** + * Split a batch into waves in which no key appears twice, keeping order: the + * n-th row for a key goes in wave n. Written one after another, the waves + * are exactly the rows written one at a time — each row meets the staleness + * gate against what the row before it left stored, and a merge overlays it + * on that. See the Postgres destination for why keeping only the newest row + * would be wrong for a merge. + */ +export function wavesByKey(stream: TableDef, rows: Row[]): { waves: Row[][]; dropped: number } { + const waves: Row[][] = [] + const seen = new Map() + let dropped = 0 + for (const row of rows) { + const key = keyValues(stream, row.data) + if (!key) { + dropped++ + continue + } + const k = key.join(KEY_SEPARATOR) + const n = seen.get(k) ?? 0 + seen.set(k, n + 1) + const wave = waves[n] ?? (waves[n] = []) + wave.push(row) + } + return { waves, dropped } +} + +export const chunk = (items: T[], size: number): T[][] => { + const out: T[][] = [] + for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size)) + return out +} + +/** + * `Row.updatedAt` as a `Date`. Postgres casts the string in SQL and a bad one + * fails the statement; here the same verdict is reached before the + * statement, with a code the error classifier reports as deterministic, so + * the event is quarantined rather than retried forever. + */ +export function parseUpdatedAt(value: string): Date { + const d = new Date(value) + if (typeof value !== 'string' || Number.isNaN(d.getTime())) { + throw Object.assign(new Error(`invalid updatedAt timestamp: ${JSON.stringify(value)}`), { + code: 'INVALID_TIMESTAMP', + }) + } + return d +} + +/** Structural equality with key order ignored, which is what `is distinct from` means for jsonb. */ +export function jsonEqual(a: unknown, b: unknown): boolean { + if (a === b) return true + if (typeof a !== typeof b || a === null || b === null || typeof a !== 'object') return false + if (Array.isArray(a) !== Array.isArray(b)) return false + if (Array.isArray(a)) { + const bb = b as unknown[] + return a.length === bb.length && a.every((v, i) => jsonEqual(v, bb[i])) + } + const ao = a as Record + const bo = b as Record + const keys = Object.keys(ao) + if (keys.length !== Object.keys(bo).length) return false + return keys.every((k) => Object.hasOwn(bo, k) && jsonEqual(ao[k], bo[k])) +} + +/** Shallow merge: the jsonb `||` of the Postgres destination. */ +const merge = (stored: Record, incoming: Record): Record => ({ + ...stored, + ...incoming, +}) + +const later = (a: Date | null, b: Date): Date => (a !== null && a.getTime() > b.getTime() ? a : b) + +export interface WriteOptions { + schema: string + batchSize?: number + /** Names this writer in `_locks`. One per process by default. */ + lockHolder?: string + lockTtlMs?: number + lockWaitMs?: number +} + +const DEFAULT_BATCH_SIZE = 500 +const PROCESS_ID = `${hostname()}:${process.pid}` + +const lockOptions = (opts: WriteOptions): LockOptions => ({ + schema: opts.schema, + holder: opts.lockHolder ?? PROCESS_ID, + ttlMs: opts.lockTtlMs, + waitMs: opts.lockWaitMs, +}) + +/** Run `fn` holding the write lock for `stream` in `env`. */ +const locked = ( + db: ChClient, + opts: WriteOptions, + stream: TableDef, + env: string, + fn: (held: HeldCheck) => Promise, +) => withStreamLock(db, lockOptions(opts), lockKey(safeIdentifier(stream.name), env), fn) +export const tableRef = (schema: string, stream: TableDef) => + `${quoteIdent(schema)}.${quoteIdent(safeIdentifier(stream.name))}` + +/** The whole key as one SQL expression: the column, or the components joined by the separator. */ +export function keyExpression(stream: TableDef): string { + const cols = stream.key.map((k) => quoteIdent(safeIdentifier(k))) + if (cols.length === 1) return cols[0]! + const parts: string[] = [] + for (const [i, col] of cols.entries()) { + if (i > 0) parts.push('char(1)') + parts.push(col) + } + return `concat(${parts.join(', ')})` +} + +interface StoredRow { + raw: Record + updatedAt: Date + deletedAt: Date | null +} + +interface Pending { + key: string[] + raw: Record + updatedAt: Date + deletedAt: Date | null +} + +/** The current state of the rows for `keys`, by joined key; keys with no row are absent. */ +async function readRows( + db: ChClient, + opts: WriteOptions, + stream: TableDef, + env: string, + keys: string[][], +): Promise> { + const rows = await db.query<{ k: string; raw: string; updated_at: string; deleted_at: string | null }>( + `SELECT ${keyExpression(stream)} AS k, ${quoteIdent(RAW)} AS raw, \`_updated_at\` AS updated_at, \`_deleted_at\` AS deleted_at + FROM ${tableRef(opts.schema, stream)} FINAL + WHERE ${quoteIdent(ENV)} = {env:String} AND ${keyExpression(stream)} IN ({keys:Array(String)})`, + { env, keys: keys.map((k) => k.join(KEY_SEPARATOR)) }, + ) + const out = new Map() + for (const row of rows) { + out.set(row.k, { + raw: JSON.parse(row.raw) as Record, + updatedAt: fromCh(row.updated_at), + deletedAt: row.deleted_at ? fromCh(row.deleted_at) : null, + }) + } + return out +} + +/** Which of `keys` already have a row, for append-only tables. */ +async function existingKeys( + db: ChClient, + opts: WriteOptions, + stream: TableDef, + env: string, + keys: string[][], +): Promise> { + const rows = await db.query<{ k: string }>( + `SELECT DISTINCT ${keyExpression(stream)} AS k FROM ${tableRef(opts.schema, stream)} + WHERE ${quoteIdent(ENV)} = {env:String} AND ${keyExpression(stream)} IN ({keys:Array(String)})`, + { env, keys: keys.map((k) => k.join(KEY_SEPARATOR)) }, + ) + return new Set(rows.map((r) => r.k)) +} + +/** Insert the decided rows as their next version. `_synced_at` is the server's clock. */ +async function writeRows(db: ChClient, opts: WriteOptions, stream: TableDef, env: string, rows: Pending[]) { + if (rows.length === 0) return + const hasDeleted = stream.mode !== 'append' + await db.insert( + tableRef(opts.schema, stream), + rows.map((r) => ({ + [ENV]: env, + [RAW]: JSON.stringify(r.raw), + _updated_at: toCh(r.updatedAt), + ...(hasDeleted ? { _deleted_at: r.deletedAt ? toCh(r.deletedAt) : null } : {}), + })), + ) +} + +/** One batch of keyed rows with their parsed timestamps, ready to be decided. */ +interface Keyed { + key: string[] + data: Record + updatedAt: Date +} + +const keyed = (stream: TableDef, rows: Row[]): Keyed[] => + rows.map((r) => ({ key: keyValues(stream, r.data)!, data: r.data, updatedAt: parseUpdatedAt(r.updatedAt) })) + +/** + * Read, decide, write: the shape of every mirror write, under the stream's + * lock. `decide` is handed each row with what is stored for it and returns + * what to write, or null to leave the row alone. + */ +async function applyBatch( + db: ChClient, + opts: WriteOptions, + stream: TableDef, + env: string, + batch: Keyed[], + decide: (row: Keyed, stored: StoredRow | undefined) => Pending | null, +): Promise { + await locked(db, opts, stream, env, async (held) => { + const stored = await readRows( + db, + opts, + stream, + env, + batch.map((r) => r.key), + ) + const pending: Pending[] = [] + for (const row of batch) { + const out = decide(row, stored.get(row.key.join(KEY_SEPARATOR))) + if (out) pending.push(out) + } + // Decided against rows read under the lock; written only if it is still ours. + if (pending.length > 0) await held() + await writeRows(db, opts, stream, env, pending) + }) +} + +export async function upsertRows( + db: ChClient, + opts: WriteOptions, + stream: TableDef, + env: string, + rows: Row[], + mode: WriteMode, +): Promise { + const stats = emptyStats() + if (rows.length === 0) return stats + const { waves, dropped } = wavesByKey(stream, rows) + // Rows we could not key (a payload with no id and no slug) are counted as + // skipped rather than silently vanishing. + stats.skipped += dropped + for (const wave of waves) { + for (const batch of chunk(keyed(stream, wave), opts.batchSize ?? DEFAULT_BATCH_SIZE)) { + if (stream.mode === 'append') { + // Immutable log. Re-delivery is a no-op, which is the whole + // idempotency story; the lock is what makes it hold for two polls + // delivering the same page at once. + await locked(db, opts, stream, env, async (held) => { + const present = await existingKeys( + db, + opts, + stream, + env, + batch.map((r) => r.key), + ) + const fresh = batch.filter((r) => !present.has(r.key.join(KEY_SEPARATOR))) + if (fresh.length > 0) await held() + await writeRows( + db, + opts, + stream, + env, + fresh.map((r) => ({ key: r.key, raw: r.data, updatedAt: r.updatedAt, deletedAt: null })), + ) + stats.created += fresh.length + stats.skipped += batch.length - fresh.length + }) + continue + } + await applyBatch(db, opts, stream, env, batch, (row, stored) => { + if (!stored) { + stats.created++ + return { key: row.key, raw: row.data, updatedAt: row.updatedAt, deletedAt: null } + } + if (row.updatedAt.getTime() < stored.updatedAt.getTime()) { + stats.skipped++ + return null + } + const raw = mode === 'merge' ? merge(stored.raw, row.data) : row.data + // An equal timestamp with nothing new is a re-delivery: no new version, + // so `_synced_at` stays and lag is not measured against a no-op. + if (row.updatedAt.getTime() === stored.updatedAt.getTime() && jsonEqual(stored.raw, raw)) { + stats.skipped++ + return null + } + stats.updated++ + // The tombstone is carried forward: a late update never resurrects a row. + return { key: row.key, raw, updatedAt: row.updatedAt, deletedAt: stored.deletedAt } + }) + } + } + return stats +} + +export async function deleteRows( + db: ChClient, + opts: WriteOptions, + stream: TableDef, + env: string, + rows: Row[], +): Promise { + const stats = emptyStats() + if (rows.length === 0) return stats + const { rows: deduped, dropped } = dedupeByKey(stream, rows) + stats.skipped += dropped + for (const batch of chunk(keyed(stream, deduped), opts.batchSize ?? DEFAULT_BATCH_SIZE)) { + await applyBatch(db, opts, stream, env, batch, (row, stored) => { + stats.deleted++ + if (!stored) return { key: row.key, raw: row.data, updatedAt: row.updatedAt, deletedAt: row.updatedAt } + // The *latest* removal is kept, not the first: a membership removed, + // re-added and removed again must carry the last removal, because that + // is what `relinkRows` compares a re-add against. The version moves to + // the deletion time, where Postgres leaves `_updated_at` alone: here + // the tombstone has to be the newest version of the row, and the + // deletion came after anything a listing could still show of it, so + // it outranks any older snapshot on the version itself — not on which + // insert happened to land last. + return { + key: row.key, + raw: merge(stored.raw, row.data), + updatedAt: later(stored.updatedAt, row.updatedAt), + deletedAt: later(stored.deletedAt, row.updatedAt), + } + }) + } + return stats +} + +/** Restore a tombstoned membership row (a re-add after a removal). */ +export async function relinkRows( + db: ChClient, + opts: WriteOptions, + stream: TableDef, + env: string, + rows: Row[], +): Promise { + const stats = emptyStats() + if (rows.length === 0) return stats + const { rows: deduped, dropped } = dedupeByKey(stream, rows) + stats.skipped += dropped + for (const batch of chunk(keyed(stream, deduped), opts.batchSize ?? DEFAULT_BATCH_SIZE)) { + await applyBatch(db, opts, stream, env, batch, (row, stored) => { + if (!stored) { + stats.created++ + return { key: row.key, raw: row.data, updatedAt: row.updatedAt, deletedAt: null } + } + // A membership is a fact that can legitimately come back: added, + // removed, added again. Unlike object deletes, a re-link clears the + // tombstone — but only a re-link at least as new as the removal. + // `_updated_at` alone is not enough: a tombstone written by `deleteRows` + // (a reconcile repair) leaves it untouched, so a link replayed from + // quarantine after that removal would pass the staleness gate and + // resurrect the membership. + const t = row.updatedAt.getTime() + if (t < stored.updatedAt.getTime() || (stored.deletedAt !== null && t < stored.deletedAt.getTime())) { + stats.skipped++ + return null + } + stats.updated++ + return { key: row.key, raw: merge(stored.raw, row.data), updatedAt: row.updatedAt, deletedAt: null } + }) + } + return stats +} + +/** + * Tombstone membership rows (a removal). The counterpart of `relinkRows`, + * and staleness-gated like it, which is what sets it apart from `deleteRows`: + * a membership can come back, so a removal is a fact with a time, not a + * terminal state, and one replayed from quarantine after a newer re-add has + * landed must not hide a membership that is active now. + */ +export async function unlinkRows( + db: ChClient, + opts: WriteOptions, + stream: TableDef, + env: string, + rows: Row[], +): Promise { + const stats = emptyStats() + if (rows.length === 0) return stats + const { rows: deduped, dropped } = dedupeByKey(stream, rows) + stats.skipped += dropped + for (const batch of chunk(keyed(stream, deduped), opts.batchSize ?? DEFAULT_BATCH_SIZE)) { + await applyBatch(db, opts, stream, env, batch, (row, stored) => { + if (!stored) { + stats.deleted++ + return { key: row.key, raw: row.data, updatedAt: row.updatedAt, deletedAt: row.updatedAt } + } + if (row.updatedAt.getTime() < stored.updatedAt.getTime()) { + stats.skipped++ + return null + } + // `_updated_at` moves to the removal time, so the next re-add is gated + // against the removal by the same comparison every membership write + // uses. `_deleted_at` keeps the latest removal for the same reason + // `deleteRows` does. + stats.deleted++ + return { + key: row.key, + raw: merge(stored.raw, row.data), + updatedAt: row.updatedAt, + deletedAt: later(stored.deletedAt, row.updatedAt), + } + }) + } + return stats +} diff --git a/packages/sync-clickhouse/src/destination.ts b/packages/sync-clickhouse/src/destination.ts new file mode 100644 index 0000000..78b2a3e --- /dev/null +++ b/packages/sync-clickhouse/src/destination.ts @@ -0,0 +1,311 @@ +/** + * The ClickHouse destination: everything `@workos/sync-engine` needs to know + * about a database, implemented over the official HTTP client. + */ +import type { + Destination, + FreshnessRow, + IdDiff, + Lease, + LeaseResult, + Logger, + RejectedEvent, + Row, + TableDef, + WorkOSEvent, + WriteMode, + WriteStats, +} from '@workos/sync-engine' +import { encodeKey } from '@workos/sync-engine' +import { connect, isoOf, type ChClient } from './db/client.js' +import { upsertRows, deleteRows, relinkRows, unlinkRows, keyExpression, tableRef, KEY_SEPARATOR } from './db/upsert.js' +import { diffIdsInClickHouse } from './db/reconcile.js' +import { acquireLease, releaseLease, getLease, getState, setState, setStateIfHeld } from './db/lease.js' +import { rejectEvents, listRejected, countRejected, clearRejected } from './db/rejected.js' +import { isTransientClickHouseError } from './db/errors.js' +import { migrate, type MigrateResult } from './schema/migrate.js' +import { jsonPathArgs, quoteIdent, safeIdentifier } from './schema/ddl.js' + +const ident = (name: string) => quoteIdent(safeIdentifier(name)) + +/** + * The columns that name a user: a membership's or sign-in's `user_id`, and + * an invitation's inviter and the user who accepted it. `redact` erases a + * row that names the subject in any of them, and an archived or quarantined + * event whose payload does. + */ +const USER_REFERENCES = ['user_id', 'inviter_user_id', 'accepted_user_id'] + +export interface ClickHouseDestinationOptions { + /** + * The HTTP interface, `https://user:password@host:8443`. Alternatively + * pass a pre-built `db`. A path in the URL is ignored: every statement + * qualifies its tables with `schema`. + */ + databaseUrl?: string + /** + * PEM-encoded CA certificate(s) the server's certificate must chain to, + * for a server whose certificate a public CA did not issue. Only with an + * `https://` `databaseUrl`; a pre-built `db` brings its own TLS settings. + * `WORKOS_SYNC_CLICKHOUSE_TLS_CA` on the CLI. + */ + sslCa?: string + /** A pre-built client, e.g. `fromClient(client)` to share the app's. Not closed by `close()`. */ + db?: ChClient + /** The database the mirror lives in, created if missing. Defaults to `workos`. */ + schema?: string + /** Identifier when several syncs share one database. Defaults to `default`. */ + syncId?: string + /** Rows per statement. Defaults to 500. */ + batchSize?: number + /** + * On `init()`, replace a `_live` view under a generated name that is not + * the generated shape. Off, `init()` fails and names it instead. `init + * --repair` on the CLI. + */ + repairViews?: boolean + /** + * During `init()`, redefine any projected column whose definition no + * longer matches the catalog. Off by default: drift is then reported at + * error level and the schema fingerprint withheld until it is fixed. Key + * columns, non-generated columns and orphans are never touched either + * way. `init --repair` on the CLI sets this together with `repairViews`. + */ + repairColumns?: boolean +} + +export class ClickHouseDestination implements Destination { + readonly schema: string + private db: ChClient | undefined + private ownsDb = false + private readonly syncId: string + private readonly batchSize: number | undefined + /** + * What the most recent `init()` did, including any column drift it found. + * The engine's `init()` returns nothing, so this is how a caller — the CLI's + * `init` command — gets the drift report back. + */ + lastMigration: MigrateResult | undefined + + constructor(private options: ClickHouseDestinationOptions) { + if (!options.db && !options.databaseUrl) throw new Error('databaseUrl or db is required') + if (options.sslCa !== undefined && (options.db || !options.databaseUrl)) { + throw new Error('sslCa applies to databaseUrl; a pre-built db carries its own TLS settings') + } + this.schema = options.schema ?? 'workos' + if (!/^[A-Za-z_][A-Za-z0-9_]{0,62}$/.test(this.schema)) { + throw new Error('schema must be a plain identifier: letters, digits and underscores, up to 63 characters') + } + this.syncId = options.syncId ?? 'default' + this.batchSize = options.batchSize + this.db = options.db + } + + private get client(): ChClient { + if (!this.db) throw new Error('destination not initialised: call init() first') + return this.db + } + + private get writeOpts() { + return { schema: this.schema, batchSize: this.batchSize } + } + + private get bookkeepingOpts() { + return { schema: this.schema, syncId: this.syncId } + } + + async init(streams: TableDef[], logger?: Logger): Promise { + if (!this.db) { + this.db = await connect(this.options.databaseUrl!, { sslCa: this.options.sslCa, logger }) + this.ownsDb = true + } + this.lastMigration = await migrate(this.db, streams, this.schema, logger, { + repair: this.options.repairColumns, + repairViews: this.options.repairViews, + }) + } + + async close(): Promise { + if (this.ownsDb && this.db) await this.db.close() + } + + upsert(stream: TableDef, env: string, rows: Row[], mode: WriteMode): Promise { + return upsertRows(this.client, this.writeOpts, stream, env, rows, mode) + } + + delete(stream: TableDef, env: string, rows: Row[]): Promise { + return deleteRows(this.client, this.writeOpts, stream, env, rows) + } + + relink(stream: TableDef, env: string, rows: Row[]): Promise { + return relinkRows(this.client, this.writeOpts, stream, env, rows) + } + + unlink(stream: TableDef, env: string, rows: Row[]): Promise { + return unlinkRows(this.client, this.writeOpts, stream, env, rows) + } + + /** `FINAL`: a key may have several versions in a mirror table, and a second copy in an append table. */ + private final(_stream: TableDef): string { + return ' FINAL' + } + + async liveIds(stream: TableDef, env: string): Promise { + const live = stream.mode === 'append' ? '' : ' AND `_deleted_at` IS NULL' + const rows = await this.client.query<{ k: string }>( + `SELECT ${keyExpression(stream)} AS k FROM ${tableRef(this.schema, stream)}${this.final(stream)} + WHERE \`_env\` = {env:String}${live} ORDER BY k`, + { env }, + ) + return rows.map((r) => encodeKey(stream.key.length === 1 ? [r.k] : r.k.split(KEY_SEPARATOR))) + } + + async rawRows(stream: TableDef, env: string): Promise[]> { + const rows = await this.client.query<{ raw: string }>( + `SELECT \`_raw\` AS raw FROM ${tableRef(this.schema, stream)}${this.final(stream)} WHERE \`_env\` = {env:String}`, + { env }, + ) + return rows.map((r) => JSON.parse(r.raw) as Record) + } + + /** Staged into tables of their own and diffed in SQL; see `db/reconcile.ts`. */ + diffIds(stream: TableDef, env: string, remote: AsyncIterable[]>): Promise { + return diffIdsInClickHouse(this.client, this.writeOpts, stream, env, remote) + } + + async freshness(stream: TableDef, env: string, lagWindowHours: number): Promise { + const hasDeleted = stream.mode !== 'append' + const recent = `\`_updated_at\` > now64(3, 'UTC') - toIntervalHour({hours:UInt64})` + const ms = (col: string) => `toUnixTimestamp64Milli(${col})` + const rows = await this.client.query<{ + n: string + deleted: string + last_synced_at: string + staleness: string + lag_s: string | null + lag_n: string + }>( + `SELECT toString(count()) AS n, + ${hasDeleted ? 'toString(countIf(`_deleted_at` IS NOT NULL))' : "'0'"} AS deleted, + max(\`_synced_at\`) AS last_synced_at, + toString((${ms(`now64(3, 'UTC')`)} - ${ms('max(`_updated_at`)')}) / 1000) AS staleness, + toString(avgIf((${ms('`_synced_at`')} - ${ms('`_updated_at`')}) / 1000, ${recent})) AS lag_s, + toString(countIf(${recent})) AS lag_n + FROM ${tableRef(this.schema, stream)}${this.final(stream)} WHERE \`_env\` = {env:String}`, + { env, hours: Math.max(0, Math.floor(lagWindowHours)) }, + ) + const row = rows[0] + const n = Number(row?.n ?? 0) + const lagN = Number(row?.lag_n ?? 0) + // Aggregates over no rows read as epoch and NaN here, not NULL. + return { + stream: stream.name, + rows: n, + deleted: Number(row?.deleted ?? 0), + lastSyncedAt: n > 0 && row ? isoOf(row.last_synced_at) : null, + stalenessSeconds: n > 0 && row ? Number(row.staleness) : null, + lagSeconds: lagN > 0 && row?.lag_s != null && Number.isFinite(Number(row.lag_s)) ? Number(row.lag_s) : null, + lagSampleSize: lagN, + } + } + + async redact(streams: TableDef[], env: string, userId: string): Promise<{ table: string; rows: number }[]> { + const out: { table: string; rows: number }[] = [] + // A delete reports no count, so each table is counted first. + const erase = async ( + label: string, + table: string, + final: string, + where: string, + params: Record, + ) => { + const counted = await this.client.query<{ n: string }>( + `SELECT toString(count()) AS n FROM ${table}${final} WHERE ${where}`, + params, + ) + const n = Number(counted[0]?.n ?? 0) + if (n === 0) return + await this.client.command(`DELETE FROM ${table} WHERE ${where}`, params) + out.push({ table: label, rows: n }) + } + for (const stream of streams) { + const refs = stream.columns.filter( + (c) => USER_REFERENCES.includes(c.name) || (c.name === 'id' && stream.name === 'users'), + ) + if (refs.length === 0) continue + await erase( + stream.name, + tableRef(this.schema, stream), + this.final(stream), + `\`_env\` = {env:String} AND (${refs.map((c) => `${ident(c.name)} = {id:String}`).join(' OR ')})`, + { env, id: userId }, + ) + } + const namedInData = USER_REFERENCES.map( + (field) => `JSONExtractString(\`_raw\`, ${jsonPathArgs(['data', field])}) = {id:String}`, + ).join(' OR ') + // Archived events embed the payload, so they must be scrubbed too. + await erase( + 'events', + `${quoteIdent(this.schema)}.\`events\``, + ' FINAL', + `\`_env\` = {env:String} AND (\`object_id\` = {id:String} OR ${namedInData})`, + { env, id: userId }, + ) + // Quarantined events embed the payload too, and they may be the only copy + // of it: an event rejected before its archive write succeeded is nowhere + // in `events`. + await erase( + '_rejected', + `${quoteIdent(this.schema)}.\`_rejected\``, + ' FINAL', + `sync_id = {syncId:String} AND env = {env:String} AND (object_id = {id:String} OR ${namedInData})`, + { syncId: this.syncId, env, id: userId }, + ) + return out + } + + reject(env: string, rows: { event: WorkOSEvent; error: string }[]): Promise { + return rejectEvents(this.client, this.bookkeepingOpts, env, rows) + } + + rejected(env: string, opts: { ids?: string[] } = {}): Promise { + return listRejected(this.client, this.bookkeepingOpts, env, opts.ids) + } + + rejectedCount(env: string): Promise { + return countRejected(this.client, this.bookkeepingOpts, env) + } + + clearRejected(env: string, ids: string[]): Promise { + return clearRejected(this.client, this.bookkeepingOpts, env, ids) + } + + isTransientError(err: unknown): boolean { + return isTransientClickHouseError(err) + } + + getState(env: string, stream: string): Promise { + return getState(this.client, this.schema, this.syncId, env, stream) + } + + setState(env: string, stream: string, state: unknown): Promise { + return setState(this.client, this.schema, this.syncId, env, stream, state) + } + + setStateIfHeld(env: string, stream: string, state: unknown, holder: string, expected: unknown): Promise { + return setStateIfHeld(this.client, this.schema, this.syncId, env, stream, state, holder, expected) + } + + acquireLease(env: string, holder: string, ttlMs: number): Promise { + return acquireLease(this.client, this.schema, this.syncId, env, holder, ttlMs) + } + + releaseLease(env: string, holder: string): Promise { + return releaseLease(this.client, this.schema, this.syncId, env, holder) + } + + getLease(env: string): Promise { + return getLease(this.client, this.schema, this.syncId, env) + } +} diff --git a/packages/sync-clickhouse/src/health.ts b/packages/sync-clickhouse/src/health.ts new file mode 100644 index 0000000..4e91bca --- /dev/null +++ b/packages/sync-clickhouse/src/health.ts @@ -0,0 +1,39 @@ +/** + * Health server settings for the `listen` command. + * + * Off unless a port is given, because the CLI also runs as one-shot commands + * and a cron `tick`, where a listening socket would be a surprise. The Docker + * image sets the port for the same reason it sets `NODE_ENV`: a container is + * the deployment where something external wants to ask whether the process is + * alive. Everything the server needs is read here, in one place, so `cli.ts` + * only has to ask for a config and hand it to `startHealthServer`. + */ +import { DEFAULT_MAX_POLL_AGE_MS, type HealthServerOptions } from '@workos/sync-engine' + +type Flags = Record + +/** + * `--health-port` or `WORKOS_SYNC_HEALTH_PORT` turns the server on; + * `WORKOS_SYNC_HEALTH_HOST` and `WORKOS_SYNC_HEALTH_MAX_POLL_AGE_MS` tune it. + * Returns null when it should stay off. + */ +export function healthOptionsFromEnv(flags: Flags, env: NodeJS.ProcessEnv = process.env): HealthServerOptions | null { + const raw = typeof flags['health-port'] === 'string' ? flags['health-port'] : env['WORKOS_SYNC_HEALTH_PORT'] + if (raw === undefined || raw.trim() === '') return null + const port = Number(raw) + if (!Number.isInteger(port) || port < 0 || port > 65535) { + throw new Error(`invalid health port "${raw}": expected an integer between 0 and 65535`) + } + const maxRaw = env['WORKOS_SYNC_HEALTH_MAX_POLL_AGE_MS'] + const maxPollAgeMs = maxRaw === undefined || maxRaw.trim() === '' ? DEFAULT_MAX_POLL_AGE_MS : Number(maxRaw) + if (!Number.isFinite(maxPollAgeMs) || maxPollAgeMs <= 0) { + throw new Error( + `invalid WORKOS_SYNC_HEALTH_MAX_POLL_AGE_MS "${maxRaw}": expected a positive number of milliseconds`, + ) + } + return { + port, + host: env['WORKOS_SYNC_HEALTH_HOST']?.trim() || '0.0.0.0', + maxPollAgeMs, + } +} diff --git a/packages/sync-clickhouse/src/index.ts b/packages/sync-clickhouse/src/index.ts new file mode 100644 index 0000000..5a553de --- /dev/null +++ b/packages/sync-clickhouse/src/index.ts @@ -0,0 +1,53 @@ +/** + * @workos/sync-clickhouse + * + * Mirror WorkOS data into your own ClickHouse: the sign-in and event logs + * that WorkOS keeps for 90 days, as tables you keep for as long as you like, + * and the identity objects beside them for the joins. + * + * The engine itself lives in `@workos/sync-engine` and is re-exported here, so this + * one package is all an application needs to import. This package adds the + * ClickHouse destination, the DDL and write path behind it, and the CLI. + * + * The mirror is a **read model**. It is eventually consistent, typically within + * seconds. Read from it for analytics, dashboards, lists and agents; keep + * writes and authoritative post-signup checks on the WorkOS API. + */ +export * from '@workos/sync-engine' + +export { ClickHouseDestination } from './destination.js' +export type { ClickHouseDestinationOptions } from './destination.js' + +export { migrate, fingerprint, DDL_VERSION } from './schema/migrate.js' +export type { MigrateOptions, MigrateResult } from './schema/migrate.js' +export { detectColumnDrift, repairColumnDrift, unrepairableReason } from './schema/drift.js' +export type { ColumnDrift, DriftKind, RepairResult } from './schema/drift.js' +export { + buildSchemaSql, + createTableSql, + columnExpression, + columnType, + generatedColumnSql, + indexSpecs, + jsonPathArgs, +} from './schema/ddl.js' +export { liveViewSql, liveViewName } from './schema/views.js' + +export { + upsertRows, + deleteRows, + relinkRows, + unlinkRows, + dedupeByKey, + wavesByKey, + keyValues, + parseUpdatedAt, + jsonEqual, +} from './db/upsert.js' +export { diffIdsInClickHouse } from './db/reconcile.js' +export type { ChDiff } from './db/reconcile.js' +export { rejectEvents, listRejected, countRejected, clearRejected } from './db/rejected.js' +export { canonicalJson } from './db/lease.js' +export { isTransientClickHouseError } from './db/errors.js' +export { connect, fromClient, clientConfig, APPLICATION_NAME, DEFAULT_SETTINGS, KEEPER_SETTINGS } from './db/client.js' +export type { ConnectOptions, ChClient } from './db/client.js' diff --git a/packages/sync-clickhouse/src/otel.ts b/packages/sync-clickhouse/src/otel.ts new file mode 100644 index 0000000..78472f2 --- /dev/null +++ b/packages/sync-clickhouse/src/otel.ts @@ -0,0 +1,75 @@ +/** + * OpenTelemetry for the command line. + * + * The engine depends on the OpenTelemetry API only, and the API does nothing + * until a process installs an SDK. The CLI is a process of its own, so it is + * where one can live: `WORKOS_SYNC_OTEL=1` starts the Node SDK, configured by + * the standard `OTEL_*` variables (exporter, endpoint, headers, service + * name), before the engine is built, and shuts it down once the command + * returns. The shutdown is the part that matters for `tick` and the sliced + * commands: they exit seconds after they start, long before the SDK's + * periodic export would have fired, and shutting down flushes. + * + * The SDK is an optional dependency of this package, so a host that embeds + * the library and runs its own SDK can leave it out; the container image has + * it. A missing module is reported as such rather than as a stack trace. + */ +export interface OtelHandle { + /** Flush and stop the SDK. Call it once the command has returned. */ + shutdown(): Promise +} + +/** The part of `@opentelemetry/sdk-node` this module uses, so nothing here types against the package. */ +export interface SdkModule { + NodeSDK: new (configuration?: { serviceName?: string }) => { start(): void; shutdown(): Promise } +} + +/** The `service.name` reported, unless `OTEL_SERVICE_NAME` says otherwise. */ +export const DEFAULT_SERVICE_NAME = 'workos-sync-clickhouse' + +const ON = new Set(['1', 'true', 'yes', 'on']) +const OFF = new Set(['0', 'false', 'no', 'off']) + +/** `WORKOS_SYNC_OTEL` is on, off, or unset, which is off. Anything else is refused up front. */ +export function otelEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + const raw = env['WORKOS_SYNC_OTEL'] + if (raw === undefined || raw.trim() === '') return false + const v = raw.trim().toLowerCase() + if (ON.has(v)) return true + if (OFF.has(v)) return false + throw new Error(`invalid WORKOS_SYNC_OTEL "${raw}": expected 1 or 0`) +} + +/** + * Start the SDK when `WORKOS_SYNC_OTEL` asks for it; null when it does not. + * The SDK reads its own configuration from the process environment; `env` + * only decides whether to start it and what to call the service. `load` is + * a test seam standing in for the import. + */ +export async function startOtel( + env: NodeJS.ProcessEnv = process.env, + load: () => Promise = () => import('@opentelemetry/sdk-node'), +): Promise { + if (!otelEnabled(env)) return null + let sdkNode: SdkModule + try { + sdkNode = await load() + } catch (err) { + if (isModuleNotFound(err)) { + throw new Error( + 'WORKOS_SYNC_OTEL is set but @opentelemetry/sdk-node is not installed: ' + + 'add it to the host (npm install @opentelemetry/sdk-node), or unset WORKOS_SYNC_OTEL', + ) + } + throw err + } + const named = Boolean(env['OTEL_SERVICE_NAME']?.trim()) + const sdk = new sdkNode.NodeSDK(named ? {} : { serviceName: DEFAULT_SERVICE_NAME }) + sdk.start() + return { shutdown: () => sdk.shutdown() } +} + +function isModuleNotFound(err: unknown): boolean { + const code = (err as { code?: unknown } | null)?.code + return code === 'ERR_MODULE_NOT_FOUND' || code === 'MODULE_NOT_FOUND' +} diff --git a/packages/sync-clickhouse/src/schema/ddl.ts b/packages/sync-clickhouse/src/schema/ddl.ts new file mode 100644 index 0000000..51e3712 --- /dev/null +++ b/packages/sync-clickhouse/src/schema/ddl.ts @@ -0,0 +1,292 @@ +/** + * DDL generation. + * + * `_raw` is the only column the engine ever writes. Every typed column is a + * `DEFAULT` expression over it, which ClickHouse computes when the engine's + * insert leaves the column out, and which `SELECT *` includes (a + * `MATERIALIZED` column would not be). A new field appearing upstream can + * never break ingest, is immediately queryable with `JSONExtract(_raw, ...)`, + * and can be promoted to a typed column later with pure DDL: a column added + * with `ALTER TABLE ... ADD COLUMN` is computed for the rows already there. + * + * Three things are ClickHouse's rather than Postgres's: + * + * - **Versions, not updates.** A mirror table is a `ReplacingMergeTree` + * versioned by `_updated_at`: every write inserts the row's new state, and + * the newest source timestamp wins at merge time and under `FINAL`. That + * is the staleness gate, enforced by the table itself whatever order the + * writes land in. The `_live` views read with `FINAL`. Append-only + * streams are a `ReplacingMergeTree` keyed by the event id with no + * version column: one row per event, never updated, and a second copy + * of one — which only a poll that outlived its lease can write, past + * the write path's own check — collapses at the next merge, or under + * `FINAL` before then. + * - **A projection that cannot be cast is NULL, not an error.** `JSONExtract` + * with a `Nullable` return type gives NULL for a missing key, a JSON null + * and a value of the wrong shape alike, and the value stays in `_raw`. + * - **Indexes are data-skipping indexes.** A column marked `index` gets a + * bloom filter (text) or a min-max index (timestamps, integers); the sort + * key is `(_env, )`, which is also what the write path looks + * rows up by. + * + * There are no foreign keys between mirror tables. Events arrive with + * children before parents routinely (a `group.user_added` before the + * `user.created`), and an FK would turn that ordering artifact into a hard + * write failure. + */ +import type { ColumnDef, TableDef } from '@workos/sync-engine' +import { quoteIdent, quoteLiteral, safeIdentifier } from './identifiers.js' +import { liveViewSql } from './views.js' + +// Re-exported so callers keep importing the identifier helpers from here. +export { quoteIdent, quoteLiteral, safeIdentifier } from './identifiers.js' + +export const RAW = '_raw' +export const ENV = '_env' + +/** Keeper path under `keeper_map_path_prefix` for a schema's bookkeeping. */ +export const keeperPath = (schema: string, table: string) => quoteLiteral(`/workos_sync/${schema}/${table}`) + +const TIMESTAMP = `DateTime64(3, 'UTC')` + +/** The JSON path as `JSONExtract` takes it: one quoted step per argument. */ +export function jsonPathArgs(path: string | string[]): string { + const parts = Array.isArray(path) ? path : [path] + return parts.map((p) => quoteLiteral(p)).join(', ') +} + +/** The ClickHouse column type for a projected column. Keys are not nullable: they sort the table. */ +export function columnType(col: ColumnDef, isKey: boolean): string { + switch (col.type) { + case 'text': + return isKey ? 'String' : 'Nullable(String)' + case 'boolean': + return 'Nullable(Bool)' + case 'bigint': + return 'Nullable(Int64)' + case 'numeric': + return 'Nullable(Decimal(38, 10))' + case 'jsonb': + return 'Nullable(String)' + case 'timestamptz': + return `Nullable(${TIMESTAMP})` + } +} + +/** The default expression for a projected field. */ +export function columnExpression(col: ColumnDef, isKey: boolean): string { + const raw = quoteIdent(RAW) + const path = jsonPathArgs(col.path) + switch (col.type) { + case 'text': + // A key reads as '' when missing; the engine never writes a keyless row. + return isKey ? `JSONExtractString(${raw}, ${path})` : `JSONExtract(${raw}, ${path}, 'Nullable(String)')` + case 'boolean': + case 'bigint': + case 'numeric': + return `JSONExtract(${raw}, ${path}, '${columnType(col, isKey)}')` + case 'jsonb': + // The subtree as JSON text; `JSONExtractRaw` alone reads '' for a missing key. + return `if(JSONHas(${raw}, ${path}), JSONExtractRaw(${raw}, ${path}), NULL)` + case 'timestamptz': { + // Every ISO-8601 shape WorkOS emits, offsets included, with the clock + // and the offset range-checked ahead of the parser; NULL for anything + // else. Upper-cased for the parser, which reads `T` and `Z` only. + const s = `JSONExtractString(${raw}, ${path})` + return `if(match(${s}, ${sqlString(ISO_8601)}), parseDateTime64BestEffortOrNull(upper(${s}), 3, 'UTC'), NULL)` + } + } +} + +/** + * The timestamps the projection accepts: the Postgres destination's grammar, + * every field range-checked in the pattern. The best-effort parser alone + * reads `24:00` as the next midnight, `23:60` as the next hour and `+00:99` + * as a zone, where the other destinations project NULL. The day of the + * month is left to the parser, which refuses `02-30` and `04-31` itself. + */ +const ISO_8601 = String.raw`^(?:[1-9]\d{3}|0[1-9]\d{2}|00[1-9]\d|000[1-9])-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12]\d|3[01])[Tt ](?:[01]\d|2[0-3]):[0-5]\d:(?:[0-5]\d|60)(?:\.\d+)?\s*(?:[Zz]|[+-](?:[01]\d|2[0-3]):?[0-5]\d)?$` + +/** A single-quoted ClickHouse string literal; backslashes are escapes there, so each is doubled. */ +const sqlString = (text: string) => `'${text.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'` + +/** + * One `` `name` type DEFAULT expr `` clause. Every path that creates a + * projected column — table creation, the additive upgrade, drift detection + * and repair — goes through here, so they cannot disagree about what a + * column should look like. The text is also what `_columns` records, and + * what drift is measured against. + */ +export function generatedColumnSql(col: ColumnDef, stream: TableDef): string { + const isKey = stream.key.includes(col.name) + return `${quoteIdent(safeIdentifier(col.name))} ${columnType(col, isKey)} DEFAULT ${columnExpression(col, isKey)}` +} + +export function createSchemaSql(schema: string): string { + return `CREATE DATABASE IF NOT EXISTS ${quoteIdent(schema)}` +} + +/** + * Engine bookkeeping. Cursors and the single-writer lease share a KeeperMap + * row per environment: KeeperMap is a key-value table held in ClickHouse + * Keeper with linearizable writes, and under `keeper_map_strict_mode` an + * `ALTER ... UPDATE` that names the row's current values is a compare-and- + * swap, which is what the lease and the fenced checkpoint need. See + * `db/lease.ts`. The applied-schema fingerprints and the record of generated + * columns live there too: an `INSERT` of an existing key is refused, which + * makes "record once" a single statement — and "take the lock" one as well, + * for the per-stream write lock in `_locks` (see `db/lock.ts`). + * + * `_rejected` holds events the tail set aside after they failed + * deterministically on two consecutive polls (see the engine's + * `sync/quarantine.ts`). It keeps the whole event, not a reference into + * `events`, because the archive write may itself have been the failure. A + * rejection written again replaces the earlier one: the table is a + * `ReplacingMergeTree` versioned by `rejected_at`. `created_at` is + * nullable and parsed in Node on the way in, so a quarantine write can + * never fail on the same bad data that caused it. + */ +export function bookkeepingSql(schema: string): [name: string, sql: string][] { + const s = quoteIdent(schema) + return [ + [ + '_sync', + `CREATE TABLE IF NOT EXISTS ${s}.\`_sync\` ( + k String, + holder String, + acquired_at ${TIMESTAMP}, + expires_at ${TIMESTAMP}, + states String +) ENGINE = KeeperMap(${keeperPath(schema, '_sync')}) PRIMARY KEY k`, + ], + [ + '_locks', + `CREATE TABLE IF NOT EXISTS ${s}.\`_locks\` ( + k String, + holder String, + expires_at ${TIMESTAMP} +) ENGINE = KeeperMap(${keeperPath(schema, '_locks')}) PRIMARY KEY k`, + ], + [ + '_migrations', + `CREATE TABLE IF NOT EXISTS ${s}.\`_migrations\` ( + fingerprint String, + applied_at ${TIMESTAMP}, + detail String +) ENGINE = KeeperMap(${keeperPath(schema, '_migrations')}) PRIMARY KEY fingerprint`, + ], + [ + '_columns', + `CREATE TABLE IF NOT EXISTS ${s}.\`_columns\` ( + k String, + table_name String, + column_name String, + ddl String +) ENGINE = KeeperMap(${keeperPath(schema, '_columns')}) PRIMARY KEY k`, + ], + [ + '_rejected', + `CREATE TABLE IF NOT EXISTS ${s}.\`_rejected\` ( + sync_id String, + env String, + event_id String, + event_type String, + object_id Nullable(String), + created_at Nullable(${TIMESTAMP}), + rejected_at ${TIMESTAMP} DEFAULT now64(3), + error String, + \`_raw\` String +) ENGINE = ReplacingMergeTree(rejected_at) ORDER BY (sync_id, env, event_id)`, + ], + ] +} + +export interface IndexSpec { + name: string + column: string + type: 'bloom_filter' | 'minmax' + granularity: number +} + +/** The data-skipping indexes a stream's table carries, by name, so `migrate` can tell which are missing. */ +export function indexSpecs(stream: TableDef): IndexSpec[] { + const tableName = safeIdentifier(stream.name) + const out: IndexSpec[] = [] + for (const c of stream.columns) { + if (!c.index) continue + const minmax = c.type === 'timestamptz' || c.type === 'bigint' || c.type === 'numeric' + out.push({ + name: safeIdentifier(`${tableName}_${c.name}_idx`), + column: safeIdentifier(c.name), + type: minmax ? 'minmax' : 'bloom_filter', + granularity: minmax ? 1 : 4, + }) + } + // Freshness and reconciliation queries scan these. + out.push({ name: safeIdentifier(`${tableName}_synced_at_idx`), column: '_synced_at', type: 'minmax', granularity: 1 }) + if (stream.mode !== 'append') { + out.push({ + name: safeIdentifier(`${tableName}_deleted_at_idx`), + column: '_deleted_at', + type: 'minmax', + granularity: 1, + }) + } + return out +} + +const indexClause = (idx: IndexSpec) => + `INDEX ${quoteIdent(idx.name)} ${quoteIdent(idx.column)} TYPE ${idx.type} GRANULARITY ${idx.granularity}` + +export function createTableSql(stream: TableDef, schema: string): string { + const s = quoteIdent(schema) + const table = quoteIdent(safeIdentifier(stream.name)) + const cols: string[] = [ + `${quoteIdent(ENV)} String`, + `${quoteIdent(RAW)} String`, + // Source-derived. The staleness gate compares this, and so does the merge. + `\`_updated_at\` ${TIMESTAMP}`, + // Destination write time. Lag is `_synced_at - _updated_at`. + `\`_synced_at\` DateTime64(6, 'UTC') DEFAULT now64(6)`, + ] + // Append-only streams are immutable logs: no tombstones. + if (stream.mode !== 'append') cols.push(`\`_deleted_at\` Nullable(${TIMESTAMP})`) + for (const c of stream.columns) cols.push(generatedColumnSql(c, stream)) + for (const idx of indexSpecs(stream)) cols.push(indexClause(idx)) + const order = [ENV, ...stream.key].map((k) => quoteIdent(safeIdentifier(k))).join(', ') + const engine = stream.mode === 'append' ? 'ReplacingMergeTree' : 'ReplacingMergeTree(`_updated_at`)' + return `CREATE TABLE IF NOT EXISTS ${s}.${table} (\n ${cols.join(',\n ')}\n) ENGINE = ${engine} ORDER BY (${order})` +} + +/** Additive DDL for a table that already exists; both are idempotent. */ +export function addColumnSql(stream: TableDef, col: ColumnDef, schema: string): string { + const table = `${quoteIdent(schema)}.${quoteIdent(safeIdentifier(stream.name))}` + return `ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS ${generatedColumnSql(col, stream)}` +} + +/** Write a column added later into the parts that predate it, so reads stop recomputing it. */ +export function materializeColumnSql(stream: TableDef, col: ColumnDef, schema: string): string { + const table = `${quoteIdent(schema)}.${quoteIdent(safeIdentifier(stream.name))}` + return `ALTER TABLE ${table} MATERIALIZE COLUMN ${quoteIdent(safeIdentifier(col.name))}` +} + +export function addIndexSql(stream: TableDef, idx: IndexSpec, schema: string): string { + const table = `${quoteIdent(schema)}.${quoteIdent(safeIdentifier(stream.name))}` + return `ALTER TABLE ${table} ADD ${indexClause(idx).replace('INDEX ', 'INDEX IF NOT EXISTS ')}` +} + +/** + * Full DDL for a set of streams, in dependency-free order: what + * `init --print-sql` prints. Every statement is idempotent, so the script + * can be applied to a database in any state. + */ +export function buildSchemaSql(streams: TableDef[], schema: string): string[] { + const out: string[] = [createSchemaSql(schema), ...bookkeepingSql(schema).map(([, sql]) => sql)] + for (const s of streams) { + out.push(createTableSql(s, schema)) + for (const c of s.columns) out.push(addColumnSql(s, c, schema)) + for (const idx of indexSpecs(s)) out.push(addIndexSql(s, idx, schema)) + out.push(...liveViewSql(s, schema)) + } + return out +} diff --git a/packages/sync-clickhouse/src/schema/drift.ts b/packages/sync-clickhouse/src/schema/drift.ts new file mode 100644 index 0000000..1cca4db --- /dev/null +++ b/packages/sync-clickhouse/src/schema/drift.ts @@ -0,0 +1,255 @@ +/** + * Column drift: a projected column whose definition no longer matches the + * catalog. + * + * Additive DDL cannot change a column that already exists, so once a + * projection is wrong — an `extraColumns` entry whose path or type was + * edited, a catalog change to a column that is already there — it stays + * wrong until something rebuilds it. This finds those columns and, when + * asked, redefines them with `ALTER TABLE ... MODIFY COLUMN`, which + * ClickHouse applies to the rows already there. + * + * The expression is compared against `_columns`, the engine's own record of + * the DDL it emitted for every projected column, not against what + * `system.columns` shows: the server stores a re-formatted expression, so + * text written by the catalog and text read back never compare equal. A + * column with no record — a schema applied by hand from `init --print-sql` + * — has only the table as a witness, so the catalog's clause is put through + * the server's own formatter (`formatQuerySingleLine`) before the two are + * compared; only a match is recorded, anything else is drift. The live type + * is compared with the catalog's directly; `system.columns` spells types + * canonically, and so does `columnType`. + * + * Never touched, only reported: key columns (they sort the table), columns + * that are not `DEFAULT` expressions (they are yours), and orphans — + * recorded columns the catalog no longer names, which still compute from + * `_raw` with their old expression and are harmless to leave. + */ +import type { Logger, TableDef } from '@workos/sync-engine' +import { KEEPER_SETTINGS, type ChClient } from '../db/client.js' +import { isKeeperConflict } from '../db/errors.js' +import { columnExpression, columnType, generatedColumnSql, quoteIdent, safeIdentifier } from './ddl.js' + +export type DriftKind = 'expression' | 'type' | 'orphan' | 'not_generated' + +export interface ColumnDrift { + stream: string + column: string + kind: DriftKind + /** The clause the catalog would emit now; null for an orphan. */ + expected: string | null + /** What is recorded or, failing that, what `system.columns` shows. */ + actual: string | null + /** Part of the sort key, so never rebuilt. */ + key: boolean +} + +export interface RepairResult { + repaired: ColumnDrift[] + failed: { drift: ColumnDrift; error: string }[] +} + +interface LiveColumn { + type: string + default_kind: string + default_expression: string +} + +async function liveColumns(db: ChClient, schema: string, table: string): Promise> { + const rows = await db.query<{ name: string; type: string; default_kind: string; default_expression: string }>( + `SELECT name, type, default_kind, default_expression FROM system.columns + WHERE database = {schema:String} AND table = {table:String}`, + { schema, table }, + ) + return new Map(rows.map((r) => [r.name, r])) +} + +async function recordedColumns(db: ChClient, schema: string, table: string): Promise> { + const rows = await db.query<{ column_name: string; ddl: string }>( + `SELECT column_name, ddl FROM ${quoteIdent(schema)}.\`_columns\` WHERE table_name = {table:String}`, + { table }, + ) + return new Map(rows.map((r) => [r.column_name, r.ddl])) +} + +const backoff = () => new Promise((r) => setTimeout(r, 5 + Math.random() * 20)) + +/** + * Record (or re-record) the clause a column was created with. Strict mode + * is off, so an existing record is overwritten. Even so, a KeeperMap + * insert decides create-or-set per key before its transaction, so two + * processes recording the same column at once can both decide "create" + * and the second is refused with "Node exists". The loser then reads what + * landed: the same clause means the record is already right, another + * clause is overwritten in place. + */ +export async function recordColumn(db: ChClient, schema: string, table: string, column: string, ddl: string) { + const ref = `${quoteIdent(schema)}.\`_columns\`` + const k = `${table}\u0001${column}` + for (let attempt = 0; attempt < 8; attempt++) { + if (attempt > 0) await backoff() + try { + await db.command( + `INSERT INTO ${ref} (k, table_name, column_name, ddl) VALUES ({k:String}, {table:String}, {column:String}, {ddl:String})`, + { k, table, column, ddl }, + { ...KEEPER_SETTINGS, keeper_map_strict_mode: 0 }, + ) + return + } catch (err) { + if (!isKeeperConflict(err)) throw err + } + const [stored] = await db.query<{ ddl: string }>(`SELECT ddl FROM ${ref} WHERE k = {k:String}`, { k }) + if (!stored) continue + if (stored.ddl === ddl) return + try { + await db.command( + `ALTER TABLE ${ref} UPDATE table_name = {table:String}, column_name = {column:String}, ddl = {ddl:String} WHERE k = {k:String}`, + { k, table, column, ddl }, + { ...KEEPER_SETTINGS, keeper_map_strict_mode: 0 }, + ) + return + } catch (err) { + if (!isKeeperConflict(err)) throw err + } + } + throw new Error(`could not record the column ${table}.${column}: the row kept changing under 8 attempts`) +} + +async function forgetColumn(db: ChClient, schema: string, table: string, column: string) { + await db.command( + `ALTER TABLE ${quoteIdent(schema)}.\`_columns\` DELETE WHERE k = {k:String}`, + { k: `${table}\u0001${column}` }, + { ...KEEPER_SETTINGS, keeper_map_strict_mode: 0 }, + ) +} + +/** + * The server's own spelling of each of a stream's generated expressions: + * `system.columns.default_expression` is the clause re-formatted by the + * server, and `formatQuerySingleLine` applies the same formatter to the + * clause the catalog would write, so the two compare equal when the column + * was created from it. A function call, not a table: `init` needs no grant + * beyond the schema's own. + */ +async function canonicalExpressions(db: ChClient, stream: TableDef): Promise> { + const names = stream.columns.map((c) => safeIdentifier(c.name)) + const exprs = stream.columns.map((c) => columnExpression(c, stream.key.includes(c.name))) + const rows = await db.query<{ name: string; expr: string }>( + `SELECT tupleElement(pair, 1) AS name, + substring(formatQuerySingleLine(concat('SELECT ', tupleElement(pair, 2))), 8) AS expr + FROM (SELECT arrayJoin(arrayZip({names:Array(String)}, {exprs:Array(String)})) AS pair)`, + { names, exprs }, + ) + return new Map(rows.map((r) => [r.name, r.expr])) +} + +export async function detectColumnDrift(db: ChClient, streams: TableDef[], schema: string): Promise { + const out: ColumnDrift[] = [] + for (const stream of streams) { + const table = safeIdentifier(stream.name) + const live = await liveColumns(db, schema, table) + const recorded = await recordedColumns(db, schema, table) + let canonical: Map | undefined + const catalog = new Set() + for (const col of stream.columns) { + const name = safeIdentifier(col.name) + catalog.add(name) + const actual = live.get(name) + // Missing entirely: `migrate` adds it, or already failed loudly trying. + if (!actual) continue + const key = stream.key.includes(col.name) + const expected = generatedColumnSql(col, stream) + if (actual.default_kind !== 'DEFAULT') { + out.push({ stream: stream.name, column: name, kind: 'not_generated', expected, actual: actual.type, key }) + continue + } + // The live type is checked on its own, before the record is consulted: + // a column retyped by hand leaves `_columns` saying what the engine + // created, and that must not hide what is actually there. + if (actual.type !== columnType(col, key)) { + out.push({ stream: stream.name, column: name, kind: 'type', expected, actual: actual.type, key }) + continue + } + const record = recorded.get(name) + if (record === undefined) { + // Nothing says what this column was created with, so the table is + // asked, in its own spelling of the clause the catalog would write. + canonical ??= await canonicalExpressions(db, stream) + if (canonical.get(name) === actual.default_expression) { + await recordColumn(db, schema, table, name, expected) + continue + } + out.push({ + stream: stream.name, + column: name, + kind: 'expression', + expected, + actual: actual.default_expression, + key, + }) + continue + } + if (record !== expected) { + out.push({ stream: stream.name, column: name, kind: 'expression', expected, actual: record, key }) + } + } + for (const [name, ddl] of recorded) { + if (catalog.has(name)) continue + if (!live.has(name)) { + // Dropped by hand: nothing to report, and the record would only mislead. + await forgetColumn(db, schema, table, name) + continue + } + out.push({ stream: stream.name, column: name, kind: 'orphan', expected: null, actual: ddl, key: false }) + } + } + return out +} + +/** Why a drifted column is not rebuilt, or null when `repairColumnDrift` may try. */ +export function unrepairableReason(d: ColumnDrift): string | null { + if (d.kind === 'orphan') return 'the catalog no longer names this column; drop it by hand if it is no longer wanted' + if (d.kind === 'not_generated') + return 'the column is not a DEFAULT expression, so it is yours: rename it or project the field under another name' + if (d.key) return 'key columns are never rebuilt: they sort the table' + return null +} + +/** + * Redefine each repairable drifted column with the catalog's type and + * expression, then recompute it for the rows already there: `MODIFY COLUMN` + * changes what future inserts compute and converts the stored values to the + * new type, but only a mutation re-evaluates the expression over `_raw`. + * Both run under `mutations_sync`, so the call returns once they are done. + */ +export async function repairColumnDrift( + db: ChClient, + drift: ColumnDrift[], + streams: TableDef[], + schema: string, + logger?: Logger, +): Promise { + const repaired: ColumnDrift[] = [] + const failed: RepairResult['failed'] = [] + for (const d of drift) { + if (unrepairableReason(d)) continue + const stream = streams.find((s) => s.name === d.stream) + const col = stream?.columns.find((c) => safeIdentifier(c.name) === d.column) + if (!stream || !col) continue + const table = safeIdentifier(stream.name) + try { + const ref = `${quoteIdent(schema)}.${quoteIdent(table)}` + const expr = columnExpression(col, false) + await db.command( + `ALTER TABLE ${ref} MODIFY COLUMN ${quoteIdent(d.column)} ${columnType(col, false)} DEFAULT ${expr}`, + ) + await db.command(`ALTER TABLE ${ref} UPDATE ${quoteIdent(d.column)} = ${expr} WHERE 1`) + await recordColumn(db, schema, table, d.column, generatedColumnSql(col, stream)) + repaired.push(d) + logger?.info('redefined drifted column from _raw', { stream: d.stream, column: d.column, kind: d.kind }) + } catch (err) { + failed.push({ drift: d, error: err instanceof Error ? err.message : String(err) }) + } + } + return { repaired, failed } +} diff --git a/packages/sync-clickhouse/src/schema/identifiers.ts b/packages/sync-clickhouse/src/schema/identifiers.ts new file mode 100644 index 0000000..b20b775 --- /dev/null +++ b/packages/sync-clickhouse/src/schema/identifiers.ts @@ -0,0 +1,27 @@ +/** + * Identifier and literal helpers shared by the table DDL and the view DDL. + * + * They live apart from `ddl.ts` so that `views.ts` can use them without + * importing the module that imports `views.ts` back for `buildSchemaSql`. + */ + +/** + * ClickHouse has no hard identifier limit, but the stream names become + * Keeper paths and file names on disk; the SQL packages' 63-byte habit is + * kept so one catalog produces one set of names everywhere. + */ +export const MAX_IDENTIFIER_LENGTH = 63 + +export function safeIdentifier(name: string): string { + if (name.length <= MAX_IDENTIFIER_LENGTH) return name + let hash = 5381 + for (let i = 0; i < name.length; i++) hash = ((hash << 5) + hash + name.charCodeAt(i)) >>> 0 + const suffix = `_${hash.toString(16).padStart(8, '0')}` + return name.slice(0, MAX_IDENTIFIER_LENGTH - suffix.length) + suffix +} + +/** Backtick-quoted, with embedded backticks and backslashes escaped. */ +export const quoteIdent = (name: string): string => `\`${name.replace(/\\/g, '\\\\').replace(/`/g, '\\`')}\`` + +/** A single-quoted SQL string literal, with backslashes and quotes escaped the way ClickHouse reads them. */ +export const quoteLiteral = (value: string): string => `'${value.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'` diff --git a/packages/sync-clickhouse/src/schema/migrate.ts b/packages/sync-clickhouse/src/schema/migrate.ts new file mode 100644 index 0000000..6665582 --- /dev/null +++ b/packages/sync-clickhouse/src/schema/migrate.ts @@ -0,0 +1,259 @@ +/** + * Schema setup. + * + * The DDL is derived from the stream catalog rather than hand-written per + * field, and a fingerprint gates the per-stream work, so an unchanged + * catalog costs one query. Every statement is idempotent — `CREATE ... IF + * NOT EXISTS`, `ADD COLUMN IF NOT EXISTS`, `ADD INDEX IF NOT EXISTS` — so + * two processes applying the schema at once, or a run that dies halfway, + * leave the same database as one clean run; there is no lock to take. + * + * The DDL is additive, which cannot change a column that already exists. So + * after it runs, and only when the fingerprint is new, the live tables are + * compared against the catalog (see `drift.ts`). The fingerprint is withheld + * while any projection is wrong: recording it would turn the next `init` + * into a silent no-op on a database that needs attention. + */ +import { createHash } from 'node:crypto' +import type { Logger, TableDef } from '@workos/sync-engine' +import { KEEPER_SETTINGS, type ChClient } from '../db/client.js' +import { isKeeperConflict } from '../db/errors.js' +import { + addColumnSql, + addIndexSql, + materializeColumnSql, + bookkeepingSql, + createSchemaSql, + createTableSql, + generatedColumnSql, + indexSpecs, + quoteIdent, + safeIdentifier, +} from './ddl.js' +import { applyLiveView, isGeneratedDefinition, liveViewName, type ApplyViewOptions } from './views.js' +import { detectColumnDrift, recordColumn, repairColumnDrift, unrepairableReason, type ColumnDrift } from './drift.js' + +/** + * Bumped whenever the DDL generated for an unchanged catalog changes: a new + * kind of object, a reworked index, a view. The fingerprint otherwise hashes + * only the catalog, so without this a database whose streams did not change + * would never run the new statements on upgrade. + */ +export const DDL_VERSION = 1 + +export function fingerprint(streams: TableDef[], schema: string): string { + const canonical = JSON.stringify({ + ddl: DDL_VERSION, + schema, + streams: streams + .map((s) => ({ + n: s.name, + k: s.key, + m: s.mode, + c: s.columns.map((c) => [c.name, c.type, c.path, c.index ?? false]), + })) + .sort((a, b) => a.n.localeCompare(b.n)), + }) + return createHash('sha256').update(canonical).digest('hex').slice(0, 16) +} + +export interface MigrateOptions extends ApplyViewOptions { + /** + * Redefine drifted columns instead of only reporting them. Key columns, + * non-generated columns and orphans are still only reported; see + * `repairColumnDrift`. + */ + repair?: boolean +} + +export interface MigrateResult { + /** DDL was run. `false` means the fingerprint was already recorded. */ + applied: boolean + fingerprint: string + statements: number + /** + * Columns that still disagree with the catalog after this run. Any + * `expression` or `type` item here means the fingerprint was *not* + * recorded, so the next run detects and reports again. Orphans are + * reported but do not hold the fingerprint back. + */ + drift: ColumnDrift[] + /** Columns redefined by this run's repair pass. Only with `repair: true`. */ + repaired: ColumnDrift[] +} + +const backoff = () => new Promise((r) => setTimeout(r, 5 + Math.random() * 20)) + +/** + * Record the fingerprint. Two processes applying the same catalog at once + * both reach this with the same value, and a KeeperMap insert that loses + * the create race is refused with "Node exists" even with strict mode off: + * the row the other process wrote is this one's, so that counts as done. + */ +async function recordFingerprint(db: ChClient, s: string, fp: string, detail: string): Promise { + for (let attempt = 0; attempt < 8; attempt++) { + if (attempt > 0) await backoff() + try { + await db.command( + `INSERT INTO ${s}.\`_migrations\` (fingerprint, applied_at, detail) VALUES ({fp:String}, now64(3, 'UTC'), {detail:String})`, + { fp, detail }, + { ...KEEPER_SETTINGS, keeper_map_strict_mode: 0 }, + ) + return + } catch (err) { + if (!isKeeperConflict(err)) throw err + } + const rows = await db.query(`SELECT 1 FROM ${s}.\`_migrations\` WHERE fingerprint = {fp:String}`, { fp }) + if (rows.length > 0) return + } + throw new Error(`could not record the schema fingerprint ${fp}: the row kept changing under 8 attempts`) +} + +const names = async (db: ChClient, sql: string, params: Record) => + new Set((await db.query<{ name: string }>(sql, params)).map((r) => r.name)) + +const typedColumns = async (db: ChClient, schema: string, table: string) => + new Map( + ( + await db.query<{ name: string; type: string }>( + `SELECT name, type FROM system.columns WHERE database = {schema:String} AND table = {table:String}`, + { schema, table }, + ) + ).map((r) => [r.name, r.type]), + ) + +export async function migrate( + db: ChClient, + streams: TableDef[], + schema: string, + logger?: Logger, + options: MigrateOptions = {}, +): Promise { + const fp = fingerprint(streams, schema) + const s = quoteIdent(schema) + + // Only when missing: creating needs a privilege the day-to-day user may not have. + const databases = await names(db, `SELECT name FROM system.databases WHERE name = {schema:String}`, { schema }) + if (!databases.has(schema)) await db.command(createSchemaSql(schema)) + // The bookkeeping tables must exist before we can check whether they say + // this fingerprint is already applied. + const tables = await names(db, `SELECT name FROM system.tables WHERE database = {schema:String}`, { schema }) + for (const [name, sql] of bookkeepingSql(schema)) if (!tables.has(name)) await db.command(sql) + + const applied = await db.query(`SELECT 1 FROM ${s}.\`_migrations\` WHERE fingerprint = {fp:String}`, { fp }) + if (applied.length > 0) { + logger?.debug('schema up to date', { fingerprint: fp }) + return { applied: false, fingerprint: fp, statements: 0, drift: [], repaired: [] } + } + + let statements = 0 + for (const stream of streams) { + const table = safeIdentifier(stream.name) + const live = await names( + db, + `SELECT name FROM system.columns WHERE database = {schema:String} AND table = {table:String}`, + { schema, table }, + ) + if (live.size === 0) { + await db.command(createTableSql(stream, schema)) + statements++ + for (const col of stream.columns) { + await recordColumn(db, schema, table, safeIdentifier(col.name), generatedColumnSql(col, stream)) + } + } else { + for (const col of stream.columns) { + if (live.has(safeIdentifier(col.name))) continue + // Computed for the rows already present, and written into their + // parts: left unmaterialised, every read of an old part would + // evaluate the expression again. + await db.command(addColumnSql(stream, col, schema)) + await db.command(materializeColumnSql(stream, col, schema)) + await recordColumn(db, schema, table, safeIdentifier(col.name), generatedColumnSql(col, stream)) + statements++ + } + const indexes = await names( + db, + `SELECT name FROM system.data_skipping_indices WHERE database = {schema:String} AND table = {table:String}`, + { schema, table }, + ) + for (const idx of indexSpecs(stream)) { + if (indexes.has(idx.name)) continue + await db.command(addIndexSql(stream, idx, schema)) + statements++ + } + } + // Last, so the view carries every column, including any added above. A + // view that is the generated shape and already does is left alone: + // `init` runs no DDL it does not need. Anything else under the name — + // a column short, or a definition that is not ours, however many + // columns it has — goes through `applyLiveView`, which refuses a + // foreign definition unless `repairViews` says to replace it. + if (stream.mode !== 'append' && live.size > 0) { + const [view] = await db.query<{ engine: string; as_select: string }>( + `SELECT engine, as_select FROM system.tables WHERE database = {schema:String} AND name = {name:String}`, + { schema, name: liveViewName(stream) }, + ) + if (view?.engine === 'View' && isGeneratedDefinition(view.as_select, schema, table)) { + // Name and type both: a view fixes its columns' types when it is + // created, so a column redefined since declares the old one there. + const viewColumns = await typedColumns(db, schema, liveViewName(stream)) + const tableColumns = await typedColumns(db, schema, table) + if ([...tableColumns].every(([c, t]) => viewColumns.get(c) === t)) continue + } + } + if (await applyLiveView(db, stream, schema, logger, options)) statements++ + } + + // Detection runs only on this path — once per catalog change, not on every + // restart — because a recorded fingerprint means it already came back clean. + let drift = await detectColumnDrift(db, streams, schema) + let repaired: ColumnDrift[] = [] + const failures = new Map() + if (options.repair && drift.some((d) => unrepairableReason(d) === null)) { + // A recorded fingerprint means "the live columns were checked against + // this catalog". Redefining a column breaks that promise for every + // catalog recorded before now, so every other fingerprint is forgotten, + // before the first statement rather than after the last, and a rollback + // re-runs detection. + await db.command( + `ALTER TABLE ${s}.\`_migrations\` DELETE WHERE fingerprint != {fp:String}`, + { fp }, + { ...KEEPER_SETTINGS, keeper_map_strict_mode: 0 }, + ) + const r = await repairColumnDrift(db, drift, streams, schema, logger) + repaired = r.repaired + for (const f of r.failed) failures.set(f.drift, f.error) + drift = drift.filter((d) => !repaired.includes(d)) + // A redefined column changes the type its view declares for it, and the + // view fixed that when it was created: the views of the streams touched + // are re-created now, after the repair rather than before it. + for (const name of new Set(repaired.map((d) => d.stream))) { + const stream = streams.find((st) => st.name === name) + if (stream && stream.mode !== 'append' && (await applyLiveView(db, stream, schema, logger, options))) statements++ + } + } + + for (const d of drift) { + const fields = { stream: d.stream, column: d.column, expected: d.expected, actual: d.actual, key: d.key } + if (d.kind === 'orphan') { + logger?.warn('orphan projected column', fields) + continue + } + const reason = failures.get(d) ?? unrepairableReason(d) ?? 'run `init --repair`, or apply the DDL by hand' + logger?.error('column drift: projection does not match the catalog', { ...fields, kind: d.kind, reason }) + } + + const unresolved = drift.filter((d) => d.kind !== 'orphan') + if (unresolved.length > 0) { + logger?.error('schema fingerprint not recorded: projected columns have drifted', { + fingerprint: fp, + columns: unresolved.length, + }) + return { applied: true, fingerprint: fp, statements, drift, repaired } + } + + await recordFingerprint(db, s, fp, JSON.stringify({ streams: streams.map((st) => st.name), statements })) + + logger?.info('schema applied', { fingerprint: fp, statements, repaired: repaired.length }) + return { applied: true, fingerprint: fp, statements, drift, repaired } +} diff --git a/packages/sync-clickhouse/src/schema/views.ts b/packages/sync-clickhouse/src/schema/views.ts new file mode 100644 index 0000000..26619af --- /dev/null +++ b/packages/sync-clickhouse/src/schema/views.ts @@ -0,0 +1,91 @@ +/** + * Live views. + * + * Every query against a mirror table has to remember `FINAL` and `WHERE + * _deleted_at IS NULL`, and forgetting either is the most common query + * mistake: `FINAL` left out counts every version a row ever had, the filter + * left out counts users who left. Ours are generated from the catalog, in + * the same pass as the tables, so they can never drift from them. + * + * ClickHouse fixes a view's column list when it is created, so a column + * promoted later only reaches the view once it is re-created; `migrate` + * does that whenever the catalog changes. The engine recognises its own + * views by the query `system.tables` stores for them: `select * from + *
final where _deleted_at is null` is ours and is replaced freely, + * anything else under that name belongs to somebody and is left alone + * unless `repairViews` says to replace it. + */ +import type { Logger, TableDef } from '@workos/sync-engine' +import type { ChClient } from '../db/client.js' +import { quoteIdent, safeIdentifier } from './identifiers.js' + +export const LIVE_VIEW_SUFFIX = '_live' + +/** `_live`, shortened the same way table names are. */ +export function liveViewName(stream: TableDef): string { + return safeIdentifier(`${stream.name}${LIVE_VIEW_SUFFIX}`) +} + +/** + * DDL for one stream's live view. Empty for append-only streams: they have + * no versions to collapse and no `_deleted_at` to filter, so a view would + * only suggest there was. + */ +export function liveViewSql(stream: TableDef, schema: string): string[] { + if (stream.mode === 'append') return [] + const s = quoteIdent(schema) + const table = quoteIdent(safeIdentifier(stream.name)) + const view = quoteIdent(liveViewName(stream)) + return [`CREATE OR REPLACE VIEW ${s}.${view} AS SELECT * FROM ${s}.${table} FINAL WHERE \`_deleted_at\` IS NULL`] +} + +export interface ApplyViewOptions { + /** + * Replace a view under a generated name that is not the generated shape. + * Off, such a view fails `init` and is reported, because replacing it + * would silently change what its users see. + */ + repairViews?: boolean +} + +/** Whether a stored view query is the generated shape over `table`. */ +export function isGeneratedDefinition(query: string, schema: string, table: string): boolean { + const normalized = query.replace(/`/g, '').replace(/\s+/g, ' ').trim().toLowerCase() + return normalized.endsWith(`from ${schema}.${table} final where _deleted_at is null`.toLowerCase()) +} + +/** + * Create or refresh a stream's live view, refusing to overwrite one that is + * not the engine's. Returns whether a statement ran. + */ +export async function applyLiveView( + db: ChClient, + stream: TableDef, + schema: string, + logger: Logger | undefined, + options: ApplyViewOptions = {}, +): Promise { + const [sql] = liveViewSql(stream, schema) + if (!sql) return false + const name = liveViewName(stream) + const existing = await db.query<{ engine: string; as_select: string }>( + `SELECT engine, as_select FROM system.tables WHERE database = {schema:String} AND name = {name:String}`, + { schema, name }, + ) + const found = existing[0] + if (found && found.engine !== 'View') { + throw new Error( + `${schema}.${name} is a table, where the engine generates a view over ${stream.name}: rename or drop it, or exclude the stream`, + ) + } + if (found && !isGeneratedDefinition(found.as_select, schema, safeIdentifier(stream.name))) { + if (!options.repairViews) { + throw new Error( + `${schema}.${name} is a view the engine did not generate; run init --repair to replace it with the generated definition, or rename it`, + ) + } + logger?.warn('replacing a view that is not the generated shape', { view: name, previous: found.as_select }) + } + await db.command(sql) + return true +} diff --git a/packages/sync-clickhouse/test/cli.test.ts b/packages/sync-clickhouse/test/cli.test.ts new file mode 100644 index 0000000..083d1cd --- /dev/null +++ b/packages/sync-clickhouse/test/cli.test.ts @@ -0,0 +1,339 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { pathToFileURL } from 'node:url' +import { SyncEngine, WorkOSClient, selectStreams, silentLogger } from '@workos/sync-engine' +import { + invokedDirectly, + main, + parseHydrate, + parseExcludeStreams, + parseLeaseTtl, + parseMaxPages, + parseSslCa, + printSchemaSql, + repairRequested, +} from '../src/cli.js' +import { fingerprint, migrate } from '../src/schema/migrate.js' +import { createTestDb, describeDb, type TestDb } from './helpers/clickhouse.js' +import { FaultyDestination } from './helpers/faulty.js' +import { TEST_CA_PEM } from './helpers/ca.js' +import { FakeWorkOS, resetIds } from '../../sync-engine/test/helpers/fake-workos.js' + +describe('entrypoint', () => { + let dir: string + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'workos-sync-clickhouse-bin-')) + }) + afterEach(() => { + rmSync(dir, { recursive: true, force: true }) + }) + + it('recognises the script itself and the symlink npm installs as the command', () => { + const script = join(dir, 'cli.js') + writeFileSync(script, '') + const url = pathToFileURL(script).href + expect(invokedDirectly(script, url)).toBe(true) + + // `npm install` links `node_modules/.bin/workos-sync-clickhouse` at dist/cli.js, + // so argv[1] is the link, not the file. + const link = join(dir, 'workos-sync-clickhouse') + symlinkSync(script, link) + expect(invokedDirectly(link, url)).toBe(true) + }) + + it('is false for an import, a missing path, or another script', () => { + const script = join(dir, 'cli.js') + writeFileSync(script, '') + const url = pathToFileURL(script).href + expect(invokedDirectly(undefined, url)).toBe(false) + expect(invokedDirectly(join(dir, 'missing'), url)).toBe(false) + const other = join(dir, 'vitest.mjs') + writeFileSync(other, '') + expect(invokedDirectly(other, url)).toBe(false) + // This test file was started by the test runner, not by the CLI. + expect(invokedDirectly(process.argv[1], url)).toBe(false) + }) +}) + +describe('WORKOS_SYNC_HYDRATE', () => { + it('leaves the engine default in place when unset', () => { + expect(parseHydrate(undefined)).toBeUndefined() + expect(parseHydrate('')).toBeUndefined() + }) + + it('accepts one mode for every stream, or per-stream overrides', () => { + expect(parseHydrate('always')).toBe('always') + expect(parseHydrate('users=off, organizations=off')).toEqual({ users: 'off', organizations: 'off' }) + }) + + it('rejects anything else loudly', () => { + expect(() => parseHydrate('sometimes')).toThrow(/invalid WORKOS_SYNC_HYDRATE/) + }) +}) + +describe('backfill --max-pages', () => { + it('means unbounded when absent and accepts a positive integer', () => { + expect(parseMaxPages(undefined)).toBeUndefined() + expect(parseMaxPages('25')).toBe(25) + }) + + it('refuses a bare flag, zero, a fraction, and anything that is not a number', () => { + for (const bad of [true, '', '0', '-3', '1.5', 'ten']) + expect(() => parseMaxPages(bad)).toThrow(/invalid --max-pages/) + }) +}) + +describe('reconcile --exclude-streams', () => { + it('means nothing excluded when absent and splits a list', () => { + expect(parseExcludeStreams(undefined)).toBeUndefined() + expect(parseExcludeStreams('users, invitations')).toEqual(['users', 'invitations']) + }) + + it('refuses a bare flag: an exclusion of nothing is the mistake it exists to prevent', () => { + expect(() => parseExcludeStreams(true)).toThrow(/invalid --exclude-streams/) + expect(() => parseExcludeStreams(' , ')).toThrow(/invalid --exclude-streams/) + }) +}) + +describe('init --repair', () => { + it('asks for repair on init and nothing else', () => { + expect(repairRequested('init', { repair: true })).toBe(true) + expect(repairRequested('reconcile', { repair: true })).toBe(false) + }) + + it('is in the help text', async () => { + const out: string[] = [] + const spy = vi.spyOn(process.stdout, 'write').mockImplementation(((chunk: string) => { + out.push(String(chunk)) + return true + }) as typeof process.stdout.write) + try { + expect(await main(['help'])).toBe(0) + } finally { + spy.mockRestore() + } + const help = out.join('') + expect(help).toContain('workos-sync-clickhouse') + expect(help).toContain('init [--repair] [--print-sql]') + expect(help).toContain('reconcile [--repair] [--resume] [--max-pages ] [--exclude-streams ]') + expect(help).toContain('WORKOS_SYNC_CLICKHOUSE_TLS_CA') + expect(help).not.toMatch(/MySQL|Postgres/) + }) +}) + +describe('WORKOS_SYNC_LEASE_TTL_MS', () => { + it('accepts a number of milliseconds and refuses a value that would expire on arrival', () => { + expect(parseLeaseTtl(undefined)).toBeUndefined() + expect(parseLeaseTtl('250')).toBe(250) + for (const bad of ['0', '-1', 'abc']) expect(() => parseLeaseTtl(bad)).toThrow(/invalid WORKOS_SYNC_LEASE_TTL_MS/) + }) +}) + +describe('WORKOS_SYNC_CLICKHOUSE_TLS_CA', () => { + it('accepts PEM as is or as base64, and refuses anything that is not a certificate', () => { + expect(parseSslCa(undefined)).toBeUndefined() + expect(parseSslCa(TEST_CA_PEM)).toBe(TEST_CA_PEM.trim()) + expect(parseSslCa(Buffer.from(TEST_CA_PEM).toString('base64'))).toBe(TEST_CA_PEM) + expect(() => parseSslCa('not a cert')).toThrow(/invalid WORKOS_SYNC_CLICKHOUSE_TLS_CA/) + }) +}) + +describe('init --print-sql', () => { + let out: string[] + let spy: ReturnType + + beforeEach(() => { + out = [] + spy = vi.spyOn(process.stdout, 'write').mockImplementation(((chunk: string) => { + out.push(String(chunk)) + return true + }) as typeof process.stdout.write) + }) + afterEach(() => { + spy.mockRestore() + }) + + it('prints the DDL init would run, without a database or an API key', async () => { + const saved = { url: process.env['DATABASE_URL'], key: process.env['WORKOS_API_KEY'] } + delete process.env['DATABASE_URL'] + delete process.env['WORKOS_API_KEY'] + try { + expect(await main(['init', '--print-sql'])).toBe(0) + } finally { + if (saved.url !== undefined) process.env['DATABASE_URL'] = saved.url + if (saved.key !== undefined) process.env['WORKOS_API_KEY'] = saved.key + } + const sql = out.join('') + expect(sql).toBe(printSchemaSql(undefined, 'workos')) + expect(sql).toContain('CREATE DATABASE IF NOT EXISTS `workos`') + expect(sql).toContain('CREATE TABLE IF NOT EXISTS `workos`.`users`') + expect(sql).toContain('CREATE OR REPLACE VIEW `workos`.`users_live`') + expect(sql).toContain(`fingerprint ${fingerprint(selectStreams(), 'workos')}`) + expect(sql).not.toMatch(/INSERT INTO [^\n]*_migrations/) + }) + + it('honors --schema and --streams the way init does', async () => { + expect(await main(['init', '--print-sql', '--schema', 'mirror', '--streams', 'users'])).toBe(0) + const sql = out.join('') + expect(sql).toContain('CREATE TABLE IF NOT EXISTS `mirror`.`users`') + expect(sql).toContain('CREATE TABLE IF NOT EXISTS `mirror`.`events`') + expect(sql).not.toContain('`organizations`') + }) + + it('refuses a stream the catalog does not know, like init', async () => { + await expect(main(['init', '--print-sql', '--streams', 'customers'])).rejects.toThrow(/unknown stream: customers/) + }) + + describeDb('applied by hand', () => { + let db: TestDb + beforeEach(async () => { + db = await createTestDb() + }) + afterEach(async () => { + await db.close() + }) + + it('prints a script a server accepts twice, after which init records the fingerprint', async () => { + expect(await main(['init', '--print-sql', '--schema', db.schema])).toBe(0) + const statements = out + .join('') + .split(/;\n\n/) + .map((s) => + s + .replace(/^--.*$/gm, '') + .trim() + .replace(/;$/, ''), + ) + .filter(Boolean) + for (const sql of statements) await db.command(sql) + for (const sql of statements) await db.command(sql) + const result = await migrate(db, selectStreams(), db.schema) + expect(result.applied).toBe(true) + expect(result.drift).toEqual([]) + const recorded = await db.query<{ fingerprint: string }>( + `SELECT fingerprint FROM \`${db.schema}\`.\`_migrations\``, + ) + expect(recorded).toEqual([{ fingerprint: fingerprint(selectStreams(), db.schema) }]) + }) + }) +}) + +describeDb('commands against a server', () => { + let db: TestDb + let fake: FakeWorkOS + let out: string[] + + beforeEach(async () => { + db = await createTestDb() + resetIds() + fake = new FakeWorkOS() + out = [] + vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => { + out.push(String(chunk)) + return true + }) + }) + afterEach(async () => { + vi.restoreAllMocks() + await db.close() + }) + + const lastJson = () => JSON.parse(out[out.length - 1]!) as Record + const makeEngine = (destination: FaultyDestination, streams = ['users']) => + new SyncEngine( + { + destination, + environments: [{ name: 'prod', apiKey: 'sk_test_1' }], + streams, + hydrate: 'off', + logger: silentLogger, + lagWindowMs: 0, + }, + { + clientFor: () => + new WorkOSClient({ apiKey: 'sk_test_1', apiUrl: 'http://fake.workos.local', fetchImpl: fake.fetchImpl }), + }, + ) + + it('exits 2 while quarantined events exist, and 0 once they replay', async () => { + fake.createUser({ email: 'good@example.com' }) + const bad = fake.createUser({ email: 'bad@example.com' }) + const destination = new FaultyDestination({ db, schema: db.schema }) + destination.poison = bad['id'] as string + const engine = makeEngine(destination) + + expect(await main(['rejected'], { engine })).toBe(0) + await engine.tick() + await engine.tick() + expect(await main(['rejected'], { engine })).toBe(2) + const listed = lastJson() as { prod: { eventId: string; eventType: string; error: string }[] } + expect(listed.prod).toHaveLength(1) + expect(listed.prod[0]!.error).toMatch(/Cannot parse string/) + + expect(await main(['rejected', '--replay'], { engine })).toBe(2) + destination.poison = null + expect(await main(['rejected', '--replay', '--id', listed.prod[0]!.eventId], { engine })).toBe(0) + expect(lastJson()).toEqual({ replayed: 1, stillRejected: [] }) + expect(await main(['rejected'], { engine })).toBe(0) + + const users = await db.query<{ email: string }>(`SELECT email FROM \`${db.schema}\`.users_live ORDER BY email`) + expect(users.map((r) => r.email)).toEqual(['bad@example.com', 'good@example.com']) + }) + + it('runs backfill, status, reconcile and streams end to end', async () => { + fake.createUser({ email: 'ada@example.com' }) + const engine = makeEngine(new FaultyDestination({ db, schema: db.schema }), ['users', 'organizations']) + + expect(await main(['backfill'], { engine })).toBe(0) + const backfill = lastJson() as unknown as { stream: string; complete: boolean }[] + expect(backfill.find((s) => s.stream === 'users')).toMatchObject({ complete: true }) + + expect(await main(['status'], { engine })).toBe(0) + const status = lastJson() as Record + expect(status['prod']!.streams.find((s) => s.stream === 'users')!.rows).toBe(1) + expect(status['prod']!.rejected).toBe(0) + + fake.users.clear() + expect(await main(['reconcile'], { engine })).toBe(2) + expect(await main(['reconcile', '--repair'], { engine })).toBe(0) + expect(await main(['reconcile'], { engine })).toBe(0) + + out = [] + expect(await main(['streams'])).toBe(0) + expect((JSON.parse(out.join('')) as { stream: string }[]).some((s) => s.stream === 'users')).toBe(true) + }) + + it('leaves out --exclude-streams on reconcile, and refuses a name the catalog does not know', async () => { + fake.createUser() + const engine = makeEngine(new FaultyDestination({ db, schema: db.schema }), ['users', 'organizations']) + await engine.backfill() + + expect(await main(['reconcile', '--exclude-streams', 'users'], { engine })).toBe(0) + const names = (lastJson() as { diffs: Record }).diffs['prod']!.map((d) => d.stream) + expect(names).not.toContain('users') + expect(names).toContain('organizations') + + await expect(main(['reconcile', '--exclude-streams', 'customers'], { engine })).rejects.toThrow( + /unknown stream: customers/, + ) + }) + + it('reports a stream reconcile abandoned as oversized, in the diff and in status', async () => { + for (let i = 0; i < 250; i++) fake.createUser() + const engine = makeEngine(new FaultyDestination({ db, schema: db.schema })) + await engine.backfill() + + // Three pages upstream, two budgeted: abandoned on the third, exit 0. + expect(await main(['reconcile', '--max-pages', '2'], { engine })).toBe(0) + const users = ( + lastJson() as { + diffs: Record + } + ).diffs['prod']!.find((d) => d.stream === 'users') + expect(users).toMatchObject({ pages: 3, complete: false, oversized: true }) + expect(await main(['status'], { engine })).toBe(0) + expect(lastJson()).toMatchObject({ prod: { oversized: ['users'] } }) + }) +}) diff --git a/packages/sync-clickhouse/test/clickhouse/config.xml b/packages/sync-clickhouse/test/clickhouse/config.xml new file mode 100644 index 0000000..0e27bdc --- /dev/null +++ b/packages/sync-clickhouse/test/clickhouse/config.xml @@ -0,0 +1,28 @@ + + + + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + 10000 + 30000 + + + 1localhost9234 + + + + localhost9181 + + /keeper_map_tables + disabled + diff --git a/packages/sync-clickhouse/test/clickhouse/users.xml b/packages/sync-clickhouse/test/clickhouse/users.xml new file mode 100644 index 0000000..a394182 --- /dev/null +++ b/packages/sync-clickhouse/test/clickhouse/users.xml @@ -0,0 +1,8 @@ + + + + + 0 + + + diff --git a/packages/sync-clickhouse/test/client.test.ts b/packages/sync-clickhouse/test/client.test.ts new file mode 100644 index 0000000..d030b4f --- /dev/null +++ b/packages/sync-clickhouse/test/client.test.ts @@ -0,0 +1,79 @@ +import { describe, it, expect } from 'vitest' +import { + clientConfig, + fromClient, + connect, + toCh, + fromCh, + isoOf, + APPLICATION_NAME, + DEFAULT_SETTINGS, +} from '../src/db/client.js' +import { describeDb, TEST_CLICKHOUSE_URL } from './helpers/clickhouse.js' + +describe('clientConfig', () => { + it('splits the URL into origin and credentials, and drops the path', () => { + const cfg = clientConfig('https://me:p%40ss@host.example:8443/some_db?x=1') + expect(cfg).toMatchObject({ + url: 'https://host.example:8443', + username: 'me', + password: 'p@ss', + application: APPLICATION_NAME, + }) + expect(cfg['tls']).toBeUndefined() + expect(cfg['clickhouse_settings']).toBe(DEFAULT_SETTINGS) + }) + + it('leaves credentials out when the URL has none', () => { + const cfg = clientConfig('http://127.0.0.1:8123') + expect(cfg['username']).toBeUndefined() + expect(cfg['password']).toBeUndefined() + }) + + it('verifies against a CA over https only', () => { + const cfg = clientConfig('https://host:8443', { sslCa: 'pem' }) + expect((cfg['tls'] as { ca_cert: Buffer }).ca_cert.toString()).toBe('pem') + expect(() => clientConfig('http://host:8123', { sslCa: 'pem' })).toThrow(/https/) + }) + + it('refuses a URL that is not the HTTP interface', () => { + expect(() => clientConfig('clickhouse://host:9000')).toThrow(/http:\/\/ or https:\/\//) + expect(() => clientConfig('not a url')).toThrow() + }) + + it('writes and reads DateTime64 in the one plain form', () => { + expect(toCh(new Date('2026-01-02T03:04:05.678Z'))).toBe('2026-01-02 03:04:05.678') + expect(fromCh('2026-01-02 03:04:05.678').toISOString()).toBe('2026-01-02T03:04:05.678Z') + expect(isoOf('2026-01-02 03:04:05.678123')).toBe('2026-01-02T03:04:05.678Z') + }) + + it('sends no request for an empty insert', async () => { + let inserted = 0 + const client = fromClient({ insert: async () => void inserted++ } as never) + await client.insert('t', []) + expect(inserted).toBe(0) + }) +}) + +describeDb('connect', () => { + it('connects, runs with the engine settings, and tolerates a path in the URL', async () => { + const url = new URL(TEST_CLICKHOUSE_URL!) + url.pathname = '/does_not_exist_' + Date.now() + const db = await connect(url.toString()) + try { + expect(await db.query<{ one: number }>('SELECT 1 AS one')).toEqual([{ one: 1 }]) + const settings = await db.query<{ name: string; value: string }>( + `SELECT name, value FROM system.settings WHERE name IN ('async_insert', 'wait_for_async_insert', 'mutations_sync', 'keeper_map_strict_mode') ORDER BY name`, + ) + expect(settings).toEqual([ + { name: 'async_insert', value: '1' }, + { name: 'keeper_map_strict_mode', value: '1' }, + { name: 'mutations_sync', value: '2' }, + { name: 'wait_for_async_insert', value: '1' }, + ]) + expect(await db.query<{ ua: string }>(`SELECT http_user_agent AS ua FROM system.query_log LIMIT 0`)).toEqual([]) + } finally { + await db.close() + } + }) +}) diff --git a/packages/sync-clickhouse/test/destination.test.ts b/packages/sync-clickhouse/test/destination.test.ts new file mode 100644 index 0000000..9abb901 --- /dev/null +++ b/packages/sync-clickhouse/test/destination.test.ts @@ -0,0 +1,32 @@ +import { describe, it, expect } from 'vitest' +import { ClickHouseDestination } from '../src/destination.js' +import type { ChClient } from '../src/db/client.js' + +const db = {} as ChClient + +describe('ClickHouseDestination options', () => { + it('needs a connection string or a client', () => { + expect(() => new ClickHouseDestination({})).toThrow(/databaseUrl or db is required/) + expect(new ClickHouseDestination({ db }).schema).toBe('workos') + expect(new ClickHouseDestination({ databaseUrl: 'https://u:p@host:8443', schema: 'mirror' }).schema).toBe('mirror') + }) + + it('takes a CA alongside an https connection string', () => { + expect(() => new ClickHouseDestination({ databaseUrl: 'https://host:8443', sslCa: 'pem' })).not.toThrow() + }) + + it('refuses a CA with a pre-built client, which carries its own TLS settings', () => { + expect(() => new ClickHouseDestination({ db, sslCa: 'pem' })).toThrow(/sslCa applies to databaseUrl/) + }) + + it('refuses a schema name that is not a plain identifier', () => { + expect(() => new ClickHouseDestination({ db, schema: '' })).toThrow(/plain identifier/) + expect(() => new ClickHouseDestination({ db, schema: 'with space' })).toThrow(/plain identifier/) + expect(() => new ClickHouseDestination({ db, schema: 'x'.repeat(64) })).toThrow(/plain identifier/) + }) + + it('refuses to be used before init', () => { + const dest = new ClickHouseDestination({ databaseUrl: 'http://localhost:8123' }) + expect(() => dest.getLease('prod')).toThrow(/call init\(\) first/) + }) +}) diff --git a/packages/sync-clickhouse/test/drift.test.ts b/packages/sync-clickhouse/test/drift.test.ts new file mode 100644 index 0000000..3f9e9db --- /dev/null +++ b/packages/sync-clickhouse/test/drift.test.ts @@ -0,0 +1,341 @@ +/** + * Projected-column drift, against a real ClickHouse. Detection compares the + * DDL the engine recorded in `_columns` with what the catalog would emit + * now, and the live type with the catalog's; only a real server can show + * that a repair redefines the column for the rows already there. + */ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { createTestDb, describeDb, type TestDb } from './helpers/clickhouse.js' +import { columnType } from '../src/schema/ddl.js' +import { migrate, fingerprint } from '../src/schema/migrate.js' +import { detectColumnDrift, unrepairableReason } from '../src/schema/drift.js' +import { main } from '../src/cli.js' +import { STREAMS, streamByName, type ColumnDef, type Logger, type StreamDef } from '@workos/sync-engine' + +const users = streamByName('users')! + +/** The stream with one column swapped out, the way a catalog release would change it. */ +const withColumn = (stream: StreamDef, col: ColumnDef): StreamDef => ({ + ...stream, + columns: stream.columns.map((c) => (c.name === col.name ? col : c)), +}) + +/** `email` projected from a different key: the classic path change. */ +const usersMovedEmail = withColumn(users, { name: 'email', type: 'text', path: 'primary_email', index: true }) + +const capture = () => { + const lines: { level: string; msg: string; fields?: Record }[] = [] + const at = (level: string) => (msg: string, fields?: Record) => { + lines.push({ level, msg, fields }) + } + const logger: Logger = { debug: at('debug'), info: at('info'), warn: at('warn'), error: at('error') } + return { logger, lines } +} + +describe('unrepairable reasons', () => { + it('names why a column is left alone', () => { + const base = { stream: 'users', column: 'x', expected: 'e', actual: 'a', key: false } + expect(unrepairableReason({ ...base, kind: 'orphan' })).toMatch(/no longer names/) + expect(unrepairableReason({ ...base, kind: 'not_generated' })).toMatch(/not a DEFAULT/) + expect(unrepairableReason({ ...base, kind: 'expression', key: true })).toMatch(/key columns/) + expect(unrepairableReason({ ...base, kind: 'expression' })).toBeNull() + expect(unrepairableReason({ ...base, kind: 'type' })).toBeNull() + }) +}) + +describeDb('column drift detection', () => { + let db: TestDb + + beforeEach(async () => { + db = await createTestDb() + }) + afterEach(async () => { + await db.close() + }) + + const insertUser = (raw: Record) => + db.insert(`\`${db.schema}\`.users`, [ + { _env: 'prod', _raw: JSON.stringify(raw), _updated_at: '2026-01-01 00:00:00.000' }, + ]) + const recorded = async (streams: StreamDef[]) => + ( + await db.query(`SELECT 1 FROM \`${db.schema}\`.\`_migrations\` WHERE fingerprint = {fp:String}`, { + fp: fingerprint(streams, db.schema), + }) + ).length > 0 + const columnExists = async (column: string) => + ( + await db.query( + `SELECT 1 FROM system.columns WHERE database = {s:String} AND table = 'users' AND name = {c:String}`, + { + s: db.schema, + c: column, + }, + ) + ).length > 0 + + it('reports nothing on a fresh schema built from the full catalog', async () => { + await migrate(db, STREAMS, db.schema) + expect(await detectColumnDrift(db, STREAMS, db.schema)).toEqual([]) + }) + + it('reports a changed path as expression drift, with both clauses', async () => { + await migrate(db, [users], db.schema) + const r = await migrate(db, [usersMovedEmail], db.schema) + expect(r.repaired).toEqual([]) + expect(r.drift).toHaveLength(1) + expect(r.drift[0]).toMatchObject({ stream: 'users', column: 'email', kind: 'expression', key: false }) + expect(r.drift[0]!.expected).toContain("'primary_email'") + expect(r.drift[0]!.actual).toContain("'email'") + }) + + it('reports a changed type as type drift', async () => { + await migrate(db, [users], db.schema) + const changed = withColumn(users, { name: 'first_name', type: 'boolean', path: 'first_name' }) + const r = await migrate(db, [changed], db.schema) + expect(r.drift).toHaveLength(1) + expect(r.drift[0]).toMatchObject({ column: 'first_name', kind: 'type', actual: 'Nullable(String)' }) + expect(r.drift[0]!.expected).toContain('`first_name` Nullable(Bool)') + }) + + it('withholds the fingerprint while drift is unrepaired, and records it once repaired', async () => { + await migrate(db, [users], db.schema) + await migrate(db, [usersMovedEmail], db.schema) + expect(await recorded([usersMovedEmail])).toBe(false) + const again = await migrate(db, [usersMovedEmail], db.schema) + expect(again.applied).toBe(true) + expect(again.drift).toHaveLength(1) + const fixed = await migrate(db, [usersMovedEmail], db.schema, undefined, { repair: true }) + expect(fixed.drift).toEqual([]) + expect(fixed.repaired).toHaveLength(1) + expect(await recorded([usersMovedEmail])).toBe(true) + expect((await migrate(db, [usersMovedEmail], db.schema)).applied).toBe(false) + }) + + it('repairs non-key drift in place and recomputes rows already present', async () => { + await migrate(db, [users], db.schema) + await insertUser({ id: 'user_1', email: 'old@example.com', primary_email: 'new@example.com', first_name: true }) + const changed = withColumn(usersMovedEmail, { name: 'first_name', type: 'boolean', path: 'first_name' }) + const r = await migrate(db, [changed], db.schema, undefined, { repair: true }) + expect(r.repaired.map((d) => [d.column, d.kind])).toEqual([ + ['email', 'expression'], + ['first_name', 'type'], + ]) + expect(r.drift).toEqual([]) + const row = await db.query<{ email: string; first_name: boolean }>( + `SELECT email, first_name FROM \`${db.schema}\`.users FINAL`, + ) + expect(row[0]).toEqual({ email: 'new@example.com', first_name: true }) + // The skipping index on the column survives a redefinition. + const idx = await db.query<{ name: string }>( + `SELECT name FROM system.data_skipping_indices WHERE database = {s:String} AND table = 'users'`, + { s: db.schema }, + ) + expect(idx.map((i) => i.name)).toContain('users_email_idx') + expect(await detectColumnDrift(db, [changed], db.schema)).toEqual([]) + expect(await recorded([changed])).toBe(true) + }) + + it('re-creates the live view after a repair changes a column type, so the view declares the new one', async () => { + await migrate(db, [users], db.schema) + const typeOf = async (table: string) => + ( + await db.query<{ type: string }>( + `SELECT type FROM system.columns WHERE database = {s:String} AND table = {t:String} AND name = 'first_name'`, + { s: db.schema, t: table }, + ) + )[0]?.type + expect(await typeOf('users_live')).toBe(await typeOf('users')) + const column = { name: 'first_name', type: 'boolean' as const, path: 'first_name' } + const r = await migrate(db, [withColumn(users, column)], db.schema, undefined, { repair: true }) + expect(r.repaired.map((d) => d.column)).toEqual(['first_name']) + expect(await typeOf('users')).toBe(columnType(column, false)) + expect(await typeOf('users_live')).toBe(await typeOf('users')) + }) + + it('judges an unrecorded column by the table itself, as after a schema applied by hand', async () => { + await migrate(db, [users], db.schema) + const forgetAll = async () => { + for (const t of ['_columns', '_migrations']) { + await db.command( + `ALTER TABLE \`${db.schema}\`.\`${t}\` DELETE WHERE 1`, + {}, + { + async_insert: 0, + keeper_map_strict_mode: 0, + }, + ) + } + } + // The same catalog again: every column checks out and is recorded. + await forgetAll() + const same = await migrate(db, [users], db.schema) + expect(same.drift).toEqual([]) + expect(await recorded([users])).toBe(true) + + // A changed path with no record to compare against. `ADD COLUMN IF NOT + // EXISTS` leaves the column as it was; the table says so, and that is + // drift to report, not a clause to record as installed. + await forgetAll() + await insertUser({ id: 'user_1', email: 'old@example.com', primary_email: 'new@example.com' }) + const moved = await migrate(db, [usersMovedEmail], db.schema) + expect(moved.drift.map((d) => [d.column, d.kind])).toEqual([['email', 'expression']]) + expect(await recorded([usersMovedEmail])).toBe(false) + const fixed = await migrate(db, [usersMovedEmail], db.schema, undefined, { repair: true }) + expect(fixed.drift).toEqual([]) + const row = await db.query<{ email: string }>(`SELECT email FROM \`${db.schema}\`.users FINAL`) + expect(row[0]!.email).toBe('new@example.com') + expect(await recorded([usersMovedEmail])).toBe(true) + }) + + it('refuses to repair a key column', async () => { + await migrate(db, [users], db.schema) + const rekeyed = withColumn(users, { name: 'id', type: 'text', path: 'uid' }) + const { logger, lines } = capture() + const r = await migrate(db, [rekeyed], db.schema, logger, { repair: true }) + expect(r.repaired).toEqual([]) + expect(r.drift).toHaveLength(1) + expect(r.drift[0]).toMatchObject({ column: 'id', kind: 'expression', key: true }) + expect(await recorded([rekeyed])).toBe(false) + const logged = lines.find((l) => l.level === 'error' && l.msg.startsWith('column drift')) + expect(String(logged?.fields?.['reason'])).toMatch(/key columns are never rebuilt/) + }) + + it('reports an orphan and leaves it alone, even when repairing', async () => { + const extended: StreamDef = { + ...users, + columns: [...users.columns, { name: 'nickname', type: 'text', path: 'nickname' }], + } + await migrate(db, [extended], db.schema) + const { logger, lines } = capture() + const r = await migrate(db, [users], db.schema, logger, { repair: true }) + expect(r.repaired).toEqual([]) + expect(r.drift).toHaveLength(1) + expect(r.drift[0]).toMatchObject({ + stream: 'users', + column: 'nickname', + kind: 'orphan', + expected: null, + key: false, + }) + expect(await columnExists('nickname')).toBe(true) + expect(lines.some((l) => l.level === 'warn' && l.msg === 'orphan projected column')).toBe(true) + expect(await recorded([users])).toBe(true) + }) + + it('forgets a record whose column was dropped by hand', async () => { + const extended: StreamDef = { + ...users, + columns: [...users.columns, { name: 'nickname', type: 'text', path: 'nickname' }], + } + await migrate(db, [extended], db.schema) + await db.command(`ALTER TABLE \`${db.schema}\`.users DROP COLUMN nickname`) + expect(await detectColumnDrift(db, [users], db.schema)).toEqual([]) + const left = await db.query(`SELECT 1 FROM \`${db.schema}\`.\`_columns\` WHERE column_name = 'nickname'`) + expect(left).toEqual([]) + }) + + it('reports a column of yours that is not a DEFAULT expression, and never touches it', async () => { + await migrate(db, [{ ...users, columns: users.columns.filter((c) => c.name !== 'locale') }], db.schema) + await db.command(`ALTER TABLE \`${db.schema}\`.users ADD COLUMN locale String`) + const r = await migrate(db, [users], db.schema, undefined, { repair: true }) + expect(r.repaired).toEqual([]) + expect(r.drift).toHaveLength(1) + expect(r.drift[0]).toMatchObject({ column: 'locale', kind: 'not_generated', actual: 'String' }) + expect(await recorded([users])).toBe(false) + }) + + it('reports a live type changed by hand, whatever the record says', async () => { + await migrate(db, [users], db.schema) + await db.command( + `ALTER TABLE \`${db.schema}\`.users MODIFY COLUMN locale String DEFAULT JSONExtractString(\`_raw\`, 'locale')`, + ) + const drift = await detectColumnDrift(db, [users], db.schema) + expect(drift).toHaveLength(1) + expect(drift[0]).toMatchObject({ column: 'locale', kind: 'type', actual: 'String' }) + await db.command( + `ALTER TABLE \`${db.schema}\`.\`_migrations\` DELETE WHERE 1`, + {}, + { async_insert: 0, keeper_map_strict_mode: 0 }, + ) + const r = await migrate(db, [users], db.schema, undefined, { repair: true }) + expect(r.repaired.map((d) => [d.column, d.kind])).toEqual([['locale', 'type']]) + expect(await detectColumnDrift(db, [users], db.schema)).toEqual([]) + expect(await recorded([users])).toBe(true) + }) + + it('forgets earlier fingerprints before a repair runs any DDL', async () => { + await migrate(db, [users], db.schema) + await migrate(db, [usersMovedEmail], db.schema, undefined, { repair: true }) + expect(await recorded([users])).toBe(false) + expect(await recorded([usersMovedEmail])).toBe(true) + // Rolled back to the old catalog: the live column is still the new one. + const r = await migrate(db, [users], db.schema) + expect(r.applied).toBe(true) + expect(r.drift.map((d) => [d.column, d.kind])).toEqual([['email', 'expression']]) + }) + + it('keeps earlier fingerprints when a change is only additive', async () => { + await migrate(db, [users], db.schema) + const extended: StreamDef = { + ...users, + columns: [...users.columns, { name: 'nickname', type: 'text', path: 'nickname' }], + } + await migrate(db, [extended], db.schema) + expect(await recorded([users])).toBe(true) + expect(await recorded([extended])).toBe(true) + }) +}) + +describeDb('init via the CLI', () => { + let db: TestDb + let out: string[] + + beforeEach(async () => { + db = await createTestDb() + out = [] + vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => { + out.push(String(chunk)) + return true + }) + process.env['WORKOS_API_KEY'] = 'sk_test_1' + process.env['WORKOS_SYNC_SCHEMA'] = db.schema + }) + afterEach(async () => { + vi.restoreAllMocks() + delete process.env['WORKOS_API_KEY'] + delete process.env['WORKOS_SYNC_SCHEMA'] + await db.close() + }) + + it('exits 2 on drift, repairs with --repair, then is clean', async () => { + expect(await main(['init', '--streams', 'users'], { db })).toBe(0) + expect(out.join('')).toContain('schema ready') + // A release moved a column: simulate by rewriting the record. + await db.command( + `ALTER TABLE \`${db.schema}\`.\`_columns\` UPDATE ddl = 'something else' WHERE column_name = 'email'`, + {}, + { async_insert: 0, keeper_map_strict_mode: 0 }, + ) + await db.command( + `ALTER TABLE \`${db.schema}\`.\`_migrations\` DELETE WHERE 1`, + {}, + { async_insert: 0, keeper_map_strict_mode: 0 }, + ) + + out = [] + expect(await main(['init', '--streams', 'users'], { db })).toBe(2) + const report = JSON.parse(out.join('')) as { drift: { column: string; kind: string }[]; repaired: unknown[] } + expect(report.drift.map((d) => [d.column, d.kind])).toEqual([['email', 'expression']]) + + out = [] + expect(await main(['init', '--streams', 'users', '--repair'], { db })).toBe(0) + const fixed = JSON.parse(out[0]!) as { drift: unknown[]; repaired: { column: string }[] } + expect(fixed.repaired.map((d) => d.column)).toEqual(['email']) + expect(out.join('')).toContain('schema ready') + + out = [] + expect(await main(['init', '--streams', 'users'], { db })).toBe(0) + expect(out.join('')).toBe('schema ready\n') + }) +}) diff --git a/packages/sync-clickhouse/test/engine.test.ts b/packages/sync-clickhouse/test/engine.test.ts new file mode 100644 index 0000000..4accbe1 --- /dev/null +++ b/packages/sync-clickhouse/test/engine.test.ts @@ -0,0 +1,673 @@ +/** + * Whole-engine tests: backfill, tail, drift and repair, driven against the + * fake WorkOS API and a real ClickHouse. The engine's own behaviour is + * covered exhaustively in `@workos/sync-postgres`; what is exercised here is + * every path through this destination, with the same fake, so that a + * ClickHouse-only difference in a write cannot hide behind a unit test. + */ +import { it, expect, beforeEach, afterEach } from 'vitest' +import { SyncEngine, WorkOSClient, LeaseHeldError, silentLogger } from '@workos/sync-engine' +import { createTestDb, describeDb, type TestDb } from './helpers/clickhouse.js' +import { FakeWorkOS, resetIds, id } from '../../sync-engine/test/helpers/fake-workos.js' +import { ClickHouseDestination } from '../src/destination.js' +import { FaultyDestination, captureLogger } from './helpers/faulty.js' + +const TIER1 = [ + 'users', + 'organizations', + 'organization_memberships', + 'organization_domains', + 'invitations', + 'events', + 'auth_attempts', +] + +function makeEngine(db: TestDb, fake: FakeWorkOS, streams = TIER1, extra = {}) { + return new SyncEngine( + { + destination: new ClickHouseDestination({ db, schema: db.schema }), + environments: [{ name: 'prod', apiKey: 'sk_test_1' }], + streams, + logger: silentLogger, + lagWindowMs: 0, + ...extra, + }, + { + clientFor: () => + new WorkOSClient({ + apiKey: 'sk_test_1', + apiUrl: 'http://fake.workos.local', + fetchImpl: fake.fetchImpl, + pageSize: 100, + }), + }, + ) +} + +const count = async (db: TestDb, table: string, final = true) => + Number( + ( + await db.query<{ n: string }>( + `SELECT toString(count()) AS n FROM \`${db.schema}\`.\`${table}\`${final ? ' FINAL' : ''}`, + ) + )[0]!.n, + ) +const emails = async (db: TestDb, table = 'users_live') => + (await db.query<{ email: string }>(`SELECT email FROM \`${db.schema}\`.\`${table}\` ORDER BY email`)).map( + (x) => x.email, + ) + +describeDb('sync engine', () => { + let db: TestDb + let fake: FakeWorkOS + + beforeEach(async () => { + db = await createTestDb() + resetIds() + fake = new FakeWorkOS() + }) + afterEach(async () => { + await db.close() + }) + + it('backfills existing state, including the field events omit', async () => { + const org = fake.createOrganization({ name: 'Acme' }) + const user = fake.createUser({ email: 'ada@acme.com' }) + fake.createMembership(user['id'] as string, org['id'] as string, 'admin') + const engine = makeEngine(db, fake) + await engine.backfill() + + expect(await count(db, 'users')).toBe(1) + expect(await count(db, 'organizations')).toBe(1) + expect(await count(db, 'organization_memberships')).toBe(1) + const u = await db.query<{ email: string; external_id: string; metadata: string }>( + `SELECT email, external_id, metadata FROM \`${db.schema}\`.users_live`, + ) + expect(u[0]!.email).toBe('ada@acme.com') + expect(u[0]!.external_id).toMatch(/^ext_/) + expect(JSON.parse(u[0]!.metadata)).toEqual({ plan: 'pro' }) + const o = await db.query<{ allow: boolean | null }>( + `SELECT allow_profiles_outside_organization AS allow FROM \`${db.schema}\`.organizations_live`, + ) + expect(o[0]!.allow).toBe(false) + const m = await db.query<{ role_slug: string }>( + `SELECT role_slug FROM \`${db.schema}\`.organization_memberships_live`, + ) + expect(m[0]!.role_slug).toBe('admin') + }) + + it('applies live events after backfill', async () => { + const user = fake.createUser({ email: 'before@example.com' }) + const engine = makeEngine(db, fake) + await engine.backfill() + fake.updateUser(user['id'] as string, { email: 'after@example.com' }) + fake.createOrganization({ name: 'New Co' }) + await engine.tick() + expect(await emails(db)).toEqual(['after@example.com']) + expect((await db.query<{ name: string }>(`SELECT name FROM \`${db.schema}\`.organizations_live`))[0]!.name).toBe( + 'New Co', + ) + }) + + it('does not null out backfilled fields when an event arrives', async () => { + const org = fake.createOrganization({ name: 'Acme' }) + const engine = makeEngine(db, fake) + await engine.backfill() + await engine.tick() + fake.updateOrganization(org['id'] as string, { name: 'Acme Corp' }) + await engine.tick() + const row = await db.query<{ name: string; allow: boolean | null }>( + `SELECT name, allow_profiles_outside_organization AS allow FROM \`${db.schema}\`.organizations_live`, + ) + expect(row[0]).toEqual({ name: 'Acme Corp', allow: false }) + }) + + it('is idempotent when the same events are delivered twice', async () => { + fake = new FakeWorkOS({ duplicateEvents: true }) + fake.createUser() + fake.createOrganization() + const engine = makeEngine(db, fake) + await engine.tick() + await engine.tick() + expect(await count(db, 'users')).toBe(1) + expect(await count(db, 'organizations')).toBe(1) + expect(await count(db, 'events', false)).toBe(2) + }) + + it('converges regardless of the order events are applied in', async () => { + const user = fake.createUser({ email: 'v1@example.com' }) + fake.updateUser(user['id'] as string, { email: 'v2@example.com' }) + fake.updateUser(user['id'] as string, { email: 'v3@example.com' }) + const shuffled = new FakeWorkOS({ shuffleEvents: true }) + shuffled.users = fake.users + shuffled.events = fake.events + const engine = makeEngine(db, shuffled) + await engine.tick() + expect(await emails(db)).toEqual(['v3@example.com']) + }) + + it('tombstones a deleted user, drops it from the live view, and keeps the base row', async () => { + const gone = fake.createUser({ email: 'gone@example.com' }) + fake.createUser({ email: 'stays@example.com' }) + const engine = makeEngine(db, fake) + await engine.backfill() + await engine.tick() + expect(await count(db, 'users_live')).toBe(2) + fake.deleteUser(gone['id'] as string) + await engine.tick() + expect(await count(db, 'users')).toBe(2) + expect(await count(db, 'users_live')).toBe(1) + expect(await emails(db)).toEqual(['stays@example.com']) + }) + + it('records sign-in attempts that exist nowhere else in the API', async () => { + const user = fake.createUser() + fake.emitAuthAttempt(user['id'] as string, true) + fake.emitAuthAttempt(user['id'] as string, false, 'password') + const engine = makeEngine(db, fake) + await engine.tick() + expect(await count(db, 'auth_attempts', false)).toBe(2) + const failed = await db.query<{ n: string }>( + `SELECT toString(countIf(status = 'failed')) AS n FROM \`${db.schema}\`.auth_attempts`, + ) + expect(Number(failed[0]!.n)).toBe(1) + }) + + it('resumes from the stored cursor rather than replaying', async () => { + fake.createUser() + const engine = makeEngine(db, fake) + await engine.tick() + const first = await engine.status() + fake.createUser() + await engine.tick() + expect((await engine.status())['prod']!.cursor).not.toBe(first['prod']!.cursor) + expect(await count(db, 'users')).toBe(2) + expect((await makeEngine(db, fake).tick()).processed).toBe(0) + }) + + it('explodes organization domains out of the organization payload', async () => { + const org = fake.createOrganization({ name: 'Acme' }) + fake.createDomain(org['id'] as string, 'acme.com') + const engine = makeEngine(db, fake) + await engine.backfill() + const d = await db.query<{ domain: string; organization_id: string }>( + `SELECT domain, organization_id FROM \`${db.schema}\`.organization_domains_live`, + ) + expect(d).toEqual([{ domain: 'acme.com', organization_id: org['id'] }]) + }) + + it('never mirrors invitation acceptance tokens, however the row arrives', async () => { + const org = fake.createOrganization() + fake.createInvitation(org['id'] as string, { email: 'new@example.com' }) + const engine = makeEngine(db, fake) + await engine.backfill() + await engine.tick() + const rows = await db.query<{ raw: string }>(`SELECT \`_raw\` AS raw FROM \`${db.schema}\`.invitations_live`) + expect(rows).toHaveLength(1) + expect(JSON.parse(rows[0]!.raw)).not.toHaveProperty('token') + const archived = await db.query<{ raw: string }>( + `SELECT \`_raw\` AS raw FROM \`${db.schema}\`.events WHERE event LIKE 'invitation.%'`, + ) + for (const e of archived) expect(JSON.parse(e.raw)['data']).not.toHaveProperty('token') + }) + + it('keeps two environments apart in one database', async () => { + fake.createUser({ email: 'prod@example.com' }) + const other = new FakeWorkOS() + other.createUser({ email: 'staging@example.com' }) + const engine = new SyncEngine( + { + destination: new ClickHouseDestination({ db, schema: db.schema }), + environments: [ + { name: 'prod', apiKey: 'sk_prod' }, + { name: 'staging', apiKey: 'sk_staging' }, + ], + streams: ['users'], + logger: silentLogger, + lagWindowMs: 0, + }, + { + clientFor: (env) => + new WorkOSClient({ + apiKey: env.apiKey, + apiUrl: 'http://fake.workos.local', + fetchImpl: env.apiKey === 'sk_prod' ? fake.fetchImpl : other.fetchImpl, + }), + }, + ) + await engine.backfill() + const rows = await db.query<{ _env: string; email: string }>( + `SELECT \`_env\`, email FROM \`${db.schema}\`.users_live ORDER BY \`_env\``, + ) + expect(rows).toEqual([ + { _env: 'prod', email: 'prod@example.com' }, + { _env: 'staging', email: 'staging@example.com' }, + ]) + expect(Object.keys(await engine.status()).sort()).toEqual(['prod', 'staging']) + }) + + it('projects extra columns out of _raw without touching the catalog', async () => { + fake.createUser({ email: 'ada@example.com', metadata: { plan: 'pro', seats: 12 } }) + const engine = makeEngine(db, fake, TIER1, { + extraColumns: { + users: [ + { name: 'plan', type: 'text', path: ['metadata', 'plan'], index: true }, + { name: 'seats', type: 'bigint', path: ['metadata', 'seats'] }, + ], + }, + }) + await engine.backfill() + const r = await db.query<{ plan: string; seats: string }>(`SELECT plan, seats FROM \`${db.schema}\`.users_live`) + expect(r[0]!.plan).toBe('pro') + expect(Number(r[0]!.seats)).toBe(12) + const idx = await db.query<{ name: string }>( + `SELECT name FROM system.data_skipping_indices WHERE database = {s:String} AND table = 'users'`, + { s: db.schema }, + ) + expect(idx.map((i) => i.name)).toContain('users_plan_idx') + }) + + it('lands a payload a typed column cannot cast, with the column NULL and the value in _raw', async () => { + fake.createUser({ email: 'good@example.com', metadata: { beta: true } }) + fake.createUser({ email: 'bad@example.com', metadata: { beta: 'maybe' } }) + const engine = makeEngine(db, fake, TIER1, { + extraColumns: { users: [{ name: 'beta', type: 'boolean', path: ['metadata', 'beta'] }] }, + }) + expect(await engine.tick()).toMatchObject({ processed: 2, rejected: 0 }) + const rows = await db.query<{ email: string; beta: boolean | null; raw: string }>( + `SELECT email, beta, \`_raw\` AS raw FROM \`${db.schema}\`.users_live ORDER BY email`, + ) + expect(rows.map((r) => [r.email, r.beta])).toEqual([ + ['bad@example.com', null], + ['good@example.com', true], + ]) + expect(JSON.parse(rows[0]!.raw)['metadata']['beta']).toBe('maybe') + }) + + it('joins mirrored identity data with the sign-in log', async () => { + const org = fake.createOrganization({ name: 'Acme' }) + const user = fake.createUser({ email: 'ada@acme.com' }) + fake.createMembership(user['id'] as string, org['id'] as string, 'admin') + fake.emitAuthAttempt(user['id'] as string, false) + fake.emitAuthAttempt(user['id'] as string, true) + const engine = makeEngine(db, fake) + await engine.backfill() + await engine.tick() + const r = await db.query<{ email: string; organization: string; failures: string }>( + `SELECT u.email AS email, o.name AS organization, toString(countIf(a.status = 'failed')) AS failures + FROM \`${db.schema}\`.auth_attempts a + JOIN \`${db.schema}\`.users_live u ON u.id = a.user_id AND u.\`_env\` = a.\`_env\` + JOIN \`${db.schema}\`.organization_memberships_live m ON m.user_id = u.id AND m.\`_env\` = u.\`_env\` + JOIN \`${db.schema}\`.organizations_live o ON o.id = m.organization_id AND o.\`_env\` = m.\`_env\` + GROUP BY email, organization`, + ) + expect(r).toEqual([{ email: 'ada@acme.com', organization: 'Acme', failures: '1' }]) + }) +}) + +describeDb('reconciliation', () => { + let db: TestDb + let fake: FakeWorkOS + + beforeEach(async () => { + db = await createTestDb() + resetIds() + fake = new FakeWorkOS() + }) + afterEach(async () => { + await db.close() + }) + + it('reports a clean mirror as matching with no drift', async () => { + fake.createUser() + fake.createOrganization() + const engine = makeEngine(db, fake) + await engine.backfill() + const { diffs } = await engine.reconcile() + expect(diffs['prod']!.find((d) => d.stream === 'users')).toMatchObject({ + matched: 1, + localOnly: [], + remoteOnly: [], + }) + }) + + it('detects a missed delete and a missed create, and repairs both', async () => { + const stale = fake.createUser({ email: 'gone@example.com' }) + const engine = makeEngine(db, fake) + await engine.backfill() + fake.users.delete(stale['id'] as string) + const added = fake.createUser({ email: 'added@example.com' }) + fake.events = [] + const first = await engine.reconcile() + const users = first.diffs['prod']!.find((d) => d.stream === 'users')! + expect(users.localOnly).toEqual([stale['id']]) + expect(users.remoteOnly).toEqual([added['id']]) + await engine.reconcile({ repair: true }) + expect(await emails(db)).toEqual(['added@example.com']) + const tombstoned = await db.query<{ email: string }>( + `SELECT email FROM \`${db.schema}\`.users FINAL WHERE \`_deleted_at\` IS NOT NULL`, + ) + expect(tombstoned[0]!.email).toBe('gone@example.com') + expect((await engine.reconcile()).diffs['prod']!.find((d) => d.stream === 'users')).toMatchObject({ + localOnly: [], + remoteOnly: [], + }) + }) + + it('keeps an oversized verdict across runs, and forgets it once the stream completes', async () => { + for (let i = 0; i < 250; i++) fake.createUser() + const engine = makeEngine(db, fake, ['users', 'invitations']) + await engine.backfill() + + // Three pages upstream and two budgeted: abandoned on the third page, + // with the verdict written to the stream's state in `_sync`. + const first = await engine.reconcile({ maxPages: 2 }) + expect(first.diffs['prod']!.find((d) => d.stream === 'users')).toMatchObject({ + pages: 3, + complete: false, + oversized: true, + }) + expect(await engine.status()).toMatchObject({ prod: { oversized: ['users'] } }) + + // A resumed slice under the same budget reads the verdict back and skips the stream. + const second = await engine.reconcile({ resume: true, maxPages: 2 }) + expect(second.diffs['prod']!.find((d) => d.stream === 'users')).toMatchObject({ pages: 0, oversized: true }) + expect(second.diffs['prod']!.find((d) => d.stream === 'invitations')).toMatchObject({ complete: true }) + + // A larger budget diffs it in full and clears the verdict. + const third = await engine.reconcile({ resume: true, maxPages: 3 }) + expect(third.diffs['prod']!.find((d) => d.stream === 'users')).toMatchObject({ complete: true, matched: 250 }) + expect(await engine.status()).toMatchObject({ prod: { oversized: [] } }) + }) + + it('reports freshness and lag per stream', async () => { + fake.createUser() + const engine = makeEngine(db, fake) + await engine.backfill() + const status = await engine.status() + const users = status['prod']!.streams.find((s) => s.stream === 'users')! + expect(users.rows).toBe(1) + expect(users.lastSyncedAt).toBeTruthy() + expect(users.lagSeconds).not.toBeNull() + }) + + it('refreshes fields that event payloads omit', async () => { + fake.createOrganization() + const engine = makeEngine(db, fake) + await engine.tick() + const read = async () => + ( + await db.query<{ allow: boolean | null }>( + `SELECT allow_profiles_outside_organization AS allow FROM \`${db.schema}\`.organizations_live`, + ) + )[0]!.allow + expect(await read()).toBeNull() + await engine.refresh() + expect(await read()).toBe(false) + }) +}) + +describeDb('single-writer lease', () => { + let db: TestDb + let fake: FakeWorkOS + + beforeEach(async () => { + db = await createTestDb() + resetIds() + fake = new FakeWorkOS() + }) + afterEach(async () => { + await db.close() + }) + + it('tick() refuses to run while another process holds the lease', async () => { + fake.createUser() + const holder = makeEngine(db, fake) + await holder.init() + const dest = new ClickHouseDestination({ db, schema: db.schema }) + await dest.init([]) + expect((await dest.acquireLease('prod', 'someone-else', 60_000)).acquired).toBe(true) + await expect(holder.tick()).rejects.toBeInstanceOf(LeaseHeldError) + await dest.releaseLease('prod', 'someone-else') + expect((await holder.tick()).processed).toBe(1) + expect(await dest.getLease('prod')).toBeNull() + }) + + it('listen() waits for the lease, takes over when it is released, and gives it back on stop', async () => { + fake.createUser() + const dest = new ClickHouseDestination({ db, schema: db.schema }) + await dest.init([]) + await dest.acquireLease('prod', 'old-process', 60_000) + const engine = makeEngine(db, fake, TIER1, { pollIntervalMs: 20 }) + await engine.init() + const listening = engine.listen() + await new Promise((r) => setTimeout(r, 200)) + expect(await count(db, 'users')).toBe(0) + await dest.releaseLease('prod', 'old-process') + const deadline = Date.now() + 10_000 + while (Date.now() < deadline && (await count(db, 'users')) < 1) await new Promise((r) => setTimeout(r, 50)) + expect(await count(db, 'users')).toBe(1) + expect((await dest.getLease('prod'))?.holder).not.toBe('old-process') + engine.stop() + await listening + expect(await dest.getLease('prod')).toBeNull() + }) +}) + +describeDb('redaction', () => { + let db: TestDb + let fake: FakeWorkOS + + beforeEach(async () => { + db = await createTestDb() + resetIds() + fake = new FakeWorkOS() + }) + afterEach(async () => { + await db.close() + }) + + it('erases a subject from the mirror, the sign-in log and the event archive', async () => { + const user = fake.createUser({ email: 'forget@example.com' }) + fake.emitAuthAttempt(user['id'] as string, true) + // Invitations name a user as the one who accepted and as the inviter. + const org = fake.createOrganization() + const accepted = fake.createInvitation(org['id'] as string) + fake.acceptInvitation(accepted['id'] as string, user['id'] as string) + fake.createInvitation(org['id'] as string, { inviter_user_id: user['id'] }) + fake.createInvitation(org['id'] as string) + const engine = makeEngine(db, fake) + await engine.backfill() + await engine.tick() + const removed = await engine.redact(user['id'] as string) + expect(removed).toContainEqual({ table: 'users', rows: 1 }) + expect(removed).toContainEqual({ table: 'auth_attempts', rows: 1 }) + expect(removed).toContainEqual({ table: 'invitations', rows: 2 }) + expect(await count(db, 'invitations')).toBe(1) + expect(removed.find((r) => r.table === 'events')!.rows).toBeGreaterThan(0) + expect(await count(db, 'users', false)).toBe(0) + const events = await db.query<{ n: string }>( + `SELECT toString(count()) AS n FROM \`${db.schema}\`.events WHERE \`_raw\` LIKE {p:String}`, + { p: `%${user['id']}%` }, + ) + expect(Number(events[0]!.n)).toBe(0) + }) +}) + +describeDb('poison events', () => { + let db: TestDb + let fake: FakeWorkOS + + beforeEach(async () => { + db = await createTestDb() + resetIds() + fake = new FakeWorkOS() + }) + afterEach(async () => { + await db.close() + }) + + const eventFor = (objectId: string) => fake.events.find((e) => e.data['id'] === objectId)! + + function makeFaulty(extra: Record = {}) { + const destination = new FaultyDestination({ db, schema: db.schema }) + const { logger, lines } = captureLogger() + const engine = new SyncEngine( + { + destination, + environments: [{ name: 'prod', apiKey: 'sk_test_1' }], + streams: TIER1, + hydrate: 'off', + logger, + lagWindowMs: 0, + ...extra, + }, + { + clientFor: () => + new WorkOSClient({ + apiKey: 'sk_test_1', + apiUrl: 'http://fake.workos.local', + fetchImpl: fake.fetchImpl, + maxRetries: 0, + }), + }, + ) + return { engine, destination, lines } + } + + it('quarantines one event after two sightings and keeps the rest of the environment moving', async () => { + fake.createUser({ email: 'good1@example.com' }) + const bad = fake.createUser({ email: 'bad@example.com' }) + fake.createUser({ email: 'good2@example.com' }) + const badEvent = eventFor(bad['id'] as string) + const { engine, destination, lines } = makeFaulty() + destination.poison = bad['id'] as string + + expect(await engine.tick()).toMatchObject({ processed: 0, rejected: 0 }) + expect(await count(db, '_rejected')).toBe(0) + expect(lines.find((l) => l.level === 'warn' && l.fields['eventId'] === badEvent.id)?.msg).toMatch( + /quarantined if it fails again/, + ) + + expect(await engine.tick()).toMatchObject({ processed: 3, rejected: 1 }) + const status = await engine.status() + expect(status['prod']!.cursor).toBe(fake.events[fake.events.length - 1]!.id) + expect(status['prod']!.rejected).toBe(1) + expect(await emails(db)).toEqual(['good1@example.com', 'good2@example.com']) + const rejected = (await engine.rejected())['prod']! + expect(rejected).toHaveLength(1) + expect(rejected[0]).toMatchObject({ + eventId: badEvent.id, + eventType: 'user.created', + objectId: bad['id'], + raw: badEvent, + }) + expect(rejected[0]!.error).toMatch(/Cannot parse string/) + expect((await engine.tick()).processed).toBe(0) + expect(await count(db, 'events', false)).toBe(3) + }) + + it('replays a quarantined event through the normal path once the cause is fixed', async () => { + fake.createUser({ email: 'good@example.com' }) + const bad = fake.createUser({ email: 'bad@example.com' }) + const { engine, destination } = makeFaulty() + destination.poison = bad['id'] as string + await engine.tick() + await engine.tick() + expect((await engine.status())['prod']!.rejected).toBe(1) + destination.fault = () => Object.assign(new Error('still broken'), { code: '36', type: 'BAD_ARGUMENTS' }) + const again = await engine.replayRejected() + expect(again.replayed).toBe(0) + expect(again.stillRejected).toEqual([{ eventId: eventFor(bad['id'] as string).id, error: 'Error: still broken' }]) + destination.poison = null + expect(await engine.replayRejected()).toEqual({ replayed: 1, stillRejected: [] }) + expect(await count(db, '_rejected')).toBe(0) + expect(await emails(db)).toEqual(['bad@example.com', 'good@example.com']) + }) + + it('never quarantines on an error the destination classifies as transient', async () => { + fake.createUser({ email: 'a@example.com' }) + const { engine, destination } = makeFaulty() + destination.failAll = true + destination.fault = () => + Object.assign(new Error('Keeper session expired'), { code: '999', type: 'KEEPER_EXCEPTION' }) + for (let i = 0; i < 3; i++) expect((await engine.tick()).processed).toBe(0) + expect(await count(db, '_rejected')).toBe(0) + destination.failAll = false + expect((await engine.tick()).processed).toBe(1) + }) + + it('does not resurrect a membership removed after a quarantined link', async () => { + const at = (ms: number) => new Date(Date.now() - 50_000 + ms).toISOString() + const pair = { group_id: 'group_01', organization_membership_id: 'om_01' } + const push = (event: string, created_at: string) => { + const e = { id: id('event_'), event, data: pair, created_at } + fake.events.push(e) + return e + } + push('group.member_added', at(0)) + push('group.member_removed', at(1_000)) + const readd = push('group.member_added', at(2_000)) + push('group.member_removed', at(3_000)) + const { engine, destination } = makeFaulty({ streams: ['group_memberships', 'events'] }) + destination.poison = readd.id + await engine.tick() + expect(await engine.tick()).toMatchObject({ processed: 4, rejected: 1 }) + expect(await count(db, 'group_memberships')).toBe(1) + expect(await count(db, 'group_memberships_live')).toBe(0) + destination.poison = null + expect(await engine.replayRejected()).toEqual({ replayed: 1, stillRejected: [] }) + expect(await count(db, 'group_memberships_live')).toBe(0) + }) + + it('does not hide a membership re-added after a quarantined removal', async () => { + const at = (ms: number) => new Date(Date.now() - 50_000 + ms).toISOString() + const pair = { group_id: 'group_01', organization_membership_id: 'om_01' } + const push = (event: string, created_at: string) => { + const e = { id: id('event_'), event, data: pair, created_at } + fake.events.push(e) + return e + } + push('group.member_added', at(0)) + const removal = push('group.member_removed', at(1_000)) + push('group.member_added', at(2_000)) + const { engine, destination } = makeFaulty({ streams: ['group_memberships', 'events'] }) + destination.poison = removal.id + await engine.tick() + expect(await engine.tick()).toMatchObject({ processed: 3, rejected: 1 }) + expect(await count(db, 'group_memberships_live')).toBe(1) + destination.poison = null + expect(await engine.replayRejected()).toEqual({ replayed: 1, stillRejected: [] }) + expect(await count(db, 'group_memberships_live')).toBe(1) + push('group.member_removed', at(3_000)) + expect(await engine.tick()).toMatchObject({ processed: 1, rejected: 0 }) + expect(await count(db, 'group_memberships_live')).toBe(0) + }) + + it('quarantines an event whose own timestamp cannot be stored, with no fault injected', async () => { + const bad = fake.createUser({ email: 'bad@example.com' }) + const badEvent = eventFor(bad['id'] as string) + badEvent.created_at = '2026-13-01T00:00:00Z' + fake.createUser({ email: 'good@example.com' }) + const { engine } = makeFaulty() + expect((await engine.tick()).processed).toBe(0) + expect(await engine.tick()).toMatchObject({ processed: 2, rejected: 1 }) + const rejected = (await engine.rejected())['prod']! + expect(rejected[0]).toMatchObject({ eventId: badEvent.id, createdAt: '2026-13-01T00:00:00Z' }) + expect(rejected[0]!.error).toMatch(/invalid updatedAt timestamp/) + expect(await emails(db)).toEqual(['good@example.com']) + }) + + it('is scrubbed by redact like the event archive', async () => { + fake.createUser({ email: 'keep@example.com' }) + const subject = fake.createUser({ email: 'forget@example.com' }) + const { engine, destination } = makeFaulty() + destination.poison = subject['id'] as string + await engine.tick() + await engine.tick() + expect(await count(db, '_rejected')).toBe(1) + const removed = await engine.redact(subject['id'] as string) + expect(removed).toContainEqual({ table: '_rejected', rows: 1 }) + expect(await count(db, '_rejected')).toBe(0) + }) +}) diff --git a/packages/sync-clickhouse/test/errors.test.ts b/packages/sync-clickhouse/test/errors.test.ts new file mode 100644 index 0000000..e9938be --- /dev/null +++ b/packages/sync-clickhouse/test/errors.test.ts @@ -0,0 +1,49 @@ +import { describe, it, expect } from 'vitest' +import { isTransientClickHouseError, isKeeperConflict } from '../src/db/errors.js' + +const err = (fields: Record, message = 'x') => Object.assign(new Error(message), fields) + +describe('isTransientClickHouseError', () => { + it('calls timeouts, lost connections, resource pressure and Keeper races transient', () => { + for (const code of ['159', '209', '210', '241', '252', '319', '394', '999', '242']) { + expect(isTransientClickHouseError(err({ code, type: 'SOMETHING' }))).toBe(true) + } + }) + + it('calls verdicts on the statement deterministic: bad values, missing objects, syntax, access', () => { + const cases: [string, string][] = [ + ['6', 'CANNOT_PARSE_TEXT'], + ['27', 'CANNOT_PARSE_INPUT_ASSERTION_FAILED'], + ['36', 'BAD_ARGUMENTS'], + ['47', 'UNKNOWN_IDENTIFIER'], + ['60', 'UNKNOWN_TABLE'], + ['62', 'SYNTAX_ERROR'], + ['497', 'ACCESS_DENIED'], + ] + for (const [code, type] of cases) expect(isTransientClickHouseError(err({ code, type }))).toBe(false) + }) + + it('recognises socket errors and the client-side failures that carry no code', () => { + for (const code of ['ECONNRESET', 'ECONNREFUSED', 'ETIMEDOUT', 'EPIPE']) { + expect(isTransientClickHouseError(err({ code }))).toBe(true) + } + expect(isTransientClickHouseError(new Error('socket hang up'))).toBe(true) + expect(isTransientClickHouseError(new Error('Timeout error.'))).toBe(true) + }) + + it('presumes anything it cannot classify is deterministic', () => { + expect(isTransientClickHouseError(new Error('plain'))).toBe(false) + expect(isTransientClickHouseError(err({ code: 'INVALID_TIMESTAMP' }))).toBe(false) + expect(isTransientClickHouseError(null)).toBe(false) + expect(isTransientClickHouseError('ECONNRESET')).toBe(false) + }) +}) + +describe('isKeeperConflict', () => { + it('names the two strict-mode refusals and nothing else', () => { + expect(isKeeperConflict(err({ code: '999' }, 'Transaction failed (Node exists): Op #0'))).toBe(true) + expect(isKeeperConflict(err({ code: '999' }, 'Transaction failed (Bad version): Op #0'))).toBe(true) + expect(isKeeperConflict(err({ code: '999' }, 'Session expired'))).toBe(false) + expect(isKeeperConflict(err({ code: '6' }, 'Node exists'))).toBe(false) + }) +}) diff --git a/packages/sync-clickhouse/test/helpers/ca.ts b/packages/sync-clickhouse/test/helpers/ca.ts new file mode 100644 index 0000000..0c48621 --- /dev/null +++ b/packages/sync-clickhouse/test/helpers/ca.ts @@ -0,0 +1,17 @@ +/** + * A self-signed certificate for tests that need a value `X509Certificate` + * accepts. Generated once with `openssl req -x509 -newkey ec ...` and its + * private key discarded on the spot, so it can sign nothing and is not a + * secret; it names no real host and is trusted by nothing. + */ +export const TEST_CA_PEM = `-----BEGIN CERTIFICATE----- +MIIBdTCCARygAwIBAgIUYA/j8vmXLoOxBg17MVsxr0WptZgwCgYIKoZIzj0EAwIw +IDEeMBwGA1UEAwwVc3luYy1wb3N0Z3JlcyB0ZXN0IENBMCAXDTI2MTAyOTE2NDE1 +OVoYDzIxMjYxMDA1MTY0MTU5WjAgMR4wHAYDVQQDDBVzeW5jLXBvc3RncmVzIHRl +c3QgQ0EwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAR/tIk0cVsFVh49bOYbZ2UB +blKkXKfCfgj3+6A5B7t+dz+phLfY0PJe/RfKCPnbvQI4ZDO8jkLctAjItiiNjbBm +ozIwMDAdBgNVHQ4EFgQUdosu7nttrBB5We4MUhtGRm7Q/gEwDwYDVR0TAQH/BAUw +AwEB/zAKBggqhkjOPQQDAgNHADBEAiBWaRIW/Ef4zN9nmK01dsPVor+p06ihoxO8 +G/oBUsR6zAIgOJSTKCW/ymLt0xjT42M7xm2AoJximsh0cy3Wbk+mohU= +-----END CERTIFICATE----- +` diff --git a/packages/sync-clickhouse/test/helpers/clickhouse.ts b/packages/sync-clickhouse/test/helpers/clickhouse.ts new file mode 100644 index 0000000..a6c6664 --- /dev/null +++ b/packages/sync-clickhouse/test/helpers/clickhouse.ts @@ -0,0 +1,49 @@ +/** + * A real ClickHouse for the tests, one throwaway database per test. + * + * There is no embedded ClickHouse with a Keeper in it, and KeeperMap needs + * one, so the suite needs a server: `TEST_CLICKHOUSE_URL=http://default@127.0.0.1:8123` + * against a server started with `test/clickhouse/config.xml`. Without it the + * database-backed tests skip themselves (`describeDb`), and + * `pnpm test:database` refuses to run. + * + * Each test gets a database named after a random suffix and drops it on + * `close()`, so tests can run in parallel workers against one server. The + * KeeperMap tables keep their Keeper paths under the database's name, so + * two tests never share a lease. + */ +import { randomBytes } from 'node:crypto' +import { describe } from 'vitest' +import { connect, type ChClient } from '../../src/db/client.js' + +export const TEST_CLICKHOUSE_URL = process.env['TEST_CLICKHOUSE_URL'] + +/** `describe` when a server is configured, `describe.skip` otherwise. */ +export const describeDb = (name: string, fn: () => void): void => { + if (TEST_CLICKHOUSE_URL) describe(name, fn) + else describe.skip(name, fn) +} + +export interface TestDb extends ChClient { + /** The throwaway database this test owns; pass it as the destination's `schema`. */ + schema: string +} + +export async function createTestDb(): Promise { + if (!TEST_CLICKHOUSE_URL) throw new Error('TEST_CLICKHOUSE_URL is not set') + const schema = `workos_test_${randomBytes(6).toString('hex')}` + const client = await connect(TEST_CLICKHOUSE_URL) + return { + schema, + query: client.query.bind(client), + command: client.command.bind(client), + insert: client.insert.bind(client), + async close() { + try { + await client.command(`DROP DATABASE IF EXISTS \`${schema}\` SYNC`) + } finally { + await client.close() + } + }, + } +} diff --git a/packages/sync-clickhouse/test/helpers/faulty.ts b/packages/sync-clickhouse/test/helpers/faulty.ts new file mode 100644 index 0000000..3f1b87c --- /dev/null +++ b/packages/sync-clickhouse/test/helpers/faulty.ts @@ -0,0 +1,65 @@ +/** + * Test doubles for the poison-event quarantine. + * + * `FaultyDestination` is the real ClickHouse destination with one write + * fault injected: it stands in for a write a payload cannot survive, or a + * router bug on one shape, without having to construct the offending data + * for every scenario. The error it throws is configurable so the same + * double can imitate a deterministic failure, a dropped connection, or a + * socket error. + */ +import type { Logger, Row, TableDef, WriteMode, WriteStats } from '@workos/sync-engine' +import { ClickHouseDestination } from '../../src/destination.js' + +export class FaultyDestination extends ClickHouseDestination { + /** Object id whose mirror write fails. `null` means no fault. */ + poison: string | null = null + /** Fail every write, as an outage would. */ + failAll = false + /** + * Fail only multi-row writes: the batch attempt fails, every event succeeds + * alone. That is what an interaction between rows, or a transient error the + * classifier does not know, looks like to the quarantine. + */ + failBatches = false + /** What the fault throws; defaults to the shape of a ClickHouse parse error. */ + fault: () => Error = () => + Object.assign(new Error("Cannot parse string 'maybe' as DateTime64: syntax error"), { + code: '6', + type: 'CANNOT_PARSE_TEXT', + }) + /** How many times the fault fired, so a test can prove isolation happened. */ + faults = 0 + + override async upsert(stream: TableDef, env: string, rows: Row[], mode: WriteMode): Promise { + const hit = + this.failAll || + (this.failBatches && rows.length > 1) || + (this.poison !== null && rows.some((r) => r.data['id'] === this.poison)) + if (hit) { + this.faults++ + throw this.fault() + } + return super.upsert(stream, env, rows, mode) + } +} + +export interface LogLine { + level: 'debug' | 'info' | 'warn' | 'error' + msg: string + fields: Record +} + +/** A logger that keeps every line, so a test can assert on what was said. */ +export function captureLogger(): { logger: Logger; lines: LogLine[] } { + const lines: LogLine[] = [] + const at = + (level: LogLine['level']) => + (msg: string, fields: Record = {}) => { + lines.push({ level, msg, fields }) + } + return { + lines, + logger: { debug: at('debug'), info: at('info'), warn: at('warn'), error: at('error') }, + } +} diff --git a/packages/sync-clickhouse/test/lease.test.ts b/packages/sync-clickhouse/test/lease.test.ts new file mode 100644 index 0000000..555e330 --- /dev/null +++ b/packages/sync-clickhouse/test/lease.test.ts @@ -0,0 +1,246 @@ +/** + * Lease and cursor semantics against a real ClickHouse Keeper, through + * KeeperMap: the compare-and-swap the engine's fenced checkpoint relies on, + * exercised with real races. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { streamByName } from '@workos/sync-engine' +import { createTestDb, describeDb, type TestDb } from './helpers/clickhouse.js' +import { ClickHouseDestination } from '../src/destination.js' +import { canonicalJson } from '../src/db/lease.js' + +describe('canonical json', () => { + it('serialises equal values equally whatever the key order', () => { + expect(canonicalJson({ b: 1, a: { d: [1, { z: 1, y: 2 }], c: null } })).toBe( + canonicalJson({ a: { c: null, d: [1, { y: 2, z: 1 }] }, b: 1 }), + ) + expect(canonicalJson({ a: 1, b: undefined })).toBe('{"a":1}') + expect(canonicalJson([3, 1])).toBe('[3,1]') + }) +}) + +describeDb('events tail lease', () => { + let db: TestDb + let dest: ClickHouseDestination + + beforeEach(async () => { + db = await createTestDb() + dest = new ClickHouseDestination({ db, schema: db.schema }) + await dest.init([streamByName('users')!]) + }) + afterEach(async () => { + await db.close() + }) + + const expire = () => + db.command( + `ALTER TABLE \`${db.schema}\`.\`_sync\` UPDATE expires_at = now64(3, 'UTC') - toIntervalSecond(1) WHERE holder != ''`, + {}, + { async_insert: 0, keeper_map_strict_mode: 0 }, + ) + + it('retries a checkpoint that lost a Keeper version race, and still refuses a cursor that moved', async () => { + await dest.acquireLease('prod', 'a', 60_000) + await dest.setState('prod', '__tail__', { cursor: 1 }) + // Refuses the first checkpoint write the way Keeper does when the row's + // version moved under it — the holder's own renewal is enough — then + // passes everything through. + let refused = 0 + const racing: TestDb = { + ...db, + async command(sql, params, settings) { + if (refused === 0 && /UPDATE states =/.test(sql)) { + refused++ + throw Object.assign(new Error('Code: 999. DB::Exception: Bad version. (KEEPER_EXCEPTION)'), { code: '999' }) + } + return db.command(sql, params, settings) + }, + } + const viaRace = new ClickHouseDestination({ db: racing, schema: db.schema }) + expect(await viaRace.setStateIfHeld('prod', '__tail__', { cursor: 2 }, 'a', { cursor: 1 })).toBe(true) + expect(refused).toBe(1) + expect(await dest.getState('prod', '__tail__')).toEqual({ cursor: 2 }) + + refused = 0 + await dest.setState('prod', '__tail__', { cursor: 5 }) + expect(await viaRace.setStateIfHeld('prod', '__tail__', { cursor: 3 }, 'a', { cursor: 2 })).toBe(false) + expect(await dest.getState('prod', '__tail__')).toEqual({ cursor: 5 }) + }) + + it('checkpoints while the holder renews, without a refusal', async () => { + await dest.acquireLease('prod', 'a', 60_000) + let cursor = 0 + await dest.setState('prod', '__tail__', { cursor }) + for (let i = 0; i < 10; i++) { + const [, ok] = await Promise.all([ + dest.acquireLease('prod', 'a', 60_000), + dest.setStateIfHeld('prod', '__tail__', { cursor: cursor + 1 }, 'a', { cursor }), + ]) + expect(ok).toBe(true) + cursor++ + } + expect(await dest.getState('prod', '__tail__')).toEqual({ cursor: 10 }) + }) + + it('is taken by the first process to ask', async () => { + const r = await dest.acquireLease('prod', 'a', 60_000) + expect(r.acquired).toBe(true) + expect(r.holder).toBe('a') + expect(Date.parse(r.expiresAt!)).toBeGreaterThan(Date.now() + 30_000) + expect(r.expiresAt).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/) + expect(await dest.getLease('prod')).toEqual({ holder: 'a', expiresAt: r.expiresAt }) + }) + + it('refuses a second process and tells it who holds the lease', async () => { + const first = await dest.acquireLease('prod', 'a', 60_000) + const second = await dest.acquireLease('prod', 'b', 60_000) + expect(second).toEqual({ acquired: false, holder: 'a', expiresAt: first.expiresAt }) + expect(await dest.getLease('prod')).toEqual({ holder: 'a', expiresAt: first.expiresAt }) + }) + + it('renews for the holder without changing hands', async () => { + const first = await dest.acquireLease('prod', 'a', 1_000) + const renewed = await dest.acquireLease('prod', 'a', 60_000) + expect(renewed.acquired).toBe(true) + expect(renewed.holder).toBe('a') + expect(Date.parse(renewed.expiresAt!)).toBeGreaterThan(Date.parse(first.expiresAt!)) + // acquired_at marks the original takeover, not the latest heartbeat. + const row = await db.query<{ same: number }>( + `SELECT acquired_at < expires_at - toIntervalSecond(30) AS same FROM \`${db.schema}\`.\`_sync\``, + ) + expect(Number(row[0]!.same)).toBe(1) + }) + + it('passes to a new process once it has expired', async () => { + const stale = await dest.acquireLease('prod', 'a', 0) + expect(stale.acquired).toBe(true) + await new Promise((r) => setTimeout(r, 20)) + expect(await dest.getLease('prod')).toBeNull() + const takeover = await dest.acquireLease('prod', 'b', 60_000) + expect(takeover.acquired).toBe(true) + expect(takeover.holder).toBe('b') + expect((await dest.acquireLease('prod', 'a', 60_000)).holder).toBe('b') + }) + + it('judges expiry by the database clock', async () => { + await dest.acquireLease('prod', 'a', 60_000) + await expire() + expect(await dest.getLease('prod')).toBeNull() + expect((await dest.acquireLease('prod', 'b', 60_000)).acquired).toBe(true) + }) + + it('is freed by release, and the cursors survive it', async () => { + await dest.acquireLease('prod', 'a', 60_000) + await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c1' }, 'a', null) + await dest.releaseLease('prod', 'a') + expect(await dest.getLease('prod')).toBeNull() + expect(await dest.getState('prod', '__tail__')).toEqual({ cursor: 'c1' }) + expect((await dest.acquireLease('prod', 'b', 60_000)).acquired).toBe(true) + }) + + it('ignores a release from anyone but the holder', async () => { + const r = await dest.acquireLease('prod', 'a', 60_000) + await dest.releaseLease('prod', 'b') + await dest.releaseLease('prod', 'a-but-older') + expect(await dest.getLease('prod')).toEqual({ holder: 'a', expiresAt: r.expiresAt }) + }) + + it('is scoped to the sync id and the environment', async () => { + await dest.acquireLease('prod', 'a', 60_000) + const other = new ClickHouseDestination({ db, schema: db.schema, syncId: 'reporting' }) + expect((await other.acquireLease('prod', 'b', 60_000)).acquired).toBe(true) + expect((await dest.acquireLease('staging', 'c', 60_000)).acquired).toBe(true) + expect(await dest.getLease('prod')).toMatchObject({ holder: 'a' }) + }) + + it('gives exactly one of many simultaneous contenders the lease', async () => { + const holders = Array.from({ length: 8 }, (_, i) => `p${i}`) + const results = await Promise.all(holders.map((h) => dest.acquireLease('prod', h, 60_000))) + const winners = results.filter((r) => r.acquired) + expect(winners).toHaveLength(1) + const winner = winners[0]!.holder + for (const r of results) if (!r.acquired) expect(r.holder).toBe(winner) + expect(await dest.getLease('prod')).toMatchObject({ holder: winner }) + }) + + it('gives exactly one of many simultaneous contenders an expired lease', async () => { + await dest.acquireLease('prod', 'old', 60_000) + await expire() + const results = await Promise.all(Array.from({ length: 8 }, (_, i) => dest.acquireLease('prod', `p${i}`, 60_000))) + expect(results.filter((r) => r.acquired)).toHaveLength(1) + }) + + describe('fences the state write', () => { + const stored = () => dest.getState>('prod', '__tail__') + + it('writes for the holder, and refuses anyone else', async () => { + await dest.acquireLease('prod', 'a', 60_000) + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c1' }, 'b', null)).toBe(false) + expect(await stored()).toBeNull() + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c1' }, 'a', null)).toBe(true) + expect(await stored()).toEqual({ cursor: 'c1' }) + }) + + it('refuses a holder whose lease has expired or been taken over', async () => { + await dest.acquireLease('prod', 'a', 60_000) + await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c1' }, 'a', null) + await expire() + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c2' }, 'a', { cursor: 'c1' })).toBe(false) + await dest.acquireLease('prod', 'b', 60_000) + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c2' }, 'a', { cursor: 'c1' })).toBe(false) + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c2' }, 'b', { cursor: 'c1' })).toBe(true) + expect(await stored()).toEqual({ cursor: 'c2' }) + }) + + it('refuses a write whose expected state is no longer the stored one', async () => { + await dest.acquireLease('prod', 'a', 60_000) + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c1' }, 'a', { cursor: 'c0' })).toBe(false) + expect(await stored()).toBeNull() + await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c1' }, 'a', null) + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c2' }, 'a', null)).toBe(false) + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c2' }, 'a', { cursor: 'c0' })).toBe(false) + expect(await stored()).toEqual({ cursor: 'c1' }) + // Equality is semantic, not textual: key order does not matter. + await dest.setState('prod', '__tail__', { cursor: 'c1', lastEventAt: null }) + expect( + await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c2' }, 'a', { lastEventAt: null, cursor: 'c1' }), + ).toBe(true) + expect(await stored()).toEqual({ cursor: 'c2' }) + }) + + it('refuses a checkpoint that races a cursor moved by someone else', async () => { + await dest.acquireLease('prod', 'a', 60_000) + await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c1' }, 'a', null) + // Two writers with the same expectation: only one can be right. + const [x, y] = await Promise.all([ + dest.setStateIfHeld('prod', '__tail__', { cursor: 'x' }, 'a', { cursor: 'c1' }), + dest.setStateIfHeld('prod', '__tail__', { cursor: 'y' }, 'a', { cursor: 'c1' }), + ]) + expect([x, y].filter(Boolean)).toHaveLength(1) + expect(await stored()).toEqual({ cursor: x ? 'x' : 'y' }) + }) + + it('keeps other streams’ cursors when writing one', async () => { + await dest.acquireLease('prod', 'a', 60_000) + await dest.setState('prod', 'users', { kind: 'backfill', status: 'complete' }) + expect(await dest.setStateIfHeld('prod', '__tail__', { cursor: 'c1' }, 'a', null)).toBe(true) + expect(await dest.getState('prod', 'users')).toEqual({ kind: 'backfill', status: 'complete' }) + expect(await stored()).toEqual({ cursor: 'c1' }) + }) + }) + + describe('plain state', () => { + it('round-trips JSON and overwrites in place', async () => { + expect(await dest.getState('prod', 'users')).toBeNull() + await dest.setState('prod', 'users', { kind: 'backfill', status: 'running', after: null }) + await dest.setState('prod', 'users', { kind: 'backfill', status: 'complete', count: 3 }) + expect(await dest.getState('prod', 'users')).toEqual({ kind: 'backfill', status: 'complete', count: 3 }) + expect(await dest.getState('staging', 'users')).toBeNull() + }) + + it('survives concurrent writers to different streams', async () => { + await Promise.all(Array.from({ length: 6 }, (_, i) => dest.setState('prod', `stream_${i}`, { i }))) + for (let i = 0; i < 6; i++) expect(await dest.getState('prod', `stream_${i}`)).toEqual({ i }) + }) + }) +}) diff --git a/packages/sync-clickhouse/test/lock.test.ts b/packages/sync-clickhouse/test/lock.test.ts new file mode 100644 index 0000000..2ad54ee --- /dev/null +++ b/packages/sync-clickhouse/test/lock.test.ts @@ -0,0 +1,133 @@ +import { it, expect, beforeEach, afterEach } from 'vitest' +import { streamByName } from '@workos/sync-engine' +import { createTestDb, describeDb, type TestDb } from './helpers/clickhouse.js' +import { migrate } from '../src/schema/migrate.js' +import { acquireStreamLock, releaseStreamLock, withStreamLock, lockKey } from '../src/db/lock.js' +import { upsertRows, deleteRows } from '../src/db/upsert.js' +import { isTransientClickHouseError } from '../src/db/errors.js' + +const users = streamByName('users')! +const events = streamByName('events')! + +describeDb('per-stream write lock', () => { + let db: TestDb + const key = lockKey('users', 'prod') + + beforeEach(async () => { + db = await createTestDb() + await migrate(db, [users, events], db.schema) + }) + afterEach(async () => { + await db.close() + }) + + it('admits one holder at a time, and the next once it is released', async () => { + const a = { schema: db.schema, holder: 'a' } + const b = { schema: db.schema, holder: 'b', waitMs: 300 } + const token = await acquireStreamLock(db, a, key) + const err = await withStreamLock(db, b, key, async () => 'ran').catch((e: unknown) => e) + expect(err).toMatchObject({ code: 'LOCK_TIMEOUT' }) + expect(isTransientClickHouseError(err)).toBe(true) + await releaseStreamLock(db, a, key, token) + expect(await withStreamLock(db, b, key, async () => 'ran')).toBe('ran') + }) + + it("is taken over once a crashed holder's lock expires, and not before", async () => { + await acquireStreamLock(db, { schema: db.schema, holder: 'crashed', ttlMs: 400 }, key) + const started = Date.now() + expect(await withStreamLock(db, { schema: db.schema, holder: 'next', waitMs: 5_000 }, key, async () => 'ran')).toBe( + 'ran', + ) + expect(Date.now() - started).toBeGreaterThanOrEqual(300) + }) + + it('refuses to write once a stalled batch has lost its lock', async () => { + const err = await withStreamLock(db, { schema: db.schema, holder: 'stalled', ttlMs: 300 }, key, async (held) => { + await new Promise((r) => setTimeout(r, 400)) + // Someone else took the expired lock and finished; the stalled batch must not write. + await withStreamLock(db, { schema: db.schema, holder: 'next' }, key, async () => {}) + await held() + }).catch((e: unknown) => e) + expect(err).toMatchObject({ code: 'LOCK_LOST' }) + expect(isTransientClickHouseError(err)).toBe(true) + }) + + it('is per table and per environment', async () => { + const a = { schema: db.schema, holder: 'a', waitMs: 300 } + await acquireStreamLock(db, a, key) + expect(await withStreamLock(db, a, lockKey('users', 'staging'), async () => 'ran')).toBe('ran') + expect(await withStreamLock(db, a, lockKey('events', 'prod'), async () => 'ran')).toBe('ran') + }) + + it('never lets two critical sections overlap, even within one process', async () => { + let active = 0 + let overlaps = 0 + let runs = 0 + await Promise.all( + Array.from({ length: 8 }, () => + withStreamLock(db, { schema: db.schema, holder: 'same' }, key, async () => { + if (active++ > 0) overlaps++ + await new Promise((r) => setTimeout(r, 20)) + active-- + runs++ + }), + ), + ) + expect(runs).toBe(8) + expect(overlaps).toBe(0) + }) + + it('serialises concurrent writers so an append-only event lands once', async () => { + const rows = Array.from({ length: 20 }, (_, i) => ({ + data: { id: `event_${i}`, event: 'user.created', created_at: '2026-01-01T00:00:00.000Z', data: {} }, + updatedAt: '2026-01-01T00:00:00.000Z', + })) + // Six polls delivering the same page at once: a lease takeover's worst case. + await Promise.all( + Array.from({ length: 6 }, (_, p) => + upsertRows(db, { schema: db.schema, lockHolder: `poll${p}` }, events, 'prod', rows, 'merge'), + ), + ) + const n = await db.query<{ n: string }>( + `SELECT toString(count()) AS n FROM \`${db.schema}\`.events WHERE \`_env\` = 'prod'`, + ) + expect(Number(n[0]!.n)).toBe(20) + }) + + it('keeps a tombstone that a concurrent snapshot write would otherwise outrank', async () => { + const payload = { object: 'user', id: 'user_1', email: 'ada@example.com', updated_at: '2026-01-01T00:00:00.000Z' } + const at = '2026-01-01T00:00:00.000Z' + await upsertRows(db, { schema: db.schema }, users, 'prod', [{ data: payload, updatedAt: at }], 'replace') + // A refresh re-listing the same snapshot with a field events omit, while + // the tail deletes the user. Both versions carry the same `_updated_at`, + // so unserialised the tie would go to whichever insert landed last. + await Promise.all([ + deleteRows(db, { schema: db.schema, lockHolder: 'tail' }, users, 'prod', [ + { data: { id: 'user_1' }, updatedAt: '2026-01-01T00:00:00.500Z' }, + ]), + upsertRows( + db, + { schema: db.schema, lockHolder: 'refresh' }, + users, + 'prod', + [{ data: { ...payload, last_sign_in_at: at }, updatedAt: at }], + 'replace', + ), + ]) + expect(await db.query(`SELECT id FROM \`${db.schema}\`.users_live`)).toEqual([]) + }) + it('versions a tombstone at the deletion time, so an older snapshot cannot outrank it however late it lands', async () => { + const at = '2026-01-01T00:00:00.000Z' + const payload = { object: 'user', id: 'user_1', email: 'ada@example.com', updated_at: at } + await upsertRows(db, { schema: db.schema }, users, 'prod', [{ data: payload, updatedAt: at }], 'replace') + await deleteRows(db, { schema: db.schema }, users, 'prod', [ + { data: { id: 'user_1' }, updatedAt: '2026-01-01T00:00:01.000Z' }, + ]) + // The row as it was before the deletion, landing after the tombstone: + // what a writer that got past the lock would insert. + await db.insert(`\`${db.schema}\`.users`, [ + { _env: 'prod', _raw: JSON.stringify(payload), _updated_at: '2026-01-01 00:00:00.000', _deleted_at: null }, + ]) + expect(await db.query(`SELECT id FROM \`${db.schema}\`.users_live`)).toEqual([]) + }) +}) diff --git a/packages/sync-clickhouse/test/reconcile.test.ts b/packages/sync-clickhouse/test/reconcile.test.ts new file mode 100644 index 0000000..73ceb7e --- /dev/null +++ b/packages/sync-clickhouse/test/reconcile.test.ts @@ -0,0 +1,232 @@ +/** + * The reconcile set difference, computed in ClickHouse rather than in Node. + * + * These exercise the database half on its own, with the remote side handed + * in as pages. The engine half — pagination, repair — is covered by the + * drift and repair cases in engine.test.ts. + */ +import { it, expect, beforeEach, afterEach } from 'vitest' +import { diffIdsInMemory, diffStream, streamByName, WorkOSClient } from '@workos/sync-engine' +import { createTestDb, describeDb, type TestDb } from './helpers/clickhouse.js' +import { migrate } from '../src/schema/migrate.js' +import { upsertRows, deleteRows } from '../src/db/upsert.js' +import { diffIdsInClickHouse } from '../src/db/reconcile.js' +import { ClickHouseDestination } from '../src/destination.js' + +const users = streamByName('users')! +const STREAMS = ['users', 'organizations', 'organization_roles'].map((n) => streamByName(n)!) + +// Zero-padded so lexical order (what the SQL sorts by) is numeric order. +const user = (n: number, over: Record = {}) => ({ + object: 'user', + id: `user_${String(n).padStart(5, '0')}`, + email: `u${n}@example.com`, + created_at: '2026-01-01T00:00:00.000Z', + updated_at: '2026-01-01T00:00:00.000Z', + ...over, +}) + +const range = (from: number, to: number) => Array.from({ length: to - from + 1 }, (_, i) => user(from + i)) + +async function* pages(rows: Record[], size = 100) { + for (let i = 0; i < rows.length; i += size) yield rows.slice(i, i + size) +} + +describeDb('diff in sql', () => { + let db: TestDb + let opts: { schema: string } + + beforeEach(async () => { + db = await createTestDb() + opts = { schema: db.schema } + await migrate(db, STREAMS, db.schema) + }) + afterEach(async () => { + await db.close() + }) + + const seed = (rows: Record[], env = 'prod') => + upsertRows( + db, + opts, + users, + env, + rows.map((data) => ({ data, updatedAt: data['updated_at'] as string })), + 'replace', + ) + + it('reports matched, localOnly and remoteOnly, with the payload intact', async () => { + await seed([user(1), user(2), user(3)]) + const four = user(4, { email: 'four@example.com', metadata: { plan: 'pro' } }) + const diff = await diffIdsInClickHouse(db, opts, users, 'prod', pages([user(2), user(3), four])) + expect(diff.matched).toBe(2) + expect(diff.localOnly).toEqual(['user_00001']) + expect(diff.remoteOnly).toEqual([four]) + expect(diff.skipped).toBe(0) + }) + + it('compares complete composite keys in SQL and in memory', async () => { + const destination = new ClickHouseDestination({ db, schema: db.schema }) + const roles = streamByName('organization_roles')! + const removed = { organization_id: 'org_1', slug: 'admin' } + const kept = { organization_id: 'org_1', slug: 'auditor' } + const elsewhere = { organization_id: 'org_2', slug: 'admin' } + const added = { organization_id: 'org_1', slug: 'member', permissions: ['read'] } + await destination.upsert( + roles, + 'prod', + [removed, kept, elsewhere].map((data) => ({ data, updatedAt: '2026-01-01T00:00:00.000Z' })), + 'replace', + ) + const remote = [ + kept, + elsewhere, + added, + { ...added, permissions: [] }, + { organization_id: 'org_1' }, + { slug: 'admin' }, + { organization_id: 'org_1', slug: '' }, + ] + const diff = await diffIdsInClickHouse(db, opts, roles, 'prod', pages(remote, 1)) + expect(diff).toEqual({ matched: 2, localOnly: ['["org_1","admin"]'], remoteOnly: [added], skipped: 3 }) + expect(await destination.liveIds(roles, 'prod')).toEqual([ + '["org_1","admin"]', + '["org_1","auditor"]', + '["org_2","admin"]', + ]) + expect(await diffIdsInMemory(destination, roles, 'prod', pages(remote, 1))).toEqual({ + matched: diff.matched, + localOnly: diff.localOnly, + remoteOnly: diff.remoteOnly, + }) + }) + + it('repairs a missing role and a missed deletion without touching sibling roles', async () => { + const destination = new ClickHouseDestination({ db, schema: db.schema }) + const roles = streamByName('organization_roles')! + const removed = { organization_id: 'org_1', slug: 'admin' } + const kept = { organization_id: 'org_1', slug: 'auditor' } + const added = { organization_id: 'org_1', slug: 'member' } + const updatedAt = '2026-01-01T00:00:00.000Z' + await destination.upsert(streamByName('organizations')!, 'prod', [{ data: { id: 'org_1' }, updatedAt }], 'replace') + await destination.upsert( + roles, + 'prod', + [removed, kept].map((data) => ({ data, updatedAt })), + 'replace', + ) + const api = new WorkOSClient({ + apiKey: 'sk_test_1', + fetchImpl: async (input) => { + expect(new URL(String(input)).pathname).toBe('/authorization/organizations/org_1/roles') + return Response.json({ data: [kept, added] }) + }, + }) + const result = await diffStream(destination, api, 'prod', roles, { repair: true }) + expect(result).toMatchObject({ + matched: 1, + localOnly: ['["org_1","admin"]'], + remoteOnly: ['["org_1","member"]'], + repaired: { deleted: 1, inserted: 1 }, + }) + const rows = await db.query<{ slug: string; deleted: number }>( + `SELECT slug, \`_deleted_at\` IS NOT NULL AS deleted FROM \`${db.schema}\`.organization_roles FINAL ORDER BY slug`, + ) + // Comparisons come back as UInt8, not Bool. + expect(rows).toEqual([ + { slug: 'admin', deleted: 1 }, + { slug: 'auditor', deleted: 0 }, + { slug: 'member', deleted: 0 }, + ]) + expect(await diffStream(destination, api, 'prod', roles)).toMatchObject({ + matched: 2, + localOnly: [], + remoteOnly: [], + }) + }) + + it('stages in batches of 500 across pages, keeping the first copy of a repeated id', async () => { + await seed(range(1, 2000)) + const remote = [...range(11, 2010), user(11, { email: 'again@example.com' })] + const diff = await diffIdsInClickHouse(db, opts, users, 'prod', pages(remote, 1200)) + expect(diff.matched).toBe(1990) + expect(diff.localOnly).toEqual(range(1, 10).map((u) => u.id)) + expect(diff.remoteOnly.map((r) => r['id'])).toEqual(range(2001, 2010).map((u) => u.id)) + expect(diff.skipped).toBe(0) + }) + + it('keeps the first copy of a remote-only id that arrives twice', async () => { + const diff = await diffIdsInClickHouse( + db, + opts, + users, + 'prod', + pages([user(1, { email: 'first@example.com' }), user(1, { email: 'second@example.com' })], 1), + ) + expect(diff.remoteOnly).toEqual([user(1, { email: 'first@example.com' })]) + }) + + it('reports a tombstoned local row that still exists upstream as remoteOnly', async () => { + await seed([user(1), user(2)]) + await deleteRows(db, opts, users, 'prod', [{ data: { id: user(2).id }, updatedAt: '2026-01-02T00:00:00.000Z' }]) + const diff = await diffIdsInClickHouse(db, opts, users, 'prod', pages([user(1), user(2)])) + expect(diff.matched).toBe(1) + expect(diff.localOnly).toEqual([]) + expect(diff.remoteOnly.map((r) => r['id'])).toEqual([user(2).id]) + }) + + it('is not fooled by a row the tail writes while the list is in progress', async () => { + await seed([user(1), user(2)]) + async function* remote() { + yield [user(1)] + await seed([user(3)]) + yield [user(2), user(4)] + } + const diff = await diffIdsInClickHouse(db, opts, users, 'prod', remote()) + expect(diff.matched).toBe(2) + expect(diff.localOnly).toEqual([]) + expect(diff.remoteOnly.map((r) => r['id'])).toEqual([user(4).id]) + }) + + it('skips remote rows without a key instead of throwing', async () => { + await seed([user(1)]) + const remote = [user(1), { object: 'user', email: 'no-id@example.com' }, user(2, { id: '' })] + const diff = await diffIdsInClickHouse(db, opts, users, 'prod', pages(remote)) + expect(diff).toMatchObject({ matched: 1, localOnly: [], remoteOnly: [], skipped: 2 }) + }) + + it('only looks at the requested environment', async () => { + await seed([user(1)], 'prod') + await seed([user(2)], 'staging') + const diff = await diffIdsInClickHouse(db, opts, users, 'prod', pages([user(1), user(2)])) + expect(diff.matched).toBe(1) + expect(diff.localOnly).toEqual([]) + expect(diff.remoteOnly.map((r) => r['id'])).toEqual([user(2).id]) + }) + + it('leaves no staging table behind, even when the listing throws', async () => { + const destination = new ClickHouseDestination({ db, schema: db.schema }) + await destination.init([users]) + await seed([user(1)]) + const staging = async () => + ( + await db.query<{ name: string }>( + `SELECT name FROM system.tables WHERE database = {schema:String} AND name LIKE '_reconcile_%'`, + { schema: db.schema }, + ) + ).map((r) => r.name) + const first = await destination.diffIds(users, 'prod', pages([user(1), user(2)])) + expect(first.remoteOnly.map((r) => r['id'])).toEqual([user(2).id]) + expect(await staging()).toEqual([]) + async function* failing() { + yield [user(1)] + throw new Error('upstream went away') + } + await expect(destination.diffIds(users, 'prod', failing())).rejects.toThrow('upstream went away') + expect(await staging()).toEqual([]) + const again = await destination.diffIds(users, 'prod', pages([user(3)])) + expect(again.matched).toBe(0) + expect(again.localOnly).toEqual([user(1).id]) + expect(again.remoteOnly.map((r) => r['id'])).toEqual([user(3).id]) + }) +}) diff --git a/packages/sync-clickhouse/test/rejected.test.ts b/packages/sync-clickhouse/test/rejected.test.ts new file mode 100644 index 0000000..8fe7a11 --- /dev/null +++ b/packages/sync-clickhouse/test/rejected.test.ts @@ -0,0 +1,100 @@ +/** + * The quarantine store on its own, against a real ClickHouse. How the engine + * fills it is covered by the poison-event cases in engine.test.ts. + */ +import { it, expect, beforeEach, afterEach } from 'vitest' +import { streamByName, type WorkOSEvent } from '@workos/sync-engine' +import { createTestDb, describeDb, type TestDb } from './helpers/clickhouse.js' +import { ClickHouseDestination } from '../src/destination.js' + +const event = (n: number, over: Partial = {}): WorkOSEvent => ({ + id: `event_${String(n).padStart(3, '0')}`, + event: 'user.created', + data: { id: `user_${n}`, email: `u${n}@example.com`, nested: { 'odd key': [1, null, 'x'] } }, + created_at: '2026-01-01T00:00:00.000Z', + ...over, +}) + +describeDb('quarantine store', () => { + let db: TestDb + let dest: ClickHouseDestination + + beforeEach(async () => { + db = await createTestDb() + dest = new ClickHouseDestination({ db, schema: db.schema }) + await dest.init([streamByName('users')!, streamByName('events')!]) + }) + afterEach(async () => { + await db.close() + }) + + it('stores whole events, lists them oldest first by id, and counts without reading them', async () => { + await dest.reject('prod', [ + { event: event(2), error: 'second' }, + { event: event(1), error: 'first' }, + ]) + expect(await dest.rejectedCount('prod')).toBe(2) + expect(await dest.rejectedCount('staging')).toBe(0) + const rows = await dest.rejected('prod') + expect(rows.map((r) => r.eventId)).toEqual(['event_001', 'event_002']) + expect(rows[0]).toMatchObject({ + eventType: 'user.created', + objectId: 'user_1', + createdAt: '2026-01-01T00:00:00.000Z', + error: 'first', + raw: event(1), + }) + expect(rows[0]!.rejectedAt).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/) + expect(Date.parse(rows[0]!.rejectedAt)).toBeGreaterThan(Date.now() - 60_000) + }) + + it('replaces the error and timestamp when an id is rejected again', async () => { + await dest.reject('prod', [{ event: event(1), error: 'first try' }]) + const before = (await dest.rejected('prod'))[0]! + await new Promise((r) => setTimeout(r, 5)) + await dest.reject('prod', [{ event: event(1, { data: { id: 'user_1', retried: true } }), error: 'second try' }]) + const rows = await dest.rejected('prod') + expect(rows).toHaveLength(1) + expect(rows[0]!.error).toBe('second try') + expect(rows[0]!.raw.data).toEqual({ id: 'user_1', retried: true }) + expect(Date.parse(rows[0]!.rejectedAt)).toBeGreaterThanOrEqual(Date.parse(before.rejectedAt)) + expect(await dest.rejectedCount('prod')).toBe(1) + }) + + it('filters by id and forgets what was replayed', async () => { + await dest.reject( + 'prod', + [1, 2, 3].map((n) => ({ event: event(n), error: 'e' })), + ) + expect((await dest.rejected('prod', { ids: ['event_002'] })).map((r) => r.eventId)).toEqual(['event_002']) + expect(await dest.rejected('prod', { ids: [] })).toEqual([]) + expect(await dest.clearRejected('prod', ['event_001', 'event_003', 'event_nope'])).toBe(2) + expect(await dest.clearRejected('prod', [])).toBe(0) + expect((await dest.rejected('prod')).map((r) => r.eventId)).toEqual(['event_002']) + expect(await dest.rejectedCount('prod')).toBe(1) + }) + + it('never fails on the event timestamp it is quarantining', async () => { + await dest.reject('prod', [{ event: event(1, { created_at: '2026-13-01T00:00:00Z' }), error: 'bad month' }]) + const rows = await dest.rejected('prod') + expect(rows[0]!.createdAt).toBe('2026-13-01T00:00:00Z') + const stored = await db.query<{ created_at: string | null }>( + `SELECT created_at FROM \`${db.schema}\`.\`_rejected\` FINAL`, + ) + expect(stored[0]!.created_at).toBeNull() + }) + + it('keeps membership events, which carry no object id', async () => { + const link = event(1, { event: 'group.member_added', data: { group_id: 'g', organization_membership_id: 'om' } }) + await dest.reject('prod', [{ event: link, error: 'e' }]) + expect((await dest.rejected('prod'))[0]!.objectId).toBeNull() + }) + + it('keeps syncs sharing a database apart', async () => { + const other = new ClickHouseDestination({ db, schema: db.schema, syncId: 'reporting' }) + await dest.reject('prod', [{ event: event(1), error: 'e' }]) + await other.reject('prod', [{ event: event(2), error: 'e' }]) + expect((await dest.rejected('prod')).map((r) => r.eventId)).toEqual(['event_001']) + expect((await other.rejected('prod')).map((r) => r.eventId)).toEqual(['event_002']) + }) +}) diff --git a/packages/sync-clickhouse/test/schema.test.ts b/packages/sync-clickhouse/test/schema.test.ts new file mode 100644 index 0000000..43dda1c --- /dev/null +++ b/packages/sync-clickhouse/test/schema.test.ts @@ -0,0 +1,411 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { createTestDb, describeDb, type TestDb } from './helpers/clickhouse.js' +import { migrate, fingerprint, DDL_VERSION } from '../src/schema/migrate.js' +import { + buildSchemaSql, + safeIdentifier, + columnExpression, + columnType, + createTableSql, + indexSpecs, + generatedColumnSql, + jsonPathArgs, +} from '../src/schema/ddl.js' +import { liveViewName, liveViewSql, isGeneratedDefinition } from '../src/schema/views.js' +import { STREAMS, streamByName, type Logger } from '@workos/sync-engine' + +const users = streamByName('users')! + +describe('DDL generation', () => { + it('projects every typed column out of _raw as a DEFAULT, never writing them directly', () => { + const sql = createTableSql(users, 'workos') + expect(sql).toContain('`_raw` String') + for (const col of users.columns) expect(sql).toMatch(new RegExp(`\`${col.name}\` [^\\n]* DEFAULT `)) + expect(sql).not.toContain('MATERIALIZED') + }) + + it('versions mirror tables by the source timestamp and keeps append logs plain', () => { + expect(createTableSql(users, 'workos')).toContain( + 'ENGINE = ReplacingMergeTree(`_updated_at`) ORDER BY (`_env`, `id`)', + ) + expect(createTableSql(users, 'workos')).toContain('`_deleted_at`') + const events = createTableSql(streamByName('events')!, 'workos') + expect(events).toContain('ENGINE = ReplacingMergeTree ORDER BY (`_env`, `id`)') + expect(events).not.toContain('`_deleted_at`') + const memberships = createTableSql(streamByName('directory_group_memberships')!, 'workos') + expect(memberships).toContain('ORDER BY (`_env`, `directory_group_id`, `directory_user_id`)') + }) + + it('keeps keys non-nullable and everything else nullable', () => { + expect(generatedColumnSql({ name: 'id', type: 'text', path: 'id' }, users)).toBe( + "`id` String DEFAULT JSONExtractString(`_raw`, 'id')", + ) + expect(generatedColumnSql({ name: 'email', type: 'text', path: 'email' }, users)).toBe( + "`email` Nullable(String) DEFAULT JSONExtract(`_raw`, 'email', 'Nullable(String)')", + ) + expect(columnType({ name: 'x', type: 'timestamptz', path: 'x' }, false)).toBe("Nullable(DateTime64(3, 'UTC'))") + expect(columnType({ name: 'x', type: 'numeric', path: 'x' }, false)).toBe('Nullable(Decimal(38, 10))') + }) + + it('walks nested paths one argument per step, quoting them', () => { + expect(jsonPathArgs(['role', 'slug'])).toBe("'role', 'slug'") + expect(jsonPathArgs(`it's`)).toBe(`'it\\'s'`) + expect(columnExpression({ name: 'x', type: 'jsonb', path: ['a', 'b'] }, false)).toBe( + "if(JSONHas(`_raw`, 'a', 'b'), JSONExtractRaw(`_raw`, 'a', 'b'), NULL)", + ) + expect(columnExpression({ name: 'x', type: 'timestamptz', path: 'created_at' }, false)).toMatch( + /^if\(match\(JSONExtractString\(`_raw`, 'created_at'\), '\^.*\$'\), parseDateTime64BestEffortOrNull\(upper\(JSONExtractString\(`_raw`, 'created_at'\)\), 3, 'UTC'\), NULL\)$/, + ) + }) + + it('turns indexed columns into skipping indexes of the right kind', () => { + expect(indexSpecs(users)).toContainEqual({ + name: 'users_email_idx', + column: 'email', + type: 'bloom_filter', + granularity: 4, + }) + expect(indexSpecs(streamByName('events')!)).toContainEqual({ + name: 'events_created_at_idx', + column: 'created_at', + type: 'minmax', + granularity: 1, + }) + expect(indexSpecs(users).map((i) => i.column)).toContain('_deleted_at') + expect(indexSpecs(streamByName('events')!).map((i) => i.column)).not.toContain('_deleted_at') + }) + + it('declares no foreign keys between mirror tables', () => { + const all = buildSchemaSql(STREAMS, 'workos').join('\n') + expect(all.toLowerCase()).not.toContain('foreign key') + for (const sql of buildSchemaSql(STREAMS, 'workos')) expect(sql).toMatch(/IF NOT EXISTS|OR REPLACE/) + }) + + it('shortens identifiers longer than the shared limit', () => { + const long = 'a'.repeat(80) + expect(safeIdentifier(long)).toHaveLength(63) + expect(safeIdentifier(long)).not.toBe(safeIdentifier(long + 'b')) + expect(safeIdentifier('users')).toBe('users') + }) + + it('changes the fingerprint when the catalog changes', () => { + const a = fingerprint([users], 'workos') + const b = fingerprint([{ ...users, columns: [...users.columns, { name: 'x', type: 'text', path: 'x' }] }], 'workos') + expect(a).not.toBe(b) + expect(fingerprint([users], 'other')).not.toBe(a) + expect(typeof DDL_VERSION).toBe('number') + }) + + it('generates a live view for every stream that has tombstones, and no other', () => { + for (const s of STREAMS) { + const sql = liveViewSql(s, 'workos') + if (s.mode === 'append') expect(sql).toEqual([]) + else { + expect(sql).toEqual([ + `CREATE OR REPLACE VIEW \`workos\`.\`${s.name}_live\` AS SELECT * FROM \`workos\`.\`${s.name}\` FINAL WHERE \`_deleted_at\` IS NULL`, + ]) + } + } + expect(liveViewName({ ...users, name: 'x'.repeat(70) })).toHaveLength(63) + }) + + it('recognises its own view definition by shape', () => { + const ours = 'SELECT * FROM workos.users FINAL WHERE _deleted_at IS NULL' + expect(isGeneratedDefinition(ours, 'workos', 'users')).toBe(true) + expect( + isGeneratedDefinition('SELECT * FROM `workos`.`users` FINAL WHERE `_deleted_at` IS NULL', 'workos', 'users'), + ).toBe(true) + expect(isGeneratedDefinition(ours.replace('IS NULL', 'IS NOT NULL'), 'workos', 'users')).toBe(false) + expect(isGeneratedDefinition(ours, 'workos', 'organizations')).toBe(false) + }) +}) + +describeDb('migrations against real ClickHouse', () => { + let db: TestDb + + beforeEach(async () => { + db = await createTestDb() + }) + afterEach(async () => { + await db.close() + }) + + const insertUser = (raw: Record) => + db.insert(`\`${db.schema}\`.users`, [ + { _env: 'prod', _raw: JSON.stringify(raw), _updated_at: '2026-01-01 00:00:00.000' }, + ]) + const tables = async () => + ( + await db.query<{ name: string }>(`SELECT name FROM system.tables WHERE database = {s:String} ORDER BY name`, { + s: db.schema, + }) + ).map((r) => r.name) + + it('creates every stream table with working projected columns, and records them', async () => { + await migrate(db, STREAMS, db.schema) + const names = await tables() + for (const s of STREAMS) expect(names).toContain(s.name) + for (const t of ['_sync', '_locks', '_migrations', '_rejected', '_columns']) expect(names).toContain(t) + const recorded = await db.query<{ n: string }>(`SELECT toString(count()) AS n FROM \`${db.schema}\`.\`_columns\``) + expect(Number(recorded[0]!.n)).toBe(STREAMS.reduce((n, s) => n + s.columns.length, 0)) + }) + + it('is a no-op on the second run', async () => { + const first = await migrate(db, STREAMS, db.schema) + const second = await migrate(db, STREAMS, db.schema) + expect(first.applied).toBe(true) + expect(second.applied).toBe(false) + expect(second.statements).toBe(0) + }) + + it('creates the database when it is missing and leaves it alone when it is not', async () => { + await db.command(`CREATE DATABASE \`${db.schema}\``) + await db.command(`CREATE TABLE \`${db.schema}\`.keep (id Int32) ENGINE = MergeTree ORDER BY id`) + await migrate(db, [users], db.schema) + expect(await tables()).toContain('keep') + }) + + it('adds a newly projected column, computed for the rows already there, and the view picks it up', async () => { + await migrate(db, [users], db.schema) + await insertUser({ id: 'user_1', email: 'a@b.com', nickname: 'ada' }) + const extended = { + ...users, + columns: [...users.columns, { name: 'nickname', type: 'text' as const, path: 'nickname' }], + } + const r = await migrate(db, [extended], db.schema) + expect(r.applied).toBe(true) + expect(r.drift).toEqual([]) + const row = await db.query<{ nickname: string }>(`SELECT nickname FROM \`${db.schema}\`.users_live`) + expect(row[0]?.nickname).toBe('ada') + }) + + it('adds a missing index on a table that already exists', async () => { + await migrate(db, [users], db.schema) + await db.command(`ALTER TABLE \`${db.schema}\`.users DROP INDEX users_email_idx`) + await db.command( + `ALTER TABLE \`${db.schema}\`.\`_migrations\` DELETE WHERE 1`, + {}, + { async_insert: 0, keeper_map_strict_mode: 0 }, + ) + const r = await migrate(db, [users], db.schema) + expect(r.applied).toBe(true) + const idx = await db.query<{ name: string }>( + `SELECT name FROM system.data_skipping_indices WHERE database = {s:String} AND table = 'users'`, + { s: db.schema }, + ) + expect(idx.map((i) => i.name)).toContain('users_email_idx') + }) + + it('parses the ISO-8601 formats WorkOS emits, and nulls anything else', async () => { + await migrate(db, [users], db.schema) + const cases: [string | null, string | null][] = [ + ['2026-01-01T00:00:00.000Z', '2026-01-01 00:00:00.000'], + ['2026-08-06T17:44:03.481Z', '2026-08-06 17:44:03.481'], + ['2026-01-01T00:00:00Z', '2026-01-01 00:00:00.000'], + ['2026-01-01T00:00:00.123456789Z', '2026-01-01 00:00:00.123'], + ['2026-01-01T05:00:00+02:00', '2026-01-01 03:00:00.000'], + ['2024-02-29T00:00:00Z', '2024-02-29 00:00:00.000'], + ['not-a-timestamp', null], + [null, null], + ['2026-02-30T12:00:00Z', null], + ['2026-13-01T00:00:00Z', null], + // The clock and the offset are range-checked ahead of the parser, which + // would otherwise read 24:00 as the next midnight and +00:99 as a zone. + ['2026-01-01T24:00:00Z', null], + ['2026-01-01T23:60:00Z', null], + ['0000-01-01T00:00:00Z', null], + ['2026-01-01T00:00:00+00:99', null], + ['2026-01-01T00:00:00-24:00', null], + ['2026-01-01T00:00:00+23:59', '2025-12-31 00:01:00.000'], + ['2026-01-01T00:00:00+0530', '2025-12-31 18:30:00.000'], + ['2026-01-01 00:00:00', '2026-01-01 00:00:00.000'], + ['2026-01-01t00:00:00z', '2026-01-01 00:00:00.000'], + ['2026-01-01', null], + ] + for (const [input, expected] of cases) { + const id = `user_${String(input)}` + await insertUser({ id, created_at: input }) + const r = await db.query<{ created_at: string | null }>( + `SELECT created_at FROM \`${db.schema}\`.users FINAL WHERE id = {id:String}`, + { id }, + ) + expect([input, r[0]?.created_at ?? null]).toEqual([input, expected]) + } + }) + + it('projects booleans, integers, decimals and JSON the way the catalog types say', async () => { + const stream = { + ...users, + columns: [ + ...users.columns, + { name: 'n', type: 'bigint' as const, path: 'n' }, + { name: 'd', type: 'numeric' as const, path: 'd' }, + { name: 'flag', type: 'boolean' as const, path: ['meta', 'flag'] }, + ], + } + await migrate(db, [stream], db.schema) + await insertUser({ + id: 'u1', + n: 9007199254740993n.toString(), + d: '12.50', + meta: { flag: false, k: [1, 'two'] }, + metadata: { a: 1 }, + }) + await insertUser({ id: 'u2', n: 'twelve', d: 'cheap', meta: { flag: 'maybe' } }) + const rows = await db.query>( + `SELECT id, toString(n) AS n, d, flag, metadata FROM \`${db.schema}\`.users FINAL ORDER BY id`, + ) + expect(String(rows[0]!['n'])).toBe('9007199254740993') + expect(Number(rows[0]!['d'])).toBe(12.5) + expect(rows[0]!['flag']).toBe(false) + expect(JSON.parse(rows[0]!['metadata'] as string)).toEqual({ a: 1 }) + expect(rows[1]).toMatchObject({ n: null, d: null, flag: null, metadata: null }) + }) + + it('finishes when a concurrent run recorded the same fingerprint and columns a moment earlier', async () => { + // A KeeperMap insert refused with "Node exists" after the other process + // created the key between this insert's existence check and its write: + // the first `_columns` and `_migrations` inserts are refused that way, + // and what the other process wrote is found in place. + const refused: string[] = [] + const racing: TestDb = { + ...db, + async command(sql, params, settings) { + const target = /INSERT INTO [^\n]*`(_columns|_migrations)`/.exec(sql)?.[1] + if (target && !refused.includes(target)) { + refused.push(target) + await db.command(sql, params, settings) + throw Object.assign(new Error('Code: 999. DB::Exception: Transaction failed (Node exists): Op #0'), { + code: '999', + }) + } + return db.command(sql, params, settings) + }, + } + const r = await migrate(racing, [users], db.schema) + expect(r.applied).toBe(true) + expect(refused).toEqual(['_columns', '_migrations']) + expect((await migrate(db, [users], db.schema)).applied).toBe(false) + const recorded = await db.query<{ n: string }>(`SELECT toString(count()) AS n FROM \`${db.schema}\`.\`_columns\``) + expect(Number(recorded[0]!.n)).toBe(users.columns.length) + }) + + it('serialises nothing: two concurrent migrations both finish with one schema', async () => { + const results = await Promise.all([migrate(db, STREAMS, db.schema), migrate(db, STREAMS, db.schema)]) + expect(results.map((r) => r.applied)).toContain(true) + expect((await migrate(db, STREAMS, db.schema)).applied).toBe(false) + }) +}) + +describeDb('live views against real ClickHouse', () => { + let db: TestDb + + beforeEach(async () => { + db = await createTestDb() + }) + afterEach(async () => { + await db.close() + }) + + const views = async () => + ( + await db.query<{ name: string }>( + `SELECT name FROM system.tables WHERE database = {s:String} AND engine = 'View' ORDER BY name`, + { s: db.schema }, + ) + ).map((r) => r.name) + const forget = () => + db.command( + `ALTER TABLE \`${db.schema}\`.\`_migrations\` DELETE WHERE 1`, + {}, + { async_insert: 0, keeper_map_strict_mode: 0 }, + ) + + it('creates a view for every mirror stream and none for the append-only logs', async () => { + await migrate(db, STREAMS, db.schema) + const names = await views() + for (const s of STREAMS) { + if (s.mode === 'append') expect(names).not.toContain(`${s.name}_live`) + else expect(names).toContain(`${s.name}_live`) + } + }) + + it('collapses versions and excludes tombstoned rows, and nothing else', async () => { + await migrate(db, [users], db.schema) + await db.insert(`\`${db.schema}\`.users`, [ + { _env: 'prod', _raw: JSON.stringify({ id: 'live', email: 'old@x' }), _updated_at: '2026-01-01 00:00:00.000' }, + { _env: 'prod', _raw: JSON.stringify({ id: 'live', email: 'new@x' }), _updated_at: '2026-01-02 00:00:00.000' }, + { + _env: 'prod', + _raw: JSON.stringify({ id: 'gone' }), + _updated_at: '2026-01-01 00:00:00.000', + _deleted_at: '2026-01-03 00:00:00.000', + }, + ]) + const live = await db.query<{ id: string; email: string }>(`SELECT id, email FROM \`${db.schema}\`.users_live`) + expect(live).toEqual([{ id: 'live', email: 'new@x' }]) + }) + + it('re-creates a view that was dropped once the schema is applied again', async () => { + await migrate(db, [users], db.schema) + await db.command(`DROP VIEW \`${db.schema}\`.users_live`) + await forget() + await migrate(db, [users], db.schema) + expect(await views()).toContain('users_live') + }) + + it('refuses to replace a view under its name that it did not generate, until repairViews', async () => { + await migrate(db, [users], db.schema) + await db.command( + `CREATE OR REPLACE VIEW \`${db.schema}\`.users_live AS SELECT id FROM \`${db.schema}\`.users WHERE email LIKE '%@acme.com'`, + ) + await forget() + await expect(migrate(db, [users], db.schema)).rejects.toThrow(/did not generate/) + const lines: string[] = [] + const logger: Logger = { debug() {}, info() {}, warn: (m) => lines.push(m), error() {} } + await migrate(db, [users], db.schema, logger, { repairViews: true }) + expect(lines).toContain('replacing a view that is not the generated shape') + const r = await db.query<{ as_select: string }>( + `SELECT as_select FROM system.tables WHERE database = {s:String} AND name = 'users_live'`, + { + s: db.schema, + }, + ) + expect(isGeneratedDefinition(r[0]!.as_select, db.schema, 'users')).toBe(true) + }) + + it('refuses a foreign view that happens to have every column, rather than mistaking it for its own', async () => { + await migrate(db, [users], db.schema) + await db.command( + `CREATE OR REPLACE VIEW \`${db.schema}\`.users_live AS SELECT * FROM \`${db.schema}\`.users WHERE email LIKE '%@acme.com'`, + ) + await forget() + await expect(migrate(db, [users], db.schema)).rejects.toThrow(/did not generate/) + await migrate(db, [users], db.schema, undefined, { repairViews: true }) + const r = await db.query<{ as_select: string }>( + `SELECT as_select FROM system.tables WHERE database = {s:String} AND name = 'users_live'`, + { s: db.schema }, + ) + expect(isGeneratedDefinition(r[0]!.as_select, db.schema, 'users')).toBe(true) + }) + + it('collapses a second copy of an append-only row under FINAL', async () => { + const events = streamByName('events')! + await migrate(db, [events], db.schema) + const row = { + _env: 'prod', + _raw: JSON.stringify({ id: 'event_1', event: 'user.created' }), + _updated_at: '2026-01-01 00:00:00.000', + } + await db.insert(`\`${db.schema}\`.events`, [row]) + await db.insert(`\`${db.schema}\`.events`, [row]) + const n = await db.query<{ n: string }>(`SELECT toString(count()) AS n FROM \`${db.schema}\`.events FINAL`) + expect(Number(n[0]!.n)).toBe(1) + }) + + it('refuses a table standing where its view would go', async () => { + await db.command(`CREATE DATABASE \`${db.schema}\``) + await db.command(`CREATE TABLE \`${db.schema}\`.users_live (id Int32) ENGINE = MergeTree ORDER BY id`) + await expect(migrate(db, [users], db.schema)).rejects.toThrow(/is a table/) + }) +}) diff --git a/packages/sync-clickhouse/test/upsert.test.ts b/packages/sync-clickhouse/test/upsert.test.ts new file mode 100644 index 0000000..b1ed6e5 --- /dev/null +++ b/packages/sync-clickhouse/test/upsert.test.ts @@ -0,0 +1,619 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { createTestDb, describeDb, type TestDb } from './helpers/clickhouse.js' +import { migrate } from '../src/schema/migrate.js' +import { + upsertRows, + deleteRows, + relinkRows, + unlinkRows, + dedupeByKey, + wavesByKey, + keyValues, + parseUpdatedAt, + jsonEqual, + keyExpression, +} from '../src/db/upsert.js' +import { isTransientClickHouseError } from '../src/db/errors.js' +import { streamByName } from '@workos/sync-engine' + +const users = streamByName('users')! + +const userPayload = (over: Record = {}) => ({ + object: 'user', + id: 'user_1', + email: 'ada@example.com', + first_name: 'Ada', + created_at: '2026-01-01T00:00:00.000Z', + updated_at: '2026-01-01T00:00:00.000Z', + ...over, +}) + +describe('key extraction', () => { + it('reads slug-keyed streams that have no id in their event payload', () => { + const roles = streamByName('roles')! + expect(keyValues(roles, { slug: 'admin', permissions: [] })).toEqual(['admin']) + expect(keyValues(roles, { permissions: [] })).toBeUndefined() + }) + + it('reads composite keys, and looks them up as one joined expression', () => { + const orgRoles = streamByName('organization_roles')! + expect(keyValues(orgRoles, { organization_id: 'org_1', slug: 'admin' })).toEqual(['org_1', 'admin']) + expect(keyExpression(orgRoles)).toBe('concat(`organization_id`, char(1), `slug`)') + expect(keyExpression(users)).toBe('`id`') + }) + + it('collapses duplicate keys within a batch, keeping the newest', () => { + const { rows, dropped } = dedupeByKey(users, [ + { data: userPayload({ email: 'old@example.com' }), updatedAt: '2026-01-01T00:00:00.000Z' }, + { data: userPayload({ email: 'new@example.com' }), updatedAt: '2026-01-02T00:00:00.000Z' }, + { data: { email: 'no-id@example.com' }, updatedAt: '2026-01-03T00:00:00.000Z' }, + ]) + expect(rows).toHaveLength(1) + expect(rows[0]!.data['email']).toBe('new@example.com') + expect(dropped).toBe(1) + }) + + it('judges the newest row by its instant, not by the text of its timestamp', () => { + // "01:00:00+01:00" sorts after "00:30:00Z" as text and is half an hour earlier as a time. + const { rows } = dedupeByKey(users, [ + { data: userPayload({ email: 'older@example.com' }), updatedAt: '2026-01-01T01:00:00+01:00' }, + { data: userPayload({ email: 'newer@example.com' }), updatedAt: '2026-01-01T00:30:00Z' }, + ]) + expect(rows.map((r) => r.data['email'])).toEqual(['newer@example.com']) + // Two rows within a millisecond tie at the gate, so the one kept here must be the newer. + const fine = dedupeByKey(users, [ + { data: userPayload({ email: 'newer@example.com' }), updatedAt: '2026-01-01T00:00:00.000900Z' }, + { data: userPayload({ email: 'older@example.com' }), updatedAt: '2026-01-01T00:00:00.000100Z' }, + ]) + expect(fine.rows.map((r) => r.data['email'])).toEqual(['newer@example.com']) + // What is not a date is ordered as text, and refused on the write. + const junk = dedupeByKey(users, [ + { data: userPayload({ email: 'a@example.com' }), updatedAt: 'later' }, + { data: userPayload({ email: 'b@example.com' }), updatedAt: 'yesterday' }, + ]) + expect(junk.rows.map((r) => r.data['email'])).toEqual(['b@example.com']) + }) + + it('splits repeated keys into ordered waves and leaves distinct keys in one', () => { + const first = { data: userPayload({ external_id: 'ext_1' }), updatedAt: '2026-01-01T00:00:00.000Z' } + const other = { data: userPayload({ id: 'user_2' }), updatedAt: '2026-01-01T00:00:00.000Z' } + const second = { data: { id: 'user_1', first_name: 'Grace' }, updatedAt: '2026-01-02T00:00:00.000Z' } + const { waves, dropped } = wavesByKey(users, [ + first, + other, + second, + { data: { email: 'no-id@example.com' }, updatedAt: '2026-01-03T00:00:00.000Z' }, + ]) + expect(waves).toEqual([[first, other], [second]]) + expect(dropped).toBe(1) + }) +}) + +describe('timestamps and equality', () => { + it('parses an ISO-8601 updatedAt and refuses anything else as a deterministic error', () => { + expect(parseUpdatedAt('2026-01-01T00:00:00.000Z').toISOString()).toBe('2026-01-01T00:00:00.000Z') + let err: unknown + try { + parseUpdatedAt('maybe') + } catch (e) { + err = e + } + expect((err as { code: string }).code).toBe('INVALID_TIMESTAMP') + expect(isTransientClickHouseError(err)).toBe(false) + }) + + it('compares JSON structurally, ignoring key order', () => { + expect(jsonEqual({ a: 1, b: [1, { c: null }] }, { b: [1, { c: null }], a: 1 })).toBe(true) + expect(jsonEqual({ a: 1 }, { a: 1, b: undefined })).toBe(false) + expect(jsonEqual({ a: [1, 2] }, { a: [2, 1] })).toBe(false) + }) +}) + +const SOME_STREAMS = ['users', 'roles', 'group_memberships', 'events'].map((n) => streamByName(n)!) + +describeDb('write path', () => { + let db: TestDb + let opts: { schema: string; batchSize?: number } + + beforeEach(async () => { + db = await createTestDb() + opts = { schema: db.schema } + await migrate(db, SOME_STREAMS, db.schema) + }) + afterEach(async () => { + await db.close() + }) + + const readUser = async (id = 'user_1') => + ( + await db.query>( + `SELECT * FROM \`${db.schema}\`.users FINAL WHERE \`_env\` = 'prod' AND id = {id:String}`, + { id }, + ) + )[0] + const versions = async (id = 'user_1') => + Number( + ( + await db.query<{ n: string }>( + `SELECT toString(count()) AS n FROM \`${db.schema}\`.users WHERE \`_env\` = 'prod' AND id = {id:String}`, + { id }, + ) + )[0]!.n, + ) + + it('inserts and reports created vs updated, keeping every version until a merge', async () => { + const first = await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload(), updatedAt: '2026-01-01T00:00:00.000Z' }], + 'replace', + ) + expect(first).toMatchObject({ created: 1, updated: 0 }) + const second = await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ email: 'changed@example.com' }), updatedAt: '2026-01-02T00:00:00.000Z' }], + 'replace', + ) + expect(second).toMatchObject({ created: 0, updated: 1 }) + expect((await readUser())!['email']).toBe('changed@example.com') + // The base table keeps both versions; FINAL and the view collapse them. + expect(await versions()).toBe(2) + }) + + it('rejects a write that is older than the stored row', async () => { + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ email: 'current@example.com' }), updatedAt: '2026-01-05T00:00:00.000Z' }], + 'replace', + ) + const stale = await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ email: 'stale@example.com' }), updatedAt: '2026-01-01T00:00:00.000Z' }], + 'replace', + ) + expect(stale).toMatchObject({ created: 0, updated: 0, skipped: 1 }) + expect((await readUser())!['email']).toBe('current@example.com') + expect(await versions()).toBe(1) + }) + + it('accepts a write with an identical timestamp, and the table keeps the later one', async () => { + const ts = '2026-01-05T00:00:00.000Z' + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ email: 'first@example.com' }), updatedAt: ts }], + 'replace', + ) + const same = await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ email: 'second@example.com' }), updatedAt: ts }], + 'replace', + ) + expect(same.updated).toBe(1) + // Equal versions: ReplacingMergeTree keeps the last inserted, which is the gate's `>=`. + expect((await readUser())!['email']).toBe('second@example.com') + }) + + it('skips a write with the same payload and timestamp, writing no version', async () => { + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload(), updatedAt: '2026-01-01T00:00:00.000Z' }], + 'replace', + ) + const again = await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload(), updatedAt: '2026-01-01T00:00:00.000Z' }], + 'replace', + ) + expect(again).toMatchObject({ created: 0, updated: 0, skipped: 1 }) + expect(await versions()).toBe(1) + }) + + it('keeps the fields of a full snapshot under a later subset in one merge batch', async () => { + const stats = await upsertRows( + db, + opts, + users, + 'prod', + [ + { + data: userPayload({ external_id: 'ext_1', metadata: { plan: 'pro' } }), + updatedAt: '2026-01-01T00:00:00.000Z', + }, + { + data: { object: 'user', id: 'user_1', email: 'ada@example.com', first_name: 'Grace' }, + updatedAt: '2026-01-02T00:00:00.000Z', + }, + ], + 'merge', + ) + expect(stats).toMatchObject({ created: 1, updated: 1, skipped: 0 }) + const row = (await readUser())! + expect(row['first_name']).toBe('Grace') + expect(row['external_id']).toBe('ext_1') + expect(JSON.parse(row['metadata'] as string)).toEqual({ plan: 'pro' }) + }) + + it('gates every row of a merge batch against the stored row, not only against the batch', async () => { + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ metadata: { plan: 'current' } }), updatedAt: '2026-01-02T00:00:00.000Z' }], + 'replace', + ) + const stats = await upsertRows( + db, + opts, + users, + 'prod', + [ + { data: { object: 'user', id: 'user_1', metadata: { plan: 'old' } }, updatedAt: '2026-01-01T00:00:00.000Z' }, + { data: { object: 'user', id: 'user_1', first_name: 'Grace' }, updatedAt: '2026-01-03T00:00:00.000Z' }, + ], + 'merge', + ) + expect(stats).toMatchObject({ created: 0, updated: 1, skipped: 1 }) + const row = (await readUser())! + expect(row['first_name']).toBe('Grace') + expect(JSON.parse(row['metadata'] as string)).toEqual({ plan: 'current' }) + expect(row['_updated_at']).toBe('2026-01-03 00:00:00.000') + }) + + it('merges shallowly: a nested object in the event replaces the stored one', async () => { + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ metadata: { plan: 'pro', seats: 5 } }), updatedAt: '2026-01-01T00:00:00.000Z' }], + 'replace', + ) + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: { object: 'user', id: 'user_1', metadata: { seats: 6 } }, updatedAt: '2026-01-02T00:00:00.000Z' }], + 'merge', + ) + expect(JSON.parse((await readUser())!['metadata'] as string)).toEqual({ seats: 6 }) + }) + + it('merges event payloads instead of nulling the fields events omit', async () => { + await upsertRows( + db, + opts, + users, + 'prod', + [ + { + data: userPayload({ external_id: 'ext_42', metadata: { plan: 'enterprise' }, locale: 'fr' }), + updatedAt: '2026-01-01T00:00:00.000Z', + }, + ], + 'replace', + ) + await upsertRows( + db, + opts, + users, + 'prod', + [ + { + data: { object: 'user', id: 'user_1', email: 'renamed@example.com', first_name: 'Ada' }, + updatedAt: '2026-01-09T00:00:00.000Z', + }, + ], + 'merge', + ) + const row = (await readUser())! + expect(row['email']).toBe('renamed@example.com') + expect(row['external_id']).toBe('ext_42') + expect(row['locale']).toBe('fr') + }) + + it('replaces the payload when the row came from the REST API', async () => { + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ locale: 'fr' }), updatedAt: '2026-01-01T00:00:00.000Z' }], + 'replace', + ) + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ updated_at: '2026-02-01T00:00:00.000Z' }), updatedAt: '2026-02-01T00:00:00.000Z' }], + 'replace', + ) + expect((await readUser())!['locale']).toBeNull() + }) + + it('projects a JSON null as NULL, a boolean as a boolean, and a bad cast as NULL with the value kept in _raw', async () => { + await upsertRows( + db, + opts, + users, + 'prod', + [ + { + data: userPayload({ id: 'user_t', first_name: null, email_verified: true }), + updatedAt: '2026-01-01T00:00:00.000Z', + }, + { data: userPayload({ id: 'user_m', email_verified: 'maybe' }), updatedAt: '2026-01-01T00:00:00.000Z' }, + ], + 'replace', + ) + const t = (await readUser('user_t'))! + expect(t['first_name']).toBeNull() + expect(t['email_verified']).toBe(true) + const m = (await readUser('user_m'))! + expect(m['email_verified']).toBeNull() + expect(JSON.parse(m['_raw'] as string)['email_verified']).toBe('maybe') + }) + + it('tombstones on delete and keeps the last known payload', async () => { + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload(), updatedAt: '2026-01-01T00:00:00.000Z' }], + 'replace', + ) + const stats = await deleteRows(db, opts, users, 'prod', [ + { data: { id: 'user_1' }, updatedAt: '2026-01-10T00:00:00.000Z' }, + ]) + expect(stats.deleted).toBe(1) + const row = (await readUser())! + expect(row['_deleted_at']).toBe('2026-01-10 00:00:00.000') + expect(row['email']).toBe('ada@example.com') + // A delete is not a staleness-gated write, but the tombstone takes the + // deletion time as its version: the newest version of the row must be it. + expect(row['_updated_at']).toBe('2026-01-10 00:00:00.000') + }) + + it('records a delete for an object it never saw created', async () => { + const stats = await deleteRows(db, opts, users, 'prod', [ + { data: { id: 'user_ghost', email: 'ghost@example.com' }, updatedAt: '2026-01-10T00:00:00.000Z' }, + ]) + expect(stats.deleted).toBe(1) + expect((await readUser('user_ghost'))!['_deleted_at']).not.toBeNull() + }) + + it('does not resurrect a deleted row when a late update arrives', async () => { + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload(), updatedAt: '2026-01-01T00:00:00.000Z' }], + 'replace', + ) + await deleteRows(db, opts, users, 'prod', [{ data: { id: 'user_1' }, updatedAt: '2026-01-10T00:00:00.000Z' }]) + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ email: 'zombie@example.com' }), updatedAt: '2026-01-20T00:00:00.000Z' }], + 'merge', + ) + const row = (await readUser())! + expect(row['_deleted_at']).not.toBeNull() + expect(row['email']).toBe('zombie@example.com') + }) + + it('keeps the latest removal when a row is deleted twice', async () => { + await deleteRows(db, opts, users, 'prod', [{ data: { id: 'user_1' }, updatedAt: '2026-01-10T00:00:00.000Z' }]) + await deleteRows(db, opts, users, 'prod', [{ data: { id: 'user_1' }, updatedAt: '2026-01-05T00:00:00.000Z' }]) + expect((await readUser())!['_deleted_at']).toBe('2026-01-10 00:00:00.000') + await deleteRows(db, opts, users, 'prod', [{ data: { id: 'user_1' }, updatedAt: '2026-01-12T00:00:00.000Z' }]) + expect((await readUser())!['_deleted_at']).toBe('2026-01-12 00:00:00.000') + }) + + it('keeps environments isolated under one database', async () => { + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ email: 'prod@example.com' }), updatedAt: '2026-01-01T00:00:00.000Z' }], + 'replace', + ) + await upsertRows( + db, + opts, + users, + 'staging', + [{ data: userPayload({ email: 'staging@example.com' }), updatedAt: '2026-01-01T00:00:00.000Z' }], + 'replace', + ) + const r = await db.query<{ _env: string; email: string }>( + `SELECT \`_env\`, email FROM \`${db.schema}\`.users FINAL ORDER BY \`_env\``, + ) + expect(r).toEqual([ + { _env: 'prod', email: 'prod@example.com' }, + { _env: 'staging', email: 'staging@example.com' }, + ]) + }) + + it('treats a re-added membership as live again', async () => { + const memberships = streamByName('group_memberships')! + const pair = { group_id: 'group_1', organization_membership_id: 'om_1' } + await relinkRows(db, opts, memberships, 'prod', [{ data: pair, updatedAt: '2026-01-01T00:00:00.000Z' }]) + await unlinkRows(db, opts, memberships, 'prod', [{ data: pair, updatedAt: '2026-01-02T00:00:00.000Z' }]) + const back = await relinkRows(db, opts, memberships, 'prod', [ + { data: pair, updatedAt: '2026-01-03T00:00:00.000Z' }, + ]) + expect(back).toMatchObject({ created: 0, updated: 1, skipped: 0 }) + const r = await db.query<{ deleted_at: string | null }>( + `SELECT \`_deleted_at\` AS deleted_at FROM \`${db.schema}\`.group_memberships FINAL WHERE group_id = 'group_1'`, + ) + expect(r[0]!.deleted_at).toBeNull() + }) + + it('ignores a re-add older than the removal, even when the row itself is older', async () => { + const memberships = streamByName('group_memberships')! + const pair = { group_id: 'group_1', organization_membership_id: 'om_1' } + await relinkRows(db, opts, memberships, 'prod', [{ data: pair, updatedAt: '2026-01-01T00:00:00.000Z' }]) + await deleteRows(db, opts, memberships, 'prod', [{ data: pair, updatedAt: '2026-01-05T00:00:00.000Z' }]) + const stale = await relinkRows(db, opts, memberships, 'prod', [ + { data: pair, updatedAt: '2026-01-03T00:00:00.000Z' }, + ]) + expect(stale).toMatchObject({ created: 0, updated: 0, skipped: 1 }) + const r = await db.query<{ deleted_at: string | null }>( + `SELECT \`_deleted_at\` AS deleted_at FROM \`${db.schema}\`.group_memberships FINAL WHERE group_id = 'group_1'`, + ) + expect(r[0]!.deleted_at).not.toBeNull() + }) + + it('ignores a membership removal older than the row, unlike an object delete', async () => { + const memberships = streamByName('group_memberships')! + const pair = { group_id: 'group_1', organization_membership_id: 'om_1' } + const deletedAt = async () => + ( + await db.query<{ deleted_at: string | null }>( + `SELECT \`_deleted_at\` AS deleted_at FROM \`${db.schema}\`.group_memberships FINAL WHERE group_id = 'group_1'`, + ) + )[0]!.deleted_at + await relinkRows(db, opts, memberships, 'prod', [{ data: pair, updatedAt: '2026-01-03T00:00:00.000Z' }]) + const stale = await unlinkRows(db, opts, memberships, 'prod', [ + { data: pair, updatedAt: '2026-01-02T00:00:00.000Z' }, + ]) + expect(stale).toMatchObject({ deleted: 0, skipped: 1 }) + expect(await deletedAt()).toBeNull() + const fresh = await unlinkRows(db, opts, memberships, 'prod', [ + { data: pair, updatedAt: '2026-01-04T00:00:00.000Z' }, + ]) + expect(fresh).toMatchObject({ deleted: 1, skipped: 0 }) + expect(await deletedAt()).not.toBeNull() + }) + + it('never writes an append-only row twice', async () => { + const events = streamByName('events')! + const e = { id: 'event_1', event: 'user.created', data: { id: 'user_1' }, created_at: '2026-01-01T00:00:00.000Z' } + const first = await upsertRows(db, opts, events, 'prod', [{ data: e, updatedAt: e.created_at }], 'replace') + expect(first).toMatchObject({ created: 1, skipped: 0 }) + const dup = await upsertRows( + db, + opts, + events, + 'prod', + [{ data: { ...e, event: 'tampered' }, updatedAt: e.created_at }], + 'replace', + ) + expect(dup).toMatchObject({ created: 0, skipped: 1 }) + const r = await db.query<{ event: string }>(`SELECT event FROM \`${db.schema}\`.events`) + expect(r).toEqual([{ event: 'user.created' }]) + }) + + it('writes large batches in chunks', async () => { + const rows = Array.from({ length: 1200 }, (_, i) => ({ + data: userPayload({ id: `user_${i}`, email: `u${i}@example.com` }), + updatedAt: '2026-01-01T00:00:00.000Z', + })) + const stats = await upsertRows(db, { schema: db.schema, batchSize: 250 }, users, 'prod', rows, 'replace') + expect(stats.created).toBe(1200) + const count = await db.query<{ n: string }>(`SELECT toString(count()) AS n FROM \`${db.schema}\`.users FINAL`) + expect(Number(count[0]!.n)).toBe(1200) + }) + + it('stores a payload whose string values look like SQL, JSON paths or parameters verbatim', async () => { + const data = userPayload({ first_name: 'O\'Brien `$.x` \\ "quoted" {id:String}', metadata: { '$.weird key': 'v' } }) + await upsertRows(db, opts, users, 'prod', [{ data, updatedAt: '2026-01-01T00:00:00.000Z' }], 'replace') + const row = (await readUser())! + expect(row['first_name']).toBe('O\'Brien `$.x` \\ "quoted" {id:String}') + expect(JSON.parse(row['metadata'] as string)).toEqual({ '$.weird key': 'v' }) + }) +}) + +describeDb('freshness reporting', () => { + let db: TestDb + + beforeEach(async () => { + db = await createTestDb() + await migrate(db, SOME_STREAMS, db.schema) + }) + afterEach(async () => { + await db.close() + }) + + it('does not report the age of old objects as sync lag', async () => { + const { ClickHouseDestination } = await import('../src/destination.js') + const destination = new ClickHouseDestination({ db, schema: db.schema }) + const opts = { schema: db.schema } + + expect(await destination.freshness(users, 'prod', 24)).toEqual({ + stream: 'users', + rows: 0, + deleted: 0, + lastSyncedAt: null, + stalenessSeconds: null, + lagSeconds: null, + lagSampleSize: 0, + }) + + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ id: 'user_old' }), updatedAt: '2024-01-01T00:00:00.000Z' }], + 'replace', + ) + const stale = await destination.freshness(users, 'prod', 24) + expect(stale.rows).toBe(1) + expect(stale.lagSampleSize).toBe(0) + expect(stale.lagSeconds).toBeNull() + expect(stale.stalenessSeconds!).toBeGreaterThan(365 * 86400) + expect(Date.parse(stale.lastSyncedAt!)).toBeGreaterThan(Date.now() - 60_000) + + await upsertRows( + db, + opts, + users, + 'prod', + [{ data: userPayload({ id: 'user_fresh' }), updatedAt: new Date(Date.now() - 2000).toISOString() }], + 'replace', + ) + await deleteRows(db, opts, users, 'prod', [{ data: { id: 'user_old' }, updatedAt: new Date().toISOString() }]) + + const fresh = await destination.freshness(users, 'prod', 24) + expect(fresh.rows).toBe(2) + expect(fresh.deleted).toBe(1) + // The deletion just now is a change inside the window too: a tombstone + // is versioned at its deletion time, so it joins the sample. + expect(fresh.lagSampleSize).toBe(2) + expect(fresh.lagSeconds).not.toBeNull() + expect(fresh.lagSeconds!).toBeLessThan(60) + expect(fresh.lagSeconds!).toBeGreaterThan(0) + }) +}) diff --git a/packages/sync-clickhouse/tsconfig.build.json b/packages/sync-clickhouse/tsconfig.build.json new file mode 100644 index 0000000..f44fac8 --- /dev/null +++ b/packages/sync-clickhouse/tsconfig.build.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "outDir": "dist", + "rootDir": "src", + "paths": {} + }, + "include": ["src/**/*"], + "exclude": ["test/**/*"] +} diff --git a/packages/sync-clickhouse/tsconfig.json b/packages/sync-clickhouse/tsconfig.json new file mode 100644 index 0000000..09ebd6a --- /dev/null +++ b/packages/sync-clickhouse/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2023"], + "strict": true, + "noUncheckedIndexedAccess": true, + "exactOptionalPropertyTypes": false, + "esModuleInterop": true, + "skipLibCheck": true, + "declaration": true, + "sourceMap": true, + "noEmit": true, + "types": ["node"], + // Typecheck and editors see @workos/sync-engine source directly, so a change + // there never needs a rebuild to show up here. The build (tsconfig.build) + // clears this and resolves the real package. + "paths": { "@workos/sync-engine": ["../sync-engine/src/index.ts"] } + }, + "include": ["src/**/*", "test/**/*"] +} diff --git a/packages/sync-clickhouse/vitest.config.ts b/packages/sync-clickhouse/vitest.config.ts new file mode 100644 index 0000000..a27ae83 --- /dev/null +++ b/packages/sync-clickhouse/vitest.config.ts @@ -0,0 +1,14 @@ +import { fileURLToPath } from 'node:url' +import { defineConfig } from 'vitest/config' +export default defineConfig({ + resolve: { + // Tests run against @workos/sync-engine source, not its last build. + alias: { '@workos/sync-engine': fileURLToPath(new URL('../sync-engine/src/index.ts', import.meta.url)) }, + }, + test: { + include: ['test/**/*.test.ts'], + testTimeout: 60_000, + hookTimeout: 60_000, + pool: 'forks', + }, +}) diff --git a/packages/sync-engine/README.md b/packages/sync-engine/README.md index 5c14a5a..c93ba93 100644 --- a/packages/sync-engine/README.md +++ b/packages/sync-engine/README.md @@ -8,11 +8,12 @@ particular database. Writes go through a small `Destination` interface. **You probably want a destination package instead.** Each one depends on this package and re-exports it, so it is the only thing to install: -| Destination | Package | -| ----------- | ------------------------------------------------------------------------------ | -| Postgres | [`@workos/sync-postgres`](https://www.npmjs.com/package/@workos/sync-postgres) | -| MongoDB | [`@workos/sync-mongodb`](https://www.npmjs.com/package/@workos/sync-mongodb) | -| MySQL | [`@workos/sync-mysql`](https://www.npmjs.com/package/@workos/sync-mysql) | +| Destination | Package | +| ----------- | ---------------------------------------------------------------------------------- | +| Postgres | [`@workos/sync-postgres`](https://www.npmjs.com/package/@workos/sync-postgres) | +| MongoDB | [`@workos/sync-mongodb`](https://www.npmjs.com/package/@workos/sync-mongodb) | +| MySQL | [`@workos/sync-mysql`](https://www.npmjs.com/package/@workos/sync-mysql) | +| ClickHouse | [`@workos/sync-clickhouse`](https://www.npmjs.com/package/@workos/sync-clickhouse) | Install this package directly only to write a new destination. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4caf8f3..22f02b1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -229,6 +229,35 @@ importers: specifier: ^7.0.2 version: 7.0.2 + packages/sync-clickhouse: + dependencies: + '@clickhouse/client': + specifier: ^1.11.0 + version: 1.25.0 + '@workos/sync-engine': + specifier: workspace:* + version: link:../sync-engine + devDependencies: + '@opentelemetry/api': + specifier: ^1.10.0 + version: 1.10.0 + '@opentelemetry/sdk-metrics': + specifier: ^3.0.0 + version: 3.0.0(@opentelemetry/api@1.10.0) + '@types/node': + specifier: ^26.6.5 + version: 26.6.5 + typescript: + specifier: ^7.0.2 + version: 7.0.2 + vitest: + specifier: ^5.0.5 + version: 5.0.5(@opentelemetry/api@1.10.0)(@types/node@26.6.5)(vite@8.3.4(@types/node@26.6.5)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.15)(yaml@2.9.1)) + optionalDependencies: + '@opentelemetry/sdk-node': + specifier: ^0.300.0 + version: 0.300.0(@opentelemetry/api@1.10.0) + packages/sync-engine: dependencies: '@opentelemetry/api': @@ -363,6 +392,10 @@ packages: resolution: {integrity: sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==} engines: {node: '>=6.9.0'} + '@clickhouse/client@1.25.0': + resolution: {integrity: sha512-sxoCDlA8iOEGpGrJzl9SeNvG7II6e5kSeo0JI4Mru2QhwhjydWuDEE55ryZg3vIJ0CSJ1bhn9+Xtnuvf1BZT/w==} + engines: {node: '>=20'} + '@commitlint/cli@21.2.4': resolution: {integrity: sha512-g3M/wrQdElqGH5vROIeL2QiYIxrzqUkhCgsDyDlWfKEyQdCt10OyMB28W2cJtpZ+VNGMvmaN923infr1hOfyrg==} engines: {node: '>=22.12.0'} @@ -2214,6 +2247,8 @@ snapshots: '@babel/helper-validator-identifier@7.29.7': {} + '@clickhouse/client@1.25.0': {} + '@commitlint/cli@21.2.4(conventional-commits-parser@7.1.4)(typescript@7.0.2)': dependencies: '@commitlint/config-conventional': 21.2.4 diff --git a/release-please-config.json b/release-please-config.json index 4c613d7..705d670 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -23,6 +23,10 @@ "packages/sync-mysql": { "release-type": "node", "component": "sync-mysql" + }, + "packages/sync-clickhouse": { + "release-type": "node", + "component": "sync-clickhouse" } } }