From a95e3e31f3665728a24a7cdfebacc3700b384d94 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Sun, 11 Oct 2026 13:06:28 -0400 Subject: [PATCH 1/5] fix(1.1.0): adopter-reported defects D1-D11, G5, G10 and release notes Flyway history filter (migrate, verify, gen advice), view replace legality decided by a rolled-back Postgres probe, --allow hints, view recreate scope, report @via dimension nullability in every port, Kotlin Names, oracle CLI providers, prerelease-link and prerelease:publish fixes, doc-writer timestamps removed. --- CHANGELOG.md | 122 ++++ docs/features/cli.md | 7 + docs/features/migrations-and-drift.md | 64 +- docs/features/prerelease.md | 4 +- docs/features/reporting.md | 44 +- docs/ports/java.md | 6 +- docs/ports/kotlin.md | 25 +- .../report-shapes.json | 6 +- scripts/ci-local.sh | 2 +- scripts/prerelease.mjs | 17 +- .../ReportRowCodegenTests.cs | 4 +- .../ReportShapeTests.cs | 88 ++- .../Generated/ProgramMinutes.g.cs | 4 +- .../Generated/SessionsByProgram.g.cs | 4 +- .../Generated/WeekShapes.g.cs | 4 +- .../Relationship/RelationshipReferences.cs | 2 +- .../MetaObjects/Core/Reporting/ReportShape.cs | 50 +- .../ai/MetaDataAIDocumentationWriter.java | 1 - .../html/MetaDataHtmlDocumentationWriter.java | 5 +- .../direct/object/BaseObjectCodeWriter.java | 3 + .../DocumentationOutputIsStableTest.java | 89 +++ .../object/GeneratedHeaderIsStableTest.java | 82 +++ .../generator/kotlin/KotlinNamesGenerator.kt | 14 +- .../kotlin/KotlinNamesGeneratorTest.kt | 32 + .../kotlin/KotlinReportTableGeneratorTest.kt | 24 +- .../generator/kotlin/ReportingInertTest.kt | 21 +- .../KotlinCodegenMatchesReferenceTest.kt | 6 +- .../kotlin/tables/ProgramMinutesView.kt | 4 +- .../kotlin/tables/SessionsByProgramView.kt | 2 +- .../kotlin/tables/WeekShapesView.kt | 2 +- .../metaobjects/reporting/ReportShape.java | 72 +- .../reporting/ReportReadModelTest.java | 2 +- .../reporting/ReportShapeTest.java | 90 ++- .../meta/core/reporting/report_shape.py | 97 ++- server/python/tests/test_report_shape.py | 161 +++++ .../packages/cli/src/commands/gen.ts | 1 + .../packages/cli/src/commands/migrate.ts | 19 +- .../packages/cli/src/commands/verify.ts | 39 +- server/typescript/packages/cli/src/index.ts | 10 + .../typescript/packages/cli/src/lib/allow.ts | 4 +- .../typescript/packages/cli/src/lib/args.ts | 31 +- .../typescript/packages/cli/src/lib/config.ts | 8 +- .../cli/src/lib/gen-migrate-advice.ts | 7 +- .../cli/test/__snapshots__/cli.test.ts.snap | 5 + .../cli/test/allow-hint-tokens.test.ts | 144 ++++ .../cli/test/migrate-flyway-history.test.ts | 194 +++++ .../cli/test/unit/args-verify.test.ts | 28 +- .../cli/test/unit/describe-change.test.ts | 19 +- .../cli/test/unit/gen-migrate-advice.test.ts | 45 ++ .../src/projection/build-projection-views.ts | 85 +-- .../projection/build-projection-views.test.ts | 5 - .../projection/report-required-joins.test.ts | 158 +++++ .../src/load-project-model.ts | 23 + .../integration-tests/src/oracle-cli.ts | 8 +- .../test/load-project-model.test.ts | 69 ++ .../test/report-views-pg.test.ts | 25 +- .../test/view-lifecycle-pg.test.ts | 12 +- .../test/view-recreate-drift-pg.test.ts | 46 +- .../test/view-replace-probe-pg.test.ts | 661 ++++++++++++++++++ .../src/core/reporting/report-shape.ts | 98 ++- .../metadata/test/report-read-model.test.ts | 2 +- .../metadata/test/report-shape.test.ts | 124 ++++ .../packages/migrate-ts/src/diff/index.ts | 219 ++++-- .../packages/migrate-ts/src/diff/status.ts | 78 ++- .../packages/migrate-ts/src/drift/drift.ts | 17 +- .../packages/migrate-ts/src/emit/postgres.ts | 41 +- .../packages/migrate-ts/src/errors.ts | 10 +- .../migrate-ts/src/expected-schema.ts | 21 +- .../packages/migrate-ts/src/index.ts | 8 +- .../migrate-ts/src/introspect/postgres.ts | 34 +- .../migrate-ts/src/snapshot/serialize.ts | 12 +- .../packages/migrate-ts/src/sql-type.ts | 18 + .../packages/migrate-ts/src/types.ts | 23 +- .../migrate-ts/src/view-column-types.ts | 110 --- .../migrate-ts/src/view-replace-probe.ts | 70 ++ .../packages/migrate-ts/src/view-replace.ts | 65 ++ .../migrate-ts/src/write-migration-flyway.ts | 14 +- .../test/check/serialize-upgrade.test.ts | 23 +- .../test/emit-drop-if-exists.test.ts | 1 - .../test/emit-postgres-create-schema.test.ts | 1 - .../expected-schema-enum-intvaluemap.test.ts | 2 +- .../integration/pg-adopt-view-239.test.ts | 4 - .../test/unit/diff-view-change-reason.test.ts | 62 +- .../unit/diff-view-recreate-columns.test.ts | 120 ++++ .../test/unit/diff-view-recreate.test.ts | 37 + .../migrate-ts/test/unit/diff.test.ts | 99 +-- .../migrate-ts/test/unit/emit-views.test.ts | 32 +- .../test/unit/view-column-types.test.ts | 115 --- .../migrate-ts/test/unit/view-replace.test.ts | 186 +++++ tools/prerelease/prerelease-link.sh | 49 +- tools/prerelease/test-prerelease-link.sh | 105 +++ 91 files changed, 3889 insertions(+), 717 deletions(-) create mode 100644 server/java/codegen-base/src/test/java/com/metaobjects/generator/direct/metadata/DocumentationOutputIsStableTest.java create mode 100644 server/java/codegen-base/src/test/java/com/metaobjects/generator/direct/object/GeneratedHeaderIsStableTest.java create mode 100644 server/typescript/packages/cli/test/allow-hint-tokens.test.ts create mode 100644 server/typescript/packages/cli/test/migrate-flyway-history.test.ts create mode 100644 server/typescript/packages/codegen-ts/test/projection/report-required-joins.test.ts create mode 100644 server/typescript/packages/integration-tests/src/load-project-model.ts create mode 100644 server/typescript/packages/integration-tests/test/load-project-model.test.ts create mode 100644 server/typescript/packages/integration-tests/test/view-replace-probe-pg.test.ts delete mode 100644 server/typescript/packages/migrate-ts/src/view-column-types.ts create mode 100644 server/typescript/packages/migrate-ts/src/view-replace-probe.ts create mode 100644 server/typescript/packages/migrate-ts/src/view-replace.ts create mode 100644 server/typescript/packages/migrate-ts/test/unit/diff-view-recreate-columns.test.ts delete mode 100644 server/typescript/packages/migrate-ts/test/unit/view-column-types.test.ts create mode 100644 server/typescript/packages/migrate-ts/test/unit/view-replace.test.ts create mode 100755 tools/prerelease/test-prerelease-link.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index e1643e34a..128c8717d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -597,6 +597,21 @@ until you regenerate. ### Changed +- **A report's `count` and `sum` columns are `bigint` (`Long` on the JVM).** Replacing a + hand-written view with a report can change its column types: a view that cast its counts to + `int` now returns `bigint`, so the replacement must be a `DROP VIEW` and a `CREATE VIEW`, not a + `CREATE OR REPLACE`, and a repository that read the column as `Int` narrows it + (`Math.toIntExact`). The API a repository serves does not have to change. +- **`@table` is the legacy name of a view's physical-name alias.** The report and projection docs + already use `@view` for a view source (`view: "v_name"`). `@table` on a view source still loads + and still names the view; new models should use `@view`. +- **Breaking when upgrading from before 1.0.13: the TypeScript plural of an entity named for a + word already ending in `s` is renamed.** The 1.0.13 pluralizer change (see its entry, "Upgrade + impact") reaches an adopter jumping straight to 1.1.0 as a renamed exported symbol, not only a + changed URL: the plural of `RealmSpecies` was `realmSpecieses` and is now `realmSpecies`, so the + generated query, hook and list names change, and code that imports the old name stops compiling + until it uses the new one. Search your imports for any entity whose name ends in `s` (`Species`, + `Stats`, `Series`) before regenerating. - **Java classes written by `entity` lose two header lines, and `mvn metaobjects:verify` no longer depends on the clock.** The header comment of every class `JavaObjectCodeGenerator` writes carried `ObjectCodeWriter:` (the writer's class name) and `Generated On:` (the @@ -954,6 +969,113 @@ until you regenerate. refused.** The preset check read `and` and `or` as field names ("references field 'or' which is not a filterable field"), so a valid composed preset failed to load. It now recurses into each composed clause and checks it like a top-level one, as TypeScript does. +- **`meta migrate --from-db --migration-format flyway` no longer proposes to drop + `flyway_schema_history`.** An adopter on a Flyway-managed database got + `DROP TABLE IF EXISTS "flyway_schema_history"` in the preview and a hint to re-run with + `--allow drop-table`; following it deletes the runner's ledger. With the flyway format the + history table is left out of the diff, since Flyway owns it, and out of a committed snapshot + that already records it (one baselined with `--from-db` before this release). That holds in + three places. `meta migrate` reads the format from `--migration-format` or from + `migrate.format` in `.metaobjects/config.json`. `meta verify --db` reported the table as + drift whichever way the format was set; it now leaves it out too, and takes the same + `--migration-format default|flyway` flag with the same precedence (flag, then config, then + `default`). `meta gen` has no such flag and reads `migrate.format` from the config: with it + set, its "now migrate" next step no longer fires for a snapshot whose only difference is the + history table. `meta migrate baseline --db` does not count the table either, so a database + holding nothing but Flyway's ledger is refused as empty instead of being baselined as already + migrated. Other formats are unchanged. +- **Postgres decides whether a view is replaced in place.** A report's `count` and `sum` columns + widen to `bigint`, and Postgres will not change a view column's type in place (`cannot change + data type of view column "calls" from integer to bigint`). It refuses at apply, after the + migration was planned. `meta migrate` no longer predicts the answer. When a live view exists + and its definition differs, a plan against the database (`--from-db`, `--apply`) creates the + new definition as a temporary view, reads the column names and types Postgres gave it, compares + them with the live view's, and rolls back. The existing columns must keep their name, type and + position, and new columns may only come last. When that holds, the plan is `CREATE OR REPLACE + VIEW`. Otherwise it is a `DROP VIEW` and a `CREATE VIEW`. `meta verify --db` asks the same + question, so its report names the change `meta migrate` would plan. + - **Planning against a live Postgres now opens a transaction and rolls it back**, where it only + read before. Nothing persists: the temporary view exists only inside the rolled-back work. + On a connection that is already in a transaction the probe uses a savepoint, and the + caller's transaction stays open and usable. + - **A replace is never planned without that proof.** When the probe cannot run, the plan is a + drop and a create: an offline `meta migrate` (the default, which diffs the committed + snapshot and has no database to ask), a role with no `TEMPORARY` privilege, a new + definition that does not compile yet because it reads a column the same migration adds, or + an `@sql` body with `SELECT *` or `NATURAL JOIN` in a migration that also adds a column (its + column list changes once the column exists, so an answer taken before would not hold). + Before, an offline plan replaced a view in place when its recorded columns looked compatible. + - **A drop and a create discards the privileges granted on the view.** `DROP VIEW` removes the + view's grants with it, and `meta migrate` writes no `GRANT`, so a role that had `SELECT` on a + projection's view loses it, with nothing reported. It needs no `--allow` and applies cleanly. + In an offline plan every changed Postgres view takes this path, including an appended field + that was a `CREATE OR REPLACE VIEW` before. A `DROP VIEW` also fails at apply if another + object depends on the view, and says which. To keep grants and dependents across the change, + plan with `--from-db`; otherwise grant the privileges again after applying. + - **Adopting a view (`--allow adopt-view`) follows the same rule**, for a derived view and for + an `@sql` one. A hand-written or pre-fingerprint view with the same columns is still replaced + in place when the probe proves it. It becomes a drop and a create when the columns differ, or + when the probe cannot run. Before, an `@sql` view, a report's view and a projection with a + computed, `first`, `any`, `all` or `collect` column were replaced without any check. + - **The comparison is exact.** `json` and `jsonb`, `smallint` and `integer`, `text` and + `character varying`, and `numeric(9,2)` and `numeric` are different types to Postgres and to + the plan. The tool's own model of view column types, which merged them, is removed. + - **The snapshot no longer records a view's columns.** Nothing reads them. The format version + stays `3`, and an older snapshot that carries them still loads; they are dropped on read. + `ExpectedView.columns` is removed from `@metaobjectsdev/codegen-ts` for the same reason. + - SQLite and D1 are unchanged. Neither has `CREATE OR REPLACE VIEW`; a replaced view is + written as `DROP VIEW` and `CREATE VIEW` there, as it always was, so there is nothing to + refuse. +- **The `--allow` hint for a blocked view change names a token the parser accepts.** It printed + `--allow replace-view`, which `meta migrate` rejects (`invalid --allow token`). It names + `adopt-view` for an unfingerprinted view and `drop-view-cascade` when the drop would cascade. + A view drop can sit behind more than one permission, and the hint names the one that blocked + this run: after `--allow drop-view-cascade` it asks for `drop-view` or `adopt-view`, not the + token you already passed. A test follows every blocked change's hints until nothing is blocked. +- **A column change recreates only the views that read the column.** On Postgres a new default or a + changed nullability no longer drops and rebuilds the views over the table (neither blocks a + view), and a type change recreates only the views whose SQL names the column (a view that + selects `*` or joins `NATURAL` reads columns it does not name, and is always recreated). SQLite and D1 still + recreate every view over a rebuilt table, and so does a `diff()` called with no `dialect`, + whose changes may be emitted for SQLite. `verify --db` and `meta migrate` now print the full + column type in a type change (`text(255) -> text`, `timestamp -> timestamptz`), so a length or + time-zone change no longer reads as `text -> text`. +- **A report dimension reached through `@via` is not nullable when every hop is a required + reference.** The view joins such a hop `INNER`, so the column is as required as its `@of` + field, but the generated Kotlin, Java, C#, Python and TypeScript types still said nullable + (`realmName: String? = null`, `.nullable()`). All five ports now derive the same answer, gated + by `report-shapes.json`. **Regenerate to see it:** a dimension of this shape changes from + nullable to non-null in the generated row type and Exposed table. A hop over an optional foreign + key, a to-one relationship whose foreign key the other entity holds, and a `@spine` report keep + their nullable columns. +- **Kotlin: a served report gets a `Names` file.** It holds the view name and one constant + per derived column, as the TypeScript and Python ports already did, so hand-written SQL over the + view has constants to use. The report's Exposed table still binds by literal. +- **The documentation writers carry no timestamp.** (The generated-class header lost its `Generated On:` line in the entry above.) The two documentation + writers in `codegen-base` stamped the clock the same way and no longer do: the HTML type + documentation's header reads `Version ` without the trailing `• Generated + `, and the AI documentation JSON has no `generationTimestamp` property in + `documentationInfo`. Nothing in the library read either; a script of yours that parses + `generationTimestamp` must stop expecting it. No other Java or Kotlin generator writes the + time of the run into its output. +- **`bun run prerelease:publish` publishes the Maven artifacts too, and `prerelease-link` pins + the right Maven version.** The default scope was npm only, so a JVM adopter could not test a + candidate without building it. The default is now `npm,java` (`--only` still picks ports). + `prerelease-link` pinned `1.1.0-rc.N` to Maven `7.1.0-rc.N`; the JVM major is the npm major + plus 7, so it is `8.1.0-rc.N`. +- **`prerelease-link` in a linked git worktree no longer leaves its `.npmrc` files committable.** + It wrote the exclude entry to a stray `home/.../info/exclude` tree inside the work tree and left + the `.npmrc` files (which can hold a registry credential) untracked but not ignored. It now asks + git for the exclude file (`git rev-parse --git-path info/exclude`) and fails if any `.npmrc` it + wrote is not ignored. +- **The independent-oracle CLI (`integration-tests`) loads a project with its providers.** It read + the metadata without the providers in `metaobjects.config.ts`, so a project that registers its + own vocabulary failed with `Unknown type`. +- **Docs: a report's columns and a cross-package model.** `fields()` on an `object.report` is + empty by design; `docs/features/reporting.md` now says to read the report's read model, that + `enforce` is on the `MetaReferenceIdentity` that `referenceIdentities()` returns, and that a bare + entity name in `@of`, `@via` and `@spine` resolves in the declaring entity's package (the + report's, for `@spine`), so another package's entity needs its package name. ## [1.0.13] — 2026-10-03 diff --git a/docs/features/cli.md b/docs/features/cli.md index a5038b14c..230950c6f 100644 --- a/docs/features/cli.md +++ b/docs/features/cli.md @@ -734,6 +734,13 @@ desyncs its `flyway_schema_history`. Generate with `meta migrate`, apply with `flyway migrate`. `--dialect d1` with `--migration-format flyway` is also refused — D1 has its own Wrangler layout and transport. +**Flyway owns `flyway_schema_history`.** Under this format the table is left out of every +schema comparison: `meta migrate` (live with `--from-db`, and a committed snapshot that +records it), `meta verify --db`, and the "now migrate" next step `meta gen` prints. +`meta verify` takes the same `--migration-format` flag with the same precedence (flag, then +`migrate.format`, then `default`); `meta gen` reads the config key. Set `migrate.format` once +and all three agree without a flag. + ```bash meta migrate --db "$DB_URL" --dialect postgres \ --migration-format flyway --slug add_program_view diff --git a/docs/features/migrations-and-drift.md b/docs/features/migrations-and-drift.md index 53c9cffb0..29bc49521 100644 --- a/docs/features/migrations-and-drift.md +++ b/docs/features/migrations-and-drift.md @@ -13,7 +13,7 @@ There are **7 drift sources**, and the toolchain has a guard for each. |---|---|---| | **Code-vs-DB** | Codegen — the generated SQL DDL is emitted from the same metadata as the entity / table code. | Build time | | **Code-vs-API-doc** | Cross-port codegen from the same metadata. | Build time | -| **DB-vs-metadata** | `meta verify --db` (TS CLI) — introspects the live DB and fails if it has drifted from metadata. Includes modeled projection **view bodies** (a changed `CREATE VIEW` is `replace-view` drift, reported with what differs: the first differing excerpt of the text on SQLite/D1, a fingerprint mismatch on Postgres). A view `meta migrate` drops and recreates only because the migration alters a table it reads, with its definition unchanged, is not drift and is not reported; a hand-authored *unmodeled* view is unmanaged and never flagged. A schema concern owned by the Node toolchain regardless of server language; on the JVM ports the runtime auto-create/validator path was removed (ADR-0015) and the `metaobjects:verify` Maven goal is not available. Cloudflare D1 has no client wire protocol, so it can't go through `--db`'s Kysely-driver introspection — use `meta verify --dialect d1 [--d1 ] [--remote]` instead (the same wrangler-shelled-out path `meta migrate --dialect d1` uses); `--remote` is required to check the *deployed* database, not the local `wrangler dev` shadow copy. Pointing `--db file:` at wrangler's local D1 state directory (`.wrangler/state/**/d1/**`) still runs, but only verifies that local copy — `verify` warns when it detects this. | CI on every PR | +| **DB-vs-metadata** | `meta verify --db` (TS CLI) — introspects the live DB and fails if it has drifted from metadata. Includes modeled projection **view bodies** (a changed `CREATE VIEW` is drift, reported with what differs: the first differing excerpt of the text on SQLite/D1, a fingerprint mismatch on Postgres). A view `meta migrate` drops and recreates only because the migration alters a table it reads, with its definition unchanged, is not drift and is not reported; a hand-authored *unmodeled* view is unmanaged and never flagged. A schema concern owned by the Node toolchain regardless of server language; on the JVM ports the runtime auto-create/validator path was removed (ADR-0015) and the `metaobjects:verify` Maven goal is not available. Cloudflare D1 has no client wire protocol, so it can't go through `--db`'s Kysely-driver introspection — use `meta verify --dialect d1 [--d1 ] [--remote]` instead (the same wrangler-shelled-out path `meta migrate --dialect d1` uses); `--remote` is required to check the *deployed* database, not the local `wrangler dev` shadow copy. Pointing `--db file:` at wrangler's local D1 state directory (`.wrangler/state/**/d1/**`) still runs, but only verifies that local copy — `verify` warns when it detects this. | CI on every PR | | **Migration-vs-metadata** | The Node `meta migrate` emits migrations FROM metadata diffs — they cannot drift from metadata by construction. Schema migrations for **every** port are owned by this Node toolchain (`@metaobjectsdev/cli migrate`, ADR-0015); the C# and Python migrate surfaces were removed. | Build time | | **Generated-edited** | `@generated` headers in emitted code + three-way merge that preserves hand-edits inside non-generated regions. | Code review | | **Prompt-vs-payload** | FR-004 `Renderer.verify` parses `{{...}}` references in templates and checks each one exists on the payload VO. | Build time + runtime | @@ -408,18 +408,51 @@ A projection's `CREATE VIEW` is generated from its `origin.*` children, so `meta owns the view. Three things are worth knowing. **Append projection fields; don't insert them.** Postgres can update a view in place -(`CREATE OR REPLACE VIEW`) only when the existing output columns are unchanged and any -new ones are added at the **end**. A view's columns come out in projection *declaration* -order, so: - -- adding a field **at the end** of a projection → non-destructive replace. Dependent - views, grants, and the view's identity all survive. -- inserting a field **in the middle**, reordering, renaming, or removing one → the view - must be **dropped and recreated**, which is destructive to anything that depends on it - and is therefore gated (`--allow drop-view`). - -Body-only changes — a different join path, an `origin.aggregate @filter`, a changed -aggregate that keeps the same result type — are always non-destructive. +(`CREATE OR REPLACE VIEW`) only when every existing output column keeps its name, type and +position and any new ones are added at the **end**. A view's columns come out in projection +*declaration* order, so: + +- adding a field **at the end** of a projection, or changing a body without changing a + column (a different join path, an `origin.aggregate @filter`) → the view can be replaced + in place. Dependent views, grants, and the view's identity all survive. +- inserting a field **in the middle**, reordering, renaming, removing or retyping one → the + view must be **dropped and recreated**, which is destructive to anything that depends on + it. + +**Postgres decides which, and only a plan against the database can ask it.** The types a +query produces are Postgres's to say: `max` of a `numeric(9,2)` column is `numeric`, +`COALESCE` of a `varchar(40)` column and a literal is a bare `character varying`, and `json` +is not `jsonb`. So `meta migrate` does not predict them. When a live view exists and its +definition differs, a plan against the database (`meta migrate --from-db`, `--apply`, and the +drift report of `meta verify --db`) creates the new definition as a **temporary view inside a +transaction it rolls back**, reads the column names and types Postgres gave it, and compares +them with the live view's. Planning on this path therefore writes inside a rolled-back +transaction (a savepoint, when the connection is already in one); nothing persists. A replace +is planned only on that proof. When the question cannot be asked, the plan is a `DROP VIEW` +and a `CREATE VIEW`: + +- the default **offline** `meta migrate`, which diffs the committed snapshot and has no + database; +- a role that may not create a temporary view (no `TEMPORARY` privilege on the database); +- a new definition that does not compile against the schema as it stands, because it reads a + column the same migration adds; +- an `@sql` body that reads columns it does not name (`SELECT *`, `NATURAL JOIN`) in a + migration that also adds a column. The probe sees the schema before the column exists, and + the body's column list changes once it does, so the answer would not hold. + +A drop and a create of the same view needs no `--allow`. It has two costs a replace does not: + +- **Privileges granted on the view are discarded.** `DROP VIEW` removes the view's grants with + it, and `meta migrate` writes no `GRANT`. A role that had `SELECT` on the view loses it, and + nothing reports that. In an offline plan every changed Postgres view takes this path. +- If something depends on the view, Postgres refuses the `DROP VIEW` at apply and names the + dependent. + +Plan with `--from-db` to get the in-place replace when it is legal, which keeps both. Otherwise +grant the privileges again after the migration is applied. + +SQLite and D1 have no `CREATE OR REPLACE VIEW`: every replaced view is written as `DROP VIEW` +and `CREATE VIEW` there, so there is no refusal to avoid and no probe. **A cascading drop is blocked, loudly.** If dropping a view would destroy dependent objects — another application's view, a materialized view — `meta migrate` blocks and @@ -440,7 +473,10 @@ closed rather than overwrite somebody's hand-written SQL: meta migrate --allow adopt-view # once per environment, after upgrading ``` -That stamps the existing views. Afterwards they converge silently. This is also what +That stamps the existing views. Afterwards they converge silently. An adopted view follows +the rule above, whether the tool derives its body or it is an `@sql` one: replaced in place +when the database shows its columns are kept, dropped and recreated otherwise, including +whenever the probe cannot run. This is also what closes the loop on the doctrine in [downstream-metadata-decisions.md](downstream-metadata-decisions.md): a hand-written view sitting where a projection expects one is now **visible to `meta verify --db`** as drift, diff --git a/docs/features/prerelease.md b/docs/features/prerelease.md index 1b472876f..3367174be 100644 --- a/docs/features/prerelease.md +++ b/docs/features/prerelease.md @@ -30,7 +30,7 @@ One canonical internal string, normalized per ecosystem in exactly one place | | canonical | npm | PyPI | NuGet | Maven | |---|---|---|---|---|---| | form | `-rc.` | `0.24.0-rc.3` | `0.24.0rc3` | `0.24.0-rc.3` | `7.24.0-rc.3` | -| why | | SemVer2 verbatim | PEP 440 canonical form | SemVer2 verbatim | same `minor.patch` on the historical major `7` | +| why | | SemVer2 verbatim | PEP 440 canonical form | SemVer2 verbatim | same `minor.patch`, major is the npm major plus 7 (`1.1.0-rc.2` is `8.1.0-rc.2`) | `` is the in-development version, read from **CHANGELOG.md's topmost `## [x.y.z]` header** — the repository's own declaration of what is being worked on, written when the @@ -100,7 +100,7 @@ cp tools/prerelease/registry.env.example tools/prerelease/registry.env Then: ```bash -bun run prerelease:publish # next iteration, npm (the default scope) +bun run prerelease:publish # next iteration, npm + Maven (the default scope) bun run prerelease:publish --only python,csharp # pick ports bun run prerelease:publish --only all # all four bun run prerelease:publish --iter 7 # pin the iteration number diff --git a/docs/features/reporting.md b/docs/features/reporting.md index 47565b62c..d281b7693 100644 --- a/docs/features/reporting.md +++ b/docs/features/reporting.md @@ -191,6 +191,15 @@ order: one per dimension, then one per measure. | time dimension `purchasedAt` at grain `day` | `purchasedAtDay` (``) | | measure `revenue` | `revenue` | +An `object.report` node has no field children, so `fields()` on it is empty. A report's +columns are its **read model**: in TypeScript `reportReadModel(report, root).fields()` (or +`reportShape(report, root).fields` for the shape with its dimension and measure nodes), in Java +and Kotlin `ReportReadModel.of(report)`, in Python `report_read_model`, in C# `ReportShapes.Of`. +Code that walks `fields()` to list a report's columns (a schema digest, say) should read the read +model. To tell a logical reference from an enforced foreign key, read `enforce` on the +`MetaReferenceIdentity` that `referenceIdentities()` returns, not on the base `MetaIdentity` that +`identities()` returns. + A `@dimensions` item is a dimension name, or `name:grain` for a time dimension (a single colon, so it cannot collide with the `::` package separator). A `@measures` item is a measure name, or `Entity.name` where `Entity` is the `@from` entity or one it extends; both forms name @@ -249,7 +258,7 @@ physical column name is your naming strategy applied to the **derived field name | Item | Column | Type | Never null? | |---|---|---|---| -| `dimension.attribute` | the dimension's name | the `@of` field's type | only when the dimension has no `@via` and the `@of` field is required (`@required: true` or a `validator.required` child) | +| `dimension.attribute` | the dimension's name | the `@of` field's type | only when the `@of` field is required (`@required: true` or a `validator.required` child) and the dimension has no `@via`, or every hop of its `@via` is a required reference the view joins `INNER` (below) | | `dimension.time` at `hour` | `Hour` | `timestamp` | same rule | | `dimension.time` at `day`, `week`, `month`, `quarter`, `year` | `` | `date`, the first day of the bucket | same rule | | `count` (with or without `@distinct`) | the measure's name | `long` | yes: a count is never null | @@ -264,9 +273,18 @@ physical column name is your naming strategy applied to the **derived field name | any measure above except `count`, declaring `@default` | the measure's name | as above | yes: it reads its `@default` instead of null | | in a `@spine` report, a dimension over a column of the spine entity | the dimension's name | the `@of` field's type | yes when the `@of` field is required (`@required` or a `validator.required` child) or one of the spine entity's `identity.primary` `@fields`; no for a dimension reached beyond the spine entity | -The last two rows are the only ways a measure other than `count`, or a dimension reached -through `@via`, is never null. The key clause applies only under `@spine`, where the key is -the row's own; without `@spine` a dimension reached through `@via` stays nullable. +The last two rows, and the `INNER` rule above, are the only ways a measure other than `count`, +or a dimension reached through `@via`, is never null. The key clause applies only under +`@spine`, where the key is the row's own. + +Without `@spine`, a dimension reached through `@via` is never null when its `@of` field is +required and **every hop is a belongs-to reference over a required foreign key**: the entity the +hop leaves holds the `identity.reference`, and its first `@fields` entry is required. The view +joins such a hop `INNER`, so every row has a match, whatever the reference's `@enforce` says (an +unenforced required reference still joins `INNER`; make its field optional to keep unmatched +rows). A hop over an optional foreign key, a to-one relationship whose foreign key the other +entity holds, or any hop under `@spine` is joined `LEFT OUTER`, and the dimension stays nullable. Every port derives the same answer +(`fixtures/persistence-conformance/report-shapes.json`). A derived column carries the type-shaping attributes of its `@of` field where they apply (`@currency`, `@values`, `@intValueMap`, `@maxLength`, `@precision`, `@scale`, `@localTime`, @@ -334,6 +352,13 @@ program. from the fact row (no `@via`, or a `@via` through another reference) is refused at load: it has no value in a row with no facts. A time dimension follows the same rule: the month a program was published is legal, the month of a purchase is not. +- **Package names in a cross-package model.** A bare entity name inside a member resolves in the + package of the entity that **declares** the member, never in `@from`'s package: that is + `@of` and `@via`. The owner of `@spine`, and the qualifier of a dotted `@measures` item, + resolve in the **report's** package. When the entity you name lives in another package, write it with its package + (`acme::session::SessionHealthSnapshot.fkSessionId`), or the load fails with `must start at + <@from>`: the bare name found nothing in that package, or found a different entity of the same + name. - **Hop names are compared as written.** A spine written with the relationship's name (`Purchase.program`) and a dimension written with the reference's name (`Purchase.fkProgram`) name the same join, and are still refused. Write the same hops in both. @@ -493,7 +518,7 @@ encodings"), applied to the column types above: | TypeScript | `.ts` (Drizzle view binding, Zod read schema, row type, descriptor, filter and sort allowlists), `.queries.ts` (the list query only), `.routes.ts` (and `.routes.hono.ts` from the Hono routes generator), `.names.ts`, `.hooks.ts` (the TanStack list hook only) with the `.meta.ts` descriptor it imports, the barrel export | | C# | `.g.cs` (keyless row class) and its `DbContext` mapping, `Routes.g.cs`, `FilterAllowlist.g.cs` | | Java | `Dto`, `Repository` (`list` and `count` only), `FilterAllowlist`, `Controller` | -| Kotlin | `Table` (Exposed), the `` data class, `FilterAllowlist`, `Controller` | +| Kotlin | `Table` (Exposed), the `` data class, `FilterAllowlist`, `Controller`, and `Names` when the opt-in `KotlinNamesGenerator` is wired | | Python | `.py` (Pydantic row model), `_filter_allowlist.py`, `_router.py`, `_names.py` | No port generates a by-id query, a `findById` on a repository seam, a create or update @@ -505,10 +530,11 @@ hook generator asks `servesClientHooks`, which is true for a served report; the generators ask `servesClientTier`, which is false for one. A hook generator you own was copied with the predicate it had then and emits no report hook until you resync it (`meta eject hooks`, then merge your edits); the other UI generators you own need -nothing. TypeScript and -Python write a names artifact because their read model flows through the names generator; -C#, Java and Kotlin bind the view and its columns by literal. These are reference helpers, -not core: copy and own them with your port's `eject`. +nothing. TypeScript, Python and Kotlin write a names artifact from the report's read model: the +view name and one constant per derived column, for hand-written SQL over the view to use. +Kotlin's is opt-in like every `Names`, and its Exposed table still binds the view and +its columns by literal, as C# and Java do, which write no names artifact for a report. These +are reference helpers, not core: copy and own them with your port's `eject`. **`meta docs`.** Every report gets a model page (kind `report`, its `@from`, its view or "Not served" with the reason, its row scope, and a column table with a definition per diff --git a/docs/ports/java.md b/docs/ports/java.md index 5bfc434c0..6a6391dec 100644 --- a/docs/ports/java.md +++ b/docs/ports/java.md @@ -370,8 +370,10 @@ like a keyless read-only projection. Four generators write one file each for a r `filter-allowlist` writes `FilterAllowlist`, and `routes` writes `Controller` (one `@GetMapping`, a `@PostMapping` answering 405, no `/{id}` mapping). Every derived field with filter operators is filterable and sortable. A DTO component that can never be null carries -`@NotNull`, as any required field does: a `count`, a measure with `@default`, and in a -`@spine` report a key or `@required` column of the spine entity (`@NotNull Long programKey`). +`@NotNull`, as any required field does: a `count`, a measure with `@default`, a dimension +reached through `@via` whose `@of` field is required and whose every hop is a belongs-to +reference over a required foreign key (the view joins it `INNER`), and in a `@spine` report a +key or `@required` column of the spine entity (`@NotNull Long programKey`). No other generator emits for a report: no names class, no entity class. `gen` fails, naming the report and the dimension or measure, when a derived field reads a `field.object`. A report with no view source, or an abstract one, diff --git a/docs/ports/kotlin.md b/docs/ports/kotlin.md index 725ca7b78..aa3e8e4d6 100644 --- a/docs/ports/kotlin.md +++ b/docs/ports/kotlin.md @@ -94,7 +94,7 @@ The 15 generators registered in `codegen-kotlin` (`GeneratorRegistry.kt`): |---|---|---| | `KotlinEntityGenerator` | `.kt` — Kotlin `data class` (Jackson-compatible; no `@Serializable`). For a value object this data class IS the template tier's payload/response type (ADR-0056); a value object a responding prompt parses into also gets `Extracted.kt` (the lenient mirror, written by the parser tier once per run, beside it) | every `object.entity`, `object.value`, and `object.projection` | | `KotlinExposedTableGenerator` | `Table.kt` — Exposed `Table` object with PK + FK + `@storage` columns | entities with `source.rdb` | -| `KotlinNamesGenerator` | `Names.kt` — physical database name constants mirroring the metadata tree (per-role source name + kind + schema, columns, identity/index names) | every object with a declared/inherited primary `source.rdb` | +| `KotlinNamesGenerator` | `Names.kt` — physical database name constants mirroring the metadata tree (per-role source name + kind + schema, columns, identity/index names) | every object with a declared/inherited primary `source.rdb`, and every served report (its view and derived columns) | | `KotlinRelationsGenerator` | `Relations.kt` — extension fns for `cardinality=many` query helpers | entities with to-many relationships | | `KotlinRepositoryGenerator` | `RepositoryBase.kt` — persistence repository base (row-mapper + CRUD + patch) | writable entities (`source.rdb @kind="table"`) | | `KotlinFilterAllowlistGenerator` | `FilterAllowlist.kt` — FR-009 filter allowlist (filterable field names + allowed ops per field) | writable entities (`source.rdb @kind="table"`) | @@ -418,9 +418,11 @@ then measures). `KotlinEntityGenerator` writes ``, an immutable data cla property per derived field and no validation annotations. `KotlinFilterAllowlistGenerator` writes `FilterAllowlist`, and `KotlinSpringControllerGenerator` writes `Controller`: one `@GetMapping` list handler, a `@PostMapping` answering 405, and no -`/{id}` mapping. Every derived field with filter operators is filterable and sortable. No -`Names` is written, and nothing from any other generator. A report with no view source, -or an abstract one, generates nothing. See [reporting](../features/reporting.md) for the vocabulary, the columns a +`/{id}` mapping. Every derived field with filter operators is filterable and sortable. A fifth, +the opt-in `KotlinNamesGenerator`, writes `Names` when you wire it: the view name and +one constant per derived column (see [`Names`](#entitynames--the-physical-names-as-constants)). +Nothing is written by any other generator. A report with no view source, or an abstract one, +generates nothing. See [reporting](../features/reporting.md) for the vocabulary, the columns a report gets and the REST contract. An excerpt of the table, under the default snake_case column naming: @@ -437,10 +439,14 @@ object ProgramMinutesTable : Table("v_program_minutes") { - A report has no identity, so the object has no `primaryKey`. List it and count it; there is no by-id read and no write. - A column is nullable exactly when the derived field can be null: a `sum`, `avg`, `min`, `max` - or ratio, and a dimension reached through `@via`. Two exceptions: a measure that declares - `@default` is never null, and in a `@spine` report neither is a dimension over a `@required` - or primary-key column of the spine entity. Each gets a column with no `.nullable()` and a - property with no `?` and no `= null`. + or ratio, a dimension over an optional field, and a dimension reached through `@via` when a + hop can leave a row without a match. Three cases are never null: a measure that declares + `@default`; a dimension reached through `@via` whose `@of` field is required and whose every + hop is a belongs-to reference over a required foreign key, which the view joins `INNER`; and + in a `@spine` report a dimension over a `@required` or primary-key column of the spine + entity. Each gets a column with no `.nullable()` and a property with no `?` and no `= null`. + The rule is the report's, not Kotlin's: see + [reporting](../features/reporting.md#a-reports-fields-are-derived-not-declared). - A derived decimal with no declared precision (an `avg`, a ratio, a `sum` of a decimal) is read as `decimal(name, 38, 18)`. Exposed rounds a decimal to the column's scale when it reads it, so the value is exact to 18 places. The object maps a view, so those numbers never reach @@ -448,7 +454,8 @@ object ProgramMinutesTable : Table("v_program_minutes") { - An enum dimension is typed by the enum class of the entity it reads (`ProgramStatus` for a dimension over `Program.status`), so it compares against the same constants as the entity's own column. No per-report enum is generated. -- The view and its columns are bound by string literal even when `useNames` is on. +- The table binds the view and its columns by string literal even when `useNames` is on; + `Names` is for your own SQL over the view. - `gen` fails, naming the report and the dimension or measure, when a derived field is named after a Kotlin hard keyword or when two derived fields land on one column property (see below for the `Column` suffix). Rename the item. diff --git a/fixtures/persistence-conformance/report-shapes.json b/fixtures/persistence-conformance/report-shapes.json index ee239f73d..ae0a753df 100644 --- a/fixtures/persistence-conformance/report-shapes.json +++ b/fixtures/persistence-conformance/report-shapes.json @@ -16,7 +16,7 @@ "name": "programTitle", "role": "dimension", "subType": "string", - "required": false, + "required": true, "typeSource": "fitness::Program.title" }, { @@ -681,7 +681,7 @@ "name": "programTitle", "role": "dimension", "subType": "string", - "required": false, + "required": true, "typeSource": "fitness::Program.title" }, { @@ -737,7 +737,7 @@ "name": "programTitle", "role": "dimension", "subType": "string", - "required": false, + "required": true, "typeSource": "fitness::Program.title" }, { diff --git a/scripts/ci-local.sh b/scripts/ci-local.sh index 0d6291e94..018542acf 100755 --- a/scripts/ci-local.sh +++ b/scripts/ci-local.sh @@ -419,7 +419,7 @@ gate_site_reference() { bun_install && bun scripts/build-site-reference.ts --che # Also runs the Maven-coordinate derivation`s own tests: the same rule serves four release # scripts and two of them used to hardcode major 7, so the 1.0 cases (never released, never # exercised) are the ones worth a test at all. See scripts/maven-coordinate.mjs. -gate_release_tag() { bun test scripts/finish-release.test.ts && node scripts/test-maven-coordinate.mjs; } +gate_release_tag() { bun test scripts/finish-release.test.ts && node scripts/test-maven-coordinate.mjs && bash tools/prerelease/test-prerelease-link.sh; } # ── repo-root scripts/ typechecks ───────────────────────────────────────────── # `bun test` transpiles per file and never typechecks, and `bun run --filter '*' diff --git a/scripts/prerelease.mjs b/scripts/prerelease.mjs index 4df25c17f..c6b5aac9c 100644 --- a/scripts/prerelease.mjs +++ b/scripts/prerelease.mjs @@ -3,7 +3,7 @@ // unreleased change can be iterated against downstream consumers without cutting a real // release on npm / PyPI / NuGet / Maven Central. // -// bun run prerelease:publish # next iteration, npm (the default scope) +// bun run prerelease:publish # next iteration, npm + Maven (the default scope) // bun run prerelease:publish --only npm,csharp # pick ports // bun run prerelease:publish --only all # all four // bun run prerelease:publish --iter 5 # pin the iteration number @@ -24,7 +24,7 @@ // npm 0.24.0-rc.3 SemVer2, verbatim // NuGet 0.24.0-rc.3 SemVer2, verbatim // PyPI 0.24.0rc3 PEP 440 canonical form -// Maven 7.24.0-rc.3 same minor.patch on the historical major 7 +// Maven 7.24.0-rc.3 same minor.patch, major = npm major + 7 (1.1.0-rc.2 -> 8.1.0-rc.2) // // The iteration counter — not a commit sha — is the collision breaker, because npm STRIPS // SemVer build metadata: `0.24.0-rc.1+aaa` and `0.24.0-rc.1+bbb` compare EQUAL, so the @@ -220,7 +220,10 @@ const V = { }; const ALL = ["npm", "python", "csharp", "java"]; -const onlyArg = flag("only", "npm"); +// The default is npm AND Maven: a JVM adopter cannot test a candidate that exists only on npm, and +// the one that tried had to build the Maven artifacts from source by hand. Python and C# stay opt-in. +const DEFAULT_ONLY = "npm,java"; +const onlyArg = flag("only", DEFAULT_ONLY); const ONLY = onlyArg === "all" ? ALL : onlyArg.split(",").map((s) => s.trim()); const unknown = ONLY.filter((p) => !ALL.includes(p)); if (unknown.length) die(`--only expects ${ALL.join("|")}|all, got '${unknown.join(",")}'`); @@ -238,6 +241,9 @@ ok(`publish target verified: ${expectedHost} (configured registry; not a public const VERSION_FILES = [ "server/typescript/packages/*/package.json", "client/web/packages/*/package.json", "server/java/pom.xml", "server/java/**/pom.xml", + // Names the reactor version in a property; check-pom-versions.sh fails + // the Java step if it lags, so the candidate bump has to reach it too. + "examples/showcase/jvm/pom.xml", "server/python/pyproject.toml", "server/csharp/Directory.Build.props", "bun.lock", ]; @@ -357,7 +363,10 @@ if (wants("java")) { // Tree-wide sed, NOT `mvn versions:set` — versions:set walks only the reactor and // silently skips the two reactor-EXCLUDED integration-test modules, whose parent // version then lags (see scripts/check-pom-versions.sh, docs/RELEASING.md). - const poms = sh(`grep -rl '${javaReleased}' --include=pom.xml server/java`).trim().split("\n").filter(Boolean); + const poms = [ + ...sh(`grep -rl '${javaReleased}' --include=pom.xml server/java`).trim().split("\n").filter(Boolean), + "examples/showcase/jvm/pom.xml", + ]; touch(...poms); sh(`sed -i 's/${javaReleased.replace(/\./g, "\\.")}/${V.maven}/g' ${poms.map((f) => `'${f}'`).join(" ")}`); sh("scripts/check-pom-versions.sh"); diff --git a/server/csharp/MetaObjects.Codegen.Tests/ReportRowCodegenTests.cs b/server/csharp/MetaObjects.Codegen.Tests/ReportRowCodegenTests.cs index 2d5a29ecb..37c83f20e 100644 --- a/server/csharp/MetaObjects.Codegen.Tests/ReportRowCodegenTests.cs +++ b/server/csharp/MetaObjects.Codegen.Tests/ReportRowCodegenTests.cs @@ -412,11 +412,11 @@ private static MetaRoot Cube() => Load(Report("SalesCube", "\"@kind\": \"view\", \"@view\": \"v_sales_cube\"", AllDimensions, AllMeasures)); [Theory] - // Dimensions: the @of field's type; nullable unless it has no @via and @of is @required. + // Dimensions: the @of field's type; nullable unless @of is @required and every @via hop is a required reference. [InlineData("public long Store { get; set; }")] [InlineData("public string? Channel { get; set; }")] [InlineData("public SalesCubeStatus Status { get; set; }")] - [InlineData("public string? StoreRegion { get; set; }")] // @required at the source, reached by @via + [InlineData("public string StoreRegion { get; set; } = default!;")] // @required at the source, reached by @via over a required reference (INNER join) [InlineData("public DateTimeOffset SoldAtHour { get; set; }")] // hour of an instant [InlineData("public DateOnly SoldAtDay { get; set; }")] // day or coarser is a date [InlineData("public DateOnly SoldAtMonth { get; set; }")] diff --git a/server/csharp/MetaObjects.Conformance.Tests/ReportShapeTests.cs b/server/csharp/MetaObjects.Conformance.Tests/ReportShapeTests.cs index 2111299e2..d61f46775 100644 --- a/server/csharp/MetaObjects.Conformance.Tests/ReportShapeTests.cs +++ b/server/csharp/MetaObjects.Conformance.Tests/ReportShapeTests.cs @@ -89,11 +89,11 @@ public void Dimensions_come_first_in_listed_order_then_measures() } [Fact] - public void A_dimension_reached_by_via_is_nullable_and_a_count_never_is() + public void A_dimension_reached_by_a_required_via_is_required_and_a_count_always_is() { var fields = Shape(LoadCanonical(), "ProgramMinutes").Fields.ToDictionary(f => f.Name); Assert.True(fields["program"].Required); // no @via, the @of field is @required - Assert.False(fields["programTitle"].Required); // reached by @via + Assert.True(fields["programTitle"].Required); // reached by @via over a required reference: an INNER join (D5) Assert.True(fields["weeks"].Required); // count Assert.False(fields["totalMinutes"].Required); // a sum of nothing is null Assert.Equal(FIELD_SUBTYPE_LONG, fields["totalMinutes"].SubType); @@ -527,4 +527,88 @@ public void A_spine_that_does_not_resolve_throws_naming_the_report(string spine) ex = Assert.Throws(() => ReportShapes.Of(report, root)); Assert.Equal($"report 'SpineSales': @spine '{spine}' does not resolve.", ex.Message); } + + // --------------------------------------------------------------------------- + // Table C, a dimension reached through @via (adopter report D5): the view joins a belongs-to + // hop over a required foreign key INNER, so the column is as required as the field it reads. + // The same model and expectations as the TypeScript, Java and Python report-shape tests. + // --------------------------------------------------------------------------- + + private static string ViaEntity(string name, string table, string children) => + "{ \"object.entity\": { \"name\": \"" + name + "\", \"children\": [" + + "{ \"source.rdb\": { \"@table\": \"" + table + "\" } }, { \"field.long\": { \"name\": \"id\" } }, " + + "{ \"identity.primary\": { \"name\": \"pk\", \"@fields\": [\"id\"] } }, " + children + "] } }"; + + private static string ViaReport(string name, string extra, string dims) => + "{ \"object.report\": { \"name\": \"" + name + "\", \"@from\": \"Snap\", " + extra + + "\"@dimensions\": [" + dims + "], \"@measures\": [\"snaps\"], " + + "\"children\": [{ \"source.rdb\": { \"@kind\": \"view\", \"@view\": \"v_" + name.ToLowerInvariant() + "\" } }] } }"; + + private static MetaRoot ViaModel() => LoadInline(AcmeFile( + ViaEntity("Realm", "realms", + "{ \"field.string\": { \"name\": \"name\", \"@required\": true } }, { \"field.string\": { \"name\": \"motto\" } }"), + ViaEntity("Session", "sessions", + "{ \"field.long\": { \"name\": \"realmId\", \"@required\": true } }, { \"field.long\": { \"name\": \"maybeRealmId\" } }, " + + "{ \"identity.reference\": { \"name\": \"fkRealm\", \"@fields\": [\"realmId\"], \"@references\": \"Realm\" } }, " + + "{ \"identity.reference\": { \"name\": \"fkMaybeRealm\", \"@fields\": [\"maybeRealmId\"], \"@references\": \"Realm\" } }, " + + "{ \"relationship.association\": { \"name\": \"realm\", \"@objectRef\": \"Realm\", \"@cardinality\": \"one\", \"@sourceRefField\": \"realmId\" } }, " + + "{ \"relationship.association\": { \"name\": \"maybeRealm\", \"@objectRef\": \"Realm\", \"@cardinality\": \"one\", \"@sourceRefField\": \"maybeRealmId\" } }"), + ViaEntity("Snap", "snaps", + "{ \"field.long\": { \"name\": \"sessionId\", \"@required\": true } }, { \"field.long\": { \"name\": \"unenforcedSessionId\", \"@required\": true } }, " + + "{ \"identity.reference\": { \"name\": \"fkSession\", \"@fields\": [\"sessionId\"], \"@references\": \"Session\" } }, " + + "{ \"identity.reference\": { \"name\": \"fkLoose\", \"@fields\": [\"unenforcedSessionId\"], \"@references\": \"Session\", \"@enforce\": false } }, " + + "{ \"relationship.association\": { \"name\": \"session\", \"@objectRef\": \"Session\", \"@cardinality\": \"one\" } }, " + + "{ \"relationship.association\": { \"name\": \"stamp\", \"@objectRef\": \"Stamp\", \"@cardinality\": \"one\" } }, " + + "{ \"measure.aggregate\": { \"name\": \"snaps\", \"@agg\": \"count\", \"@of\": \"Snap.id\" } }, " + + "{ \"dimension.attribute\": { \"name\": \"realmName\", \"@of\": \"Realm.name\", \"@via\": \"Snap.fkSession.fkRealm\" } }, " + + "{ \"dimension.attribute\": { \"name\": \"realmMotto\", \"@of\": \"Realm.motto\", \"@via\": \"Snap.fkSession.fkRealm\" } }, " + + "{ \"dimension.attribute\": { \"name\": \"maybeRealmName\", \"@of\": \"Realm.name\", \"@via\": \"Snap.fkSession.fkMaybeRealm\" } }, " + + "{ \"dimension.attribute\": { \"name\": \"looseRealmName\", \"@of\": \"Realm.name\", \"@via\": \"Snap.fkLoose.fkRealm\" } }, " + + "{ \"dimension.attribute\": { \"name\": \"sessionRealmId\", \"@of\": \"Session.realmId\", \"@via\": \"Snap.fkSession\" } }, " + + "{ \"dimension.attribute\": { \"name\": \"relRealmName\", \"@of\": \"Realm.name\", \"@via\": \"Snap.session.realm\" } }, " + + "{ \"dimension.attribute\": { \"name\": \"relMaybeRealmName\", \"@of\": \"Realm.name\", \"@via\": \"Snap.session.maybeRealm\" } }, " + + "{ \"dimension.attribute\": { \"name\": \"relSessionRealmId\", \"@of\": \"Session.realmId\", \"@via\": \"Snap.session\" } }, " + + "{ \"dimension.attribute\": { \"name\": \"mixedRealmName\", \"@of\": \"Realm.name\", \"@via\": \"Snap.session.fkRealm\" } }, " + + "{ \"dimension.attribute\": { \"name\": \"stampLabel\", \"@of\": \"Stamp.label\", \"@via\": \"Snap.stamp\" } }"), + // The foreign key between Snap and Stamp is held by Stamp, the far side of Snap's `stamp`. + ViaEntity("Stamp", "stamps", + "{ \"field.long\": { \"name\": \"snapId\", \"@required\": true } }, { \"field.string\": { \"name\": \"label\", \"@required\": true } }, " + + "{ \"identity.reference\": { \"name\": \"fkSnap\", \"@fields\": [\"snapId\"], \"@references\": \"Snap\" } }"), + ViaReport("Related", "", + "\"relRealmName\", \"relMaybeRealmName\", \"relSessionRealmId\", \"mixedRealmName\", \"stampLabel\""), + ViaReport("Plain", "", + "\"realmName\", \"realmMotto\", \"maybeRealmName\", \"looseRealmName\", \"sessionRealmId\""), + ViaReport("Spined", "\"@spine\": \"Snap.fkSession\", ", "\"realmName\", \"sessionRealmId\""))); + + [Fact] + public void A_via_dimension_is_required_when_every_hop_is_a_required_belongs_to() + { + var root = ViaModel(); + var plain = RequiredOf(root, "Plain"); + Assert.True(plain["realmName"]); // two required hops over a required field + Assert.True(plain["sessionRealmId"]); // one required hop + Assert.True(plain["looseRealmName"]); // @enforce: false does not change the join type + Assert.False(plain["realmMotto"]); // the target field is optional + Assert.False(plain["maybeRealmName"]); // a hop over a nullable foreign key is a LEFT join + + var spined = RequiredOf(root, "Spined"); + Assert.False(spined["realmName"]); // under @spine every join beyond the spine is LEFT + Assert.True(spined["sessionRealmId"]); // on the spine, a required field + } + + // The same rule when a hop is a @cardinality: one relationship instead of the reference it + // crosses: the relationship resolves to that reference (by name pairing for `session`, by + // @sourceRefField for the two onto Realm), and the foreign key it holds decides the join. + [Fact] + public void A_to_one_relationship_hop_follows_the_reference_it_crosses() + { + var related = RequiredOf(ViaModel(), "Related"); + Assert.True(related["relRealmName"]); // two relationship hops, both over a required foreign key + Assert.True(related["relSessionRealmId"]); // one relationship hop + Assert.True(related["mixedRealmName"]); // a relationship hop, then a reference hop + Assert.False(related["relMaybeRealmName"]); // the second relationship crosses a nullable foreign key + // The other entity holds the foreign key: a Snap with no Stamp has no match, so the join is + // LEFT whatever that key and the target field require. + Assert.False(related["stampLabel"]); + } } diff --git a/server/csharp/MetaObjects.IntegrationTests/Generated/ProgramMinutes.g.cs b/server/csharp/MetaObjects.IntegrationTests/Generated/ProgramMinutes.g.cs index 86d9ac20c..42f262166 100644 --- a/server/csharp/MetaObjects.IntegrationTests/Generated/ProgramMinutes.g.cs +++ b/server/csharp/MetaObjects.IntegrationTests/Generated/ProgramMinutes.g.cs @@ -13,8 +13,10 @@ public class ProgramMinutes [Column("program")] public long Program { get; set; } [Column("programTitle")] + [Required(AllowEmptyStrings = true)] [MaxLength(200)] - public string? ProgramTitle { get; set; } + [MinLength(1)] + public string ProgramTitle { get; set; } = default!; [Column("weeks")] public long Weeks { get; set; } [Column("longWeeks")] diff --git a/server/csharp/MetaObjects.IntegrationTests/Generated/SessionsByProgram.g.cs b/server/csharp/MetaObjects.IntegrationTests/Generated/SessionsByProgram.g.cs index 257778100..0ca11873c 100644 --- a/server/csharp/MetaObjects.IntegrationTests/Generated/SessionsByProgram.g.cs +++ b/server/csharp/MetaObjects.IntegrationTests/Generated/SessionsByProgram.g.cs @@ -11,8 +11,10 @@ namespace MetaObjects.IntegrationTests.Generated; public class SessionsByProgram { [Column("programTitle")] + [Required(AllowEmptyStrings = true)] [MaxLength(200)] - public string? ProgramTitle { get; set; } + [MinLength(1)] + public string ProgramTitle { get; set; } = default!; [Column("weekKey")] public long? WeekKey { get; set; } [Column("sessions")] diff --git a/server/csharp/MetaObjects.IntegrationTests/Generated/WeekShapes.g.cs b/server/csharp/MetaObjects.IntegrationTests/Generated/WeekShapes.g.cs index 5ea76d960..f2d3feab5 100644 --- a/server/csharp/MetaObjects.IntegrationTests/Generated/WeekShapes.g.cs +++ b/server/csharp/MetaObjects.IntegrationTests/Generated/WeekShapes.g.cs @@ -11,8 +11,10 @@ namespace MetaObjects.IntegrationTests.Generated; public class WeekShapes { [Column("programTitle")] + [Required(AllowEmptyStrings = true)] [MaxLength(200)] - public string? ProgramTitle { get; set; } + [MinLength(1)] + public string ProgramTitle { get; set; } = default!; [Column("weeks")] public long Weeks { get; set; } [Column("unlabelled")] diff --git a/server/csharp/MetaObjects/Core/Relationship/RelationshipReferences.cs b/server/csharp/MetaObjects/Core/Relationship/RelationshipReferences.cs index 595b31cff..a8987af59 100644 --- a/server/csharp/MetaObjects/Core/Relationship/RelationshipReferences.cs +++ b/server/csharp/MetaObjects/Core/Relationship/RelationshipReferences.cs @@ -46,7 +46,7 @@ private static string StripOneSuffix(string value) } /// The FK field a reference is anchored on (first field; composite FKs pair on their first column). - private static string? RefFkField(MetaReferenceIdentity reference) + internal static string? RefFkField(MetaReferenceIdentity reference) { var fields = reference.Fields; return fields.Count > 0 ? fields[0] : null; diff --git a/server/csharp/MetaObjects/Core/Reporting/ReportShape.cs b/server/csharp/MetaObjects/Core/Reporting/ReportShape.cs index b181dd77a..726d4868d 100644 --- a/server/csharp/MetaObjects/Core/Reporting/ReportShape.cs +++ b/server/csharp/MetaObjects/Core/Reporting/ReportShape.cs @@ -9,6 +9,7 @@ // artifact every port byte-matches; see ReportShapeTests). using System.Text; +using MetaObjects.Core.Relationship; using MetaObjects.Loader; using MetaObjects.Meta; @@ -156,8 +157,44 @@ private static bool IsPrimaryKeyField(MetaObject entity, MetaField field) } /// - /// Table C (required of a dimension). Without @spine: only a dimension with - /// no @via over an @of field that is effectively required (@required or a validator.required child). With + /// True when every hop of , walked from , is a + /// belongs-to hop over a required foreign key: the hops the view joins INNER, so every row + /// of the report has a match. This is the join-type rule of the view lowering (TypeScript + /// walkViaPath, #209): the reference is held by the entity the hop leaves, its first + /// FK field is required, and every earlier hop is INNER too. @enforce: false does not + /// change it. A hop that does not resolve, an ambiguous reference or a has-many hop is not INNER. + /// + private static bool ViaJoinsInner(IReadOnlyList hops, MetaObject from, MetaRoot root) + { + var current = from; + foreach (string hopName in hops) + { + // ADR-0039: resolving Relationships()/ReferenceIdentities(), so an inherited hop is found. + var rel = current.Relationships().FirstOrDefault(r => r.Name == hopName); + var refNode = rel is null ? current.ReferenceIdentities().FirstOrDefault(r => r.Name == hopName) : null; + string? targetRef = rel is not null ? rel.ObjectRef : refNode?.ReferencesRaw; + if (string.IsNullOrEmpty(targetRef)) return false; + if (NamingRefs.ResolveObjectRef(root, targetRef, NamingRefs.EffectivePackage(current)) is not MetaObject target) + return false; + + // The reference the hop crosses, as the lowering resolves it. A has-many hop, whose FK + // sits on the child, matches none of the current entity's own references. + var reference = refNode + ?? RelationshipReferences.ResolveRelationshipReference(current, hopName, target.Name, rel!.SourceRefField); + if (reference is null) return false; + string? fkName = RelationshipReferences.RefFkField(reference); + var fk = fkName is null ? null : current.FindField(fkName); + if (fk is null || !fk.IsRequired) return false; + current = target; + } + return true; + } + + /// + /// Table C (required of a dimension). Without @spine: only a dimension over an + /// @of field that is effectively required (@required or a validator.required child) + /// and, when it has a @via, only when every hop is a required belongs-to reference, the + /// hops the view joins INNER (). With /// @spine: only a dimension whose @via hops equal the spine's (a column of /// the spine entity itself, whose rows are the report's rows) over an @of field that /// is required (same rule) or a primary-key column of the entity @of names. A dimension @@ -170,7 +207,14 @@ private static bool DimensionRequired( string? via = dim.Via(); // Effective required-ness, resolving (ADR-0039): the @required attr OR a validator.required child. bool ofRequired = of.IsRequired; - if (spine is null) return via is null && ofRequired; + if (spine is null) + { + if (via is null) return ofRequired; + if (!ofRequired) return false; + var viaHops = ReportingViaHops(via, ReportingMemberOwner(dim, from), from, root) + ?? throw Unresolved(reportName, $"dimension '{dim.Name}' @via '{via}'"); + return ViaJoinsInner(viaHops, from, root); + } if (via is null) return false; // a column of @from: null in a spine row with no facts var hops = ReportingViaHops(via, ReportingMemberOwner(dim, from), from, root) ?? throw Unresolved(reportName, $"dimension '{dim.Name}' @via '{via}'"); diff --git a/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/metadata/ai/MetaDataAIDocumentationWriter.java b/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/metadata/ai/MetaDataAIDocumentationWriter.java index 53127321e..82c9ab64c 100644 --- a/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/metadata/ai/MetaDataAIDocumentationWriter.java +++ b/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/metadata/ai/MetaDataAIDocumentationWriter.java @@ -157,7 +157,6 @@ private JsonObject createDocumentationInfo() { JsonObject info = new JsonObject(); info.addProperty("version", version); info.addProperty("generatedFrom", "TypeDefinition Registry"); - info.addProperty("generationTimestamp", System.currentTimeMillis()); info.addProperty("aiOptimized", true); info.addProperty("inheritanceSupport", includeInheritance); info.addProperty("totalTypes", typeRegistry.getRegisteredTypes().size()); diff --git a/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/metadata/html/MetaDataHtmlDocumentationWriter.java b/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/metadata/html/MetaDataHtmlDocumentationWriter.java index e6f713b0b..63a18877c 100644 --- a/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/metadata/html/MetaDataHtmlDocumentationWriter.java +++ b/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/metadata/html/MetaDataHtmlDocumentationWriter.java @@ -18,8 +18,6 @@ import java.io.PrintWriter; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; import java.util.*; import java.util.stream.Collectors; @@ -378,8 +376,7 @@ private void generateHeader() { writer.println("
"); writer.println("

" + escapeHtml(title) + "

"); writer.println("
Comprehensive Type System Documentation
"); - writer.println("
Version " + escapeHtml(version) + " • Generated " + - LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm")) + "
"); + writer.println("
Version " + escapeHtml(version) + "
"); writer.println("
"); } diff --git a/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/object/BaseObjectCodeWriter.java b/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/object/BaseObjectCodeWriter.java index c722fdb4f..1d2da67e6 100644 --- a/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/object/BaseObjectCodeWriter.java +++ b/server/java/codegen-base/src/main/java/com/metaobjects/generator/direct/object/BaseObjectCodeWriter.java @@ -216,6 +216,9 @@ public String writeObject(MetaObject mo) throws GeneratorIOException { // from hand-written code, so a sibling generator emitting at the same path // declined to overwrite them and the adopter silently kept the wrong type. // Deleting this line is the documented gesture for taking ownership of the file. + // + // No timestamp: a "Generated On" line made every regeneration of an unchanged model + // rewrite every generated file. The header is a function of the metadata alone. List docs = Arrays.asList( "GENERATED — regenerated from metadata. Delete this line to own this file.", "MetaObject: " + mo.getName(), diff --git a/server/java/codegen-base/src/test/java/com/metaobjects/generator/direct/metadata/DocumentationOutputIsStableTest.java b/server/java/codegen-base/src/test/java/com/metaobjects/generator/direct/metadata/DocumentationOutputIsStableTest.java new file mode 100644 index 000000000..21f734b28 --- /dev/null +++ b/server/java/codegen-base/src/test/java/com/metaobjects/generator/direct/metadata/DocumentationOutputIsStableTest.java @@ -0,0 +1,89 @@ +package com.metaobjects.generator.direct.metadata; + +import com.metaobjects.generator.GeneratorTestBase; +import com.metaobjects.generator.direct.metadata.ai.MetaDataAIDocumentationWriter; +import com.metaobjects.generator.direct.metadata.html.MetaDataHtmlDocumentationWriter; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.TimeZone; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * Adopter report G10, in the documentation writers: output generated twice from the same + * registry must be byte-identical, so regenerating an unchanged model rewrites no file. Both + * writers stamped the time of the run: the HTML header carried + * {@code Generated } and the AI JSON a {@code generationTimestamp} in epoch + * milliseconds. + * + *

The two runs must differ by more than each stamp's resolution. A pause covers the + * millisecond one. The HTML one read the local wall clock to the minute, so the second run + * moves the JVM's default zone by five and a half hours instead of waiting out a minute: the + * local time that stamp rendered changes in both its hour and its minute. + */ +public class DocumentationOutputIsStableTest extends GeneratorTestBase { + + private interface Generation { + byte[] run() throws Exception; + } + + private static byte[][] twiceAcrossAClockChange(Generation generation) throws Exception { + TimeZone original = TimeZone.getDefault(); + try { + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + byte[] first = generation.run(); + Thread.sleep(50); + TimeZone.setDefault(TimeZone.getTimeZone("Asia/Kolkata")); + return new byte[][] { first, generation.run() }; + } finally { + TimeZone.setDefault(original); + } + } + + private byte[] html() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + MetaDataHtmlDocumentationWriter writer = new MetaDataHtmlDocumentationWriter(getSharedLoader(), out) + .withTitle("Stable Documentation") + .withVersion("1.2.3") + .withInheritance(true) + .withExamples(true) + .withExtensionGuide(true); + writer.writeHtml(); + writer.close(); + return out.toByteArray(); + } + + private static byte[] aiJson() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + MetaDataAIDocumentationWriter writer = new MetaDataAIDocumentationWriter(null, out); + writer.withVersion("1.2.3") + .withInheritance(true) + .withImplementationDetails(true) + .withExtensionGuidance(true) + .withCrossLanguageInfo(true); + writer.writeJson(); + writer.close(); + return out.toByteArray(); + } + + private static String text(byte[] bytes) { + return new String(bytes, StandardCharsets.UTF_8); + } + + @Test + public void htmlDocumentationIsIdenticalAcrossRuns() throws Exception { + byte[][] runs = twiceAcrossAClockChange(this::html); + assertFalse("the writer wrote nothing", runs[0].length == 0); + assertEquals(text(runs[0]), text(runs[1])); + } + + @Test + public void aiDocumentationIsIdenticalAcrossRuns() throws Exception { + byte[][] runs = twiceAcrossAClockChange(DocumentationOutputIsStableTest::aiJson); + assertFalse("the writer wrote nothing", runs[0].length == 0); + assertEquals(text(runs[0]), text(runs[1])); + } +} diff --git a/server/java/codegen-base/src/test/java/com/metaobjects/generator/direct/object/GeneratedHeaderIsStableTest.java b/server/java/codegen-base/src/test/java/com/metaobjects/generator/direct/object/GeneratedHeaderIsStableTest.java new file mode 100644 index 000000000..99c49c851 --- /dev/null +++ b/server/java/codegen-base/src/test/java/com/metaobjects/generator/direct/object/GeneratedHeaderIsStableTest.java @@ -0,0 +1,82 @@ +package com.metaobjects.generator.direct.object; + +import com.metaobjects.generator.direct.object.javacode.JavaObjectCodeGenerator; +import com.metaobjects.loader.InMemoryStringSource; +import com.metaobjects.loader.LoaderOptions; +import com.metaobjects.loader.MetaDataLoader; +import com.metaobjects.registry.ObjectClassRegistry; +import org.junit.After; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.*; + +/** + * Adopter report G10: the header of a generated class stamped the wall-clock time + * ({@code Generated On: Sat Oct 10 15:17:47 EDT 2026}), so regenerating an unchanged model + * rewrote every generated file and put a full tree of one-line diffs in the adopter's commit. + * The header must be a function of the metadata alone. + */ +public class GeneratedHeaderIsStableTest { + + private static final String META = "{ \"metadata.root\": {" + + " \"package\": \"acme::payload\"," + + " \"children\": [" + + " { \"object.value\": { \"name\": \"Address\", \"children\": [" + + " { \"field.string\": { \"name\": \"city\" } }" + + " ]}}" + + " ]" + + "}}"; + + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + @After + public void resetRegistry() { + ObjectClassRegistry.resetGlobal(); + } + + private Map generate(String dir) throws Exception { + MetaDataLoader loader = new MetaDataLoader( + LoaderOptions.create(false, false, true), MetaDataLoader.SUBTYPE_MANUAL, "stable-header"); + loader.init(); + loader.load(List.of(new InMemoryStringSource(META, "stable-header/meta.json"))); + Path out = tmp.newFolder(dir).toPath(); + Map args = new HashMap<>(); + args.put("outputDir", out.toString()); + args.put("type", "class"); + args.put("flavor", "valueObject"); + JavaObjectCodeGenerator generator = new JavaObjectCodeGenerator(); + generator.setArgs(args); + generator.execute(loader); + Map files = new TreeMap<>(); + try (Stream s = Files.walk(out)) { + for (Path p : s.filter(Files::isRegularFile).collect(Collectors.toList())) { + files.put(out.relativize(p).toString(), Files.readString(p)); + } + } + return files; + } + + @Test + public void regeneratingAnUnchangedModelWritesIdenticalFiles() throws Exception { + Map first = generate("first"); + // Past the one-second resolution of the old wall-clock stamp. + Thread.sleep(1100); + Map second = generate("second"); + assertFalse("the generator wrote nothing", first.isEmpty()); + assertEquals(first, second); + first.values().forEach(src -> + assertFalse("the header must carry no timestamp:\n" + src, src.contains("Generated On"))); + } +} diff --git a/server/java/codegen-kotlin/src/main/kotlin/com/metaobjects/generator/kotlin/KotlinNamesGenerator.kt b/server/java/codegen-kotlin/src/main/kotlin/com/metaobjects/generator/kotlin/KotlinNamesGenerator.kt index cf41a8e8b..d4451df91 100644 --- a/server/java/codegen-kotlin/src/main/kotlin/com/metaobjects/generator/kotlin/KotlinNamesGenerator.kt +++ b/server/java/codegen-kotlin/src/main/kotlin/com/metaobjects/generator/kotlin/KotlinNamesGenerator.kt @@ -14,6 +14,8 @@ import com.metaobjects.generator.GeneratorIOWriter import com.metaobjects.generator.direct.MultiFileDirectGeneratorBase import com.metaobjects.generator.util.GeneratedFileWriter import com.metaobjects.generator.util.GeneratorUtil +import com.metaobjects.generator.util.RestSurfaceGate +import com.metaobjects.reporting.ReportReadModel import com.metaobjects.loader.MetaDataLoader import com.metaobjects.`object`.MetaObject import java.io.OutputStream @@ -95,9 +97,15 @@ open class KotlinNamesGenerator : // Pass 1 — every object that participates in the database (#248). val emitted = mutableSetOf() for (entity in loader.metaObjects) { - // FR-044: a report gets no names artifact. Its Exposed table (the one thing Kotlin - // generates for a view-backed report) binds its view and columns by literal. - if (GeneratorUtil.isReport(entity)) continue + // FR-044: a served report (a view-backed one) gets a names artifact built from its + // READ MODEL, so the derived columns have constants for hand-written SQL over the + // view to use; its Exposed table still binds by literal. A report that is not + // served has no view, so there is nothing to name. + if (GeneratorUtil.isReport(entity)) { + if (RestSurfaceGate.isServedReport(entity) && + emit(ReportReadModel.of(entity), outRoot, strategy)) emitted += entity.name + continue + } if (emit(entity, outRoot, strategy)) emitted += entity.name } // Pass 2 — the abstract bases those participants EXTEND, each carrying the columns diff --git a/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/KotlinNamesGeneratorTest.kt b/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/KotlinNamesGeneratorTest.kt index 356b28b4b..b217288d7 100644 --- a/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/KotlinNamesGeneratorTest.kt +++ b/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/KotlinNamesGeneratorTest.kt @@ -185,6 +185,38 @@ class KotlinNamesGeneratorTest { assertFalse("READ_ONLY" in src, src) } + @Test fun `a served report gets a names artifact built from its read model`() { + // D7: a hand-written query over a report's view had no constant to name the view or + // its derived columns. The artifact is the report's READ MODEL (its derived fields), + // because the declared report node has no field children. + val model = """{ + "metadata.root": { "package": "acme", "children": [ + { "object.entity": { "name": "Sale", "children": [ + { "source.rdb": { "@table": "sales" } }, + { "field.long": { "name": "id" } }, + { "field.string": { "name": "channel" } }, + { "identity.primary": { "name": "pk", "@fields": ["id"] } }, + { "dimension.attribute": { "name": "channel", "@of": "Sale.channel" } }, + { "measure.aggregate": { "name": "sales", "@agg": "count", "@of": "Sale.id" } } + ] } }, + { "object.report": { "name": "SalesByChannel", "@from": "Sale", + "@dimensions": ["channel"], "@measures": ["sales"], "children": [ + { "source.rdb": { "@kind": "view", "@view": "v_sales_by_channel" } } + ] } }, + { "object.report": { "name": "Unserved", "@from": "Sale", + "@dimensions": ["channel"], "@measures": ["sales"] } } + ] } + }""".trimIndent() + val files = emit(model) + val src = files.getValue("acme/SalesByChannelNames.kt") + assertTrue("const val SOURCE_PRIMARY_VIEW: String = \"v_sales_by_channel\"" in src, src) + assertTrue("const val SUB_TYPE: String = \"report\"" in src, src) + assertTrue("const val CHANNEL_COLUMN: String = \"channel\"" in src, src) + assertTrue("const val SALES_COLUMN: String = \"sales\"" in src, src) + // A report with no view has nothing to name. + assertFalse("acme/UnservedNames.kt" in files, files.keys.toString()) + } + @Test fun `an inherited field and its inherited @column both resolve`() { // ADR-0039: entity.metaFields (getMetaFields(), which defaults to // includeParentData=true) must see a field AND @column declared on an abstract diff --git a/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/KotlinReportTableGeneratorTest.kt b/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/KotlinReportTableGeneratorTest.kt index 94cae0746..992b42ba3 100644 --- a/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/KotlinReportTableGeneratorTest.kt +++ b/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/KotlinReportTableGeneratorTest.kt @@ -96,7 +96,7 @@ class KotlinReportTableGeneratorTest { |/** GENERATED — do not hand-edit. Regenerated from metadata. */ |object ProgramMinutesTable : Table("v_program_minutes") { | val program = long("program") - | val programTitle = varchar("programTitle", 200).nullable() + | val programTitle = varchar("programTitle", 200) | val weeks = long("weeks") | val longWeeks = long("longWeeks") | val labels = long("labels") @@ -162,7 +162,7 @@ class KotlinReportTableGeneratorTest { fun `the naming strategy applies to the derived field name`() { val src = canonical(emptyMap()).getValue("fitness/ProgramMinutesTable.kt") // snake_case default assertTrue(" val avgMinutes = decimal(\"avg_minutes\", 38, 18).nullable()\n" in src, src) - assertTrue(" val programTitle = varchar(\"program_title\", 200).nullable()\n" in src, src) + assertTrue(" val programTitle = varchar(\"program_title\", 200)\n" in src, src) } @Test @@ -178,7 +178,7 @@ class KotlinReportTableGeneratorTest { } @Test - fun `with the names generator in the run a report still binds by literal and gets no names artifact`() { + fun `with the names generator in the run a report still binds by literal and gets a names artifact`() { val files = emit( loadDirectory("report-table-names", canonicalDir()), mapOf("columnNaming" to "literal", "useNames" to "true"), @@ -190,7 +190,10 @@ class KotlinReportTableGeneratorTest { assertTrue("object ProgramMinutesTable : Table(\"v_program_minutes\") {" in src, src) assertTrue(" val weeks = long(\"weeks\")\n" in src, src) assertFalse("Names" in src, src) - assertFalse(files.keys.any { it.endsWith("ProgramMinutesNames.kt") }, files.keys.toString()) + // D7: the view and the derived columns have constants for hand-written SQL. + val names = files.getValue("fitness/ProgramMinutesNames.kt") + assertTrue("const val SOURCE_PRIMARY_VIEW: String = \"v_program_minutes\"" in names, names) + assertTrue("const val PROGRAM_TITLE_COLUMN: String = \"programTitle\"" in names, names) } // --- @spine and @default: what is nullable (Table C) ----------------------------------- @@ -221,11 +224,11 @@ class KotlinReportTableGeneratorTest { |""".trimMargin(), canonical().getValue("fitness/ProgramRosterTable.kt"), ) - // The same Program.title dimension in ProgramMinutes, which has no @spine, stays - // nullable: without @spine Table C changes nothing, and a dimension reached by @via - // is never required there. + // The same Program.title dimension in ProgramMinutes, which has no @spine, is required: + // without @spine, a dimension reached by @via is required when every hop is a + // required reference (an INNER join) and the @of field is required (D5). assertTrue( - " val programTitle = varchar(\"programTitle\", 200).nullable()\n" in + " val programTitle = varchar(\"programTitle\", 200)\n" in canonical().getValue("fitness/ProgramMinutesTable.kt"), ) // With no dimensions and no @spine, a defaulted sum and ratio are still non-null. @@ -524,9 +527,10 @@ class KotlinReportTableGeneratorTest { generators = listOf(KotlinEntityGenerator(), KotlinExposedTableGenerator()), ) val src = files.getValue("acme/shop/SalesByTierTable.kt") - // Store's class, and nullable: a dimension reached by @via can be null. + // Store's class, and not nullable: Store.tier is required and the only hop is a required + // reference, an INNER join (D5). assertTrue( - " val storeTier = enumerationByName(\"store_tier\", ${KotlinTypeMapper.ENUM_VARCHAR_LEN}, StoreTier::class).nullable()\n" in src, + " val storeTier = enumerationByName(\"store_tier\", ${KotlinTypeMapper.ENUM_VARCHAR_LEN}, StoreTier::class)\n" in src, src, ) // An int-backed enum keeps its mapping, typed by Sale's class. diff --git a/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/ReportingInertTest.kt b/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/ReportingInertTest.kt index fd1bfc4ad..69914f0ff 100644 --- a/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/ReportingInertTest.kt +++ b/server/java/codegen-kotlin/src/test/kotlin/com/metaobjects/generator/kotlin/ReportingInertTest.kt @@ -23,13 +23,14 @@ import kotlin.test.assertTrue * `dimension.*`, `measure.*` and `segment.*` generate nothing anywhere. An `object.report` * with no read-only source generates nothing anywhere. A report that declares a read-only * `source.rdb @kind: view` is lowered to that view (by TypeScript migrate), and Kotlin - * generates exactly four files for it (FR-044 Plan 3), one from each of four generators: + * generates exactly five files for it (FR-044 Plan 3), one from each of five generators: * its read-only Exposed table ([KotlinExposedTableGenerator]), its row data class - * ([KotlinEntityGenerator]), its filter allowlist ([KotlinFilterAllowlistGenerator]) and its - * read-only controller ([KotlinSpringControllerGenerator]). Every other generator in - * [GENERATOR_REGISTRY] — names, relations, repository, validator, the prompt tier — emits + * ([KotlinEntityGenerator]), its filter allowlist ([KotlinFilterAllowlistGenerator]), its + * read-only controller ([KotlinSpringControllerGenerator]) and its physical names + * ([KotlinNamesGenerator]). Every other generator in + * [GENERATOR_REGISTRY] — relations, repository, validator, the prompt tier — emits * for a model that USES the vocabulary exactly what it emits for the same model without it, - * byte for byte, and those four emit nothing else new. + * byte for byte, and those five emit nothing else new. * * The model pair is `fixtures/codegen-noop/reporting/{with,without}`, shared with the other * four ports' copies of this test. `with/` carries three sourceless reports @@ -110,7 +111,7 @@ class ReportingInertTest { return if (differing.isEmpty()) null else "$label: $differing differ once reporting nodes are declared" } - /** What a generator may add for the with-model: one file of the view-backed report, from four generators. */ + /** What a generator may add for the with-model: one file of the view-backed report, from five generators. */ private fun allowedFor(info: GeneratorInfo): Set = setOfNotNull(STORE_TOTALS_FILES[info.name]) @Test @@ -125,7 +126,7 @@ class ReportingInertTest { } @Test - fun `only four generators emit for a report, each one file, for the view-backed one`() { + fun `only five generators emit for a report, each one file, for the view-backed one`() { // Every generator is compared before anything is asserted, so one red run names // every leak rather than the first. val leaks = GENERATOR_REGISTRY.values.mapNotNull { info -> @@ -143,7 +144,7 @@ class ReportingInertTest { } @Test - fun `the view-backed report emits exactly one file from each of the four generators`() { + fun `the view-backed report emits exactly one file from each of the five generators`() { // Else the allowance above is vacuous: each file really is emitted. for ((name, path) in STORE_TOTALS_FILES) { val info = GENERATOR_REGISTRY.getValue(name) @@ -160,7 +161,7 @@ class ReportingInertTest { val hits = files.filter { (path, text) -> report in path || report in text }.keys assertTrue(hits.isEmpty(), "$report leaked into $hits") } - // The view-backed report is named by its four files and by nothing else. + // The view-backed report is named by its five files and by nothing else. val hits = files.filter { (path, text) -> "StoreTotals" in path || "StoreTotals" in text }.keys assertEquals(STORE_TOTALS_FILES.values.toSet(), hits) } @@ -244,6 +245,8 @@ class ReportingInertTest { "entity" to "acme/shop/StoreTotals.kt", "filter-allowlist" to "acme/shop/StoreTotalsFilterAllowlist.kt", "routes" to "acme/shop/StoreTotalsController.kt", + // D7: the served report's view and derived columns get names constants too. + "names" to "acme/shop/StoreTotalsNames.kt", ) /** `StoreTotals`: three measures over `Purchase` — two counts and a sum of a currency. */ diff --git a/server/java/integration-tests-kotlin/src/test/kotlin/com/metaobjects/integration/kotlin/KotlinCodegenMatchesReferenceTest.kt b/server/java/integration-tests-kotlin/src/test/kotlin/com/metaobjects/integration/kotlin/KotlinCodegenMatchesReferenceTest.kt index 80537b780..9fa70d158 100644 --- a/server/java/integration-tests-kotlin/src/test/kotlin/com/metaobjects/integration/kotlin/KotlinCodegenMatchesReferenceTest.kt +++ b/server/java/integration-tests-kotlin/src/test/kotlin/com/metaobjects/integration/kotlin/KotlinCodegenMatchesReferenceTest.kt @@ -113,7 +113,7 @@ internal class KotlinCodegenMatchesReferenceTest { "ProgramMinutes" to EntityExpectation( columns = listOf( ExpectedColumn("program", families = setOf("long"), nullable = false), - ExpectedColumn("programTitle", families = setOf("varchar"), nullable = true), + ExpectedColumn("programTitle", families = setOf("varchar"), nullable = false), ExpectedColumn("weeks", families = setOf("long"), nullable = false), ExpectedColumn("longWeeks", families = setOf("long"), nullable = false), ExpectedColumn("labels", families = setOf("long"), nullable = false), @@ -275,7 +275,7 @@ internal class KotlinCodegenMatchesReferenceTest { ), "WeekShapes" to EntityExpectation( columns = listOf( - ExpectedColumn("programTitle", families = setOf("varchar"), nullable = true), + ExpectedColumn("programTitle", families = setOf("varchar"), nullable = false), ExpectedColumn("weeks", families = setOf("long"), nullable = false), ExpectedColumn("unlabelled", families = setOf("long"), nullable = false), ExpectedColumn("labelledMinutes", families = setOf("long"), nullable = true), @@ -287,7 +287,7 @@ internal class KotlinCodegenMatchesReferenceTest { ), "SessionsByProgram" to EntityExpectation( columns = listOf( - ExpectedColumn("programTitle", families = setOf("varchar"), nullable = true), + ExpectedColumn("programTitle", families = setOf("varchar"), nullable = false), ExpectedColumn("weekKey", families = setOf("long"), nullable = true), ExpectedColumn("sessions", families = setOf("long"), nullable = false), ExpectedColumn("sessionMinutes", families = setOf("long"), nullable = true), diff --git a/server/java/integration-tests-kotlin/src/test/kotlin/com/metaobjects/integration/kotlin/tables/ProgramMinutesView.kt b/server/java/integration-tests-kotlin/src/test/kotlin/com/metaobjects/integration/kotlin/tables/ProgramMinutesView.kt index 2fd7108eb..012e43c5c 100644 --- a/server/java/integration-tests-kotlin/src/test/kotlin/com/metaobjects/integration/kotlin/tables/ProgramMinutesView.kt +++ b/server/java/integration-tests-kotlin/src/test/kotlin/com/metaobjects/integration/kotlin/tables/ProgramMinutesView.kt @@ -14,7 +14,7 @@ import org.jetbrains.exposed.sql.Table * column type and nullable exactly where `fixtures/persistence-conformance/report-shapes.json` * says `required: false`: * - `program` = Week.programId (required FK) → BIGINT → `long` - * - `programTitle` = Program.title, reached by @via → VARCHAR → `varchar`, nullable + * - `programTitle` = Program.title, required, reached by @via over a required reference (an INNER join) → VARCHAR → `varchar` * - a `count`, with or without @distinct → BIGINT → `long` (never null) * - `sum` of an int, cast by the view → BIGINT → `long`, nullable * - `avg`, and a ratio → NUMERIC → `decimal`, nullable @@ -27,7 +27,7 @@ import org.jetbrains.exposed.sql.Table */ object ProgramMinutesView : Table("v_program_minutes") { val program = long("program") - val programTitle = varchar("programTitle", 200).nullable() + val programTitle = varchar("programTitle", 200) val weeks = long("weeks") val longWeeks = long("longWeeks") val labels = long("labels") diff --git a/server/java/integration-tests-kotlin/src/test/kotlin/com/metaobjects/integration/kotlin/tables/SessionsByProgramView.kt b/server/java/integration-tests-kotlin/src/test/kotlin/com/metaobjects/integration/kotlin/tables/SessionsByProgramView.kt index b702a9be2..128794c8d 100644 --- a/server/java/integration-tests-kotlin/src/test/kotlin/com/metaobjects/integration/kotlin/tables/SessionsByProgramView.kt +++ b/server/java/integration-tests-kotlin/src/test/kotlin/com/metaobjects/integration/kotlin/tables/SessionsByProgramView.kt @@ -16,7 +16,7 @@ import org.jetbrains.exposed.sql.Table * fetched by id. Column names are the derived field names (the corpus's `literal` naming). */ object SessionsByProgramView : Table("v_sessions_by_program") { - val programTitle = varchar("programTitle", 200).nullable() + val programTitle = varchar("programTitle", 200) val weekKey = long("weekKey").nullable() val sessions = long("sessions") val sessionMinutes = long("sessionMinutes").nullable() diff --git a/server/java/integration-tests-kotlin/src/test/kotlin/com/metaobjects/integration/kotlin/tables/WeekShapesView.kt b/server/java/integration-tests-kotlin/src/test/kotlin/com/metaobjects/integration/kotlin/tables/WeekShapesView.kt index 34ecef5f5..8cfa88cb1 100644 --- a/server/java/integration-tests-kotlin/src/test/kotlin/com/metaobjects/integration/kotlin/tables/WeekShapesView.kt +++ b/server/java/integration-tests-kotlin/src/test/kotlin/com/metaobjects/integration/kotlin/tables/WeekShapesView.kt @@ -16,7 +16,7 @@ import org.jetbrains.exposed.sql.Table * fetched by id. Column names are the derived field names (the corpus's `literal` naming). */ object WeekShapesView : Table("v_week_shapes") { - val programTitle = varchar("programTitle", 200).nullable() + val programTitle = varchar("programTitle", 200) val weeks = long("weeks") val unlabelled = long("unlabelled") val labelledMinutes = long("labelledMinutes").nullable() diff --git a/server/java/metadata/src/main/java/com/metaobjects/reporting/ReportShape.java b/server/java/metadata/src/main/java/com/metaobjects/reporting/ReportShape.java index c091d0692..598e0c349 100644 --- a/server/java/metadata/src/main/java/com/metaobjects/reporting/ReportShape.java +++ b/server/java/metadata/src/main/java/com/metaobjects/reporting/ReportShape.java @@ -27,9 +27,13 @@ import com.metaobjects.field.LongField; import com.metaobjects.field.MetaField; import com.metaobjects.field.TimestampField; +import com.metaobjects.identity.MetaIdentity; import com.metaobjects.identity.PrimaryIdentity; +import com.metaobjects.identity.ReferenceIdentity; import com.metaobjects.loader.ValidationPhase; import com.metaobjects.object.MetaObject; +import com.metaobjects.relationship.MetaRelationship; +import com.metaobjects.relationship.RelationshipReferences; import com.metaobjects.reporting.ReportAccessors.ReportDimensionItem; import com.metaobjects.source.MetaSource; import com.metaobjects.validator.RequiredValidator; @@ -337,8 +341,60 @@ private static boolean isPrimaryKeyField(MetaObject entity, MetaField field) } /** - * Table C ({@code required} of a dimension). Without {@code @spine}: only a dimension with - * no {@code @via} over an {@code @of} field whose effective {@code @required} is true. With + * True when every hop of {@code hops}, walked from {@code from}, is a belongs-to hop over a + * required foreign key: the hops the view joins INNER, so every row of the report has a + * match. This is the join-type rule of the view lowering (TypeScript {@code walkViaPath}, + * #209): the reference is held by the entity the hop leaves, its first FK field is required, + * and every earlier hop is INNER too. {@code @enforce: false} does not change it. A hop that + * does not resolve, an ambiguous reference or a has-many hop is not INNER. + */ + private static boolean viaJoinsInner(List hops, MetaObject from, MetaRoot root) { + MetaObject current = from; + for (String hopName : hops) { + // ADR-0039: resolving getRelationships()/getIdentities(), so an inherited hop is found. + MetaRelationship rel = null; + for (MetaRelationship r : current.getRelationships()) { + if (hopName.equals(r.getShortName())) { rel = r; break; } + } + ReferenceIdentity refNode = null; + if (rel == null) { + for (MetaIdentity id : current.getIdentities()) { + if (id instanceof ReferenceIdentity && hopName.equals(id.getShortName())) { + refNode = (ReferenceIdentity) id; + break; + } + } + } + String targetRef = rel != null ? rel.getObjectRef() : refNode != null ? refNode.getTargetEntity() : null; + if (targetRef == null || targetRef.isEmpty()) return false; + MetaObject target = ValidationPhase.resolveRootObject(root, targetRef, packageOf(current)); + if (target == null) return false; + + // The reference the hop crosses, as the lowering resolves it. A has-many hop, whose FK + // sits on the child, matches none of the current entity's own references. + ReferenceIdentity reference = refNode != null + ? refNode + : RelationshipReferences.resolveRelationshipReference( + current, rel.getShortName(), target.getShortName(), rel.getSourceRefField()); + if (reference == null || reference.getFields().isEmpty()) return false; + MetaField fk = current.findMetaField(reference.getFields().get(0)).orElse(null); + if (fk == null || !isRequiredField(fk)) return false; + current = target; + } + return true; + } + + /** Effective required-ness, resolving (ADR-0039): the {@code @required} attr OR a validator.required child. */ + private static boolean isRequiredField(MetaField field) { + return ReportingAttrs.isTrue(field, MetaField.ATTR_REQUIRED) + || field.getValidators().stream().anyMatch(v -> RequiredValidator.SUBTYPE_REQUIRED.equals(v.getSubType())); + } + + /** + * Table C ({@code required} of a dimension). Without {@code @spine}: only a dimension over an + * {@code @of} field whose effective {@code @required} is true and, when it has a {@code @via}, + * only when every hop is a required belongs-to reference, the hops the view joins INNER + * ({@link #viaJoinsInner}). With * {@code @spine}: only a dimension whose {@code @via} hops equal the spine's (a column of * the spine entity itself, whose rows are the report's rows) over an {@code @of} field that * is {@code @required} or a primary-key column of the entity {@code @of} names. A dimension @@ -347,10 +403,14 @@ private static boolean isPrimaryKeyField(MetaObject entity, MetaField field) private static boolean dimensionRequired(MetaDimension dim, MetaObject named, MetaField of, MetaObject from, MetaRoot root, List spine, MetaObject report) { String via = dim.getVia(); - // Effective required-ness, resolving (ADR-0039): the @required attr OR a validator.required child. - boolean ofRequired = ReportingAttrs.isTrue(of, MetaField.ATTR_REQUIRED) - || of.getValidators().stream().anyMatch(v -> RequiredValidator.SUBTYPE_REQUIRED.equals(v.getSubType())); - if (spine == null) return via == null && ofRequired; + boolean ofRequired = isRequiredField(of); + if (spine == null) { + if (via == null) return ofRequired; + if (!ofRequired) return false; + List hops = reportingViaHops(via, memberOwner(dim, from), from, root); + if (hops == null) throw unresolved(report, "dimension '" + dim.getShortName() + "' @via '" + via + "'"); + return viaJoinsInner(hops, from, root); + } if (via == null) return false; // a column of @from: null in a spine row with no facts List hops = reportingViaHops(via, memberOwner(dim, from), from, root); if (hops == null) throw unresolved(report, "dimension '" + dim.getShortName() + "' @via '" + via + "'"); diff --git a/server/java/metadata/src/test/java/com/metaobjects/reporting/ReportReadModelTest.java b/server/java/metadata/src/test/java/com/metaobjects/reporting/ReportReadModelTest.java index d8a378b77..16e83701e 100644 --- a/server/java/metadata/src/test/java/com/metaobjects/reporting/ReportReadModelTest.java +++ b/server/java/metadata/src/test/java/com/metaobjects/reporting/ReportReadModelTest.java @@ -125,7 +125,7 @@ public void carriesOneRealFieldPerTableBRowInOrder() { public void requiredComesFromTheShapeNotFromTheTypeSource() { ReportReadModel model = ReportReadModel.of(object(canonical, "ProgramMinutes"), canonical); assertTrue(required(model.getMetaField("program"))); - assertFalse("reached by @via", required(model.getMetaField("programTitle"))); + assertTrue("reached by @via over a required reference (an INNER join)", required(model.getMetaField("programTitle"))); assertTrue("a count is never null", required(model.getMetaField("weeks"))); // Week.durationMinutes is @required, yet a min over it is nullable (an empty group). assertFalse(required(model.getMetaField("minMinutes"))); diff --git a/server/java/metadata/src/test/java/com/metaobjects/reporting/ReportShapeTest.java b/server/java/metadata/src/test/java/com/metaobjects/reporting/ReportShapeTest.java index 2946f2f8a..25bef70be 100644 --- a/server/java/metadata/src/test/java/com/metaobjects/reporting/ReportShapeTest.java +++ b/server/java/metadata/src/test/java/com/metaobjects/reporting/ReportShapeTest.java @@ -163,7 +163,7 @@ public void fieldsAreDimensionsThenMeasuresInListedOrder() { } @Test - public void attributeDimensionKeepsTheOfFieldsSubtypeAndIsRequiredOnlyWithoutVia() { + public void attributeDimensionKeepsTheOfFieldsSubtypeAndIsRequiredWhenEveryJoinIsInner() { ReportShape shape = ReportShape.of(object(canonical, "ProgramMinutes"), canonical); ReportShape.Field program = field(shape, "program"); @@ -177,7 +177,7 @@ public void attributeDimensionKeepsTheOfFieldsSubtypeAndIsRequiredOnlyWithoutVia ReportShape.Field title = field(shape, "programTitle"); assertEquals("string", title.subType()); - assertFalse("reached by @via, so nullable", title.required()); + assertTrue("reached by @via over a required reference (an INNER join) to a required field", title.required()); assertEquals("fitness::Program.title", title.typeSourceKey()); } @@ -734,4 +734,90 @@ public void aSpineThatDoesNotResolveThrowsNamingTheReport() { assertUnresolved(expected, report, root); } } + + // --------------------------------------------------------------------------- + // Table C, a dimension reached through @via (adopter report D5): the view joins a + // belongs-to hop over a required foreign key INNER, so the column is as required as the + // field it reads. The same model and expectations as the TypeScript report-shape test. + // --------------------------------------------------------------------------- + + private static final String VIA_MODEL = "{\"metadata.root\":{\"package\":\"a\",\"children\":[" + + entityJson("Realm", "realms", + "{\"field.string\":{\"name\":\"name\",\"@required\":true}},{\"field.string\":{\"name\":\"motto\"}}") + + "," + entityJson("Session", "sessions", + "{\"field.long\":{\"name\":\"realmId\",\"@required\":true}},{\"field.long\":{\"name\":\"maybeRealmId\"}}," + + "{\"identity.reference\":{\"name\":\"fkRealm\",\"@fields\":[\"realmId\"],\"@references\":\"Realm\"}}," + + "{\"identity.reference\":{\"name\":\"fkMaybeRealm\",\"@fields\":[\"maybeRealmId\"],\"@references\":\"Realm\"}}," + + "{\"relationship.association\":{\"name\":\"realm\",\"@objectRef\":\"Realm\",\"@cardinality\":\"one\",\"@sourceRefField\":\"realmId\"}}," + + "{\"relationship.association\":{\"name\":\"maybeRealm\",\"@objectRef\":\"Realm\",\"@cardinality\":\"one\",\"@sourceRefField\":\"maybeRealmId\"}}") + + "," + entityJson("Snap", "snaps", + "{\"field.long\":{\"name\":\"sessionId\",\"@required\":true}},{\"field.long\":{\"name\":\"unenforcedSessionId\",\"@required\":true}}," + + "{\"identity.reference\":{\"name\":\"fkSession\",\"@fields\":[\"sessionId\"],\"@references\":\"Session\"}}," + + "{\"identity.reference\":{\"name\":\"fkLoose\",\"@fields\":[\"unenforcedSessionId\"],\"@references\":\"Session\",\"@enforce\":false}}," + + "{\"relationship.association\":{\"name\":\"session\",\"@objectRef\":\"Session\",\"@cardinality\":\"one\"}}," + + "{\"relationship.association\":{\"name\":\"stamp\",\"@objectRef\":\"Stamp\",\"@cardinality\":\"one\"}}," + + "{\"measure.aggregate\":{\"name\":\"snaps\",\"@agg\":\"count\",\"@of\":\"Snap.id\"}}," + + "{\"dimension.attribute\":{\"name\":\"realmName\",\"@of\":\"Realm.name\",\"@via\":\"Snap.fkSession.fkRealm\"}}," + + "{\"dimension.attribute\":{\"name\":\"realmMotto\",\"@of\":\"Realm.motto\",\"@via\":\"Snap.fkSession.fkRealm\"}}," + + "{\"dimension.attribute\":{\"name\":\"maybeRealmName\",\"@of\":\"Realm.name\",\"@via\":\"Snap.fkSession.fkMaybeRealm\"}}," + + "{\"dimension.attribute\":{\"name\":\"looseRealmName\",\"@of\":\"Realm.name\",\"@via\":\"Snap.fkLoose.fkRealm\"}}," + + "{\"dimension.attribute\":{\"name\":\"sessionRealmId\",\"@of\":\"Session.realmId\",\"@via\":\"Snap.fkSession\"}}," + + "{\"dimension.attribute\":{\"name\":\"relRealmName\",\"@of\":\"Realm.name\",\"@via\":\"Snap.session.realm\"}}," + + "{\"dimension.attribute\":{\"name\":\"relMaybeRealmName\",\"@of\":\"Realm.name\",\"@via\":\"Snap.session.maybeRealm\"}}," + + "{\"dimension.attribute\":{\"name\":\"relSessionRealmId\",\"@of\":\"Session.realmId\",\"@via\":\"Snap.session\"}}," + + "{\"dimension.attribute\":{\"name\":\"mixedRealmName\",\"@of\":\"Realm.name\",\"@via\":\"Snap.session.fkRealm\"}}," + + "{\"dimension.attribute\":{\"name\":\"stampLabel\",\"@of\":\"Stamp.label\",\"@via\":\"Snap.stamp\"}}") + // The foreign key between Snap and Stamp is held by Stamp, the far side of Snap's `stamp`. + + "," + entityJson("Stamp", "stamps", + "{\"field.long\":{\"name\":\"snapId\",\"@required\":true}},{\"field.string\":{\"name\":\"label\",\"@required\":true}}," + + "{\"identity.reference\":{\"name\":\"fkSnap\",\"@fields\":[\"snapId\"],\"@references\":\"Snap\"}}") + + ",{\"object.report\":{\"name\":\"Related\",\"@from\":\"Snap\"," + + "\"@dimensions\":[\"relRealmName\",\"relMaybeRealmName\",\"relSessionRealmId\",\"mixedRealmName\",\"stampLabel\"]," + + "\"@measures\":[\"snaps\"],\"children\":[{\"source.rdb\":{\"@kind\":\"view\",\"@view\":\"v_related\"}}]}}" + + ",{\"object.report\":{\"name\":\"Plain\",\"@from\":\"Snap\"," + + "\"@dimensions\":[\"realmName\",\"realmMotto\",\"maybeRealmName\",\"looseRealmName\",\"sessionRealmId\"]," + + "\"@measures\":[\"snaps\"],\"children\":[{\"source.rdb\":{\"@kind\":\"view\",\"@view\":\"v_plain\"}}]}}," + + "{\"object.report\":{\"name\":\"Spined\",\"@from\":\"Snap\",\"@spine\":\"Snap.fkSession\"," + + "\"@dimensions\":[\"realmName\",\"sessionRealmId\"]," + + "\"@measures\":[\"snaps\"],\"children\":[{\"source.rdb\":{\"@kind\":\"view\",\"@view\":\"v_spined\"}}]}}" + + "]}}"; + + private static String entityJson(String name, String table, String children) { + return "{\"object.entity\":{\"name\":\"" + name + "\",\"children\":[" + + "{\"source.rdb\":{\"@table\":\"" + table + "\"}},{\"field.long\":{\"name\":\"id\"}}," + + "{\"identity.primary\":{\"name\":\"pk\",\"@fields\":[\"id\"]}}," + children + "]}}"; + } + + @Test + public void viaDimensionIsRequiredWhenEveryHopIsARequiredBelongsTo() { + MetaRoot root = loadJson(VIA_MODEL); + ReportShape plain = ReportShape.of(object(root, "Plain"), root); + assertTrue("two required hops over a required field", field(plain, "realmName").required()); + assertTrue("one required hop", field(plain, "sessionRealmId").required()); + assertTrue("@enforce: false does not change the join type", field(plain, "looseRealmName").required()); + assertFalse("the target field is optional", field(plain, "realmMotto").required()); + assertFalse("a hop over a nullable foreign key is a LEFT join", field(plain, "maybeRealmName").required()); + + ReportShape spined = ReportShape.of(object(root, "Spined"), root); + assertFalse("under @spine every join beyond the spine is LEFT", field(spined, "realmName").required()); + assertTrue("on the spine, a required field", field(spined, "sessionRealmId").required()); + } + + /** + * The same rule when a hop is a {@code @cardinality: one} relationship instead of the + * reference it crosses: the relationship resolves to that reference (by name pairing for + * {@code session}, by {@code @sourceRefField} for the two onto Realm), and the foreign key + * it holds decides the join. + */ + @Test + public void toOneRelationshipHopFollowsTheReferenceItCrosses() { + MetaRoot root = loadJson(VIA_MODEL); + ReportShape related = ReportShape.of(object(root, "Related"), root); + assertTrue("two relationship hops, both over a required foreign key", field(related, "relRealmName").required()); + assertTrue("one relationship hop", field(related, "relSessionRealmId").required()); + assertTrue("a relationship hop, then a reference hop", field(related, "mixedRealmName").required()); + assertFalse("the second relationship crosses a nullable foreign key", field(related, "relMaybeRealmName").required()); + // A Snap with no Stamp has no match, so the join is LEFT whatever that key and the target field require. + assertFalse("the other entity holds the foreign key", field(related, "stampLabel").required()); + } } diff --git a/server/python/src/metaobjects/meta/core/reporting/report_shape.py b/server/python/src/metaobjects/meta/core/reporting/report_shape.py index 420227293..3f1a1e8a0 100644 --- a/server/python/src/metaobjects/meta/core/reporting/report_shape.py +++ b/server/python/src/metaobjects/meta/core/reporting/report_shape.py @@ -29,8 +29,20 @@ FIELD_SUBTYPE_TIMESTAMP, ) from ..field.meta_field import MetaField -from ..identity.identity_constants import IDENTITY_ATTR_FIELDS +from ..identity.identity_constants import ( + IDENTITY_ATTR_FIELDS, + IDENTITY_REFERENCE_ATTR_REFERENCES, + IDENTITY_SUBTYPE_REFERENCE, +) from ..object.meta_object import MetaObject +from ..relationship.relationship_constants import ( + RELATIONSHIP_ATTR_OBJECT_REF, + RELATIONSHIP_ATTR_SOURCE_REF_FIELD, +) +from ..relationship.relationship_references import ( + reference_fields, + resolve_relationship_reference, +) from .meta_dimension import MetaDimension from .meta_measure import MetaMeasure from .report_accessors import ( @@ -216,6 +228,68 @@ def _declared_member(from_: MetaObject, type_: str, name: str, cls: type[MetaDat ) +def _is_required_field(field: MetaField) -> bool: + """ADR-0039 resolving: the field's effective required-ness, the ``@required`` attr OR a + ``validator.required`` child (``children()`` is the effective set, own + inherited).""" + return field.get_meta_attr(FIELD_ATTR_REQUIRED) is True or any( + c.type == "validator" and c.sub_type == "required" for c in field.children() + ) + + +def _via_joins_inner(hops: list[str], from_: MetaObject, root: MetaRoot) -> bool: + """True when every hop of ``hops``, walked from ``from_``, is a belongs-to hop over a + required foreign key: the hops the view joins INNER, so every row of the report has a + match. This is the join-type rule of the view lowering (TypeScript ``walkViaPath``, #209): + the reference is held by the entity the hop leaves, its first FK field is required, and + every earlier hop is INNER too. ``@enforce: false`` does not change it. A hop that does + not resolve, an ambiguous reference or a has-many hop is not INNER.""" + current = from_ + for hop_name in hops: + # ADR-0039: resolving children(), so a hop inherited through extends is found. + rel = next((c for c in current.children() if c.type == "relationship" and c.name == hop_name), None) + ref_node = None + if rel is None: + ref_node = next( + ( + c + for c in current.children() + if c.type == "identity" and c.sub_type == IDENTITY_SUBTYPE_REFERENCE and c.name == hop_name + ), + None, + ) + hop = rel if rel is not None else ref_node + if hop is None: + return False + target_ref = hop.get_meta_attr( + RELATIONSHIP_ATTR_OBJECT_REF if rel is not None else IDENTITY_REFERENCE_ATTR_REFERENCES + ) + if not isinstance(target_ref, str): + return False + target = resolve_object_ref(root, target_ref, _package_of_key(current.resolution_key())) + if not isinstance(target, MetaObject): + return False + # The reference the hop crosses, as the lowering resolves it. A has-many hop, whose + # FK sits on the child, matches none of the current entity's own references. + if ref_node is not None: + reference = ref_node + else: + source_ref_field = hop.get_meta_attr(RELATIONSHIP_ATTR_SOURCE_REF_FIELD) + reference = resolve_relationship_reference( + current, + hop_name, + target.name, + source_ref_field if isinstance(source_ref_field, str) else None, + ) + if reference is None: + return False + fk_fields = reference_fields(reference) + fk = current.find_field(fk_fields[0]) if fk_fields else None + if fk is None or not _is_required_field(fk): + return False + current = target + return True + + def _dimension_required( dim: MetaDimension, named: MetaObject, @@ -225,20 +299,25 @@ def _dimension_required( spine: list[str] | None, report_name: str, ) -> bool: - """Table C (``required`` of a dimension). Without ``@spine``: only a dimension with no - ``@via`` over an ``@of`` field whose effective ``@required`` is true. With ``@spine``: + """Table C (``required`` of a dimension). Without ``@spine``: only a dimension over an + ``@of`` field whose effective ``@required`` is true and, when it has a ``@via``, only + when every hop is a required belongs-to reference, the hops the view joins INNER + (:func:`_via_joins_inner`). With ``@spine``: only a dimension whose ``@via`` hops equal the spine's (a column of the spine entity itself, whose rows are the report's rows) over an ``@of`` field that is ``@required`` or a primary-key column of the entity ``@of`` names. A dimension beyond the spine is reached by a LEFT OUTER join.""" via = dim.via() - # ADR-0039 resolving: the @of field's effective required-ness, the @required attr OR a - # validator.required child (children() is the effective set, own + inherited). - of_required = of.get_meta_attr(FIELD_ATTR_REQUIRED) is True or any( - c.type == "validator" and c.sub_type == "required" for c in of.children() - ) + of_required = _is_required_field(of) if spine is None: - return via is None and of_required + if via is None: + return of_required + if not of_required: + return False + hops = reporting_via_hops(via, reporting_member_owner(dim, from_), from_, root) + if hops is None: + raise _unresolved(report_name, f"dimension '{dim.name}' @via '{via}'") + return _via_joins_inner(hops, from_, root) if via is None: return False # a column of @from: null in a spine row with no facts hops = reporting_via_hops(via, reporting_member_owner(dim, from_), from_, root) diff --git a/server/python/tests/test_report_shape.py b/server/python/tests/test_report_shape.py index 7cefee7f8..8497ae864 100644 --- a/server/python/tests/test_report_shape.py +++ b/server/python/tests/test_report_shape.py @@ -656,3 +656,164 @@ def test_a_spine_that_does_not_resolve_raises_naming_the_report(spine: str) -> N with pytest.raises(ValueError) as e: report_shape(report, root) assert str(e.value) == f"report 'SpineSales': @spine '{spine}' does not resolve." + + +# --------------------------------------------------------------------------- +# Table C, a dimension reached through @via (adopter report D5): the view joins a belongs-to +# hop over a required foreign key INNER, so the column is as required as the field it reads. +# The same model and expectations as the TypeScript and Java report-shape tests. +# --------------------------------------------------------------------------- + + +def _via_entity(name: str, table: str, children: list) -> dict: + return { + "object.entity": { + "name": name, + "children": [ + {"source.rdb": {"@table": table}}, + {"field.long": {"name": "id"}}, + {"identity.primary": {"name": "pk", "@fields": ["id"]}}, + *children, + ], + } + } + + +def _via_report(name: str, **extra: Any) -> dict: + return { + "object.report": { + "name": name, + "@from": "Snap", + "@dimensions": ["realmName", "realmMotto", "maybeRealmName", "looseRealmName", "sessionRealmId"], + "@measures": ["snaps"], + **extra, + "children": [{"source.rdb": {"@kind": "view", "@view": f"v_{name.lower()}"}}], + } + } + + +def _via_model(*reports: dict): + realm = _via_entity( + "Realm", + "realms", + [ + {"field.string": {"name": "name", "@required": True}}, + {"field.string": {"name": "motto"}}, + ], + ) + session = _via_entity( + "Session", + "sessions", + [ + {"field.long": {"name": "realmId", "@required": True}}, + {"field.long": {"name": "maybeRealmId"}}, + {"identity.reference": {"name": "fkRealm", "@fields": ["realmId"], "@references": "Realm"}}, + {"identity.reference": {"name": "fkMaybeRealm", "@fields": ["maybeRealmId"], "@references": "Realm"}}, + { + "relationship.association": { + "name": "realm", + "@objectRef": "Realm", + "@cardinality": "one", + "@sourceRefField": "realmId", + } + }, + { + "relationship.association": { + "name": "maybeRealm", + "@objectRef": "Realm", + "@cardinality": "one", + "@sourceRefField": "maybeRealmId", + } + }, + ], + ) + snap = _via_entity( + "Snap", + "snaps", + [ + {"field.long": {"name": "sessionId", "@required": True}}, + {"field.long": {"name": "unenforcedSessionId", "@required": True}}, + {"identity.reference": {"name": "fkSession", "@fields": ["sessionId"], "@references": "Session"}}, + { + "identity.reference": { + "name": "fkLoose", + "@fields": ["unenforcedSessionId"], + "@references": "Session", + "@enforce": False, + } + }, + {"relationship.association": {"name": "session", "@objectRef": "Session", "@cardinality": "one"}}, + {"relationship.association": {"name": "stamp", "@objectRef": "Stamp", "@cardinality": "one"}}, + {"measure.aggregate": {"name": "snaps", "@agg": "count", "@of": "Snap.id"}}, + {"dimension.attribute": {"name": "realmName", "@of": "Realm.name", "@via": "Snap.fkSession.fkRealm"}}, + {"dimension.attribute": {"name": "realmMotto", "@of": "Realm.motto", "@via": "Snap.fkSession.fkRealm"}}, + { + "dimension.attribute": { + "name": "maybeRealmName", + "@of": "Realm.name", + "@via": "Snap.fkSession.fkMaybeRealm", + } + }, + {"dimension.attribute": {"name": "looseRealmName", "@of": "Realm.name", "@via": "Snap.fkLoose.fkRealm"}}, + {"dimension.attribute": {"name": "sessionRealmId", "@of": "Session.realmId", "@via": "Snap.fkSession"}}, + {"dimension.attribute": {"name": "relRealmName", "@of": "Realm.name", "@via": "Snap.session.realm"}}, + { + "dimension.attribute": { + "name": "relMaybeRealmName", + "@of": "Realm.name", + "@via": "Snap.session.maybeRealm", + } + }, + {"dimension.attribute": {"name": "relSessionRealmId", "@of": "Session.realmId", "@via": "Snap.session"}}, + {"dimension.attribute": {"name": "mixedRealmName", "@of": "Realm.name", "@via": "Snap.session.fkRealm"}}, + {"dimension.attribute": {"name": "stampLabel", "@of": "Stamp.label", "@via": "Snap.stamp"}}, + ], + ) + # The foreign key between Snap and Stamp is held by Stamp, the far side of Snap's `stamp`. + stamp = _via_entity( + "Stamp", + "stamps", + [ + {"field.long": {"name": "snapId", "@required": True}}, + {"field.string": {"name": "label", "@required": True}}, + {"identity.reference": {"name": "fkSnap", "@fields": ["snapId"], "@references": "Snap"}}, + ], + ) + return _load(_file("a", [realm, session, snap, stamp, *reports])) + + +def _via_required_of(root, name: str) -> dict[str, bool]: + return {f.name: f.required for f in report_shape(_object(root, name), root).fields} + + +def test_a_via_dimension_is_required_when_every_hop_is_a_required_belongs_to() -> None: + root = _via_model(_via_report("Plain")) + req = _via_required_of(root, "Plain") + assert req["realmName"] is True # two required hops over a required field + assert req["sessionRealmId"] is True # one required hop + assert req["looseRealmName"] is True # @enforce: false does not change the join type + assert req["realmMotto"] is False # the target field is optional + assert req["maybeRealmName"] is False # a hop over a nullable foreign key is a LEFT join + + +def test_a_via_dimension_under_spine_keeps_the_spine_rule() -> None: + root = _via_model(_via_report("Spined", **{"@spine": "Snap.fkSession", "@dimensions": ["realmName", "sessionRealmId"]})) + req = _via_required_of(root, "Spined") + assert req["realmName"] is False # every join beyond the spine is LEFT + assert req["sessionRealmId"] is True # on the spine, a required field + + +def test_a_to_one_relationship_hop_follows_the_reference_it_crosses() -> None: + # The same rule when a hop is a @cardinality: one relationship instead of the reference it + # crosses: the relationship resolves to that reference (by name pairing for `session`, by + # @sourceRefField for the two onto Realm), and the foreign key it holds decides the join. + dims = ["relRealmName", "relMaybeRealmName", "relSessionRealmId", "mixedRealmName", "stampLabel"] + root = _via_model(_via_report("Related", **{"@dimensions": dims})) + req = _via_required_of(root, "Related") + assert req["relRealmName"] is True # two relationship hops, both over a required foreign key + assert req["relSessionRealmId"] is True # one relationship hop + assert req["mixedRealmName"] is True # a relationship hop, then a reference hop + assert req["relMaybeRealmName"] is False # the second relationship crosses a nullable foreign key + # The other entity holds the foreign key: a Snap with no Stamp has no match, so the join is + # LEFT whatever that key and the target field require. + assert req["stampLabel"] is False diff --git a/server/typescript/packages/cli/src/commands/gen.ts b/server/typescript/packages/cli/src/commands/gen.ts index 515bb0f69..e1b1b8c23 100644 --- a/server/typescript/packages/cli/src/commands/gen.ts +++ b/server/typescript/packages/cli/src/commands/gen.ts @@ -268,6 +268,7 @@ export async function genCommand(args: string[], cwd: string, fmt: OutputFormat columnNamingStrategy: forgeConfig.columnNamingStrategy, projectRoot, migrateOutDir: migrateDefaults.outDir, + migrateFormat: migrateDefaults.format, changedFiles: result.files.filter((f) => CHANGED_STATUSES.has(f.status)).map((f) => f.path), }); diff --git a/server/typescript/packages/cli/src/commands/migrate.ts b/server/typescript/packages/cli/src/commands/migrate.ts index d1ec403b0..3693b9191 100644 --- a/server/typescript/packages/cli/src/commands/migrate.ts +++ b/server/typescript/packages/cli/src/commands/migrate.ts @@ -39,6 +39,7 @@ import { writeMigrationD1, writeMigrationFlyway, introspectD1, + withoutFlywayHistory, applyPending, rollbackTo, findWranglerConfig, @@ -54,6 +55,7 @@ import { type SchemaProvenance, type SchemaSnapshot, type TableDescriptor, + postgresViewReplaceProbe, } from "@metaobjectsdev/migrate-ts"; import { buildWranglerExecuteArgs, @@ -398,15 +400,19 @@ function nextStepsDetail(dialect: string | undefined): string { } /** Table count of the target DB, or undefined when it can't be reached/introspected - * (any failure ⇒ unknown ⇒ never block; the caller warns instead of refusing). */ + * (any failure ⇒ unknown ⇒ never block; the caller warns instead of refusing). Under the + * flyway format Flyway's own history table is not counted: a database that holds only that + * table has none of the project's tables. */ async function countLiveTables( url: string, dialect: ResolvedMigrateConfig["dialect"], + format: ResolvedMigrateConfig["format"], ): Promise { try { const kysely = await buildKyselyFromUrl(url, dialect); try { - return (await introspect(kysely.db, kysely.dialect)).tables.length; + const live = await introspect(kysely.db, kysely.dialect); + return (format === "flyway" ? withoutFlywayHistory(live) : live).tables.length; } finally { await kysely.close(); } @@ -790,6 +796,8 @@ export async function migrateCommand( let actual; try { actual = await introspect(kysely.db, kysely.dialect); + // Flyway owns its history table; diffing it proposes DROP TABLE flyway_schema_history. + if (config.format === "flyway") actual = withoutFlywayHistory(actual); } catch (err) { log.error(`migrate: failed to connect to ${kysely.displayUrl}: ${describeError(err)}`); await kysely.close(); @@ -810,6 +818,9 @@ export async function migrateCommand( ...scopedDiffInputs(scoped, collectUnmanagedNames(metadata)), actual, dialect: kysely.dialect, + // Postgres decides whether a changed view can be replaced in place: the probe + // creates the new definition as a temporary view in a transaction it rolls back. + ...(kysely.dialect === "postgres" ? { viewReplaceProbe: postgresViewReplaceProbe(kysely.db) } : {}), allow: tokensToAllowOptions(config.allow), renames: config.renames, // #258 — adopting a live DB whose PRIMARY KEY differs from the metadata identity @@ -1133,6 +1144,7 @@ export async function runBaseline( } try { snapshot = await introspect(kysely.db, kysely.dialect); + if (config.format === "flyway") snapshot = withoutFlywayHistory(snapshot); } finally { await kysely.close(); } @@ -1145,7 +1157,7 @@ export async function runBaseline( // couldn't reach — we still write the snapshot but WARN: an offline baseline is // only correct when the database already matches the metadata, and emits no DDL. const liveTableCount = config.databaseUrl !== undefined - ? await countLiveTables(config.databaseUrl, config.dialect) + ? await countLiveTables(config.databaseUrl, config.dialect, config.format) : undefined; if (liveTableCount === 0) { // `liveTableCount === 0` ⇒ databaseUrl was defined (countLiveTables only @@ -1395,6 +1407,7 @@ export async function runOfflineGenerate( emitStructuredError("no schema snapshot", nextStepsDetail(config.dialect), fmt); return 2; } + if (config.format === "flyway") snapshot = withoutFlywayHistory(snapshot); const collectedAmbiguous: AmbiguousChange[] = []; const onAmbiguousResolution = mapOnAmbiguous(config.onAmbiguous); diff --git a/server/typescript/packages/cli/src/commands/verify.ts b/server/typescript/packages/cli/src/commands/verify.ts index 600e2e551..527d49b7e 100644 --- a/server/typescript/packages/cli/src/commands/verify.ts +++ b/server/typescript/packages/cli/src/commands/verify.ts @@ -43,7 +43,7 @@ import { lintNodeNames } from "../lib/name-lint.js"; import { lintDeprecatedReferences } from "../lib/deprecation-lint.js"; import { lintDuplicateFields, lintReferenceFields } from "../lib/field-lint.js"; import { FileSource } from "@metaobjectsdev/metadata/core"; -import { resolveD1Config, resolveMigrateConfig } from "../lib/config.js"; +import { resolveD1Config, resolveMigrateConfig, type ResolvedMigrateConfig } from "../lib/config.js"; import { buildWranglerExecuteArgs, defaultWranglerRunner, @@ -82,6 +82,8 @@ import { type D1Binding, type D1Runner, type DriftResult, + postgresViewReplaceProbe, + withoutFlywayHistory, } from "@metaobjectsdev/migrate-ts"; import { DEFAULT_METAOBJECTS_DIR, @@ -121,9 +123,11 @@ const ERR_UNKNOWN_ATTR = "ERR_UNKNOWN_ATTR"; /** * A no-flags MigrateFlags, so `resolveMigrateConfig` yields exactly what `meta migrate` - * would use with nothing passed on the command line — config value, else default. + * would use with nothing passed on the command line — config value, else default. The one + * migrate flag verify also takes, `--migration-format`, is laid over it per invocation, so + * the format resolves flag > config > default exactly as it does for `meta migrate`. * - * verify consumes `outDir` (#292) and — for the replay gate ONLY — `dialect`. The #292 + * verify consumes `outDir` (#292), `format` and — for the replay gate ONLY — `dialect`. The #292 * restriction that reading anything else "would be reaching into migrate's decisions" * was written about the DRIFT gate, whose dialect comes from the live `--db` URL. The * replay gate has no `--db` at all, and the dialect a committed chain was EMITTED for @@ -174,6 +178,7 @@ export async function verifyCommand( emitStructured({ error: msg, hint: "run `meta verify --help` for the accepted flags" }, fmt); return 2; } + const migrateFlags: MigrateFlags = { ...EMPTY_MIGRATE_FLAGS, format: flags.migrationFormat }; // ADR-0021 D2 — explicit verify subverbs. Each flag selects one drift mode; // any combination runs each and the overall exit code is the MAX (non-zero on @@ -501,7 +506,7 @@ export async function verifyCommand( // Resolve the migrations directory and the chain's dialect through MIGRATE's own // precedence, never a second derivation — verify must not look somewhere migrate // does not write, nor assume a dialect the chain was not emitted for. - const migrateConfig = await resolveMigrateConfig(EMPTY_MIGRATE_FLAGS, projectRoot); + const migrateConfig = await resolveMigrateConfig(migrateFlags, projectRoot); if (migrateConfig.format === "flyway") { log.error( @@ -1292,6 +1297,11 @@ export async function verifyCommand( try { const allow = tokensToAllowOptions(flags.allow); + // Resolve the migrate config once for this whole path: `tryLoadConfig` is uncached, + // and the flyway filter below and checkCommittedSnapshot's migrations-dir read want + // the same value — resolving twice would read and parse `.metaobjects/config.json` + // twice per `meta verify --db`. + const migrateConfig = await resolveMigrateConfig(migrateFlags, projectRoot); // Expected views from the single view-SQL source (codegen-ts), so view-body // drift is detected against the live DB. const viewStrategy = forgeConfig?.columnNamingStrategy ?? "snake_case"; @@ -1303,9 +1313,15 @@ export async function verifyCommand( // `actual` this drift comparison uses, and re-introspecting for it would both // cost a second round trip and open a window where the two could disagree. actual = await introspect(kysely.db, kysely.dialect); + // Flyway owns its history table; it is no drift under the flyway format, as in `meta migrate`. + if (migrateConfig.format === "flyway") { + actual = withoutFlywayHistory(actual); + } driftResult = await computeDriftFromActual(actual, kysely.dialect, root, { allow, views: expectedViews, + // The same question `meta migrate` asks, so the report names the change it plans. + ...(kysely.dialect === "postgres" ? { viewReplaceProbe: postgresViewReplaceProbe(kysely.db) } : {}), ...(schemaScope !== undefined ? { inScope: schemaScope } : {}), // The import exclusion reaches scopeExpectedSchema through this option bag — // there is no direct call on this path to attach it to. @@ -1319,7 +1335,7 @@ export async function verifyCommand( const snapshotDrift = driftResult.changes.length === 0 - ? await checkCommittedSnapshot(actual, kysely.dialect, kysely.displayUrl, driftResult) + ? await checkCommittedSnapshot(actual, kysely.dialect, kysely.displayUrl, driftResult, migrateConfig) : []; return reportSchemaDrift(driftResult, [...ledgerDrift, ...snapshotDrift], kysely.displayUrl); @@ -1444,13 +1460,15 @@ export async function verifyCommand( dialect: Dialect, displayUrl: string, governed: GovernedScope, + // Resolved by the caller and threaded through — see the `verify --db` branch: the same + // run's flyway filter reads this config too, and `tryLoadConfig` does not cache. + migrateConfig: ResolvedMigrateConfig, ): Promise { if (dialect === "d1") return []; // d1 keeps migrations Wrangler-native; no offline snapshot - // Resolve the migrations dir through migrate's OWN precedence (flag > config > - // default) rather than re-deriving it, so verify can never look somewhere migrate - // does not write. Only `outDir` is consumed; the rest of the resolved config is - // migrate's business. - const migrateConfig = await resolveMigrateConfig(EMPTY_MIGRATE_FLAGS, projectRoot); + // The migrations dir comes from migrate's OWN precedence (flag > config > default) + // rather than a re-derivation here, so verify can never look somewhere migrate does + // not write. Only `outDir` and `format` are consumed; the rest of the resolved config + // is migrate's business. const dir = resolvePath(projectRoot, migrateConfig.outDir); let snapshot: SchemaSnapshot | null; try { @@ -1459,6 +1477,7 @@ export async function verifyCommand( return []; } if (snapshot === null) return []; + if (migrateConfig.format === "flyway") snapshot = withoutFlywayHistory(snapshot); // Out-of-scope objects leave BOTH sides of this comparison, and the schema pin // comes from the scope decision the DRIFT comparison already made — one door diff --git a/server/typescript/packages/cli/src/index.ts b/server/typescript/packages/cli/src/index.ts index 4cd1cfae2..65c7c7703 100644 --- a/server/typescript/packages/cli/src/index.ts +++ b/server/typescript/packages/cli/src/index.ts @@ -135,6 +135,11 @@ VERIFY FLAGS (ADR-0021 D2 — explicit subverbs; combine any; exit 1 on ANY drif --allow Accepted for parity with 'migrate'; does NOT affect the verify drift gate (the gate fails on ANY detected change) --skip-schema Skip the schema-drift gate even when --db is present + --migration-format default|flyway + Migration file layout (default: default), as for 'meta migrate'. + Under 'flyway' the schema-drift gate leaves flyway_schema_history + out of the comparison, because Flyway owns it. Also settable + once as migrate.format in .metaobjects/config.json. --d1 D1 binding name from wrangler.toml (only with --dialect d1) --remote Target remote D1 instead of local (only with --dialect d1) — the ONLY way to verify the actual deployed D1 database @@ -338,6 +343,11 @@ FLAGS: --dialect sqlite|postgres|d1 Optional override (auto-detected from --db URL scheme) --allow Accepted for parity with 'migrate'; does NOT affect the drift gate --skip-schema Skip the schema-drift gate even when --db is present + --migration-format default|flyway + Migration file layout (default: default), as for 'meta migrate'. + Under 'flyway' the schema-drift gate leaves flyway_schema_history + out of the comparison, because Flyway owns it. Also settable + once as migrate.format in .metaobjects/config.json. --d1 D1 binding name from wrangler.toml (only with --dialect d1) --remote Target remote D1 instead of local (only with --dialect d1) — the ONLY way to verify the actual deployed D1 database diff --git a/server/typescript/packages/cli/src/lib/allow.ts b/server/typescript/packages/cli/src/lib/allow.ts index 67acc9961..24e5ad8df 100644 --- a/server/typescript/packages/cli/src/lib/allow.ts +++ b/server/typescript/packages/cli/src/lib/allow.ts @@ -2,7 +2,7 @@ // both `meta migrate` and `meta verify --db`. Keeping a single copy avoids the // two commands drifting on which `--allow` tokens exist or how a change reads. -import { allowOptionFor, isViewRecreateOnly, suggestColumnRenames } from "@metaobjectsdev/migrate-ts"; +import { allowOptionFor, describeSqlType, isViewRecreateOnly, suggestColumnRenames } from "@metaobjectsdev/migrate-ts"; import type { AllowOptions, Change, ColumnRenameSuggestion, ViewChangeReason } from "@metaobjectsdev/migrate-ts"; import type { BlockedEntry } from "./output.js"; @@ -71,7 +71,7 @@ export function describeChange(c: Change): string { const from = c.fromDefault?.value ?? "none"; const to = c.toDefault?.value ?? "none"; const dflt = from === to && c.fromDefault?.kind === c.toDefault?.kind ? "" : `, default ${from} → ${to}`; - return `${c.table}.${c.column} (${c.from.kind} → ${c.to.kind}${dflt})`; + return `${c.table}.${c.column} (${describeSqlType(c.from)} → ${describeSqlType(c.to)}${dflt})`; } case "change-column-nullable": return `${c.table}.${c.column} (${c.from ? "NULL" : "NOT NULL"} → ${c.to ? "NULL" : "NOT NULL"})`; case "change-column-default": return `${c.table}.${c.column}`; diff --git a/server/typescript/packages/cli/src/lib/args.ts b/server/typescript/packages/cli/src/lib/args.ts index 54cb051b7..97954a8ab 100644 --- a/server/typescript/packages/cli/src/lib/args.ts +++ b/server/typescript/packages/cli/src/lib/args.ts @@ -251,6 +251,20 @@ function assertSchemaDialect(dialect: string | undefined): void { export const MIGRATE_FORMATS = ["default", "flyway"] as const; export type MigrateFormat = (typeof MIGRATE_FORMATS)[number]; +/** + * A `--migration-format` value, validated. One parser for `meta migrate` and `meta verify`, + * so the two accept the same formats and refuse a bad one with the same message. + * + * The flag is --migration-format, not --format: `--format` is already a GLOBAL cli flag + * selecting output rendering (toon|json|text), consumed in index.ts. + */ +function parseMigrationFormat(raw: string | undefined): MigrateFormat | undefined { + if (raw !== undefined && !MIGRATE_FORMATS.includes(raw as MigrateFormat)) { + throw new Error(`invalid --migration-format '${raw}'; expected: ${MIGRATE_FORMATS.join(", ")}`); + } + return raw as MigrateFormat | undefined; +} + // Exported (not just module-local) so allow-tokens-pinned.test.ts can pin it // against sdk's AllowTokenEnum (config.json's static migrate.allow validator) // — the two lists drifted silently before that test existed: sdk's enum was @@ -308,6 +322,12 @@ export interface VerifyFlags { allow: AllowToken[]; /** Skip the schema-drift gate even when --db is present. */ skipSchema: boolean; + /** + * The migration file layout the project uses, spelled as `meta migrate` spells it and + * resolved with the same precedence (flag > `.metaobjects/config.json` > default). Under + * `flyway` the schema-drift gate leaves Flyway's own history table out of the comparison. + */ + migrationFormat: MigrateFormat | undefined; /** * #225 — D1 binding name from wrangler.toml, spelled/documented the same way * `meta migrate --d1 ` does. Gated on `--dialect d1` (D1 has no URL @@ -458,6 +478,7 @@ export const VERIFY_OPTIONS = { dialect: { type: "string" }, allow: { type: "string", multiple: true }, "skip-schema": { type: "boolean", default: false }, + "migration-format": { type: "string" }, templates: { type: "boolean", default: false }, codegen: { type: "boolean", default: false }, "forbid-hand-edits": { type: "boolean", default: false }, @@ -545,6 +566,7 @@ export function parseVerifyArgs(argv: string[]): VerifyFlags { dialect: dialect as Dialect | undefined, allow: allowTokens as AllowToken[], skipSchema: !!values["skip-schema"], + migrationFormat: parseMigrationFormat(values["migration-format"] as string | undefined), templates, codegen, forbidHandEdits, @@ -678,12 +700,7 @@ export function parseMigrateArgs(argv: string[]): MigrateFlags { const dialect = values.dialect as string | undefined; assertSchemaDialect(dialect); - // NOTE: the flag is --migration-format, not --format: `--format` is already a - // GLOBAL cli flag selecting output rendering (toon|json|text), consumed in index.ts. - const format = values["migration-format"] as string | undefined; - if (format !== undefined && !MIGRATE_FORMATS.includes(format as MigrateFormat)) { - throw new Error(`invalid --migration-format '${format}'; expected: ${MIGRATE_FORMATS.join(", ")}`); - } + const format = parseMigrationFormat(values["migration-format"] as string | undefined); const onAmb = values["on-ambiguous"] as string | undefined; if (onAmb !== undefined && !ON_AMBIGUOUS.includes(onAmb as OnAmbiguous)) { @@ -695,7 +712,7 @@ export function parseMigrateArgs(argv: string[]): MigrateFlags { return { db: values.db as string | undefined, dialect: dialect as Dialect | undefined, - format: format as MigrateFormat | undefined, + format, outDir: values["out-dir"] as string | undefined, slug: values.slug as string | undefined, allow: allowTokens as AllowToken[], diff --git a/server/typescript/packages/cli/src/lib/config.ts b/server/typescript/packages/cli/src/lib/config.ts index 43a9ef06b..cb16a37c2 100644 --- a/server/typescript/packages/cli/src/lib/config.ts +++ b/server/typescript/packages/cli/src/lib/config.ts @@ -83,16 +83,18 @@ async function tryLoadConfig(metaRoot: string): Promise { // --------------------------------------------------------------------------- /** - * The migrations directory and dialect `meta migrate` would use with no flags — what - * `meta gen` needs to word its "now migrate" next step and find the committed snapshot. + * The migrations directory, dialect and format `meta migrate` would use with no flags — what + * `meta gen` needs to word its "now migrate" next step and read the committed snapshot the + * way `meta migrate` reads it. */ export async function resolveMigrateDefaults( metaRoot: string, -): Promise<{ outDir: string; dialect: Dialect | undefined }> { +): Promise<{ outDir: string; dialect: Dialect | undefined; format: MigrateFormat }> { const config = await tryLoadConfig(metaRoot); return { outDir: config?.migrate?.outDir ?? MIGRATE_DEFAULTS.outDir, dialect: config?.migrate?.dialect ?? MIGRATE_DEFAULTS.dialect, + format: config?.migrate?.format ?? MIGRATE_DEFAULTS.format, }; } diff --git a/server/typescript/packages/cli/src/lib/gen-migrate-advice.ts b/server/typescript/packages/cli/src/lib/gen-migrate-advice.ts index f03016226..61f47e5b3 100644 --- a/server/typescript/packages/cli/src/lib/gen-migrate-advice.ts +++ b/server/typescript/packages/cli/src/lib/gen-migrate-advice.ts @@ -26,6 +26,7 @@ import { parseWranglerConfig, readSnapshot, snapshotPath, + withoutFlywayHistory, type Change, type D1Binding, } from "@metaobjectsdev/migrate-ts"; @@ -47,6 +48,9 @@ export interface GenMigrateAdviceInput { projectRoot: string; /** The migrations directory `meta migrate` uses (holds the committed snapshot). */ migrateOutDir: string; + /** `.metaobjects/config.json` `migrate.format`. Under `flyway` the snapshot is read without + * Flyway's history table, as `meta migrate` reads it. */ + migrateFormat: "default" | "flyway"; /** Paths (any form) of the files this run wrote, merged or removed. */ changedFiles: readonly string[]; } @@ -75,7 +79,8 @@ export async function genMigrateAdvice(input: GenMigrateAdviceInput): Promise VIEW_CHANGE_KINDS.has(c.kind))) return undefined; diff --git a/server/typescript/packages/cli/test/__snapshots__/cli.test.ts.snap b/server/typescript/packages/cli/test/__snapshots__/cli.test.ts.snap index 8e96556ff..cfc7eae18 100644 --- a/server/typescript/packages/cli/test/__snapshots__/cli.test.ts.snap +++ b/server/typescript/packages/cli/test/__snapshots__/cli.test.ts.snap @@ -115,6 +115,11 @@ VERIFY FLAGS (ADR-0021 D2 — explicit subverbs; combine any; exit 1 on ANY drif --allow Accepted for parity with 'migrate'; does NOT affect the verify drift gate (the gate fails on ANY detected change) --skip-schema Skip the schema-drift gate even when --db is present + --migration-format default|flyway + Migration file layout (default: default), as for 'meta migrate'. + Under 'flyway' the schema-drift gate leaves flyway_schema_history + out of the comparison, because Flyway owns it. Also settable + once as migrate.format in .metaobjects/config.json. --d1 D1 binding name from wrangler.toml (only with --dialect d1) --remote Target remote D1 instead of local (only with --dialect d1) — the ONLY way to verify the actual deployed D1 database diff --git a/server/typescript/packages/cli/test/allow-hint-tokens.test.ts b/server/typescript/packages/cli/test/allow-hint-tokens.test.ts new file mode 100644 index 000000000..f0c9cf94a --- /dev/null +++ b/server/typescript/packages/cli/test/allow-hint-tokens.test.ts @@ -0,0 +1,144 @@ +/** + * Every `--allow` hint names a token the parser accepts (adopter report D3). + * + * `allowFlagFor` fell back to the change's own kind when it had no mapped permission, so a + * blocked view adoption printed `--allow replace-view`, which `parseArgs` rejects. This builds + * a blocked change of each shape `diff` can block and checks the hint against ALLOW_TOKENS. + * + * A hint must also name what blocked the change NOW. A view drop sits behind up to three gates + * (cascade, adoption, the removal itself), and a hint chosen from the change's shape repeated a + * token the user had already passed. So each scenario is replayed the way a user would follow + * it: grant what the hints name, diff again, until nothing is blocked. + */ +import { describe, test, expect } from "bun:test"; +import { + diff, type Change, type SchemaSnapshot, type TableDescriptor, type ViewDescriptor, type ViewReplaceProbe, +} from "@metaobjectsdev/migrate-ts"; +import { ALLOW_TOKENS } from "../src/lib/args.js"; +import { allowFlagFor, blockedEntriesFor, blockedHintLines, tokensToAllowOptions } from "../src/lib/allow.js"; + +const T = (over: Partial = {}): TableDescriptor => ({ + name: "t", + columns: [ + { name: "id", sqlType: { kind: "integer", bits: 64 }, nullable: false }, + { name: "label", sqlType: { kind: "text" }, nullable: true }, + ], + indexes: [], foreignKeys: [], primaryKey: ["id"], checks: [], ...over, +}); +const snap = (tables: TableDescriptor[], views: ViewDescriptor[] = []): SchemaSnapshot => ({ tables, views }); +const view = (over: Partial = {}): ViewDescriptor => + ({ name: "v", sql: "SELECT id FROM t", fingerprint: "a".repeat(64), dependsOn: ["t"], ...over }); +const unstamped: ViewDescriptor = { name: "v", sql: " SELECT t.id FROM t;" }; + +/** Somebody else's view built on `v`: a DROP of `v` needs CASCADE and destroys it. */ +const downstream: NonNullable = [{ schema: "public", name: "downstream", relkind: "v", managed: false }]; + +/** `provenReplace`: the database shows the live view's columns are the expected view's, so the + * change is planned as a replace. Without it nothing proves the replace, and it is a drop + create. */ +type Scenario = { expected: SchemaSnapshot; actual: SchemaSnapshot; provenReplace?: true }; + +const sameColumns: ViewReplaceProbe = async () => { + const columns = [{ name: "id", type: "bigint", collation: "0" }]; + return { expected: columns, live: columns }; +}; + +async function blocked({ expected, actual, provenReplace }: Scenario, granted: string[] = []): Promise { + return (await diff(expected, actual, { + dialect: "postgres", + allow: tokensToAllowOptions(granted), + ...(provenReplace === true ? { viewReplaceProbe: sameColumns } : {}), + })).blocked; +} + +const hints = (changes: readonly Change[]): string[] => [...new Set(changes.map(allowFlagFor))]; + +const SCENARIOS: Record = { + "drop-table": { expected: snap([]), actual: snap([T()]) }, + "drop-column": { expected: snap([T({ columns: [T().columns[0]!] })]), actual: snap([T()]) }, + "lossy type change": { + expected: snap([T({ columns: [T().columns[0]!, { name: "label", sqlType: { kind: "integer", bits: 32 }, nullable: true }] })]), + actual: snap([T()]), + }, + "nullable to not null": { + expected: snap([T({ columns: [T().columns[0]!, { name: "label", sqlType: { kind: "text" }, nullable: false }] })]), + actual: snap([T()]), + }, + "drop-index": { + expected: snap([T()]), + actual: snap([T({ indexes: [{ name: "ix", columns: ["label"], unique: false }] })]), + }, + "drop-fk": { + expected: snap([T()]), + actual: snap([T({ foreignKeys: [{ name: "fk", columns: ["label"], refTable: "t", refColumns: ["id"] }] })]), + }, + "drop-check": { + expected: snap([T()]), + actual: snap([T({ checks: [{ name: "ck", expression: "id > 0" }] })]), + }, + "drop-view (undeclared)": { expected: snap([T()]), actual: snap([T()], [view()]) }, + "drop-view (undeclared) with external dependents": { + expected: snap([T()]), + actual: snap([T()], [view({ dependents: downstream })]), + }, + "replace-view adopting an unstamped view": { + expected: snap([T()], [view()]), actual: snap([T()], [unstamped]), provenReplace: true, + }, + "drop-view half of a structural adoption": { + expected: snap([T()], [view()]), + actual: snap([T()], [unstamped]), + }, + "structural adoption with external dependents": { + expected: snap([T()], [view()]), + actual: snap([T()], [{ ...unstamped, dependents: downstream }]), + }, +}; + +describe("every blocked change hints at a token the parser accepts", () => { + for (const [name, scenario] of Object.entries(SCENARIOS)) { + test(name, async () => { + const changes = await blocked(scenario); + expect(changes.length).toBeGreaterThan(0); + for (const c of changes) { + expect(ALLOW_TOKENS as readonly string[]).toContain(allowFlagFor(c)); + } + }); + } + + test("a blocked view adoption names adopt-view", async () => { + const [c] = await blocked(SCENARIOS["replace-view adopting an unstamped view"]!); + expect(c).toMatchObject({ kind: "replace-view" }); + const [entry] = blockedEntriesFor([c!], [c!]); + expect(blockedHintLines(entry!).join("\n")).toContain("--allow adopt-view"); + }); +}); + +describe("following the hints unblocks the change", () => { + for (const [name, scenario] of Object.entries(SCENARIOS)) { + test(name, async () => { + const granted: string[] = []; + let pending = await blocked(scenario); + expect(pending.length).toBeGreaterThan(0); + while (pending.length > 0) { + const hinted = hints(pending); + // A hint that names a token already passed leaves the user looping on it. + for (const token of hinted) expect(granted).not.toContain(token); + granted.push(...hinted); + pending = await blocked(scenario, granted); + } + }); + } + + test("a view drop with dependents asks for the cascade, then for the drop", async () => { + const scenario = SCENARIOS["drop-view (undeclared) with external dependents"]!; + expect(hints(await blocked(scenario))).toEqual(["drop-view-cascade"]); + expect(hints(await blocked(scenario, ["drop-view-cascade"]))).toEqual(["drop-view"]); + expect(await blocked(scenario, ["drop-view-cascade", "drop-view"])).toEqual([]); + }); + + test("an adoption with dependents asks for the cascade, then for the adoption", async () => { + const scenario = SCENARIOS["structural adoption with external dependents"]!; + expect(hints(await blocked(scenario))).toEqual(["drop-view-cascade"]); + expect(hints(await blocked(scenario, ["drop-view-cascade"]))).toEqual(["adopt-view"]); + expect(await blocked(scenario, ["drop-view-cascade", "adopt-view"])).toEqual([]); + }); +}); diff --git a/server/typescript/packages/cli/test/migrate-flyway-history.test.ts b/server/typescript/packages/cli/test/migrate-flyway-history.test.ts new file mode 100644 index 000000000..6fb7b6b9c --- /dev/null +++ b/server/typescript/packages/cli/test/migrate-flyway-history.test.ts @@ -0,0 +1,194 @@ +/** + * Flyway owns `flyway_schema_history` (adopter report D1). + * + * `meta migrate --from-db` diffs metadata against the live database. On a Flyway-managed + * database the runner's own ledger is a live table the metadata never declares, so the diff + * proposed `DROP TABLE flyway_schema_history`, and its hint said to re-run with + * `--allow drop-table`. Following the hint deletes the runner's history. + */ +import { describe, test, expect, afterAll } from "bun:test"; +import { mkdtemp, rm, mkdir, writeFile, readdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { sql } from "kysely"; +import { readSnapshot, snapshotPath } from "@metaobjectsdev/migrate-ts"; +import { migrateCommand } from "../src/commands/migrate.js"; +import { verifyCommand } from "../src/commands/verify.js"; +import { buildKyselyFromUrl } from "../src/lib/kysely.js"; + +const dirs: string[] = []; +afterAll(async () => { for (const d of dirs) await rm(d, { recursive: true, force: true }); }); + +const MODEL = JSON.stringify({ + "metadata.root": { + package: "acme::platform", + children: [{ + "object.entity": { + name: "Job", + children: [ + { "source.rdb": { name: "src", "@table": "jobs" } }, + { "field.long": { name: "id" } }, + { "identity.primary": { name: "pk", "@fields": ["id"], "@generation": "increment" } }, + ], + }, + }], + }, +}); + +async function fixture(opts: { jobs: boolean } = { jobs: true }): Promise<{ root: string; db: string }> { + const root = await mkdtemp(join(tmpdir(), "flyway-history-")); + dirs.push(root); + await mkdir(join(root, "metaobjects"), { recursive: true }); + await writeFile(join(root, "metaobjects", "meta.platform.json"), MODEL, "utf8"); + await mkdir(join(root, ".metaobjects"), { recursive: true }); + await writeFile( + join(root, ".metaobjects", "config.json"), + JSON.stringify({ schema_version: 1, migrate: { dialect: "sqlite" } }), + "utf8", + ); + const db = join(root, "t.db"); + const k = await buildKyselyFromUrl(`file:${db}`, "sqlite"); + try { + if (opts.jobs) await sql`CREATE TABLE "jobs" (id INTEGER NOT NULL PRIMARY KEY)`.execute(k.db); + await sql`CREATE TABLE "flyway_schema_history" (installed_rank INTEGER NOT NULL PRIMARY KEY)`.execute(k.db); + } finally { + await k.close(); + } + return { root, db }; +} + +const files = async (dir: string): Promise => + (await readdir(dir).catch(() => [] as string[])).filter((e) => !e.startsWith(".")); + +describe("meta migrate --from-db and Flyway's history table", () => { + test("--migration-format flyway leaves flyway_schema_history out of the diff", async () => { + const f = await fixture(); + const out = join(f.root, "flyway-out"); + // No --allow: a proposed DROP TABLE would be blocked and exit 1. + const code = await migrateCommand( + ["--from-db", "--db", `file:${f.db}`, "--dialect", "sqlite", "--migration-format", "flyway", + "--out-dir", out, "--slug", "x"], + f.root, + ); + expect(code).toBe(0); + expect(await files(out)).toEqual([]); + }); + + test("without the flyway format the table is still an unmodeled live table", async () => { + const f = await fixture(); + const code = await migrateCommand( + ["--from-db", "--db", `file:${f.db}`, "--dialect", "sqlite", "--slug", "x"], + f.root, + ); + expect(code).toBe(1); + }); + + test("meta verify --db agrees: the history table is no drift under the flyway format", async () => { + const f = await fixture(); + await writeFile( + join(f.root, ".metaobjects", "config.json"), + JSON.stringify({ schema_version: 1, migrate: { dialect: "sqlite", format: "flyway" } }), + "utf8", + ); + expect(await verifyCommand(["--db", `file:${f.db}`, "--dialect", "sqlite"], f.root)).toBe(0); + }); + + // A project baselined on a Flyway-managed database before the table was filtered at + // introspection carries it in its committed snapshot, and that snapshot is one side of + // both the offline diff and `verify --db`'s snapshot comparison. + async function fixtureWithHistoryInSnapshot(): Promise<{ root: string; db: string }> { + const f = await fixture(); + expect(await migrateCommand(["baseline", "--from-db", "--db", `file:${f.db}`, "--dialect", "sqlite"], f.root)).toBe(0); + const snapshot = await readSnapshot(snapshotPath(join(f.root, ".metaobjects", "migrations"), "sqlite")); + expect(snapshot?.tables.map((t) => t.name)).toContain("flyway_schema_history"); + return f; + } + + test("offline --migration-format flyway does not propose dropping a history table the snapshot records", async () => { + const f = await fixtureWithHistoryInSnapshot(); + const code = await migrateCommand( + ["--dialect", "sqlite", "--migration-format", "flyway", "--slug", "x"], + f.root, + ); + expect(code).toBe(0); + expect(await files(join(f.root, "src", "main", "resources", "db", "migration"))).toEqual([]); + }); + + test("offline without the flyway format still proposes the drop", async () => { + const f = await fixtureWithHistoryInSnapshot(); + expect(await migrateCommand(["--dialect", "sqlite", "--slug", "x"], f.root)).toBe(1); + }); + + test("meta verify --db agrees with a snapshot that records the history table", async () => { + const f = await fixtureWithHistoryInSnapshot(); + await writeFile( + join(f.root, ".metaobjects", "config.json"), + JSON.stringify({ schema_version: 1, migrate: { dialect: "sqlite", format: "flyway" } }), + "utf8", + ); + expect(await verifyCommand(["--db", `file:${f.db}`, "--dialect", "sqlite"], f.root)).toBe(0); + }); + + // The format reaches `meta verify` the way it reaches `meta migrate`: by flag, with no + // `migrate.format` in the config. The same run without the flag is the drift case below. + test("meta verify --db --migration-format flyway (flag, no config) leaves the history table out", async () => { + const f = await fixture(); + expect(await verifyCommand( + ["--db", `file:${f.db}`, "--dialect", "sqlite", "--migration-format", "flyway"], f.root, + )).toBe(0); + }); + + test("the flag also covers a snapshot that records the history table", async () => { + const f = await fixtureWithHistoryInSnapshot(); + expect(await verifyCommand( + ["--db", `file:${f.db}`, "--dialect", "sqlite", "--migration-format", "flyway"], f.root, + )).toBe(0); + }); + + test("the flag wins over the config key, as it does for meta migrate", async () => { + const f = await fixture(); + await writeFile( + join(f.root, ".metaobjects", "config.json"), + JSON.stringify({ schema_version: 1, migrate: { dialect: "sqlite", format: "flyway" } }), + "utf8", + ); + expect(await verifyCommand( + ["--db", `file:${f.db}`, "--dialect", "sqlite", "--migration-format", "default"], f.root, + )).toBe(1); + }); + + test("meta verify refuses a migration format meta migrate does not accept", async () => { + const f = await fixture(); + expect(await verifyCommand( + ["--db", `file:${f.db}`, "--dialect", "sqlite", "--migration-format", "liquibase"], f.root, + )).toBe(2); + }); + + // Flyway creates its history table on its first run, before any project table exists. An + // offline baseline on such a database would record the metadata as already applied, so no + // CREATE TABLE would ever be emitted: the empty-database refusal has to see it as empty. + describe("migrate baseline on a database that holds only Flyway's history table", () => { + const committedSnapshot = (root: string) => + readSnapshot(snapshotPath(join(root, ".metaobjects", "migrations"), "sqlite")); + + test("under the flyway format it is an empty database: refused, no snapshot written", async () => { + const f = await fixture({ jobs: false }); + const code = await migrateCommand( + ["baseline", "--db", `file:${f.db}`, "--dialect", "sqlite", "--migration-format", "flyway"], f.root, + ); + expect(code).toBe(2); + expect(await committedSnapshot(f.root)).toBeNull(); + }); + + test("without the flyway format the table still counts as a live table", async () => { + const f = await fixture({ jobs: false }); + expect(await migrateCommand(["baseline", "--db", `file:${f.db}`, "--dialect", "sqlite"], f.root)).toBe(0); + expect((await committedSnapshot(f.root))?.tables.map((t) => t.name)).toEqual(["jobs"]); + }); + }); + + test("meta verify --db without the flyway format reports it as drift", async () => { + const f = await fixture(); + expect(await verifyCommand(["--db", `file:${f.db}`, "--dialect", "sqlite"], f.root)).toBe(1); + }); +}); diff --git a/server/typescript/packages/cli/test/unit/args-verify.test.ts b/server/typescript/packages/cli/test/unit/args-verify.test.ts index 300528fce..fe758ff27 100644 --- a/server/typescript/packages/cli/test/unit/args-verify.test.ts +++ b/server/typescript/packages/cli/test/unit/args-verify.test.ts @@ -1,11 +1,11 @@ import { describe, test, expect } from "bun:test"; -import { parseVerifyArgs } from "../../src/lib/args.js"; +import { parseMigrateArgs, parseVerifyArgs } from "../../src/lib/args.js"; import { DEFAULT_ADVISORY_LIMIT } from "../../src/lib/advisory.js"; describe("parseVerifyArgs", () => { test("defaults: prompts/db/dialect undefined, allow empty, skipSchema false, no explicit subverb", () => { expect(parseVerifyArgs([])).toEqual({ - prompts: undefined, db: undefined, dialect: undefined, allow: [], skipSchema: false, + prompts: undefined, db: undefined, dialect: undefined, allow: [], skipSchema: false, migrationFormat: undefined, templates: false, codegen: false, forbidHandEdits: false, docs: false, deps: false, anyExplicit: false, noAntipatterns: false, noRequirementLint: false, requireImplementers: false, noOverlayLint: false, noNameLint: false, noDeprecationLint: false, noFieldLint: false, lax: false, replay: false, replaySnapshot: false, d1: undefined, remote: false, limit: DEFAULT_ADVISORY_LIMIT, @@ -13,7 +13,7 @@ describe("parseVerifyArgs", () => { }); test("--prompts

is captured", () => { expect(parseVerifyArgs(["--prompts", "templates"])).toEqual({ - prompts: "templates", db: undefined, dialect: undefined, allow: [], skipSchema: false, + prompts: "templates", db: undefined, dialect: undefined, allow: [], skipSchema: false, migrationFormat: undefined, templates: false, codegen: false, forbidHandEdits: false, docs: false, deps: false, anyExplicit: false, noAntipatterns: false, noRequirementLint: false, requireImplementers: false, noOverlayLint: false, noNameLint: false, noDeprecationLint: false, noFieldLint: false, lax: false, replay: false, replaySnapshot: false, d1: undefined, remote: false, limit: DEFAULT_ADVISORY_LIMIT, @@ -21,7 +21,7 @@ describe("parseVerifyArgs", () => { }); test("--db / --dialect / --skip-schema are captured", () => { expect(parseVerifyArgs(["--db", "file:x.db", "--dialect", "sqlite", "--skip-schema"])).toEqual({ - prompts: undefined, db: "file:x.db", dialect: "sqlite", allow: [], skipSchema: true, + prompts: undefined, db: "file:x.db", dialect: "sqlite", allow: [], skipSchema: true, migrationFormat: undefined, templates: false, codegen: false, forbidHandEdits: false, docs: false, deps: false, anyExplicit: true, noAntipatterns: false, noRequirementLint: false, requireImplementers: false, noOverlayLint: false, noNameLint: false, noDeprecationLint: false, noFieldLint: false, lax: false, replay: false, replaySnapshot: false, d1: undefined, remote: false, limit: DEFAULT_ADVISORY_LIMIT, @@ -31,7 +31,7 @@ describe("parseVerifyArgs", () => { // --dialect d1 alone is already an explicit subverb selector (D1 has no --db URL). test("--dialect d1 --d1 --remote are captured; --dialect d1 alone is an explicit subverb", () => { expect(parseVerifyArgs(["--dialect", "d1", "--d1", "DB", "--remote"])).toEqual({ - prompts: undefined, db: undefined, dialect: "d1", allow: [], skipSchema: false, + prompts: undefined, db: undefined, dialect: "d1", allow: [], skipSchema: false, migrationFormat: undefined, templates: false, codegen: false, forbidHandEdits: false, docs: false, deps: false, anyExplicit: true, noAntipatterns: false, noRequirementLint: false, requireImplementers: false, noOverlayLint: false, noNameLint: false, noDeprecationLint: false, noFieldLint: false, lax: false, replay: false, replaySnapshot: false, d1: "DB", remote: true, limit: DEFAULT_ADVISORY_LIMIT, @@ -41,6 +41,22 @@ describe("parseVerifyArgs", () => { expect(bare.d1).toBeUndefined(); expect(bare.remote).toBe(false); }); + // One format vocabulary for `meta migrate` and `meta verify`: the same values, and the same + // refusal for anything else. + test("--migration-format takes meta migrate's values and refuses the rest with its message", () => { + expect(parseVerifyArgs([]).migrationFormat).toBeUndefined(); + expect(parseVerifyArgs(["--migration-format", "flyway"]).migrationFormat).toBe("flyway"); + expect(parseVerifyArgs(["--migration-format", "default"]).migrationFormat).toBe("default"); + expect(parseVerifyArgs(["--migration-format", "flyway"]).anyExplicit).toBe(false); + const refusal = (parse: () => unknown): string => { + try { parse(); } catch (err) { return (err as Error).message; } + return "(accepted)"; + }; + expect(refusal(() => parseVerifyArgs(["--migration-format", "liquibase"]))) + .toBe("invalid --migration-format 'liquibase'; expected: default, flyway"); + expect(refusal(() => parseVerifyArgs(["--migration-format", "liquibase"]))) + .toBe(refusal(() => parseMigrateArgs(["--migration-format", "liquibase"]))); + }); test("--no-antipatterns is captured", () => { expect(parseVerifyArgs(["--no-antipatterns"]).noAntipatterns).toBe(true); expect(parseVerifyArgs([]).noAntipatterns).toBe(false); @@ -60,7 +76,7 @@ describe("parseVerifyArgs", () => { test("--allow csv is parsed into tokens", () => { expect(parseVerifyArgs(["--allow", "drop-column,drop-table"])).toEqual({ prompts: undefined, db: undefined, dialect: undefined, - allow: ["drop-column", "drop-table"], skipSchema: false, + allow: ["drop-column", "drop-table"], skipSchema: false, migrationFormat: undefined, templates: false, codegen: false, forbidHandEdits: false, docs: false, deps: false, anyExplicit: false, noAntipatterns: false, noRequirementLint: false, requireImplementers: false, noOverlayLint: false, noNameLint: false, noDeprecationLint: false, noFieldLint: false, lax: false, replay: false, replaySnapshot: false, d1: undefined, remote: false, limit: DEFAULT_ADVISORY_LIMIT, diff --git a/server/typescript/packages/cli/test/unit/describe-change.test.ts b/server/typescript/packages/cli/test/unit/describe-change.test.ts index 27e2cbe3e..f4f486a4c 100644 --- a/server/typescript/packages/cli/test/unit/describe-change.test.ts +++ b/server/typescript/packages/cli/test/unit/describe-change.test.ts @@ -21,7 +21,7 @@ test("an unchanged default is not mentioned", () => { from: { kind: "integer", bits: 32 }, to: { kind: "integer", bits: 64 }, fromDefault: zero, toDefault: zero, status: { state: "allowed" }, }; - expect(describeChange(c)).toBe("orders.qty (integer → integer)"); + expect(describeChange(c)).toBe("orders.qty (integer → bigint)"); }); // A blocked CHECK drop used to fall through to a JSON dump of the whole change object. @@ -48,7 +48,7 @@ test("every change kind reads as text, never JSON", () => { { kind: "rename-column", table: "t", from: "a", to: "b", status: { state: "allowed" } }, { kind: "change-column-default", table: "t", column: "a", status: { state: "allowed" } }, { kind: "change-column-nullable", table: "t", column: "a", from: true, to: false, status: { state: "allowed" } }, - { kind: "replace-view", view: { name: "v", columns: [] }, status: { state: "allowed" } }, + { kind: "replace-view", view: { name: "v" }, status: { state: "allowed" } }, ]; for (const c of kinds) expect(describeChange(c)).not.toMatch(/^\{/); }); @@ -94,3 +94,18 @@ test("a view change says why it was planned", () => { ]; for (const [c, text] of cases) expect(describeChange(c)).toBe(text); }); + +// A length or time-zone change used to print as "text → text" / "timestamp → timestamp" (D4). +test("a type change that only differs in length or time zone says so", () => { + const len: Change = { + kind: "change-column-type", table: "games", column: "channel", + from: { kind: "text", maxLength: 255 }, to: { kind: "text" }, status: { state: "allowed" }, + }; + expect(describeChange(len)).toBe("games.channel (text(255) → text)"); + const tz: Change = { + kind: "change-column-type", table: "games", column: "at", + from: { kind: "timestamp", withTimezone: false }, to: { kind: "timestamp", withTimezone: true }, + status: { state: "allowed" }, + }; + expect(describeChange(tz)).toBe("games.at (timestamp → timestamptz)"); +}); diff --git a/server/typescript/packages/cli/test/unit/gen-migrate-advice.test.ts b/server/typescript/packages/cli/test/unit/gen-migrate-advice.test.ts index d135c6e7a..cef0fabff 100644 --- a/server/typescript/packages/cli/test/unit/gen-migrate-advice.test.ts +++ b/server/typescript/packages/cli/test/unit/gen-migrate-advice.test.ts @@ -48,6 +48,7 @@ function input(over: Partial & Pick { expect(advice).toContain("meta migrate --db --dialect sqlite --slug --apply"); }); + // A snapshot baselined from a Flyway-managed database records Flyway's own history table. + // Under the flyway format `meta migrate` leaves that table out and reports no changes, so + // advising a migrate here would send the adopter to a command with nothing to do. + describe("a committed snapshot that records flyway_schema_history", () => { + const withHistory = async (metadata: MetaRoot): Promise => { + const root = project(); + mkdirSync(join(root, OUT_DIR), { recursive: true }); + const expected = buildExpectedSchema(metadata, { dialect: "sqlite" }); + await writeSnapshot(snapshotPath(join(root, OUT_DIR), "sqlite"), { + ...expected, + tables: [...expected.tables, { + name: "flyway_schema_history", + columns: [{ name: "installed_rank", sqlType: { kind: "integer", bits: 32 }, nullable: false }], + indexes: [], foreignKeys: [], primaryKey: ["installed_rank"], checks: [], + }], + }); + return root; + }; + + test("flyway format → no advice: the history table is not a schema change", async () => { + const metadata = await load([PET]); + expect(await genMigrateAdvice(input({ metadata, projectRoot: await withHistory(metadata), migrateFormat: "flyway" }))) + .toBeUndefined(); + }); + + test("default format → the table is an unmodeled live table, and migrate has something to say", async () => { + const metadata = await load([PET]); + expect(await genMigrateAdvice(input({ metadata, projectRoot: await withHistory(metadata) }))) + .toContain("meta migrate --db --dialect sqlite --slug --apply"); + }); + + test("flyway format still advises when the metadata moved ahead of the snapshot", async () => { + const root = await withHistory(await load([PET])); + const grown = await load([{ + "object.entity": { + ...PET["object.entity"], + children: [...PET["object.entity"].children, { "field.string": { name: "microchip" } }], + }, + }]); + expect(await genMigrateAdvice(input({ metadata: grown, projectRoot: root, migrateFormat: "flyway" }))) + .toContain("meta migrate --db "); + }); + }); + describe("a D1 project (wrangler config with a D1 binding)", () => { const d1Project = (): string => { const root = project(); diff --git a/server/typescript/packages/codegen-ts/src/projection/build-projection-views.ts b/server/typescript/packages/codegen-ts/src/projection/build-projection-views.ts index 4d8a046b9..c58986bb8 100644 --- a/server/typescript/packages/codegen-ts/src/projection/build-projection-views.ts +++ b/server/typescript/packages/codegen-ts/src/projection/build-projection-views.ts @@ -9,7 +9,6 @@ // drift / snapshot as the `views` input. This is the ONE place view SQL is produced. import { - type AggregateFunction, type MetaData, MetaObject, type MetaRoot, @@ -37,25 +36,7 @@ import { emitViewDdl } from "./view-ddl-emit.js"; import type { JoinNode, ViewSpec } from "./view-spec.js"; import type { ColumnNamingStrategy } from "../metaobjects-config.js"; -/** Structurally matches migrate-ts's `ViewDescriptor` (name + body sql + optional schema). */ -/** - * One output column of the view, in SELECT order, described PHYSICALLY (table + - * column, not entity + field). - * - * migrate-ts resolves each of these to a `SqlType` against its own expected table - * descriptors. Deliberately no SqlType here: codegen-ts stays ignorant of migrate - * concerns, and migrate-ts stays ignorant of metadata traversal — the existing - * layering (migrate-ts never imports codegen-ts; the CLI threads views in). - * - * Postgres allows a non-destructive `CREATE OR REPLACE VIEW` only when the existing - * output columns are a PREFIX of the new ones (same names, same types, same order). - * That decision cannot be made without knowing the column list, which is why it is - * carried here. - */ -export type ExpectedViewColumn = - | { kind: "passthrough"; name: string; sourceTable: string; sourceColumn: string } - | { kind: "aggregate"; name: string; sourceTable: string; sourceColumn: string; agg: AggregateFunction }; - +/** Structurally matches migrate-ts's `ExpectedViewInput` (name + body sql + optional schema). */ export interface ExpectedView { name: string; schema?: string; @@ -75,19 +56,6 @@ export interface ExpectedView { * so the view must be dropped before and recreated after. */ dependsOn: string[]; - /** - * The view's output columns, in SELECT order — i.e. DECLARATION order, since - * extractViewSpec walks the projection's children in order. That is exactly the - * order Postgres's OR-REPLACE prefix rule wants: a field appended to the - * projection lands last, so the change stays non-destructive. (Re-canonicalizing - * to, say, alphabetical order would be strictly WORSE — it would scatter an - * appended field into the middle and force a destructive drop+create.) - * - * OMITTED for an `@sql` (#208) view: the body is opaque — the tool never parses it, - * so its output columns are unknown. migrate-ts reads absent columns as "unknown" - * and fails safe to a gated drop+create instead of an illegal CREATE OR REPLACE. - */ - columns?: ExpectedViewColumn[]; } export interface BuildProjectionViewsOptions { @@ -222,7 +190,6 @@ export function buildReportViews(root: MetaData, opts: BuildReportViewsOptions): dependsOn: reportDependsOn(spec, baseTableName, joinTables), fqn: report.resolutionKey(), ...(schema !== undefined ? { schema } : {}), - // `columns` omitted on purpose: unknown, so migrate takes the fail-safe drop+create (Table F). }); } return out; @@ -297,12 +264,10 @@ function emitViewFor( const body = emitViewDdl(spec, { dialect, baseTableName, joinTables, bodyOnly: true }); const schema = resolveTableSchema(host); const dependsOn = collectDependsOn(spec, baseTableName, joinTables); - const columns = collectViewColumns(spec, baseTableName, joinTables); out.push({ name: spec.viewName, sql: body, dependsOn, - columns, fqn: host.resolutionKey(), ...(schema !== undefined ? { schema } : {}), }); @@ -314,8 +279,6 @@ function emitViewFor( * new machinery: * - buildExpectedSchema Pass 4 fingerprints `v.sql` (hash of the body); * - emit stamps the COMMENT marker; author re-indentation does not re-stamp; - * - `columns` is OMITTED (the body is opaque — the tool never parses it), so the diff - * fails safe to a gated drop+create instead of a wrong-but-confident OR REPLACE; * - a pre-existing unstamped view at this name → `replace-view` blocked pending * `migrate --allow adopt-view` (the one-time adoption ceremony). * `dependsOn` is derived from the host's own writable table (a write-through host) plus @@ -346,7 +309,6 @@ function emitSqlView( sql: source.sqlBody!, // verbatim — never parsed, never re-wrapped dependsOn, fqn: host.resolutionKey(), - // columns OMITTED → "unknown" → gated drop+create fail-safe. ...(schema !== undefined ? { schema } : {}), }); } @@ -400,51 +362,6 @@ function collectSqlDependsOn( return [...tables]; } -/** - * The SELECT list as PHYSICAL (table, column) pairs, in emitted order. - * - * SelectColumn carries a join ALIAS, which means nothing outside this module — so - * resolve every alias to its physical table first. - */ -function collectViewColumns( - spec: ViewSpec, - baseTableName: string, - joinTables: Readonly>, -): ExpectedViewColumn[] { - const aliasToTable = new Map([[spec.joinTree.baseAlias, baseTableName]]); - const walk = (node: JoinNode): void => { - const t = joinTables[node.targetEntity]; - if (t) aliasToTable.set(node.alias, t); - for (const child of node.children) walk(child); - }; - for (const j of spec.joinTree.joins) walk(j); - - const out: ExpectedViewColumn[] = []; - for (const c of spec.selectSpec.columns) { - // #195/#335: the five non-scalar origin column kinds (predicateAgg / collectAgg / - // collectObjectAgg / computed / first) do not resolve to a single (table, column) - // SqlType via the prefix rule — computed is an expression, first is a correlated - // subquery, and the array/jsonb/boolean aggregate result types are richer than the - // OR-REPLACE prefix check models. Per this module's - // fail-safe doctrine (unknown → drop+create, never a wrong-but-confident replace), - // an unknown column drops the whole list so migrate routes through a gated - // drop+create. Precise native typing is a later phase. - if (c.kind !== "passthrough" && c.kind !== "aggregate") return []; - const sourceTable = aliasToTable.get(c.sourceAlias); - // An unresolvable alias would make the column list a lie, and a wrong list would - // make the diff propose an ILLEGAL `CREATE OR REPLACE VIEW` that fails at apply. - // Drop the whole list instead: migrate-ts treats an absent list as "unknown" and - // fails safe to a gated drop+create. - if (sourceTable === undefined) return []; - out.push( - c.kind === "aggregate" - ? { kind: "aggregate", name: c.dbColAlias, sourceTable, sourceColumn: c.sourceColumn, agg: c.agg } - : { kind: "passthrough", name: c.dbColAlias, sourceTable, sourceColumn: c.sourceColumn }, - ); - } - return out; -} - /** * Is this projection's view DERIVED from the model — i.e. does migrate own its DDL? * diff --git a/server/typescript/packages/codegen-ts/test/projection/build-projection-views.test.ts b/server/typescript/packages/codegen-ts/test/projection/build-projection-views.test.ts index bca095cfd..be346a002 100644 --- a/server/typescript/packages/codegen-ts/test/projection/build-projection-views.test.ts +++ b/server/typescript/packages/codegen-ts/test/projection/build-projection-views.test.ts @@ -244,7 +244,6 @@ describe("buildProjectionViews — #208 @sql / @unmanaged DDL-ownership escape v expect(v).toBeDefined(); expect(v!.sql).toBe(RECURSIVE_BODY); // verbatim body, NOT a synthesized "SELECT nodes.id AS id …" expect(v!.sql).toContain("WITH RECURSIVE"); - expect(v!.columns).toBeUndefined(); // opaque — column list unknown → diff fails safe to drop+create // dependsOn is derived from the extends-bound anchor entity (Node → nodes), D7. expect(v!.dependsOn).toEqual(["nodes"]); }); @@ -286,7 +285,6 @@ describe("buildProjectionViews — #208 @sql / @unmanaged DDL-ownership escape v expect(v).toBeDefined(); expect(v!.sql).toContain("rank() OVER"); // verbatim @sql body expect(v!.dependsOn).toEqual(["programs"]); // the host's own table, not [] - expect(v!.columns).toBeUndefined(); }); test("@sql on a non-view kind (materializedView) hard-errors at migrate lowering (D4)", async () => { @@ -329,8 +327,6 @@ describe("buildReportViews — view-backed reports (FR-044 Plan 2, Table A)", () const v = views[0]!; expect(v.fqn).toBe("acme::shop::StoreTotals"); expect(v.dependsOn).toEqual(["purchases"]); - expect(v.columns).toBeUndefined(); - expect("columns" in v).toBe(false); expect(v.sql).toContain("FROM \"purchases\" p"); expect(v.sql).toContain('COUNT(p."id") FILTER (WHERE p."status" = \'active\') AS "purchases"'); expect(v.sql).not.toContain("CREATE VIEW"); @@ -380,7 +376,6 @@ describe("buildReportViews — view-backed reports (FR-044 Plan 2, Table A)", () expect(views[0]!.sql).toBe(body); expect(views[0]!.dependsOn).toEqual(["purchases"]); expect(views[0]!.fqn).toBe("acme::shop::StoreTotals"); - expect("columns" in views[0]!).toBe(false); }); test("the projection loop does not see an @sql report either: one view, not two", async () => { diff --git a/server/typescript/packages/codegen-ts/test/projection/report-required-joins.test.ts b/server/typescript/packages/codegen-ts/test/projection/report-required-joins.test.ts new file mode 100644 index 000000000..27fc1db9e --- /dev/null +++ b/server/typescript/packages/codegen-ts/test/projection/report-required-joins.test.ts @@ -0,0 +1,158 @@ +// A report dimension's `required` and the join type of its `@via` path are one rule stated in +// two packages (adopter report D5): `reportShape` in `@metaobjectsdev/metadata` decides whether +// the generated row type is nullable, and the view lowering here decides INNER or LEFT OUTER +// (#209). Nothing shares code between them, so a change to one can leave the other behind, and +// that failure is silent: a row type that says non-null over a column the view null-extends. +// +// This reads the join types out of the view SQL the lowering emits and holds `reportShape` to +// them. The expected table is derived by hand from the #209 rule: a hop is INNER when the +// entity it leaves holds a required foreign key and every hop before it is INNER. +// A dimension's `@via` crosses only identity.reference and `@cardinality: one` hops (a to-many +// hop does not load), so the hop that is LEFT OUTER for a reason other than a nullable key is +// the to-one relationship whose foreign key sits on the entity it reaches. + +import { describe, test, expect } from "bun:test"; +import { MetaDataLoader, InMemoryStringSource, reportShape, type MetaObject, type MetaRoot } from "@metaobjectsdev/metadata"; +import { buildProjectionViews } from "../../src/projection/build-projection-views.js"; +import { extractReportSpec } from "../../src/projection/extract-report-spec.js"; +import type { JoinNode } from "../../src/projection/view-spec.js"; + +type JoinKeyword = "INNER" | "LEFT OUTER"; + +const entity = (name: string, table: string, children: unknown[]) => ({ + "object.entity": { + name, + children: [ + { "source.rdb": { "@table": table } }, + { "field.long": { name: "id" } }, + { "identity.primary": { name: "pk", "@fields": ["id"] } }, + ...children, + ], + }, +}); + +const realm = entity("Realm", "realms", [ + { "field.string": { name: "name", "@required": true } }, + { "field.string": { name: "motto" } }, +]); +const session = entity("Session", "sessions", [ + { "field.long": { name: "realmId", "@required": true } }, + { "field.long": { name: "maybeRealmId" } }, + { "identity.reference": { name: "fkRealm", "@fields": ["realmId"], "@references": "Realm" } }, + { "identity.reference": { name: "fkMaybeRealm", "@fields": ["maybeRealmId"], "@references": "Realm" } }, + { "relationship.association": { name: "realm", "@objectRef": "Realm", "@cardinality": "one", "@sourceRefField": "realmId" } }, + { + "relationship.association": { + name: "maybeRealm", "@objectRef": "Realm", "@cardinality": "one", "@sourceRefField": "maybeRealmId", + }, + }, +]); + +// The foreign key between Snap and Stamp is held by Stamp, the far side of Snap's `stamp`. +const stamp = entity("Stamp", "stamps", [ + { "field.long": { name: "snapId", "@required": true } }, + { "field.string": { name: "label", "@required": true } }, + { "identity.reference": { name: "fkSnap", "@fields": ["snapId"], "@references": "Snap" } }, +]); + +/** Each dimension: its `@of`, its `@via`, the joins its path must render, and its `required`. */ +const DIMENSIONS: Record = { + // No @via: a column of the @from entity itself. + sessionKey: { of: "Snap.sessionId", joins: [], required: true }, + // identity.reference hops. + sessionRealmId: { of: "Session.realmId", via: "Snap.fkSession", joins: ["INNER"], required: true }, + realmName: { of: "Realm.name", via: "Snap.fkSession.fkRealm", joins: ["INNER", "INNER"], required: true }, + looseRealmName: { of: "Realm.name", via: "Snap.fkLoose.fkRealm", joins: ["INNER", "INNER"], required: true }, + maybeRealmName: { of: "Realm.name", via: "Snap.fkSession.fkMaybeRealm", joins: ["INNER", "LEFT OUTER"], required: false }, + // INNER all the way, over a field that is itself optional. + realmMotto: { of: "Realm.motto", via: "Snap.fkSession.fkRealm", joins: ["INNER", "INNER"], required: false }, + // @cardinality: one relationship hops, and one path that mixes the two hop kinds. + relSessionRealmId: { of: "Session.realmId", via: "Snap.session", joins: ["INNER"], required: true }, + relRealmName: { of: "Realm.name", via: "Snap.session.realm", joins: ["INNER", "INNER"], required: true }, + mixedRealmName: { of: "Realm.name", via: "Snap.session.fkRealm", joins: ["INNER", "INNER"], required: true }, + relMaybeRealmName: { of: "Realm.name", via: "Snap.session.maybeRealm", joins: ["INNER", "LEFT OUTER"], required: false }, + // A to-one relationship whose foreign key the OTHER entity holds: a Snap with no Stamp has no + // match, so the hop is LEFT OUTER whatever the far side's key and the target field require. + stampLabel: { of: "Stamp.label", via: "Snap.stamp", joins: ["LEFT OUTER"], required: false }, +}; + +const snap = entity("Snap", "snaps", [ + { "field.long": { name: "sessionId", "@required": true } }, + { "field.long": { name: "unenforcedSessionId", "@required": true } }, + { "identity.reference": { name: "fkSession", "@fields": ["sessionId"], "@references": "Session" } }, + { + "identity.reference": { + name: "fkLoose", "@fields": ["unenforcedSessionId"], "@references": "Session", "@enforce": false, + }, + }, + { "relationship.association": { name: "session", "@objectRef": "Session", "@cardinality": "one" } }, + { "relationship.association": { name: "stamp", "@objectRef": "Stamp", "@cardinality": "one" } }, + { "measure.aggregate": { name: "snaps", "@agg": "count", "@of": "Snap.id" } }, + ...Object.entries(DIMENSIONS).map(([name, d]) => ({ + "dimension.attribute": { name, "@of": d.of, ...(d.via !== undefined ? { "@via": d.via } : {}) }, + })), +]); + +const VIEW = "v_snap_realms"; +const report = { + "object.report": { + name: "SnapRealms", + "@from": "Snap", + "@dimensions": Object.keys(DIMENSIONS), + "@measures": ["snaps"], + children: [{ "source.rdb": { "@kind": "view", "@view": VIEW } }], + }, +}; + +async function load(): Promise<{ root: MetaRoot; report: MetaObject }> { + const { root, errors } = await new MetaDataLoader().load([ + new InMemoryStringSource(JSON.stringify({ "metadata.root": { package: "a", children: [realm, session, snap, stamp, report] } })), + ]); + expect(errors).toEqual([]); + return { root, report: root.objects().find((o) => o.name === "SnapRealms")! }; +} + +/** alias → the join keyword the emitted view SQL joins it with. */ +function joinKeywordsOf(sql: string): Map { + const out = new Map(); + for (const m of sql.matchAll(/\b(INNER|LEFT OUTER) JOIN "[^"]+" (\w+) ON\b/g)) out.set(m[2]!, m[1] as JoinKeyword); + return out; +} + +/** The aliases joined on the way from the base table to `alias`, outermost first. */ +function aliasPath(joins: readonly JoinNode[], alias: string): string[] | undefined { + for (const node of joins) { + if (node.alias === alias) return [node.alias]; + const below = aliasPath(node.children, alias); + if (below !== undefined) return [node.alias, ...below]; + } + return undefined; +} + +describe("a report dimension's required agrees with the joins its view renders", () => { + test("every dimension: the emitted joins, and required only over an all-INNER path", async () => { + const { root, report: snapRealms } = await load(); + const views = buildProjectionViews(root, { dialect: "postgres", columnNamingStrategy: "snake_case" }); + const sql = views.find((v) => v.name === VIEW)!.sql!; + const keywords = joinKeywordsOf(sql); + const spec = extractReportSpec(snapRealms, root, { columnNamingStrategy: "snake_case" }); + const required = Object.fromEntries(reportShape(snapRealms, root).fields.map((f) => [f.name, f.required])); + + const seen: Record = {}; + for (const column of spec.columns) { + if (column.kind !== "dimension") continue; + const alias = column.ref.slice(0, column.ref.indexOf(".")); + const path = alias === spec.joinTree.baseAlias ? [] : aliasPath(spec.joinTree.joins, alias); + expect([column.fieldName, path === undefined ? "no join path" : "join path"]).toEqual([column.fieldName, "join path"]); + const joins = path!.map((a) => keywords.get(a)); + seen[column.fieldName] = { joins, required: required[column.fieldName] }; + // The direction that fails silently: non-null typed over a join that can null-extend. + if (required[column.fieldName] === true) { + expect([column.fieldName, joins.every((j) => j === "INNER")]).toEqual([column.fieldName, true]); + } + } + expect(seen).toEqual( + Object.fromEntries(Object.entries(DIMENSIONS).map(([name, d]) => [name, { joins: d.joins, required: d.required }])), + ); + }); +}); diff --git a/server/typescript/packages/integration-tests/src/load-project-model.ts b/server/typescript/packages/integration-tests/src/load-project-model.ts new file mode 100644 index 000000000..05663dbd1 --- /dev/null +++ b/server/typescript/packages/integration-tests/src/load-project-model.ts @@ -0,0 +1,23 @@ +// load-project-model.ts — load a real project's metadata the way the `meta` CLI loads it. +// +// `loadMemory(dir)` alone reads the declared sources and opted-in libraries, and nothing else. +// A project that registers its own vocabulary (a `layout.picker`, say) does so through the +// `providers` in its `metaobjects.config.ts`, so a bare load of that project stops at +// `Unknown type`. This reads the config first and hands the providers to `loadMemory`, which +// is what every `meta` command does. + +import { existsSync } from "node:fs"; +import { join, resolve } from "node:path"; +import type { MetaRoot } from "@metaobjectsdev/metadata"; +import { loadMemory } from "@metaobjectsdev/sdk"; +import { loadMemoryOptionsFrom, loadMetaobjectsConfig, resolveGenConfigDir } from "../../cli/src/lib/load-metaobjects-config.ts"; + +const CONFIG_FILE = "metaobjects.config.ts"; + +export async function loadProjectModel(projectDir: string): Promise { + const dir = resolve(projectDir); + // The nearest `metaobjects.config.ts` governs, exactly as it does for `meta gen`. + const configDir = resolveGenConfigDir(dir, dir); + const config = existsSync(join(configDir, CONFIG_FILE)) ? await loadMetaobjectsConfig(configDir) : undefined; + return loadMemory(dir, loadMemoryOptionsFrom(config)); +} diff --git a/server/typescript/packages/integration-tests/src/oracle-cli.ts b/server/typescript/packages/integration-tests/src/oracle-cli.ts index 334e27ef3..7a0e6e2e4 100644 --- a/server/typescript/packages/integration-tests/src/oracle-cli.ts +++ b/server/typescript/packages/integration-tests/src/oracle-cli.ts @@ -2,8 +2,8 @@ // // Ask the independent oracle (independent-oracle.ts) of a real project and a real database: // does every FK and column the project's metadata declares exist in that database? The -// metadata is loaded the way the `meta` CLI loads it (`loadMemory`: declared sources plus -// opted-in libraries); the answer comes from pg_catalog, never from migrate-ts. +// metadata is loaded the way the `meta` CLI loads it (declared sources, opted-in libraries and +// the providers its `metaobjects.config.ts` registers); the answer comes from pg_catalog, never from migrate-ts. // // An adopter's estate runs this against its release-candidate database (the G3d run). // Exit 0 when nothing declared is missing, 1 when something is, 2 on bad usage. FKs the @@ -12,7 +12,7 @@ import { resolve } from "node:path"; import type { ColumnNamingStrategy } from "@metaobjectsdev/metadata"; -import { loadMemory } from "@metaobjectsdev/sdk"; +import { loadProjectModel } from "./load-project-model.ts"; import { actualColumns, actualForeignKeys, expectedColumns, expectedForeignKeys, fkKey, missingColumns, } from "./independent-oracle.ts"; @@ -33,7 +33,7 @@ if (naming === undefined || projectDir === undefined || dbUrl === undefined) { process.exit(2); } -const root = await loadMemory(resolve(projectDir)); +const root = await loadProjectModel(resolve(projectDir)); const expected = expectedForeignKeys(root, naming).map(fkKey); const actual = (await actualForeignKeys(dbUrl)).map(fkKey); const missingFks = expected.filter((k) => !actual.includes(k)); diff --git a/server/typescript/packages/integration-tests/test/load-project-model.test.ts b/server/typescript/packages/integration-tests/test/load-project-model.test.ts new file mode 100644 index 000000000..14ad1d7a2 --- /dev/null +++ b/server/typescript/packages/integration-tests/test/load-project-model.test.ts @@ -0,0 +1,69 @@ +// The oracle CLI (D8) loads a project the way `meta` does: through the providers its +// `metaobjects.config.ts` registers. A bare `loadMemory` stops at `Unknown type`. +import { describe, test, expect, afterAll } from "bun:test"; +import { mkdtemp, rm, mkdir, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { loadMemory } from "@metaobjectsdev/sdk"; +import { loadProjectModel } from "../src/load-project-model.ts"; + +const dirs: string[] = []; +afterAll(async () => { for (const d of dirs) await rm(d, { recursive: true, force: true }); }); + +const CONFIG = [ + `import { defineConfig } from "@metaobjectsdev/codegen-ts";`, + `import { TypeId, TYPE_VALIDATOR, MetaValidator } from "@metaobjectsdev/metadata";`, + `const geoProvider = {`, + ` id: "test-geocheck",`, + ` dependencies: ["metaobjects-core-types"],`, + ` registerTypes(registry) {`, + ` registry.register({`, + ` typeId: new TypeId(TYPE_VALIDATOR, "geocheck"),`, + ` description: "A custom validator",`, + ` factory: (typeId, name) => new MetaValidator(typeId, name),`, + ` childRules: [],`, + ` attributes: [],`, + ` });`, + ` },`, + `};`, + `export default defineConfig({ outDir: "out", dialect: "postgres", generators: [], providers: [geoProvider] });`, +].join("\n"); + +const META = { + "metadata.root": { + package: "acme::geo", + children: [{ + "object.entity": { + name: "Place", + children: [ + { "field.long": { name: "id" } }, + { "field.string": { name: "name", children: [{ "validator.geocheck": { name: "chk" } }] } }, + { "source.rdb": { name: "src", "@table": "places" } }, + { "identity.primary": { name: "pk", "@fields": ["id"], "@generation": "increment" } }, + ], + }, + }], + }, +}; + +async function project(withConfig: boolean): Promise { + const root = await mkdtemp(join(tmpdir(), "load-project-model-")); + dirs.push(root); + await mkdir(join(root, "metaobjects"), { recursive: true }); + await writeFile(join(root, "metaobjects", "meta.json"), JSON.stringify(META), "utf8"); + if (withConfig) await writeFile(join(root, "metaobjects.config.ts"), CONFIG, "utf8"); + return root; +} + +describe("loadProjectModel", () => { + test("a bare loadMemory cannot load a project whose vocabulary comes from a config provider", async () => { + const root = await project(true); + await expect(loadMemory(root)).rejects.toThrow(/geocheck/); + }); + + test("loads that project through the providers its config registers", async () => { + const root = await project(true); + const model = await loadProjectModel(root); + expect(model.objects().map((o) => o.name)).toContain("Place"); + }); +}); diff --git a/server/typescript/packages/integration-tests/test/report-views-pg.test.ts b/server/typescript/packages/integration-tests/test/report-views-pg.test.ts index f34d1d7fb..2b7b2a204 100644 --- a/server/typescript/packages/integration-tests/test/report-views-pg.test.ts +++ b/server/typescript/packages/integration-tests/test/report-views-pg.test.ts @@ -18,7 +18,7 @@ import { describe, test, expect, beforeAll, afterAll, beforeEach } from "bun:test"; import { - buildExpectedSchema, diff, emit, introspectPostgres, collectUnmanagedNames, + buildExpectedSchema, diff, emit, introspectPostgres, postgresViewReplaceProbe, collectUnmanagedNames, type AllowOptions, type SchemaSnapshot, } from "@metaobjectsdev/migrate-ts"; import { buildProjectionViews } from "@metaobjectsdev/codegen-ts"; @@ -79,7 +79,7 @@ function expectedFor(root: MetaRoot): SchemaSnapshot { async function migrate(root: MetaRoot, allow: AllowOptions = {}) { const expected = expectedFor(root); const unmanagedNames = collectUnmanagedNames(root); - const result = await diff({ expected, actual: await introspectPostgres(k), dialect: "postgres", allow, unmanagedNames }); + const result = await diff({ expected, actual: await introspectPostgres(k), dialect: "postgres", viewReplaceProbe: postgresViewReplaceProbe(k), allow, unmanagedNames }); const emittable = result.changes.filter((c) => c.status.state !== "blocked"); const { up } = emittable.length === 0 ? { up: "" } : emit(emittable, { dialect: "postgres" }); if (result.blocked.length === 0 && up.trim().length > 0) await applyRaw(up); @@ -88,7 +88,7 @@ async function migrate(root: MetaRoot, allow: AllowOptions = {}) { /** THE gate: re-diffing the just-migrated database proposes nothing. */ async function assertConverged(expected: SchemaSnapshot, unmanagedNames: string[] = []): Promise { - const followup = await diff({ expected, actual: await introspectPostgres(k), dialect: "postgres", unmanagedNames }); + const followup = await diff({ expected, actual: await introspectPostgres(k), dialect: "postgres", viewReplaceProbe: postgresViewReplaceProbe(k), unmanagedNames }); if (followup.changes.length > 0) { console.error("NOT CONVERGED — a further `meta migrate` would emit:"); for (const c of followup.changes) console.error(" -", c.kind, JSON.stringify(c).slice(0, 200)); @@ -639,23 +639,26 @@ describe("report views — inline models on real Postgres", () => { ]); }, 60_000); - test("CHANGE A REPORT: adding a measure is a drop and a create of that view, and then converges", async () => { + test("CHANGE A REPORT: a measure added at the end replaces the view in place, and then converges", async () => { const before = await loadInline(metricModel(["samples"])); const first = await migrate(before); await assertConverged(first.expected, first.unmanagedNames); + const oid = async (): Promise => + String((await select(`SELECT 'public."v_metrics_by_kind"'::regclass::oid::text AS oid`))[0]!.oid); + const created = await oid(); const after = await loadInline(metricModel(["samples", "total"])); - // A plain `meta migrate`, no --allow: the drop is PAIRED with the create of the same view, - // which is not a destructive change. Passing dropView here would hide a broken pairing. const { result, up, expected, unmanagedNames } = await migrate(after); expect(result.blocked).toEqual([]); + // The existing columns are a prefix of the new ones, by name and by type, which is the + // one change Postgres accepts as CREATE OR REPLACE. Applying it is the proof the report's + // column types were derived correctly: Postgres refuses the statement otherwise. const viewChanges = result.changes.filter((c) => c.kind.endsWith("-view")); - expect(viewChanges.map((c) => c.kind).sort()).toEqual(["create-view", "drop-view"]); - expect(up).toContain(`DROP VIEW`); - expect(up).toContain(`CREATE VIEW "v_metrics_by_kind" AS`); - // A fail-safe drop and create, not CREATE OR REPLACE (the report carries no column list). - expect(up).not.toContain("CREATE OR REPLACE"); + expect(viewChanges.map((c) => c.kind)).toEqual(["replace-view"]); + expect(up).toContain(`CREATE OR REPLACE VIEW "v_metrics_by_kind" AS`); + expect(up).not.toContain("DROP VIEW"); + expect(await oid()).toBe(created); const cols = await sql.raw( `SELECT column_name FROM information_schema.columns WHERE table_name = 'v_metrics_by_kind' ORDER BY ordinal_position`, diff --git a/server/typescript/packages/integration-tests/test/view-lifecycle-pg.test.ts b/server/typescript/packages/integration-tests/test/view-lifecycle-pg.test.ts index bec18dd73..480eb3953 100644 --- a/server/typescript/packages/integration-tests/test/view-lifecycle-pg.test.ts +++ b/server/typescript/packages/integration-tests/test/view-lifecycle-pg.test.ts @@ -22,7 +22,7 @@ import { describe, test, expect, beforeAll, afterAll, beforeEach } from "bun:test"; import { - buildExpectedSchema, diff, emit, introspectPostgres, collectUnmanagedNames, + buildExpectedSchema, diff, emit, introspectPostgres, postgresViewReplaceProbe, collectUnmanagedNames, type AllowOptions, type SchemaSnapshot, } from "@metaobjectsdev/migrate-ts"; import { buildProjectionViews } from "@metaobjectsdev/codegen-ts"; @@ -209,7 +209,7 @@ async function migrate(metaJson: string, allow: AllowOptions = {}) { views: buildProjectionViews(root, { dialect: "postgres", columnNamingStrategy: "literal" }), }); const unmanagedNames = collectUnmanagedNames(root); - const result = await diff({ expected, actual: await introspectPostgres(k), dialect: "postgres", allow, unmanagedNames }); + const result = await diff({ expected, actual: await introspectPostgres(k), dialect: "postgres", viewReplaceProbe: postgresViewReplaceProbe(k), allow, unmanagedNames }); const emittable = result.changes.filter((c) => c.status.state !== "blocked"); const { up, down } = emittable.length === 0 ? { up: "", down: "" } @@ -220,7 +220,7 @@ async function migrate(metaJson: string, allow: AllowOptions = {}) { /** THE gate: a second `meta migrate` against the just-migrated DB must be a no-op. */ async function assertConverged(expected: SchemaSnapshot, allow: AllowOptions = {}, unmanagedNames: string[] = []): Promise { - const followup = await diff({ expected, actual: await introspectPostgres(k), dialect: "postgres", allow, unmanagedNames }); + const followup = await diff({ expected, actual: await introspectPostgres(k), dialect: "postgres", viewReplaceProbe: postgresViewReplaceProbe(k), allow, unmanagedNames }); if (followup.changes.length > 0) { console.error("NOT CONVERGED — a second `meta migrate` would emit:"); for (const c of followup.changes) console.error(" -", c.kind, JSON.stringify(c).slice(0, 200)); @@ -590,7 +590,7 @@ describe("view lifecycle — real Postgres", () => { const m2 = meta({ summaryFields: [PASSTHROUGH_TITLE, PASSTHROUGH_STATUS, AGG_WEEK_COUNT] }); const expected = await expectedFor(m2); const result = await diff({ - expected, actual: await introspectPostgres(k), dialect: "postgres", + expected, actual: await introspectPostgres(k), dialect: "postgres", viewReplaceProbe: postgresViewReplaceProbe(k), allow: { dropView: true }, // even WITH drop-view, the cascade is still gated }); @@ -636,7 +636,7 @@ describe("view lifecycle — real Postgres", () => { await sql.raw(`COMMENT ON VIEW "v_program_summary" IS NULL`).execute(k); const expected = await expectedFor(m); - const result = await diff({ expected, actual: await introspectPostgres(k), dialect: "postgres" }); + const result = await diff({ expected, actual: await introspectPostgres(k), dialect: "postgres", viewReplaceProbe: postgresViewReplaceProbe(k) }); const blocked = result.blocked.find((c) => c.kind === "replace-view"); expect(blocked).toBeDefined(); expect(blocked!.status.blockedReason).toContain("no MetaObjects fingerprint"); @@ -704,7 +704,7 @@ describe("view lifecycle — real Postgres", () => { await sql.raw(`COMMENT ON VIEW "v_active_programs" IS NULL`).execute(k); const expected = await expectedFor(m); - const result = await diff({ expected, actual: await introspectPostgres(k), dialect: "postgres" }); + const result = await diff({ expected, actual: await introspectPostgres(k), dialect: "postgres", viewReplaceProbe: postgresViewReplaceProbe(k) }); const blocked = result.blocked.find((c) => c.kind === "replace-view"); expect(blocked).toBeDefined(); expect(blocked!.status.blockedReason).toContain("allow.adoptView"); diff --git a/server/typescript/packages/integration-tests/test/view-recreate-drift-pg.test.ts b/server/typescript/packages/integration-tests/test/view-recreate-drift-pg.test.ts index c9a2678bb..668308e8a 100644 --- a/server/typescript/packages/integration-tests/test/view-recreate-drift-pg.test.ts +++ b/server/typescript/packages/integration-tests/test/view-recreate-drift-pg.test.ts @@ -8,7 +8,7 @@ */ import { describe, test, expect, beforeAll, afterAll, beforeEach } from "bun:test"; import { - buildExpectedSchema, collectUnmanagedNames, computeDriftFromActual, diff, emit, introspectPostgres, + buildExpectedSchema, collectUnmanagedNames, computeDriftFromActual, diff, emit, introspectPostgres, postgresViewReplaceProbe, isViewRecreateOnly, type SchemaSnapshot, } from "@metaobjectsdev/migrate-ts"; import { buildProjectionViews } from "@metaobjectsdev/codegen-ts"; @@ -47,6 +47,21 @@ const views = () => buildProjectionViews(canonical, { dialect: "postgres", colum const expectedSchema = (): SchemaSnapshot => buildExpectedSchema(canonical, { columnNamingStrategy: STRATEGY, views: views() }); +/** + * The expected schema once the metadata widens `weeks.durationMinutes` to a bigint. Postgres blocks + * a type change on a column a view reads, so this is the change that strands every view over it + * (a nullability or default change does not: it alters in place). The views' SQL is untouched, so + * their fingerprints still match. + */ +let widened = false; +const expectedForRun = (): SchemaSnapshot => { + const schema = expectedSchema(); + if (!widened) return schema; + const weeks = schema.tables.find((t) => t.name === "weeks")!; + weeks.columns.find((c) => c.name === "durationMinutes")!.sqlType = { kind: "integer", bits: 64 }; + return schema; +}; + async function applyRaw(text: string): Promise { for (const stmt of text.split(/;\s*\n/).map((s) => s.trim()).filter(Boolean)) { await sql.raw(stmt.endsWith(";") ? stmt : `${stmt};`).execute(k); @@ -54,35 +69,41 @@ async function applyRaw(text: string): Promise { } async function migrate(allow = {}) { - const expected = expectedSchema(); + const expected = expectedForRun(); const unmanagedNames = collectUnmanagedNames(canonical); - const result = await diff({ expected, actual: await introspectPostgres(k), dialect: "postgres", allow, unmanagedNames }); + const result = await diff({ expected, actual: await introspectPostgres(k), dialect: "postgres", viewReplaceProbe: postgresViewReplaceProbe(k), allow, unmanagedNames }); expect(result.blocked).toEqual([]); const { up } = result.changes.length === 0 ? { up: "" } : emit(result.changes, { dialect: "postgres" }); if (up.trim().length > 0) await applyRaw(up); return { result, up }; } -/** The canonical model migrated from empty, then `weeks.durationMinutes` made nullable by hand. */ -async function migratedWithNullableWeeks(): Promise { +/** The canonical model migrated from empty, then the metadata widens `weeks.durationMinutes`. */ +async function migratedThenWidened(): Promise { + widened = false; await migrate(); - await sql.raw(`ALTER TABLE "weeks" ALTER COLUMN "durationMinutes" DROP NOT NULL`).execute(k); + widened = true; } describe("D3 — a view recreated only around a column change is not drift (postgres)", () => { test("drift reports the column and no view", async () => { - await migratedWithNullableWeeks(); - const drift = await computeDriftFromActual(await introspectPostgres(k), "postgres", canonical, { + await migratedThenWidened(); + // computeDriftFromActual builds its own expected schema from the metadata, so mirror the + // widening on the other side: the database column reads as the wider type. + const actual = await introspectPostgres(k); + actual.tables.find((t) => t.name === "weeks")!.columns.find((c) => c.name === "durationMinutes")!.sqlType = + { kind: "integer", bits: 64 }; + const drift = await computeDriftFromActual(actual, "postgres", canonical, { columnNamingStrategy: STRATEGY, views: views(), }); - expect(drift.changes.map((c) => c.kind)).toEqual(["change-column-nullable"]); + expect(drift.changes.map((c) => c.kind)).toEqual(["change-column-type"]); expect(drift.blocked.filter(isViewChange)).toEqual([]); }, 60_000); test("the diff plans each pair marked unchanged; it applies and converges", async () => { - await migratedWithNullableWeeks(); - const { result, up } = await migrate({ nullableToNotNull: true }); + await migratedThenWidened(); + const { result, up } = await migrate({ typeChange: true }); const pairs = result.changes.filter(isViewChange); expect(pairs.length).toBeGreaterThan(0); for (const c of pairs) { @@ -92,9 +113,10 @@ describe("D3 — a view recreated only around a column change is not drift (post expect(up).toContain(`CREATE VIEW "v_program_minutes"`); const followup = await diff({ - expected: expectedSchema(), + expected: expectedForRun(), actual: await introspectPostgres(k), dialect: "postgres", + viewReplaceProbe: postgresViewReplaceProbe(k), unmanagedNames: collectUnmanagedNames(canonical), }); expect(followup.changes).toEqual([]); diff --git a/server/typescript/packages/integration-tests/test/view-replace-probe-pg.test.ts b/server/typescript/packages/integration-tests/test/view-replace-probe-pg.test.ts new file mode 100644 index 000000000..6b8ddac1e --- /dev/null +++ b/server/typescript/packages/integration-tests/test/view-replace-probe-pg.test.ts @@ -0,0 +1,661 @@ +/** + * Replacing a view in place, against a REAL Postgres (adopter report D2). + * + * Postgres replaces a view (`CREATE OR REPLACE VIEW`) only when every existing column keeps + * its name, type and position and new columns come last. It refuses anything else when the + * migration is APPLIED (`cannot change data type of view column`). So the planner asks the + * database before it plans: `postgresViewReplaceProbe` creates the expected definition as a + * temporary view in a transaction it rolls back, reads the columns Postgres gave it, and + * compares them with the live view's. A replace is planned only on that evidence; with no + * evidence the plan is DROP + CREATE. + * + * Postgres is the oracle throughout. For a view the plan drops, the test first asks Postgres + * directly and it refuses the replace. For a view the plan replaces, the replace is applied + * and the view keeps its OID. The cases cover every kind of derived column (each `origin.*` + * on a projection, each column kind of a report) and the type pairs a coarser type model + * cannot tell apart: json and jsonb, smallint and integer, text and character varying. + */ + +import { describe, test, expect, beforeAll, afterAll, beforeEach } from "bun:test"; +import { randomUUID } from "node:crypto"; +import { + buildExpectedSchema, diff, emit, introspectPostgres, postgresViewReplaceProbe, + type AllowOptions, type SchemaSnapshot, type ViewDescriptor, type ViewReplaceProbe, +} from "@metaobjectsdev/migrate-ts"; +import { buildProjectionViews } from "@metaobjectsdev/codegen-ts"; +import { MetaDataLoader, InMemoryStringSource, type MetaRoot } from "@metaobjectsdev/metadata"; +import { Kysely, PostgresDialect, sql } from "kysely"; +import { Pool } from "pg"; +import { startPostgres, type RunningPg } from "../src/postgres-container.ts"; + +type Db = Kysely>; + +let pg: RunningPg; +let k: Db; + +const connect = (connectionString: string): Db => + new Kysely>({ dialect: new PostgresDialect({ pool: new Pool({ connectionString }) }) }); + +beforeAll(async () => { + pg = await startPostgres(); + k = connect(pg.connectionUri); +}, 120_000); + +afterAll(async () => { + await k.destroy(); + await pg.stop(); +}, 60_000); + +beforeEach(async () => { + await sql.raw("DROP SCHEMA public CASCADE").execute(k); + await sql.raw("CREATE SCHEMA public").execute(k); +}); + +const STRATEGY = "literal" as const; +const PROBE_RELATION = "mo_view_replace_probe"; + +const program = (extra: unknown[] = []) => ({ + "object.entity": { name: "Program", children: [ + { "source.rdb": { "@table": "programs" } }, + { "field.long": { name: "id" } }, + { "field.string": { name: "status", "@required": true, "@maxLength": 20 } }, + { "field.string": { name: "nickname", "@maxLength": 40 } }, + { "field.string": { name: "notes" } }, + ...extra, + { "identity.primary": { name: "id", "@fields": "id", "@generation": "increment" } }, + { "relationship.aggregation": { name: "weeks", "@cardinality": "many", "@objectRef": "Week" } }, + ] }, +}); +const WEEK = { + "object.entity": { name: "Week", children: [ + { "source.rdb": { "@table": "weeks" } }, + { "field.long": { name: "id" } }, + { "field.long": { name: "programId", "@required": true } }, + { "field.string": { name: "label", "@required": true, "@maxLength": 40 } }, + { "field.int": { name: "durationMinutes", "@required": true } }, + { "field.decimal": { name: "score", "@precision": 9, "@scale": 2 } }, + { "field.timestamp": { name: "createdAt", "@required": true } }, + { "field.date": { name: "heldOn" } }, + { "identity.primary": { name: "id", "@fields": "id", "@generation": "increment" } }, + { "identity.reference": { name: "ref_program", "@fields": "programId", "@references": "Program" } }, + { "dimension.attribute": { name: "program", "@of": "Week.programId" } }, + { "dimension.attribute": { name: "label", "@of": "Week.label" } }, + { "dimension.time": { name: "createdAt", "@of": "Week.createdAt", "@grains": ["hour", "month"] } }, + { "dimension.time": { name: "heldOn", "@of": "Week.heldOn", "@grains": ["day", "month"] } }, + { "measure.aggregate": { name: "weeks", "@agg": "count", "@of": "Week.id" } }, + { "measure.aggregate": { name: "minutes", "@agg": "sum", "@of": "Week.durationMinutes", "@default": 0 } }, + { "measure.aggregate": { name: "points", "@agg": "sum", "@of": "Week.score" } }, + { "measure.aggregate": { name: "longest", "@agg": "max", "@of": "Week.durationMinutes" } }, + { "measure.aggregate": { name: "bestScore", "@agg": "max", "@of": "Week.score" } }, + { "measure.aggregate": { name: "typical", "@agg": "avg", "@of": "Week.durationMinutes" } }, + { "measure.ratio": { name: "minutesPerWeek", "@numerator": "minutes", "@denominator": "weeks", "@default": 0 } }, + ] }, +}; +const WEEK_BRIEF = { + "object.value": { name: "WeekBrief", children: [ + { "field.long": { name: "id" } }, + { "field.string": { name: "label" } }, + ] }, +}; + +const VIA = { "@via": "Program.weeks" }; +const SUMMARY_VIEW = "v_program_summary"; + +/** A projection over Program with its key and the given derived fields. */ +function summary(...fields: unknown[]): unknown { + return { + "object.projection": { name: "ProgramSummary", children: [ + { "source.rdb": { "@kind": "view", "@table": SUMMARY_VIEW } }, + { "identity.primary": { name: "id", extends: "Program.id", "@fields": "id" } }, + { "field.long": { name: "id", extends: "Program.id", children: [{ "origin.passthrough": { "@from": "Program.id" } }] } }, + ...fields, + ] }, + }; +} + +const REPORT = { + "object.report": { + name: "WeekStats", + "@from": "Week", + "@dimensions": ["program", "label", "createdAt:hour", "createdAt:month", "heldOn:day", "heldOn:month"], + "@measures": ["weeks", "minutes", "points", "longest", "bestScore", "typical", "minutesPerWeek"], + children: [{ "source.rdb": { "@kind": "view", "@table": "v_week_stats" } }], + }, +}; + +const computed = (name: string, subType: string, expr: unknown): Record => + ({ [`field.${subType}`]: { name, children: [{ "origin.computed": { "@expr": expr } }] } }); +const COALESCE_NICKNAME = computed("displayName", "string", { fn: "coalesce", args: [{ field: "nickname" }, { value: "unnamed" }] }); +const COALESCE_NOTES = computed("displayName", "string", { fn: "coalesce", args: [{ field: "notes" }, { value: "unnamed" }] }); + +interface Case { + /** The kind of derived column under test. */ + readonly kind: string; + /** The object that declares the view. */ + readonly host: unknown; + readonly view: string; + /** The column the incompatible hand-written view types differently. */ + readonly column: string; + /** That column in the hand-written view, given a reference to the derived value. */ + readonly retyped: (ref: string) => string; +} + +const cast = (type: string) => (ref: string): string => `CAST(${ref} AS ${type})`; +const projection = (kind: string, field: Record, retyped: Case["retyped"]): Case => { + const [declared] = Object.values(field) as { name: string }[]; + return { kind, host: summary(field), view: SUMMARY_VIEW, column: declared!.name, retyped }; +}; + +const CASES: readonly Case[] = [ + projection("passthrough", + { "field.string": { name: "status", children: [{ "origin.passthrough": { "@from": "Program.status" } }] } }, + cast("text")), + // The adopter's case: a hand-written view that cast its count to int. + projection("aggregate count", + { "field.long": { name: "weekCount", children: [{ "origin.aggregate": { "@agg": "count", "@of": "Week.id", ...VIA } }] } }, + cast("integer")), + projection("aggregate sum of an integer", + { "field.long": { name: "totalMinutes", children: [ + { "origin.aggregate": { "@agg": "sum", "@of": "Week.durationMinutes", ...VIA } }] } }, + cast("integer")), + // An aggregate never reports its argument's length or precision. + projection("aggregate sum of a decimal", + { "field.decimal": { name: "totalScore", children: [{ "origin.aggregate": { "@agg": "sum", "@of": "Week.score", ...VIA } }] } }, + cast("numeric(9,2)")), + projection("aggregate min of a bounded string", + { "field.string": { name: "firstLabel", children: [{ "origin.aggregate": { "@agg": "min", "@of": "Week.label", ...VIA } }] } }, + cast("varchar(40)")), + projection("aggregate max of a decimal", + { "field.decimal": { name: "bestScore", children: [{ "origin.aggregate": { "@agg": "max", "@of": "Week.score", ...VIA } }] } }, + cast("numeric(9,2)")), + // smallint against integer. + projection("aggregate min of an integer", + { "field.int": { name: "shortest", children: [ + { "origin.aggregate": { "@agg": "min", "@of": "Week.durationMinutes", ...VIA } }] } }, + cast("smallint")), + projection("aggregate avg", + { "field.decimal": { name: "typicalMinutes", children: [ + { "origin.aggregate": { "@agg": "avg", "@of": "Week.durationMinutes", ...VIA } }] } }, + cast("double precision")), + projection("aggregate any", + { "field.boolean": { name: "anyLongWeek", children: [ + { "origin.aggregate": { "@agg": "any", ...VIA, "@filter": { durationMinutes: { gt: 60 } } } }] } }, + cast("integer")), + projection("aggregate all", + { "field.boolean": { name: "allLongWeeks", children: [ + { "origin.aggregate": { "@agg": "all", ...VIA, "@filter": { durationMinutes: { gt: 60 } } } }] } }, + cast("integer")), + // character varying[] (the collect of a bounded string) against text[]. + projection("aggregate collect", + { "field.string": { name: "weekLabels", isArray: true, children: [ + { "origin.aggregate": { "@agg": "collect", "@of": "Week.label", ...VIA } }] } }, + cast("text[]")), + // jsonb against json. + projection("aggregate collect of a whole object", + { "field.object": { name: "weekBriefs", isArray: true, "@objectRef": "WeekBrief", children: [ + { "origin.aggregate": { "@agg": "collect", ...VIA } }] } }, + cast("json")), + // A scalar subquery keeps its column's length, where an aggregate over it does not. + projection("first", + { "field.string": { name: "latestWeekLabel", children: [ + { "origin.first": { "@of": "Week.label", ...VIA, "@orderBy": ["createdAt:desc"] } }] } }, + cast("text")), + projection("computed comparison", + computed("isPublished", "boolean", { op: "eq", left: { field: "status" }, right: { value: "PUBLISHED" } }), + cast("integer")), + projection("computed null test", + computed("isNamed", "boolean", { op: "isNotNull", arg: { field: "nickname" } }), + cast("integer")), + projection("computed column reference", computed("alias", "string", { field: "nickname" }), cast("text")), + // COALESCE(varchar(40), 'x') is a bare character varying, which is not text. + projection("computed coalesce with a literal", COALESCE_NICKNAME, cast("text")), + ...([ + ["report count", "weeks", "integer"], + ["report sum with a default", "minutes", "integer"], + ["report sum of a decimal", "points", "numeric(9,2)"], + ["report max of a decimal", "bestScore", "numeric(9,2)"], + ["report avg", "typical", "double precision"], + ["report ratio", "minutesPerWeek", "double precision"], + ["report dimension", "label", "text"], + ["report hour grain", "createdAtHour", "timestamp"], + ["report month grain", "createdAtMonth", "timestamp"], + ] as const).map(([kind, column, type]): Case => ({ kind, host: REPORT, view: "v_week_stats", column, retyped: cast(type) })), +]; + +async function load(host: unknown, programFields: unknown[] = []): Promise { + const json = JSON.stringify({ + "metadata.root": { package: "acme", children: [program(programFields), WEEK, WEEK_BRIEF, host] }, + }); + const { root, errors } = await new MetaDataLoader().load([new InMemoryStringSource(json)]); + expect(errors).toEqual([]); + return root; +} + +async function applyRaw(text: string, db: Db = k): Promise { + for (const stmt of text.split(/;\s*\n/).map((s) => s.trim()).filter(Boolean)) { + await sql.raw(stmt.endsWith(";") ? stmt : `${stmt};`).execute(db); + } +} + +interface Model { expected: SchemaSnapshot; view: ViewDescriptor } + +/** The expected schema of a model, and the one view under test. */ +async function model(host: unknown, viewName: string, programFields: unknown[] = []): Promise { + const root = await load(host, programFields); + const expected = buildExpectedSchema(root, { + columnNamingStrategy: STRATEGY, + views: buildProjectionViews(root, { dialect: "postgres", columnNamingStrategy: STRATEGY }), + }); + const view = expected.views.find((v) => v.name === viewName); + if (view === undefined) throw new Error(`no view '${viewName}' was derived`); + return { expected, view }; +} + +/** A plan on the live path: introspect, then diff with the probe. `probe: null` plans offline-style, with none. */ +async function plan(expected: SchemaSnapshot, allow: AllowOptions = {}, probe: ViewReplaceProbe | null = postgresViewReplaceProbe(k)) { + return diff({ + expected, actual: await introspectPostgres(k), dialect: "postgres", allow, + ...(probe !== null ? { viewReplaceProbe: probe } : {}), + }); +} + +async function apply(changes: Awaited>["changes"]): Promise { + const { up } = emit(changes, { dialect: "postgres" }); + await applyRaw(up); + return up; +} + +/** Create the model's tables and no view: the state a hand-written view is created in. */ +async function createTables(expected: SchemaSnapshot): Promise { + const tables = await plan({ ...expected, views: [] }); + expect(tables.blocked).toEqual([]); + await apply(tables.changes); +} + +/** The whole model, views included and stamped. */ +async function migrate(expected: SchemaSnapshot): Promise { + const all = await plan(expected); + expect(all.blocked).toEqual([]); + await apply(all.changes); + expect((await plan(expected)).changes).toEqual([]); +} + +/** The column names of the derived body, in order, as Postgres reports them. */ +async function columnNames(view: ViewDescriptor): Promise { + await sql.raw(`CREATE VIEW "mo_column_names" AS ${view.sql}`).execute(k); + const r = await sql.raw( + `SELECT column_name FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'mo_column_names' ORDER BY ordinal_position`, + ).execute(k); + await sql.raw(`DROP VIEW "mo_column_names"`).execute(k); + return (r.rows as { column_name: string }[]).map((row) => row.column_name); +} + +/** + * Create a view somebody wrote by hand: other SQL, the derived view's column names in its + * order. `retype` gives one column another type; `withoutLast` leaves the last column out. + */ +async function createHandWritten( + view: ViewDescriptor, + shape: { retype?: Pick; withoutLast?: true } = {}, +): Promise { + const all = await columnNames(view); + const names = shape.withoutLast === true ? all.slice(0, -1) : all; + expect(names.length).toBeGreaterThan(0); + if (shape.retype !== undefined) expect(names).toContain(shape.retype.column); + const columns = names.map((name) => { + const ref = `q."${name}"`; + return shape.retype?.column === name ? `${shape.retype.retyped(ref)} AS "${name}"` : ref; + }); + await sql.raw(`CREATE VIEW "${view.name}" AS SELECT ${columns.join(", ")} FROM (${view.sql}) q`).execute(k); +} + +async function scalar(query: string, db: Db = k): Promise { + const r = await sql.raw(query).execute(db); + return Object.values(r.rows[0] as Record)[0]; +} +const viewOid = (name: string) => scalar(`SELECT 'public."${name}"'::regclass::oid::text`); +const viewComment = (name: string) => scalar(`SELECT obj_description('public."${name}"'::regclass, 'pg_class')`); +const probeRelations = (db: Db = k) => scalar(`SELECT count(*)::int FROM pg_class WHERE relname = '${PROBE_RELATION}'`, db); +const columnTypes = async (name: string): Promise> => { + const r = await sql.raw( + `SELECT a.attname, format_type(a.atttypid, a.atttypmod) AS type FROM pg_attribute a + WHERE a.attrelid = 'public."${name}"'::regclass AND a.attnum > 0 ORDER BY a.attnum`, + ).execute(k); + return Object.fromEntries((r.rows as { attname: string; type: string }[]).map((row) => [row.attname, row.type])); +}; + +const STAMP = /^metaobjects:v1:sha256:[0-9a-f]{64}$/; +const kindsOf = (changes: readonly { kind: string }[]): string[] => changes.map((c) => c.kind).sort(); +const refusesReplace = (view: ViewDescriptor) => + expect(sql.raw(`CREATE OR REPLACE VIEW "${view.name}" AS ${view.sql}`).execute(k)).rejects.toThrow( + /cannot (change data type|change name|drop columns)/, + ); + +describe("adopting a hand-written view (postgres)", () => { + for (const c of CASES) { + describe(c.kind, () => { + test("the same columns: replaced in place", async () => { + const { expected, view } = await model(c.host, c.view); + await createTables(expected); + await createHandWritten(view); + const before = await viewOid(c.view); + + // Unfingerprinted, so the adoption is gated whichever way it is planned. + expect(kindsOf((await plan(expected)).blocked)).toEqual(["replace-view"]); + + const adopted = await plan(expected, { adoptView: true }); + expect(adopted.blocked).toEqual([]); + expect(kindsOf(adopted.changes)).toEqual(["replace-view"]); + expect(await apply(adopted.changes)).toContain(`CREATE OR REPLACE VIEW "${c.view}" AS`); + + // The same view object: nothing was dropped. + expect(await viewOid(c.view)).toBe(before); + expect(await viewComment(c.view)).toMatch(STAMP); + expect((await plan(expected)).changes).toEqual([]); + }, 60_000); + + test("the derived view appends a column: replaced in place", async () => { + const { expected, view } = await model(c.host, c.view); + await createTables(expected); + await createHandWritten(view, { withoutLast: true }); + const before = await viewOid(c.view); + + const adopted = await plan(expected, { adoptView: true }); + expect(adopted.blocked).toEqual([]); + expect(kindsOf(adopted.changes)).toEqual(["replace-view"]); + // Undoing an append removes a column, which a replace cannot do. + expect(adopted.changes[0]).toMatchObject({ proven: { restorable: false } }); + await apply(adopted.changes); + + expect(await viewOid(c.view)).toBe(before); + expect((await plan(expected)).changes).toEqual([]); + }, 60_000); + + test("one column of another type: dropped and created", async () => { + const { expected, view } = await model(c.host, c.view); + await createTables(expected); + await createHandWritten(view, { retype: c }); + + // Postgres will not replace this view with the derived one. + await refusesReplace(view); + + expect(kindsOf((await plan(expected)).blocked)).toContain("drop-view"); + + const adopted = await plan(expected, { adoptView: true }); + expect(adopted.blocked).toEqual([]); + expect(kindsOf(adopted.changes)).toEqual(["create-view", "drop-view"]); + expect(await apply(adopted.changes)).not.toContain("CREATE OR REPLACE"); + + expect(await viewComment(c.view)).toMatch(STAMP); + expect((await plan(expected)).changes).toEqual([]); + }, 60_000); + }); + } +}); + +describe("changing a managed view (postgres)", () => { + // A stamped projection whose coalesce moves from a varchar(40) field to an unbounded one. + // The column keeps its name and is still a string, and its type goes from character + // varying to text. + test("a column whose type changes is dropped and created, and applies", async () => { + const before = await model(summary(COALESCE_NICKNAME), SUMMARY_VIEW); + await migrate(before.expected); + expect(await columnTypes(SUMMARY_VIEW)).toEqual({ id: "bigint", displayName: "character varying" }); + + const after = await model(summary(COALESCE_NOTES), SUMMARY_VIEW); + await refusesReplace(after.view); + + const changed = await plan(after.expected); + expect(changed.blocked).toEqual([]); + expect(kindsOf(changed.changes)).toEqual(["create-view", "drop-view"]); + await apply(changed.changes); + + expect(await columnTypes(SUMMARY_VIEW)).toEqual({ id: "bigint", displayName: "text" }); + expect((await plan(after.expected)).changes).toEqual([]); + }, 60_000); + + test("an appended field is replaced in place, and the down migration drops and recreates", async () => { + const before = await model(summary(COALESCE_NICKNAME), SUMMARY_VIEW); + await migrate(before.expected); + const oid = await viewOid(SUMMARY_VIEW); + + const after = await model( + summary(COALESCE_NICKNAME, computed("isNamed", "boolean", { op: "isNotNull", arg: { field: "nickname" } })), + SUMMARY_VIEW, + ); + const changed = await plan(after.expected); + expect(kindsOf(changed.changes)).toEqual(["replace-view"]); + const { up, down } = emit(changed.changes, { dialect: "postgres" }); + await applyRaw(up); + expect(await viewOid(SUMMARY_VIEW)).toBe(oid); + expect(Object.keys(await columnTypes(SUMMARY_VIEW))).toEqual(["id", "displayName", "isNamed"]); + + // The down removes a column: Postgres refuses that as a replace, so it is a drop and a create. + expect(down).not.toContain("CREATE OR REPLACE"); + await applyRaw(down); + expect(Object.keys(await columnTypes(SUMMARY_VIEW))).toEqual(["id", "displayName"]); + expect((await plan(before.expected)).changes).toEqual([]); + }, 60_000); + + test("a change that keeps every column is replaced in place, and so is its down migration", async () => { + const before = await model(summary(computed("flag", "boolean", { op: "isNull", arg: { field: "nickname" } })), SUMMARY_VIEW); + await migrate(before.expected); + const oid = await viewOid(SUMMARY_VIEW); + + const after = await model(summary(computed("flag", "boolean", { op: "isNull", arg: { field: "notes" } })), SUMMARY_VIEW); + const changed = await plan(after.expected); + expect(kindsOf(changed.changes)).toEqual(["replace-view"]); + expect(changed.changes[0]).toMatchObject({ proven: { restorable: true } }); + const { up, down } = emit(changed.changes, { dialect: "postgres" }); + await applyRaw(up); + expect(await viewOid(SUMMARY_VIEW)).toBe(oid); + + expect(down).toContain(`CREATE OR REPLACE VIEW "${SUMMARY_VIEW}" AS`); + await applyRaw(down); + expect(await viewOid(SUMMARY_VIEW)).toBe(oid); + expect((await plan(before.expected)).changes).toEqual([]); + }, 60_000); +}); + +// The probe sees the schema as it stands; the replace runs after the plan's table changes. +// A hand-written `@sql` body with `SELECT *` re-expands then, so a column the same plan adds +// lands in the middle of the view's column list. +describe("an @sql body that reads every column (postgres)", () => { + const starBody = (extra: number): string => `SELECT p.*, ${extra} AS "extra" FROM "programs" p`; + const starView = (extra: number): unknown => ({ + "object.projection": { name: "ProgramSummary", children: [ + { "source.rdb": { "@kind": "view", "@view": SUMMARY_VIEW, "@sql": starBody(extra) } }, + { "identity.primary": { name: "id", extends: "Program.id", "@fields": "id" } }, + { "field.long": { name: "id", extends: "Program.id" } }, + ] }, + }); + const TAGLINE = [{ "field.string": { name: "tagline" } }]; + + test("a body change beside an added column is dropped and created, and applies", async () => { + await migrate((await model(starView(1), SUMMARY_VIEW)).expected); + const after = await model(starView(2), SUMMARY_VIEW, TAGLINE); + + // Postgres accepts the replace today and refuses it once the column exists. + const rollback = new Error("rollback"); + await expect(k.transaction().execute(async (trx) => { + await sql.raw(`CREATE OR REPLACE VIEW "${SUMMARY_VIEW}" AS ${after.view.sql}`).execute(trx); + throw rollback; + })).rejects.toBe(rollback); + await expect(k.transaction().execute(async (trx) => { + await sql.raw(`ALTER TABLE "programs" ADD COLUMN "tagline" TEXT`).execute(trx); + await sql.raw(`CREATE OR REPLACE VIEW "${SUMMARY_VIEW}" AS ${after.view.sql}`).execute(trx); + })).rejects.toThrow(/cannot change name of view column "extra" to "tagline"/); + + const changed = await plan(after.expected); + expect(changed.blocked).toEqual([]); + expect(kindsOf(changed.changes)).toEqual(["add-column", "create-view", "drop-view"]); + await apply(changed.changes); + + expect(Object.keys(await columnTypes(SUMMARY_VIEW))).toEqual(["id", "status", "nickname", "notes", "tagline", "extra"]); + expect((await plan(after.expected)).changes).toEqual([]); + }, 60_000); + + test("a body change with no added column is replaced in place", async () => { + await migrate((await model(starView(1), SUMMARY_VIEW)).expected); + const oid = await viewOid(SUMMARY_VIEW); + const after = await model(starView(2), SUMMARY_VIEW); + + const changed = await plan(after.expected); + expect(kindsOf(changed.changes)).toEqual(["replace-view"]); + await apply(changed.changes); + expect(await viewOid(SUMMARY_VIEW)).toBe(oid); + expect((await plan(after.expected)).changes).toEqual([]); + }, 60_000); +}); + +describe("when the probe cannot answer, the plan is a drop and a create (postgres)", () => { + const compatible = async (): Promise => { + const m = await model(summary(COALESCE_NICKNAME), SUMMARY_VIEW); + await createTables(m.expected); + await createHandWritten(m.view); + return m; + }; + + test("no probe at all (an offline plan)", async () => { + const { expected } = await compatible(); + const adopted = await plan(expected, { adoptView: true }, null); + expect(kindsOf(adopted.changes)).toEqual(["create-view", "drop-view"]); + await apply(adopted.changes); + expect((await plan(expected)).changes).toEqual([]); + }, 60_000); + + // The new definition reads a column the same migration adds, so it does not compile + // against the schema as it stands. + test("the expected definition does not compile yet", async () => { + const before = await model(summary(COALESCE_NICKNAME), SUMMARY_VIEW); + await migrate(before.expected); + + const after = await model( + summary(COALESCE_NICKNAME, { "field.string": { name: "tagline", children: [{ "origin.passthrough": { "@from": "Program.tagline" } }] } }), + SUMMARY_VIEW, + [{ "field.string": { name: "tagline" } }], + ); + const live = (await introspectPostgres(k)).views.find((v) => v.name === SUMMARY_VIEW)!; + expect(await postgresViewReplaceProbe(k)(after.view, live)).toBeUndefined(); + + const changed = await plan(after.expected); + expect(changed.blocked).toEqual([]); + expect(kindsOf(changed.changes)).toEqual(["add-column", "create-view", "drop-view"]); + await apply(changed.changes); + expect((await plan(after.expected)).changes).toEqual([]); + }, 60_000); + + test("the connection may not create a temporary view", async () => { + const { expected, view } = await compatible(); + const role = `mo_limited_${randomUUID().slice(0, 8)}`; + const database = String(await scalar("SELECT current_database()")); + await sql.raw(`CREATE ROLE "${role}" LOGIN PASSWORD 'limited'`).execute(k); + await sql.raw(`REVOKE TEMPORARY ON DATABASE "${database}" FROM PUBLIC`).execute(k); + await sql.raw(`GRANT USAGE ON SCHEMA public TO "${role}"`).execute(k); + await sql.raw(`GRANT SELECT ON ALL TABLES IN SCHEMA public TO "${role}"`).execute(k); + const uri = new URL(pg.connectionUri); + uri.username = role; + uri.password = "limited"; + const limited = connect(uri.toString()); + try { + const live = (await introspectPostgres(k)).views.find((v) => v.name === SUMMARY_VIEW)!; + // It can read the model's tables, so only the temporary view is refused. + expect(await scalar(`SELECT count(*)::int FROM "programs"`, limited)).toBe(0); + expect(await postgresViewReplaceProbe(limited)(view, live)).toBeUndefined(); + + const adopted = await plan(expected, { adoptView: true }, postgresViewReplaceProbe(limited)); + expect(kindsOf(adopted.changes)).toEqual(["create-view", "drop-view"]); + await apply(adopted.changes); + expect((await plan(expected)).changes).toEqual([]); + } finally { + await limited.destroy(); + await sql.raw(`GRANT TEMPORARY ON DATABASE "${database}" TO PUBLIC`).execute(k); + await sql.raw(`DROP OWNED BY "${role}"`).execute(k); + await sql.raw(`DROP ROLE "${role}"`).execute(k); + } + }, 60_000); +}); + +describe("the probe itself (postgres)", () => { + const TYPED_VIEW = `SELECT p."id", p."nickname", CAST(NULL AS jsonb) AS "doc", CAST(NULL AS json) AS "raw", + CAST(1 AS smallint) AS "small", CAST(NULL AS text[]) AS "names", CAST(1 AS numeric(9,2)) AS "price" + FROM "programs" p`; + + /** A stamped derived view, and a second definition of it to ask the probe about. */ + async function arranged(): Promise<{ expected: ViewDescriptor; live: ViewDescriptor }> { + const m = await model(summary(COALESCE_NICKNAME), SUMMARY_VIEW); + await migrate(m.expected); + const live = (await introspectPostgres(k)).views.find((v) => v.name === SUMMARY_VIEW)!; + return { expected: { name: SUMMARY_VIEW, sql: TYPED_VIEW }, live }; + } + + test("reports each view's exact column names and types, in order", async () => { + const { expected, live } = await arranged(); + const evidence = await postgresViewReplaceProbe(k)(expected, live); + expect(evidence?.expected.map((c) => [c.name, c.type])).toEqual([ + ["id", "bigint"], + ["nickname", "character varying(40)"], + ["doc", "jsonb"], + ["raw", "json"], + ["small", "smallint"], + ["names", "text[]"], + ["price", "numeric(9,2)"], + ]); + expect(evidence?.live.map((c) => [c.name, c.type])).toEqual([ + ["id", "bigint"], + ["displayName", "character varying"], + ]); + }, 60_000); + + test("leaves nothing behind, whether it answers or not", async () => { + const { expected, live } = await arranged(); + const oid = await viewOid(SUMMARY_VIEW); + const comment = await viewComment(SUMMARY_VIEW); + const relations = await scalar(`SELECT count(*)::int FROM pg_class`); + + expect(await postgresViewReplaceProbe(k)(expected, live)).toBeDefined(); + expect(await postgresViewReplaceProbe(k)({ name: SUMMARY_VIEW, sql: `SELECT "nope" FROM "programs"` }, live)).toBeUndefined(); + expect(await postgresViewReplaceProbe(k)({ name: SUMMARY_VIEW }, live)).toBeUndefined(); + + expect(await probeRelations()).toBe(0); + expect(await scalar(`SELECT count(*)::int FROM pg_class`)).toBe(relations); + expect(await viewOid(SUMMARY_VIEW)).toBe(oid); + expect(await viewComment(SUMMARY_VIEW)).toBe(comment); + }, 60_000); + + // A caller that plans inside its own transaction: the probe must undo only its own work, + // and a probe that fails must not abort the caller's transaction. + test("inside a caller's open transaction it uses a savepoint and leaves the transaction usable", async () => { + const { expected, live } = await arranged(); + await k.transaction().execute(async (trx) => { + await sql.raw(`INSERT INTO "programs" ("id", "status") VALUES (7, 'DRAFT')`).execute(trx); + + const evidence = await postgresViewReplaceProbe(trx)(expected, live); + expect(evidence?.expected.map((c) => c.name)).toEqual(["id", "nickname", "doc", "raw", "small", "names", "price"]); + expect(await probeRelations(trx)).toBe(0); + expect(await scalar(`SELECT count(*)::int FROM "programs"`, trx)).toBe(1); + + // A statement that fails aborts a Postgres transaction unless it is inside a savepoint. + expect(await postgresViewReplaceProbe(trx)({ name: SUMMARY_VIEW, sql: `SELECT "nope" FROM "programs"` }, live)).toBeUndefined(); + expect(await scalar(`SELECT count(*)::int FROM "programs"`, trx)).toBe(1); + await sql.raw(`INSERT INTO "programs" ("id", "status") VALUES (8, 'DRAFT')`).execute(trx); + }); + // The caller's own work committed; the probe's did not. + expect(await scalar(`SELECT count(*)::int FROM "programs"`)).toBe(2); + expect(await probeRelations()).toBe(0); + }, 60_000); + + test("a caller's transaction that rolls back takes nothing of the probe's with it but its own work", async () => { + const { expected, live } = await arranged(); + const rollback = new Error("rollback"); + await expect(k.transaction().execute(async (trx) => { + await sql.raw(`INSERT INTO "programs" ("id", "status") VALUES (9, 'DRAFT')`).execute(trx); + expect(await postgresViewReplaceProbe(trx)(expected, live)).toBeDefined(); + throw rollback; + })).rejects.toBe(rollback); + expect(await scalar(`SELECT count(*)::int FROM "programs"`)).toBe(0); + expect(await probeRelations()).toBe(0); + expect(await viewComment(SUMMARY_VIEW)).toMatch(STAMP); + }, 60_000); +}); diff --git a/server/typescript/packages/metadata/src/core/reporting/report-shape.ts b/server/typescript/packages/metadata/src/core/reporting/report-shape.ts index 64a5b29ce..88dfcd8dc 100644 --- a/server/typescript/packages/metadata/src/core/reporting/report-shape.ts +++ b/server/typescript/packages/metadata/src/core/reporting/report-shape.ts @@ -22,8 +22,10 @@ import { FIELD_SUBTYPE_TIMESTAMP, } from "../field/field-constants.js"; import { TYPE_IDENTITY, TYPE_RELATIONSHIP } from "../../shared/base-types.js"; +import type { MetaReferenceIdentity } from "../identity/meta-identity.js"; import { IDENTITY_REFERENCE_ATTR_REFERENCES, IDENTITY_SUBTYPE_REFERENCE } from "../identity/identity-constants.js"; -import { RELATIONSHIP_ATTR_OBJECT_REF } from "../relationship/relationship-constants.js"; +import { RELATIONSHIP_ATTR_OBJECT_REF, RELATIONSHIP_ATTR_SOURCE_REF_FIELD } from "../relationship/relationship-constants.js"; +import { resolveRelationshipReference } from "../relationship/resolve-relationship-reference.js"; import { MetaDimension } from "./meta-dimension.js"; import { MetaMeasure } from "./meta-measure.js"; import { identityEffectiveFields } from "../identity/validate-identity-passthrough.js"; @@ -187,27 +189,47 @@ export function reportSpineHops(report: MetaObject, from: MetaObject, root: Meta return hops; } +/** + * One hop of a `@via`/`@spine` path, as the loader's rule R8 walks it: the `relationship.*` + * (read through its `@objectRef`) or `identity.reference` (through its `@references`) of + * `current` named `hopName`, with its target resolved in `current`'s package. Returns + * `target: undefined` for an unresolvable hop or a ref attr that is not a string / does not + * reach a metadata object. + * ADR-0039: resolving children(), so a hop inherited through extends is found. + */ +function resolveViaHop( + current: MetaObject, + hopName: string, + root: MetaRoot, +): { hop: MetaData; isReference: boolean; target: MetaObject | undefined } | undefined { + const rel = current.children().find((c) => c.type === TYPE_RELATIONSHIP && c.name === hopName); + const refNode = rel !== undefined + ? undefined + : current.children().find( + (c) => c.type === TYPE_IDENTITY && c.subType === IDENTITY_SUBTYPE_REFERENCE && c.name === hopName, + ); + const hop = rel ?? refNode; + if (hop === undefined) return undefined; + const isReference = rel === undefined; + const ref = hop.attr(isReference ? IDENTITY_REFERENCE_ATTR_REFERENCES : RELATIONSHIP_ATTR_OBJECT_REF); + const node = typeof ref === "string" ? resolveObjectRef(root, ref, packageOfKey(current.resolutionKey())).node : undefined; + return { hop, isReference, target: isMetaObject(node) ? node : undefined }; +} + /** * The spine entity: the object at the end of a report's `@spine`, whose rows are the - * report's rows. Walked hop by hop from `from` as the loader's rule R8 walks it: each hop is - * a `relationship.*` (its `@objectRef`) or an `identity.reference` (its `@references`) of the - * entity reached so far, and its target resolves in that entity's package. Undefined when the - * report declares no `@spine`; throws, as {@link reportSpineHops} does, when one does not - * resolve (a report that passed `validateReporting` always resolves). + * report's rows. Walked hop by hop from `from` as the loader's rule R8 walks it via + * {@link resolveViaHop}; an unresolved hop throws, as {@link reportSpineHops} does (a report + * that passed `validateReporting` always resolves). Undefined when the report declares no + * `@spine`. */ export function reportSpineEntity(report: MetaObject, from: MetaObject, root: MetaRoot): MetaObject | undefined { const hops = reportSpineHops(report, from, root); if (hops === undefined) return undefined; let current: MetaObject = from; for (const hop of hops) { - // ADR-0039: resolving children(), so a relationship or reference inherited through - // extends is a hop, as the loader's walk finds it. - const node = - current.children().find((c) => c.type === TYPE_RELATIONSHIP && c.name === hop) ?? - current.children().find((c) => c.type === TYPE_IDENTITY && c.subType === IDENTITY_SUBTYPE_REFERENCE && c.name === hop); - const ref = node?.attr(node.type === TYPE_IDENTITY ? IDENTITY_REFERENCE_ATTR_REFERENCES : RELATIONSHIP_ATTR_OBJECT_REF); - const target = typeof ref === "string" ? resolveObjectRef(root, ref, packageOfKey(current.resolutionKey())).node : undefined; - if (!isMetaObject(target)) throw unresolved(report.name, `@spine '${reportSpine(report) ?? ""}' hop '${hop}'`); + const target = resolveViaHop(current, hop, root)?.target; + if (target === undefined) throw unresolved(report.name, `@spine '${reportSpine(report) ?? ""}' hop '${hop}'`); current = target; } return current; @@ -236,8 +258,44 @@ function isTimeGrain(grain: string | undefined): grain is TimeGrain { } /** - * Table C (`required` of a dimension). Without `@spine`: only a dimension with no `@via` - * over an `@of` field that is effectively required (`@required`, or a `validator.required` child). With `@spine`: only a dimension + * True when every hop of `hops`, walked from `from`, is a belongs-to hop over a required + * foreign key: the hops the view joins INNER, so every row of the report has a match. This + * is the join-type rule of the view lowering (codegen-ts `walkViaPath`, #209): the reference + * is held by the entity the hop leaves, its first FK field is required, and every earlier hop + * is INNER too. `@enforce: false` does not change it. A hop that does not resolve, an + * ambiguous reference or a has-many hop is not INNER. + */ +function viaJoinsInner(hops: readonly string[], from: MetaObject, root: MetaRoot): boolean { + let current = from; + for (const hopName of hops) { + const resolved = resolveViaHop(current, hopName, root); + if (resolved === undefined || resolved.target === undefined) return false; + const { hop, isReference, target } = resolved; + + // The reference the hop crosses, as the lowering resolves it. A has-many hop, whose FK + // sits on the child, matches none of the current entity's own references. + let reference: MetaReferenceIdentity | undefined; + if (isReference) { + reference = hop as unknown as MetaReferenceIdentity; + } else { + const sourceRefField = hop.attr(RELATIONSHIP_ATTR_SOURCE_REF_FIELD); + reference = resolveRelationshipReference( + current, hopName, target.name, typeof sourceRefField === "string" ? sourceRefField : undefined, + ); + } + if (reference === undefined) return false; + const fkName = reference.fields[0]; + const fk = fkName === undefined ? undefined : current.findField(fkName); + if (fk === undefined || !fk.isRequired) return false; + current = target; + } + return true; +} + +/** + * Table C (`required` of a dimension). Without `@spine`: only a dimension over an `@of` field + * that is effectively required (`@required`, or a `validator.required` child), and, when it has a `@via`, only when every hop is a + * required belongs-to reference, the hops the view joins INNER ({@link viaJoinsInner}). With `@spine`: only a dimension * whose `@via` hops equal the spine's (a column of the spine entity itself, whose rows are * the report's rows) over an `@of` field that is `@required` or a primary-key column of the * entity `@of` names. A dimension beyond the spine is reached by a LEFT OUTER join. @@ -252,7 +310,13 @@ function dimensionRequired( reportName: string, ): boolean { const via = dim.via(); - if (spine === undefined) return via === undefined && of.isRequired; + if (spine === undefined) { + if (via === undefined) return of.isRequired; + if (!of.isRequired) return false; + const hops = reportingViaHops(via, reportingMemberOwner(dim, from), from, root); + if (hops === undefined) throw unresolved(reportName, `dimension '${dim.name}' @via '${via}'`); + return viaJoinsInner(hops, from, root); + } if (via === undefined) return false; // a column of @from: null in a spine row with no facts const hops = reportingViaHops(via, reportingMemberOwner(dim, from), from, root); if (hops === undefined) throw unresolved(reportName, `dimension '${dim.name}' @via '${via}'`); diff --git a/server/typescript/packages/metadata/test/report-read-model.test.ts b/server/typescript/packages/metadata/test/report-read-model.test.ts index c895d9db9..661009162 100644 --- a/server/typescript/packages/metadata/test/report-read-model.test.ts +++ b/server/typescript/packages/metadata/test/report-read-model.test.ts @@ -80,7 +80,7 @@ describe("reportReadModel (FR-044 Table B as a detached read model)", () => { const m = model(root, "ProgramMinutes"); const required = Object.fromEntries(m.fields().map((f) => [f.name, f.attr(FIELD_ATTR_REQUIRED)])); expect(required["program"]).toBe(true); // no @via, @of required - expect(required["programTitle"]).toBe(false); // reached by @via + expect(required["programTitle"]).toBe(true); // reached by @via over a required reference, an INNER join (D5) expect(required["weeks"]).toBe(true); // a count is never null expect(required["minMinutes"]).toBe(false); // Week.durationMinutes is required; a min is not }); diff --git a/server/typescript/packages/metadata/test/report-shape.test.ts b/server/typescript/packages/metadata/test/report-shape.test.ts index e4cb78ffc..291aac5d3 100644 --- a/server/typescript/packages/metadata/test/report-shape.test.ts +++ b/server/typescript/packages/metadata/test/report-shape.test.ts @@ -12,6 +12,7 @@ import { loadUris, measureDerivedSubType, reportMeasureItemName, + reportReadModel, reportShape, reportSpineEntity, reportSpineHops, @@ -559,3 +560,126 @@ describe("measureDerivedSubType (Table B for one measure, listed in a report or } }); }); + +// --------------------------------------------------------------------------- +// Table C, a dimension reached through `@via` (adopter report D5). The view joins a +// belongs-to hop over a required foreign key INNER, so every row has a match and the column +// is as required as the field it reads. +// --------------------------------------------------------------------------- + +describe("reportShape required of a dimension reached through @via", () => { + const entity = (name: string, table: string, children: unknown[]) => ({ + "object.entity": { + name, + children: [ + { "source.rdb": { "@table": table } }, + { "field.long": { name: "id" } }, + { "identity.primary": { name: "pk", "@fields": ["id"] } }, + ...children, + ], + }, + }); + const realm = entity("Realm", "realms", [ + { "field.string": { name: "name", "@required": true } }, + { "field.string": { name: "motto" } }, + ]); + const session = entity("Session", "sessions", [ + { "field.long": { name: "realmId", "@required": true } }, + { "field.long": { name: "maybeRealmId" } }, + { "identity.reference": { name: "fkRealm", "@fields": ["realmId"], "@references": "Realm" } }, + { "identity.reference": { name: "fkMaybeRealm", "@fields": ["maybeRealmId"], "@references": "Realm" } }, + { "relationship.association": { name: "realm", "@objectRef": "Realm", "@cardinality": "one", "@sourceRefField": "realmId" } }, + { + "relationship.association": { + name: "maybeRealm", "@objectRef": "Realm", "@cardinality": "one", "@sourceRefField": "maybeRealmId", + }, + }, + ]); + const snap = entity("Snap", "snaps", [ + { "field.long": { name: "sessionId", "@required": true } }, + { "field.long": { name: "unenforcedSessionId", "@required": true } }, + { "identity.reference": { name: "fkSession", "@fields": ["sessionId"], "@references": "Session" } }, + { + "identity.reference": { + name: "fkLoose", "@fields": ["unenforcedSessionId"], "@references": "Session", "@enforce": false, + }, + }, + { "relationship.association": { name: "session", "@objectRef": "Session", "@cardinality": "one" } }, + { "relationship.association": { name: "stamp", "@objectRef": "Stamp", "@cardinality": "one" } }, + { "measure.aggregate": { name: "snaps", "@agg": "count", "@of": "Snap.id" } }, + { "dimension.attribute": { name: "realmName", "@of": "Realm.name", "@via": "Snap.fkSession.fkRealm" } }, + { "dimension.attribute": { name: "realmMotto", "@of": "Realm.motto", "@via": "Snap.fkSession.fkRealm" } }, + { "dimension.attribute": { name: "maybeRealmName", "@of": "Realm.name", "@via": "Snap.fkSession.fkMaybeRealm" } }, + { "dimension.attribute": { name: "looseRealmName", "@of": "Realm.name", "@via": "Snap.fkLoose.fkRealm" } }, + { "dimension.attribute": { name: "sessionRealmId", "@of": "Session.realmId", "@via": "Snap.fkSession" } }, + { "dimension.attribute": { name: "relRealmName", "@of": "Realm.name", "@via": "Snap.session.realm" } }, + { "dimension.attribute": { name: "relMaybeRealmName", "@of": "Realm.name", "@via": "Snap.session.maybeRealm" } }, + { "dimension.attribute": { name: "relSessionRealmId", "@of": "Session.realmId", "@via": "Snap.session" } }, + { "dimension.attribute": { name: "mixedRealmName", "@of": "Realm.name", "@via": "Snap.session.fkRealm" } }, + { "dimension.attribute": { name: "stampLabel", "@of": "Stamp.label", "@via": "Snap.stamp" } }, + ]); + // The foreign key between Snap and Stamp is held by Stamp, the far side of Snap's `stamp`. + const stamp = entity("Stamp", "stamps", [ + { "field.long": { name: "snapId", "@required": true } }, + { "field.string": { name: "label", "@required": true } }, + { "identity.reference": { name: "fkSnap", "@fields": ["snapId"], "@references": "Snap" } }, + ]); + const realmReport = (name: string, extra: Record = {}) => ({ + "object.report": { + name, + "@from": "Snap", + "@dimensions": ["realmName", "realmMotto", "maybeRealmName", "looseRealmName", "sessionRealmId"], + "@measures": ["snaps"], + ...extra, + children: [{ "source.rdb": { "@kind": "view", "@view": `v_${name.toLowerCase()}` } }], + }, + }); + const requiredOf = (root: MetaRoot, name: string): Record => + Object.fromEntries(reportShape(report(root, name), root).fields.map((f) => [f.name, f.required])); + + test("required when the target field and every hop's foreign key are required", async () => { + const root = await loadInline([file("a", [realm, session, snap, stamp, realmReport("Plain")])]); + const req = requiredOf(root, "Plain"); + expect(req["realmName"]).toBe(true); // two required hops over a required field + expect(req["sessionRealmId"]).toBe(true); // one required hop + expect(req["looseRealmName"]).toBe(true); // @enforce: false does not change the join type + expect(req["realmMotto"]).toBe(false); // the target field is optional + expect(req["maybeRealmName"]).toBe(false); // a hop over a nullable foreign key is a LEFT join + }); + + // The same rule when a hop is a `@cardinality: one` relationship instead of the reference it + // crosses: the relationship resolves to that reference (by name pairing for `session`, by + // `@sourceRefField` for the two onto Realm), and the foreign key it holds decides the join. + test("a to-one relationship hop follows the reference it crosses", async () => { + const related = realmReport("Related", { + "@dimensions": ["relRealmName", "relMaybeRealmName", "relSessionRealmId", "mixedRealmName", "stampLabel"], + }); + const root = await loadInline([file("a", [realm, session, snap, stamp, related])]); + const req = requiredOf(root, "Related"); + expect(req["relRealmName"]).toBe(true); // two relationship hops, both over a required foreign key + expect(req["relSessionRealmId"]).toBe(true); // one relationship hop + expect(req["mixedRealmName"]).toBe(true); // a relationship hop, then a reference hop + expect(req["relMaybeRealmName"]).toBe(false); // the second relationship crosses a nullable foreign key + // The other entity holds the foreign key: a Snap with no Stamp has no match, so the join is + // LEFT whatever that key and the target field require. + expect(req["stampLabel"]).toBe(false); + }); + + // D6: the two accessors an adopter's schema digest needs, pinned so the docs stay true. + test("fields() on a report is empty; the read model carries its columns; referenceIdentities() exposes enforce", async () => { + const root = await loadInline([file("a", [realm, session, snap, stamp, realmReport("Plain")])]); + expect(report(root, "Plain").fields()).toEqual([]); + expect(reportReadModel(report(root, "Plain"), root).fields().map((f) => f.name)).toEqual( + ["realmName", "realmMotto", "maybeRealmName", "looseRealmName", "sessionRealmId", "snaps"], + ); + const refs = Object.fromEntries(report(root, "Snap").referenceIdentities().map((r) => [r.name, r.enforce])); + expect(refs).toEqual({ fkSession: true, fkLoose: false }); + }); + + test("a @spine report keeps its own rule: every join is LEFT", async () => { + const root = await loadInline([file("a", [realm, session, snap, stamp, realmReport("Spined", { "@spine": "Snap.fkSession", "@dimensions": ["realmName", "sessionRealmId"] })])]); + const req = requiredOf(root, "Spined"); + expect(req["realmName"]).toBe(false); + expect(req["sessionRealmId"]).toBe(true); // on the spine, a required field + }); +}); diff --git a/server/typescript/packages/migrate-ts/src/diff/index.ts b/server/typescript/packages/migrate-ts/src/diff/index.ts index 3d34d7781..92676d8b5 100644 --- a/server/typescript/packages/migrate-ts/src/diff/index.ts +++ b/server/typescript/packages/migrate-ts/src/diff/index.ts @@ -11,7 +11,7 @@ import { applyStatus } from "./status.js"; import { PrimaryKeyChangeError } from "../errors.js"; import { detectColumnRenames, detectTableRenames } from "./rename-heuristic.js"; import { firstViewSqlDifference, viewSqlEquals } from "../view-sql-compare.js"; -import { viewReplaceIsLegal } from "../view-column-types.js"; +import { viewReplaceIsLegal, type ViewReplaceProbe } from "../view-replace.js"; import { checkExprEquals, checkExprRespelledNullSafe, normalizeCheckExpr, renameExprIdentifiers } from "../check-expr-compare.js"; import { isPgAutoSequenceDefault } from "../pg-identity-default.js"; import { DEFAULT_DB_SCHEMA_POSTGRES } from "@metaobjectsdev/metadata"; @@ -71,6 +71,13 @@ export interface DiffArgs { unmanagedNames?: string[]; /** Dialect; CHECK-constraint evolution on existing tables is emitted for postgres only. */ dialect?: Dialect; + /** + * Postgres: asks the live database whether a changed view can be replaced in place + * (`postgresViewReplaceProbe`). A view is planned as `replace-view` only when the probe + * proves `CREATE OR REPLACE VIEW` legal. Without a probe (an offline plan against the + * committed snapshot), or when it cannot answer, the plan is DROP + CREATE. + */ + viewReplaceProbe?: ViewReplaceProbe; /** * #258 — refuse (throw {@link PrimaryKeyChangeError}) when an existing table's live * PRIMARY KEY differs from the metadata identity. There is no primary-key change kind @@ -254,7 +261,7 @@ export async function diff( const actualViewsInScope = args.actual.views.filter( (v) => inScope(v.schema) && !unmanaged.has(viewIdentity(v)), ); - diffViews(expectedViewsInScope, actualViewsInScope, changes, args.dialect); + await diffViews(expectedViewsInScope, actualViewsInScope, changes, args.dialect, args.viewReplaceProbe); // Pass 2c: a change that disturbs the table under a view forces the view to be // dropped before and recreated after — postgres blocks ALTER on a column a view @@ -876,10 +883,11 @@ const viewIdentity = qualifiedDbName; * An unknown dialect keeps the old body comparison (legacy positional callers hold * hand-built snapshots; changing their semantics silently is worse than leaving them). */ -function diffViews( +async function diffViews( expected: ViewDescriptor[], actual: ViewDescriptor[], changes: Change[], dialect: Dialect | undefined, -): void { + probe: ViewReplaceProbe | undefined, +): Promise { const exp = new Map(expected.map((v) => [viewIdentity(v), v] as const)); const act = new Map(actual.map((v) => [viewIdentity(v), v] as const)); @@ -904,11 +912,11 @@ function diffViews( // closed: propose the adoption but BLOCK it pending `allow.adoptView`. // // #239: route through the SAME legal/illegal OR-REPLACE decision the managed - // path makes — a structural change (rename/reorder/mid-insert) is not a legal - // CREATE OR REPLACE and must be drop+create. `unmanaged: true` carries the + // path makes — a structural change (rename/reorder/mid-insert/retype) is not a + // legal CREATE OR REPLACE and must be drop+create. `unmanaged: true` carries the // adopt-view gate onto whichever change kind we emit. if (a.fingerprint === undefined) { - pushViewUpdate(v, a, changes, /* unmanaged */ true); + await pushViewUpdate(v, a, changes, probe, /* unmanaged */ true); continue; } @@ -916,9 +924,9 @@ function diffViews( // no-op on an unchanged schema. if (a.fingerprint === v.fingerprint) continue; - // Stamped and different → the view genuinely changed. Prefer a non-destructive - // CREATE OR REPLACE; fall back to drop+create only when Postgres would refuse it. - pushViewUpdate(v, a, changes); + // Stamped and different → the view genuinely changed. A non-destructive + // CREATE OR REPLACE when Postgres is shown to accept it; drop+create otherwise. + await pushViewUpdate(v, a, changes, probe); continue; } @@ -944,48 +952,52 @@ function diffViews( } /** - * Emit either a non-destructive replace, or the drop+create pair Postgres forces. + * Emit either a non-destructive replace, or the drop+create pair. * * Replace is strictly better when it is legal: dependent views, grants, and the - * object's OID all survive. And it IS legal for the common case by construction — a - * view's columns come out in projection DECLARATION order, so a field APPENDED to a - * projection lands last, which is exactly what Postgres's prefix rule permits. + * object's OID all survive. Whether it is legal is Postgres's to say, so the probe asks + * it (view-replace.ts): the live view's columns must be a prefix of the expected view's, + * by name, type and collation. A view's columns come out in projection DECLARATION + * order, so a field APPENDED to a projection lands last, which is what the rule permits. + * + * A replace is planned ONLY on that evidence. No probe (an offline plan), or a probe + * that could not answer (no permission, a definition that does not compile against the + * schema as it stands), plans the drop+create pair: it always applies, and it is loud + * about what depends on the view. This holds for a managed change and for an adoption, + * and for a derived body and an opaque `@sql` one alike. + * + * The probe answers for the schema as it stands, and the replace runs after this plan's + * table changes. A body that reads columns it does not name (`SELECT *`) has a different + * column list once the plan adds a column, so the answer would not hold: it is not asked. + * Every other table change that alters a view's columns is a type change, a drop or a + * rename, which Pass 2c turns into the pair. */ -function pushViewUpdate( +async function pushViewUpdate( expected: ViewDescriptor, actual: ViewDescriptor, changes: Change[], + probe: ViewReplaceProbe | undefined, // #239: when the ACTUAL view is unmanaged (no fingerprint), this is an ADOPTION — // stamp the adopt-view gate (`unmanagedActual`) onto whichever change we emit so // status.ts fails closed on `allow.adoptView`, legal-replace and drop+create alike. unmanaged = false, -): void { +): Promise { const sx = schemaSpread(expected.schema); const adopt = unmanaged ? { unmanagedActual: true as const } : {}; // Postgres compares fingerprints, never text: an unstamped view cannot be compared at all. const reason: ViewChangeReason = unmanaged ? { kind: "unfingerprinted" } : { kind: "definition", compared: "fingerprint" }; - // #239/#240: emit a replace only when it is a legal CREATE OR REPLACE. The decision - // keys on the EXPECTED (desired) view's column knowledge, NOT on both sides: - // - EXPECTED columns KNOWN (a projection): run viewReplaceIsLegal. It fails safe to - // `false` (→ drop+create) when the ACTUAL columns are unknown too — which is - // exactly the OFFLINE adopt case (#240): a pre-fingerprint snapshot records no - // view columns, yet the projection's desired shape IS known, so a structural - // change must still drop+create, never an illegal OR REPLACE. (Also fails safe - // when actual is known but non-prefix — the #239 online case.) - // - EXPECTED columns UNKNOWN (an opaque `@sql` body): legality is unprovable. A - // MANAGED change drops+creates (a real body change is safest rebuilt); an ADOPTION - // keeps the non-destructive replace (the common "re-stamp an identical, - // pre-fingerprint hand-written view" case, #208 — its behavior before #239). - const legal = - expected.columns !== undefined - ? viewReplaceIsLegal(expected.columns, actual.columns) - : unmanaged; - if (legal) { - changes.push({ kind: "replace-view", view: expected, ...sx, restore: actual, ...adopt, reason, status: ALLOWED }); + const columnsMayGrow = expected.sql !== undefined && viewReadsEveryColumn(expected.sql) + && changes.some((c) => c.kind === "add-column"); + const evidence = probe === undefined || columnsMayGrow ? undefined : await probe(expected, actual); + if (evidence !== undefined && viewReplaceIsLegal(evidence.expected, evidence.live)) { + changes.push({ + kind: "replace-view", view: expected, ...sx, restore: actual, ...adopt, reason, + proven: { restorable: viewReplaceIsLegal(evidence.live, evidence.expected) }, + status: ALLOWED, + }); return; } - // The column list changed shape (removed / renamed / reordered / retyped), so - // Postgres refuses OR REPLACE. The view must be dropped and rebuilt — which is + // Not shown to be a legal replace. The view is dropped and rebuilt — which is // destructive to anything depending on it (annotateViewDropDependents makes that loud). changes.push({ kind: "drop-view", view: expected.name, ...sx, restore: actual, ...adopt, reason, status: ALLOWED }); changes.push({ kind: "create-view", view: expected, ...sx, reason, status: ALLOWED }); @@ -1036,13 +1048,22 @@ function annotateViewDropDependents(actual: readonly ViewDescriptor[], changes: } /** - * Postgres: a view reading a column blocks `ALTER … ALTER COLUMN`, so a column-altering - * change forces the view to be dropped-before / recreated-after. FK/CHECK changes are + * Postgres: a view reading a column blocks `ALTER … ALTER COLUMN TYPE`, a drop and a rename + * of it, so those force the view to be dropped-before / recreated-after. A new default or a + * changed nullability alters in place under a view. FK/CHECK changes are * `ALTER TABLE ADD/DROP CONSTRAINT` — no table rebuild — so a dependent view is unaffected. */ const VIEW_RECREATE_TRIGGERS_PG = new Set([ - "change-column-type", "change-column-nullable", "change-column-default", - "drop-column", "rename-column", + "change-column-type", "drop-column", "rename-column", +]); +/** + * No dialect: the caller has not said which emitter runs, and it may be SQLite's, which + * rebuilds the table for a nullable or default change too. Every column-altering kind + * recreates every view over the table. + */ +const VIEW_RECREATE_TRIGGERS_UNKNOWN_DIALECT = new Set([ + ...VIEW_RECREATE_TRIGGERS_PG, + "change-column-nullable", "change-column-default", ]); /** * SQLite / D1: recreate-and-copy DROPs + RENAMEs the whole table for ANY of these @@ -1051,39 +1072,131 @@ const VIEW_RECREATE_TRIGGERS_PG = new Set([ * CHECK, or an FK change. So the FK/CHECK kinds join the column-altering set here. (#243) */ const VIEW_RECREATE_TRIGGERS_SQLITE = new Set([ - ...VIEW_RECREATE_TRIGGERS_PG, + ...VIEW_RECREATE_TRIGGERS_UNKNOWN_DIALECT, "add-fk", "drop-fk", "add-check", "drop-check", ]); +/** The column names a change touches, or undefined for a change that is table-wide. */ +function changedColumns(c: Change): string[] | undefined { + switch (c.kind) { + case "change-column-type": + case "change-column-nullable": + case "change-column-default": + case "drop-column": + return [c.column]; + case "rename-column": + return [c.from, c.to]; + default: + return undefined; + } +} + +/** `count(*)` has a star but does not read every column; strip it before the `*` test. */ +const COUNT_STAR_RE = /count\s*\(\s*\*\s*\)/gi; +/** `NATURAL JOIN` joins on every column the two tables share, naming none of them. */ +const NATURAL_JOIN_RE = /\bNATURAL\b/i; + +/** + * Does the body read columns it does not name? A `*` in the select list reads every column, + * and a `NATURAL JOIN` reads each shared column. Both are a property of the body. Generated + * projection and report bodies use neither; a hand-written `@sql` body may. + */ +function viewReadsEveryColumn(sql: string): boolean { + return /\*/.test(sql.replace(COUNT_STAR_RE, "")) || NATURAL_JOIN_RE.test(sql); +} + +/** + * Per-column identifier matches, compiled once per column name: a wide schema diff revisits + * the same column names across tables and views, and recompiling a RegExp per test would + * rebuild identical state over and over. + */ +const columnIdentifierRes = new Map(); +function columnIdentifierRe(column: string): RegExp { + let re = columnIdentifierRes.get(column); + if (re === undefined) { + const escaped = column.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + re = new RegExp(`(?, readsEverything: boolean): boolean { + if (readsEverything) return true; + for (const col of cols) { + if (columnIdentifierRe(col).test(sql)) return true; + } + return false; +} + function recreateViewsDependingOnChangedTables( expectedViews: ViewDescriptor[], actualViews: ViewDescriptor[], changes: Change[], dialect: Dialect | undefined, ): void { - // Only the recreate-and-copy dialects rebuild the table (and strand the view) for an - // FK/CHECK change; postgres and the dialect-less legacy path keep the narrow set. - const triggers = dialect === "sqlite" || dialect === "d1" - ? VIEW_RECREATE_TRIGGERS_SQLITE - : VIEW_RECREATE_TRIGGERS_PG; + // Only Postgres is known to alter a column in place, so only there is a view stranded by + // the columns it reads. Everywhere else a triggering change strands every view over the + // table: the recreate-and-copy dialects rebuild it, and without a dialect they may. + const columnScoped = dialect === "postgres"; + const triggers = dialect === "postgres" + ? VIEW_RECREATE_TRIGGERS_PG + : dialect === undefined ? VIEW_RECREATE_TRIGGERS_UNKNOWN_DIALECT : VIEW_RECREATE_TRIGGERS_SQLITE; const actualById = new Map(actualViews.map((v) => [viewIdentity(v), v] as const)); - const alteredTables = new Set(); + // table -> the columns altered on it, or `null` when a change rebuilds the whole table. + const alteredTables = new Map | null>(); for (const c of changes) { - if (triggers.has(c.kind)) { - const t = (c as { table?: string }).table; - if (typeof t === "string") alteredTables.add(t); + if (!triggers.has(c.kind)) continue; + const t = (c as { table?: string }).table; + if (typeof t !== "string") continue; + const cols = changedColumns(c); + if (cols === undefined || !columnScoped) { + alteredTables.set(t, null); // a rebuild strands every view over the table + } else if (alteredTables.get(t) !== null) { + const set = alteredTables.get(t) ?? new Set(); + for (const col of cols) set.add(col); + alteredTables.set(t, set); } } if (alteredTables.size === 0) return; + // View identities already queued for a definition change, collected once instead of + // rescanning `changes` per expected view over a diff that can carry hundreds of changes. + const definitionQueued = new Set(); + const createQueued = new Set(); + for (const c of changes) { + if (c.kind !== "replace-view" && c.kind !== "create-view") continue; + const queuedId = viewIdentity(c.view); + definitionQueued.add(queuedId); + if (c.kind === "create-view") createQueued.add(queuedId); + } + for (const v of expectedViews) { - const tables = (v.dependsOn ?? []).filter((t) => alteredTables.has(t)); - if (tables.length === 0) continue; const id = viewIdentity(v); + const viewChanges = definitionQueued.has(id); + const sql = v.sql; + const readsEverything = sql !== undefined && viewReadsEveryColumn(sql); + const tables = (v.dependsOn ?? []).filter((t) => { + if (!alteredTables.has(t)) return false; + // A view whose own definition is unchanged is stranded only if it READS an altered + // column. One already queued for a replace (its definition differs) is recreated + // regardless: the old definition may read a column the new one does not. + const cols = alteredTables.get(t) ?? null; + if (cols === null || viewChanges || sql === undefined) return true; + return viewReadsAnyColumn(sql, cols, readsEverything); + }); + if (tables.length === 0) continue; // Brand-new view (create-view already queued): the DB has no prior view to // block the ALTER, and the queued create-view already builds it post-change. - if (changes.some((c) => c.kind === "create-view" && viewIdentity(c.view) === id)) continue; + if (createQueued.has(id)) continue; // Supersede any replace-view (CREATE OR REPLACE can't run mid-ALTER) with an // explicit drop(before)/create(after) pair — the emit STAGE_ORDER sequences diff --git a/server/typescript/packages/migrate-ts/src/diff/status.ts b/server/typescript/packages/migrate-ts/src/diff/status.ts index f232cede2..a545e6b61 100644 --- a/server/typescript/packages/migrate-ts/src/diff/status.ts +++ b/server/typescript/packages/migrate-ts/src/diff/status.ts @@ -33,39 +33,46 @@ export function applyStatus(changes: Change[], allow: AllowOptions = {}): void { } } for (const c of changes) { - const blockedReason = blockedReasonFor(c, allow, recreatedViews); - if (blockedReason !== null) { - c.status = { state: "blocked", blockedReason }; - } else { - c.status = { state: "allowed" }; - } + const block = blockFor(c, allow, recreatedViews); + c.status = block !== null ? { state: "blocked", ...block } : { state: "allowed" }; } } -function blockedReasonFor( +/** Why a change is blocked, and the one `AllowOptions` field that lifts that block. */ +interface Block { + blockedBy: keyof AllowOptions; + blockedReason: string; +} + +/** Null when `allow[blockedBy]` is granted, else the block naming it. */ +function gate(allow: AllowOptions, blockedBy: keyof AllowOptions, blockedReason: string): Block | null { + return allow[blockedBy] ? null : { blockedBy, blockedReason }; +} + +function blockFor( c: Change, allow: AllowOptions, recreatedViews: ReadonlySet, -): string | null { +): Block | null { switch (c.kind) { case "drop-column": - return allow.dropColumn ? null : "destructive: drop-column not allowed (pass allow.dropColumn)"; + return gate(allow, "dropColumn", "destructive: drop-column not allowed (pass allow.dropColumn)"); case "drop-table": - return allow.dropTable ? null : "destructive: drop-table not allowed (pass allow.dropTable)"; + return gate(allow, "dropTable", "destructive: drop-table not allowed (pass allow.dropTable)"); case "drop-index": // Carried through a rename-column: the same rule over the same data — nothing is lost. if (c.carriedByRename === true) return null; - return allow.dropIndex ? null : "destructive: drop-index not allowed (pass allow.dropIndex)"; + return gate(allow, "dropIndex", "destructive: drop-index not allowed (pass allow.dropIndex)"); case "drop-fk": // Carried through a rename-column: the same rule over the same data — nothing is lost. if (c.carriedByRename === true) return null; - return allow.dropFk ? null : "destructive: drop-fk not allowed (pass allow.dropFk)"; + return gate(allow, "dropFk", "destructive: drop-fk not allowed (pass allow.dropFk)"); case "drop-check": // Carried through a rename-column: the same rule over the same data — nothing is lost. if (c.carriedByRename === true) return null; // Re-spelled for older SQLite: the same rule comes straight back. if (c.respelled === true) return null; - return allow.dropCheck ? null : "destructive: drop-check not allowed (pass allow.dropCheck)"; + return gate(allow, "dropCheck", "destructive: drop-check not allowed (pass allow.dropCheck)"); case "drop-view": { // A drop that would destroy relations we do NOT manage is gated INDEPENDENTLY of // whether our own view survives. Even the recreate pair — where our view comes @@ -74,10 +81,13 @@ function blockedReasonFor( // FIRST: it is the one thing `allow.dropView` must never be able to wave through. const external = c.dependents ?? []; if (external.length > 0 && !allow.dropViewCascade) { - return `destructive: dropping view "${c.view}" would CASCADE into ${external.length} object(s) ` - + `MetaObjects does not manage and cannot restore ` - + `(${external.map((d) => `${d.schema}.${d.name}`).join(", ")}) ` - + `— pass allow.dropViewCascade to destroy them, or migrate them off this view first`; + return { + blockedBy: "dropViewCascade", + blockedReason: `destructive: dropping view "${c.view}" would CASCADE into ${external.length} object(s) ` + + `MetaObjects does not manage and cannot restore ` + + `(${external.map((d) => `${d.schema}.${d.name}`).join(", ")}) ` + + `— pass allow.dropViewCascade to destroy them, or migrate them off this view first`, + }; } // #239: the drop half of ADOPTING an unmanaged view (no fingerprint) whose shape // changed too much for a legal CREATE OR REPLACE. It is paired with a create-view, @@ -85,13 +95,16 @@ function blockedReasonFor( // hand-written) SQL needs explicit consent, exactly like the replace-view adopt // path. Gate on allow.adoptView BEFORE the recreate-pair auto-allow. if (c.unmanagedActual && !allow.adoptView) { - return `existing view "${c.view}" carries no MetaObjects fingerprint — it is hand-written, ` - + `or was created before view fingerprinting. Its shape changed, so adopting it means ` - + `DROP + rebuild, which takes ownership and cannot be undone. Pass allow.adoptView to adopt it`; + return { + blockedBy: "adoptView", + blockedReason: `existing view "${c.view}" carries no MetaObjects fingerprint — it is hand-written, ` + + `or was created before view fingerprinting. Its shape changed, so adopting it means ` + + `DROP + rebuild, which takes ownership and cannot be undone. Pass allow.adoptView to adopt it`, + }; } // Identity, not bare name — see viewIdentity(). if (recreatedViews.has(viewIdentity(c.view, c.schema))) return null; // recreate pair — view survives - return allow.dropView ? null : "destructive: drop-view not allowed (pass allow.dropView)"; + return gate(allow, "dropView", "destructive: drop-view not allowed (pass allow.dropView)"); } case "replace-view": @@ -103,9 +116,12 @@ function blockedReasonFor( // Every environment upgrading from a pre-fingerprint toolchain hits this exactly // once, then its views are stamped and it never fires again. if (c.unmanagedActual && !allow.adoptView) { - return `existing view "${c.view.name}" carries no MetaObjects fingerprint — it is hand-written, ` - + `or was created before view fingerprinting. Overwriting it takes ownership and cannot be undone. ` - + `Pass allow.adoptView to adopt it`; + return { + blockedBy: "adoptView", + blockedReason: `existing view "${c.view.name}" carries no MetaObjects fingerprint — it is hand-written, ` + + `or was created before view fingerprinting. Overwriting it takes ownership and cannot be undone. ` + + `Pass allow.adoptView to adopt it`, + }; } return null; @@ -115,7 +131,7 @@ function blockedReasonFor( const refusal = autoSequenceDropRefusal(c.table, c.column, c.fromDefault, c.toDefault, allow); if (refusal !== null) return refusal; if (isWidening(c.from, c.to)) return null; // widening always allowed - return allow.typeChange ? null : "lossy type change (pass allow.typeChange)"; + return gate(allow, "typeChange", "lossy type change (pass allow.typeChange)"); } case "change-column-nullable": @@ -123,7 +139,7 @@ function blockedReasonFor( // notnull (false) → nullable (true): allowed // nullable (true) → notnull (false): requires flag (existing data must satisfy) if (c.from === false && c.to === true) return null; - return allow.nullableToNotNull ? null : "nullable→notnull requires existing data to satisfy (pass allow.nullableToNotNull)"; + return gate(allow, "nullableToNotNull", "nullable→notnull requires existing data to satisfy (pass allow.nullableToNotNull)"); case "change-column-default": { // Ordinary default changes (literal→literal, adding/removing a plain @@ -165,14 +181,14 @@ function blockedReasonFor( */ function autoSequenceDropRefusal( table: string, column: string, from: ColumnDefault | undefined, to: ColumnDefault | undefined, - allow: { dropIdentityDefault?: boolean }, -): string | null { - if (!dropsAutoSequenceDefault(from, to) || allow.dropIdentityDefault) return null; - return `column "${table}"."${column}" has a live Postgres auto-increment default ` + allow: AllowOptions, +): Block | null { + if (!dropsAutoSequenceDefault(from, to)) return null; + return gate(allow, "dropIdentityDefault", `column "${table}"."${column}" has a live Postgres auto-increment default ` + `(${from!.value}) but its metadata declares no @generation — this is ambiguous: it ` + `could mean @generation was never declared, or that auto-increment is being removed on ` + `purpose. Dropping the default is destructive (every insert that omits the column starts ` + `failing), so this refuses rather than guessing. Declare @generation: increment on the ` + `identity to keep the sequence, or pass --allow drop-identity-default if removing it is ` - + `intentional`; + + `intentional`); } diff --git a/server/typescript/packages/migrate-ts/src/drift/drift.ts b/server/typescript/packages/migrate-ts/src/drift/drift.ts index ffab2090a..06679841f 100644 --- a/server/typescript/packages/migrate-ts/src/drift/drift.ts +++ b/server/typescript/packages/migrate-ts/src/drift/drift.ts @@ -23,6 +23,8 @@ import { collectUnmanagedNames } from "../unmanaged.js"; import { withoutViewRecreates } from "../view-change-reason.js"; import { scopeExpectedSchema, scopedDiffInputs, type ObjectScopePredicate } from "../scope.js"; import type { AllowOptions, Dialect, DiffResult, SchemaSnapshot } from "../types.js"; +import { postgresViewReplaceProbe } from "../view-replace-probe.js"; +import type { ViewReplaceProbe } from "../view-replace.js"; export interface ComputeDriftOptions { /** @@ -48,6 +50,13 @@ export interface ComputeDriftOptions { * itself; pass these so view drift is detected. Defaults to none. */ views?: readonly import("../expected-schema.js").ExpectedViewInput[]; + /** + * Postgres: the live-database probe that decides whether a changed view is reported as + * a replace or as a drop and a create (passed through to `diff`). `computeDrift` + * supplies it from its connection; a caller of `computeDriftFromActual` passes its own + * so the drift report names the change `meta migrate` would plan. + */ + viewReplaceProbe?: ViewReplaceProbe; /** * Per-command scope (`migrate.scope`): objects whose declaring FQN this predicate * rejects are governed by somebody else. They leave the expected side AND are @@ -139,6 +148,7 @@ export async function computeDriftFromActual( dialect, allow: opts?.allow ?? {}, ...(opts?.ignoreTables !== undefined ? { ignoreTables: opts.ignoreTables } : {}), + ...(opts?.viewReplaceProbe !== undefined ? { viewReplaceProbe: opts.viewReplaceProbe } : {}), }); return { ...result, @@ -166,5 +176,10 @@ export async function computeDrift( opts?: ComputeDriftOptions, ): Promise { const actual = await introspect(db, dialect); - return computeDriftFromActual(actual, dialect, metadata, opts); + return computeDriftFromActual(actual, dialect, metadata, { + ...opts, + ...(dialect === "postgres" && opts?.viewReplaceProbe === undefined + ? { viewReplaceProbe: postgresViewReplaceProbe(db) } + : {}), + }); } diff --git a/server/typescript/packages/migrate-ts/src/emit/postgres.ts b/server/typescript/packages/migrate-ts/src/emit/postgres.ts index 4da45c7b9..aaf653f02 100644 --- a/server/typescript/packages/migrate-ts/src/emit/postgres.ts +++ b/server/typescript/packages/migrate-ts/src/emit/postgres.ts @@ -5,7 +5,6 @@ import type { import type { SqlType } from "../sql-type.js"; import { DEFAULT_DB_SCHEMA_POSTGRES } from "@metaobjectsdev/metadata"; import { renderFingerprintMarker, viewFingerprint } from "../view-fingerprint.js"; -import { viewReplaceIsLegal } from "../view-column-types.js"; import { columnDefaultsEqual } from "../column-default.js"; import { dataHazardOf } from "../hazards.js"; @@ -176,7 +175,7 @@ function renderUp(c: Change): string { case "drop-check": return `ALTER TABLE IF EXISTS ${quoteQualified(c.table, c.schema)} DROP CONSTRAINT IF EXISTS ${quote(c.check)};`; case "create-view": return renderCreateView(c.view, c.schema, /* orReplace */ false); case "drop-view": return renderDropView(c); - case "replace-view": return renderCreateView(c.view, c.schema, /* orReplace */ true); + case "replace-view": return renderReplaceView(c); } } @@ -285,8 +284,9 @@ function renderDown(c: Change): string { : `-- WARNING: down migration cannot restore the original CHECK definition`; case "create-view": return `DROP VIEW ${quoteQualifiedView(c.view.name, c.schema)};`; // The deparsed body introspection captured IS a valid restore payload. - case "drop-view": return renderRestoreView(c.restore, c.view, c.schema); - case "replace-view": return renderRestoreView(c.restore, c.view.name, c.schema, c.view); + case "drop-view": return renderRestoreView(c.restore, c.view, c.schema, "create"); + case "replace-view": + return renderRestoreView(c.restore, c.view.name, c.schema, c.proven?.restorable === true ? "replace" : "drop-and-create"); } } @@ -626,6 +626,19 @@ function renderCreateView(v: ViewDescriptor, schema: string | undefined, orRepla return `${create}\n${renderViewComment(qualified, renderFingerprintMarker(fingerprint))}`; } +/** + * `CREATE OR REPLACE VIEW` only for a replace the planner proved against the live database + * (`proven`). Postgres refuses a replace that changes an existing column, and it refuses + * at apply, so an unproven one is written as DROP + CREATE, which always applies. The + * DROP is not `CASCADE`: if something depends on the view, Postgres says so instead of + * destroying it. + */ +function renderReplaceView(c: Extract): string { + if (c.proven !== undefined) return renderCreateView(c.view, c.schema, /* orReplace */ true); + const qualified = quoteQualifiedView(c.view.name, c.schema); + return `DROP VIEW IF EXISTS ${qualified};\n${renderCreateView(c.view, c.schema, /* orReplace */ false)}`; +} + function renderViewComment(qualifiedView: string, comment: string | null): string { if (comment === null) return `COMMENT ON VIEW ${qualifiedView} IS NULL;`; return `COMMENT ON VIEW ${qualifiedView} IS '${comment.replace(/'/g, "''")}';`; @@ -678,8 +691,12 @@ function renderRestoreView( restore: ViewDescriptor | undefined, name: string, schema: string | undefined, - /** The view the UP migration left in place — i.e. what the down is replacing. */ - current?: ViewDescriptor, + /** + * `create`: the UP migration dropped the view, so nothing stands at the name. + * `replace`: the planner proved the view the UP migration left has the restored view's + * columns, so it can be replaced in place. `drop-and-create`: it cannot. + */ + how: "create" | "replace" | "drop-and-create", ): string { if (restore?.sql === undefined || restore.sql.trim().length === 0) { return `-- WARNING: down migration cannot restore the original view definition`; @@ -690,10 +707,10 @@ function renderRestoreView( // Postgres's OR-REPLACE prefix rule applies to the down migration too, and usually // REFUSES: undoing an appended field means REMOVING a view column, and // `CREATE OR REPLACE VIEW` cannot drop columns ("cannot drop columns from view"). - // So ask the same question the forward path asks, with the arguments swapped — and - // fall back to DROP + CREATE when the answer is no. + // The planner asked that question with the forward one (`proven.restorable`), and + // `how` carries the answer. // - // The fallback DROP is deliberately NOT `CASCADE`: if a dependent has since been built + // The DROP is deliberately NOT `CASCADE`: if a dependent has since been built // on the newer shape, Postgres refuses the down migration rather than silently // destroying that dependent. Loud beats convenient. const stamp = restore.fingerprint !== undefined @@ -703,8 +720,8 @@ function renderRestoreView( // does not have, and the next migrate would believe the stamp and skip it. : renderViewComment(qualified, null); - if (current !== undefined && !viewReplaceIsLegal(restore.columns, current.columns)) { - return `DROP VIEW ${qualified};\nCREATE VIEW ${qualified} AS\n${body};\n${stamp}`; - } + const create = `CREATE VIEW ${qualified} AS\n${body};\n${stamp}`; + if (how === "create") return create; + if (how === "drop-and-create") return `DROP VIEW ${qualified};\n${create}`; return `CREATE OR REPLACE VIEW ${qualified} AS\n${body};\n${stamp}`; } diff --git a/server/typescript/packages/migrate-ts/src/errors.ts b/server/typescript/packages/migrate-ts/src/errors.ts index d636c6e3a..9e0d8fb77 100644 --- a/server/typescript/packages/migrate-ts/src/errors.ts +++ b/server/typescript/packages/migrate-ts/src/errors.ts @@ -51,11 +51,15 @@ const ALLOW_OPTION_BY_KIND: Partial> = { /** * The `AllowOptions` field that unblocks `c`, chosen by what BLOCKED it rather than by its kind - * alone. A type change carries its column's default change, so one that would drop a live - * auto-sequence default is blocked by THAT, and `typeChange` would not unblock it. The single - * source for this error's hints and the CLI's `--allow` token. + * alone: `diff` records the gate that refused the change on its status (`blockedBy`), and one + * change can sit behind several gates in turn (a view drop: cascade, then adoption, then the + * removal itself). A status that records no gate falls back to the change's kind, where a type + * change carries its column's default change, so one that would drop a live auto-sequence + * default is blocked by THAT, and `typeChange` would not unblock it. The single source for this + * error's hints and the CLI's `--allow` token. */ export function allowOptionFor(c: Change): keyof AllowOptions | undefined { + if (c.status.blockedBy !== undefined) return c.status.blockedBy; if (c.kind === "change-column-type" && dropsAutoSequenceDefault(c.fromDefault, c.toDefault)) { return "dropIdentityDefault"; } diff --git a/server/typescript/packages/migrate-ts/src/expected-schema.ts b/server/typescript/packages/migrate-ts/src/expected-schema.ts index b600b1ea0..c960b1aac 100644 --- a/server/typescript/packages/migrate-ts/src/expected-schema.ts +++ b/server/typescript/packages/migrate-ts/src/expected-schema.ts @@ -66,7 +66,6 @@ import type { import { qualifiedDbName } from "./qualified-name.js"; import { SQLITE_ISO_NOW } from "./column-default.js"; import { viewFingerprint } from "./view-fingerprint.js"; -import { resolveViewColumns, type ExpectedViewColumnInput } from "./view-column-types.js"; import { resolveReferentialActions, findReferentialActionConflict, @@ -100,9 +99,7 @@ export interface BuildExpectedSchemaOptions { * code generator); view SQL has a single source, `emitViewDdl` in codegen-ts. * Defaults to none. * - * The caller supplies each view's output columns PHYSICALLY but untyped - * (codegen-ts knows nothing of SqlType); Pass 4 resolves them against the - * expected tables and computes the view's fingerprint. + * Pass 4 computes each view's fingerprint. */ views?: readonly ExpectedViewInput[]; /** @@ -117,14 +114,13 @@ export interface BuildExpectedSchemaOptions { /** * A view as the caller (codegen-ts `buildProjectionViews`) produces it — structurally * an `ExpectedView`. It becomes a full `ViewDescriptor` in Pass 4, which is where the - * fingerprint is computed and the column types are resolved. + * fingerprint is computed. */ export interface ExpectedViewInput { name: string; schema?: string; sql?: string; dependsOn?: readonly string[]; - columns?: readonly ExpectedViewColumnInput[]; /** * `resolutionKey()` of the object that declared this view — its PROVENANCE. * Recorded in the provenance map and deliberately NEVER copied onto the @@ -373,21 +369,16 @@ export function buildExpectedSchemaWithProvenance( // via codegen-ts's buildProjectionViews, the single view-SQL source). migrate-ts // never generates view DDL itself, keeping it free of a codegen-ts dependency. // - // Two things are derived here rather than by the caller: - // - the FINGERPRINT, a hash of the generated body. It is the only sound way to - // compare a view against Postgres, which deparses view SQL and so can never - // hand back the text we wrote (see view-fingerprint.ts). Computed here so the - // producer and the parser of the marker live in one package. - // - the column SQL TYPES, resolved against the tables built above. They decide - // whether a view change can use a non-destructive CREATE OR REPLACE. + // The FINGERPRINT is derived here rather than by the caller: a hash of the generated + // body. It is the only sound way to compare a view against Postgres, which deparses + // view SQL and so can never hand back the text we wrote (see view-fingerprint.ts). + // Computed here so the producer and the parser of the marker live in one package. const views: ViewDescriptor[] = (opts?.views ?? []).map((v) => { - const columns = resolveViewColumns(v.columns, tables); const descriptor: ViewDescriptor = { name: v.name, ...(v.schema !== undefined ? { schema: v.schema } : {}), ...(v.sql !== undefined ? { sql: v.sql, fingerprint: viewFingerprint(v.sql) } : {}), ...(v.dependsOn !== undefined ? { dependsOn: v.dependsOn } : {}), - ...(columns !== undefined ? { columns } : {}), }; // The declaring FQN goes to the provenance map ONLY — never onto the descriptor, // which is what the committed snapshot serializes (see ExpectedViewInput.fqn). diff --git a/server/typescript/packages/migrate-ts/src/index.ts b/server/typescript/packages/migrate-ts/src/index.ts index 704b5103f..fcd2a29ee 100644 --- a/server/typescript/packages/migrate-ts/src/index.ts +++ b/server/typescript/packages/migrate-ts/src/index.ts @@ -38,7 +38,7 @@ export { emit } from "./emit/index.js"; export { columnTypeSql } from "./column-type-sql.js"; export { writeMigration } from "./write-migration.js"; export { writeMigrationD1 } from "./write-migration-d1.js"; -export { writeMigrationFlyway } from "./write-migration-flyway.js"; +export { writeMigrationFlyway, withoutFlywayHistory, FLYWAY_HISTORY_TABLE } from "./write-migration-flyway.js"; // Reference-snapshot generation (offline, deterministic). export { @@ -66,7 +66,7 @@ export { suggestColumnRenames } from "./diff/rename-suggestions.js"; export type { ColumnRenameSuggestion } from "./diff/rename-suggestions.js"; // SqlType helpers (rarely needed but useful for advanced consumers) -export { isWidening, sqlTypeEquals } from "./sql-type.js"; +export { describeSqlType, isWidening, sqlTypeEquals } from "./sql-type.js"; // Types export type { SqlType } from "./sql-type.js"; @@ -90,6 +90,10 @@ export type { WriteMigrationFlywayOptions, WriteMigrationFlywayResult } from "./ // / view-ddl-{postgres,sqlite} stack was folded into the schema-diff path). export { normalizeViewSql, viewSqlEquals } from "./view-sql-compare.js"; export { isViewRecreateOnly, withoutViewRecreates } from "./view-change-reason.js"; +// Whether a live Postgres view can be replaced in place is asked of the database, in a +// transaction that is always rolled back. Pass the probe to `diff` on a live plan. +export { postgresViewReplaceProbe } from "./view-replace-probe.js"; +export type { ExactViewColumn, ViewReplaceEvidence, ViewReplaceProbe } from "./view-replace.js"; // D1 dialect emitter + safety pass. // renderD1 is exported directly (unlike renderSqlite/renderPostgres) so diff --git a/server/typescript/packages/migrate-ts/src/introspect/postgres.ts b/server/typescript/packages/migrate-ts/src/introspect/postgres.ts index 13dd017a2..1f0d84358 100644 --- a/server/typescript/packages/migrate-ts/src/introspect/postgres.ts +++ b/server/typescript/packages/migrate-ts/src/introspect/postgres.ts @@ -327,13 +327,6 @@ async function readPgViews(k: RawKysely): Promise { // no MetaObjects stamp ⇒ hand-written or pre-fingerprint ⇒ the diff fails closed. const marker = parseFingerprintMarker(r.view_comment); if (marker !== null) view.fingerprint = marker.fingerprint; - // A view is a relation: information_schema.columns describes its output columns - // exactly like a table's, so the same reader gives us the list (and the types) - // that decide whether a CREATE OR REPLACE is legal. - view.columns = (await readColumns(k, r.table_schema, r.table_name)).map((c) => ({ - name: c.name, - sqlType: c.sqlType, - })); views.push(view); } @@ -352,6 +345,33 @@ async function readPgViews(k: RawKysely): Promise { } } +/** + * A view's output columns exactly as Postgres holds them, in order: the name, the type with + * its modifier (`format_type`) and the collation. These three are what `CREATE OR REPLACE + * VIEW` compares, so they are read from the catalog and never mapped to the canonical + * `SqlType`, which merges types the comparison keeps apart (`json` and `jsonb`, `smallint` + * and `integer`, `text` and `character varying`). + * + * `relation` is a qualified, quoted name (`"public"."v_x"`, `pg_temp."probe"`). A relation + * that does not exist rejects. + */ +export async function readPgViewColumns( + db: Kysely>, + relation: string, +): Promise<{ name: string; type: string; collation: string }[]> { + const rows = await sql<{ name: string; type: string; collation: string }>` + SELECT a.attname AS name, + format_type(a.atttypid, a.atttypmod) AS type, + a.attcollation::text AS collation + FROM pg_attribute a + WHERE a.attrelid = ${relation}::regclass + AND a.attnum > 0 + AND NOT a.attisdropped + ORDER BY a.attnum + `.execute(db); + return rows.rows.map((r) => ({ name: r.name, type: r.type, collation: r.collation })); +} + function viewKey(schema: string | undefined, name: string): string { return `${schema ?? DEFAULT_DB_SCHEMA_POSTGRES}.${name}`; } diff --git a/server/typescript/packages/migrate-ts/src/snapshot/serialize.ts b/server/typescript/packages/migrate-ts/src/snapshot/serialize.ts index 8f7b93896..4849395ee 100644 --- a/server/typescript/packages/migrate-ts/src/snapshot/serialize.ts +++ b/server/typescript/packages/migrate-ts/src/snapshot/serialize.ts @@ -6,7 +6,7 @@ import type { SchemaSnapshot } from "../types.js"; * SchemaSnapshot descriptor gains a field (a DDL-coverage feature); add the * matching upgrade branch in parseSnapshot at the same time. * - * v3 — ViewDescriptor gained `fingerprint`, `columns` and `dependents`. The + * v3 — ViewDescriptor gained `fingerprint` and `dependents`. The * fingerprint is how Postgres decides whether a view is up to date (Postgres deparses * view SQL, so the emitted text can never be compared against what it stores). A * toolchain that does not understand these fields would silently fall back to the old @@ -15,10 +15,11 @@ import type { SchemaSnapshot } from "../types.js"; * snapshot stays fine: the missing fields read as `undefined`, which every consumer * treats as "unknown" and fails safe on. * - * NOTE for anyone touching canonicalize(): a view's `columns` array is ORDER-SENSITIVE. - * Postgres allows a non-destructive CREATE OR REPLACE only when the old column list is - * a PREFIX of the new one, so sorting that array would destroy the very information the - * decision rests on. + * A v3 snapshot written before 1.1.0 also records a view's `columns` (the tool's guess at + * the view's column types). Nothing reads it any more: whether a view can be replaced in + * place is asked of the live database (view-replace.ts). It is no longer written, and + * `parseSnapshot` drops it. The version does not move: a reader that still looks for + * the field finds it absent, which it already treats as unknown. */ export const SNAPSHOT_FORMAT_VERSION = 3; @@ -91,5 +92,6 @@ export function parseSnapshot(text: string): SchemaSnapshot { } } } + for (const v of file.snapshot.views ?? []) delete (v as { columns?: unknown }).columns; return file.snapshot; } diff --git a/server/typescript/packages/migrate-ts/src/sql-type.ts b/server/typescript/packages/migrate-ts/src/sql-type.ts index a24a9f081..63bdfd6e6 100644 --- a/server/typescript/packages/migrate-ts/src/sql-type.ts +++ b/server/typescript/packages/migrate-ts/src/sql-type.ts @@ -21,6 +21,24 @@ export type SqlType = | { kind: "inet" } // Postgres-native inet (IPv4/IPv6) — field.inet (ADR-0036/0037 Wave 3) | { kind: "array"; element: SqlType }; // native SQL array (e.g. uuid[], text[]) +/** + * A short human spelling of `t` that keeps what `sqlTypeEquals` compares, so two types that + * differ never print alike: `text(255)` against `text`, `timestamptz` against `timestamp`. + * For messages only; the dialect renderers own the SQL. + */ +export function describeSqlType(t: SqlType): string { + switch (t.kind) { + case "text": return t.maxLength === undefined ? "text" : `text(${t.maxLength})`; + case "integer": return t.bits === 64 ? "bigint" : "integer"; + case "numeric": + return t.precision === undefined ? "numeric" + : `numeric(${t.precision}${t.scale === undefined ? "" : `,${t.scale}`})`; + case "timestamp": return t.withTimezone ? "timestamptz" : "timestamp"; + case "array": return `${describeSqlType(t.element)}[]`; + default: return t.kind; + } +} + /** Structural equality on SqlType. */ export function sqlTypeEquals(a: SqlType, b: SqlType): boolean { if (a.kind !== b.kind) return false; diff --git a/server/typescript/packages/migrate-ts/src/types.ts b/server/typescript/packages/migrate-ts/src/types.ts index 48bbc4e1e..3693a3b63 100644 --- a/server/typescript/packages/migrate-ts/src/types.ts +++ b/server/typescript/packages/migrate-ts/src/types.ts @@ -131,12 +131,6 @@ export interface FkDescriptor { export type FkAction = "cascade" | "set-null" | "restrict" | "no-action"; -/** One output column of a view, in SELECT order. */ -export interface ViewColumnDescriptor { - name: string; - sqlType: SqlType; -} - /** A relation that depends on a view — i.e. what a `DROP ... CASCADE` would destroy. */ export interface DependentRelation { schema: string; @@ -190,13 +184,6 @@ export interface ViewDescriptor { * `allow.adoptView`. */ fingerprint?: string; - /** - * The view's output columns, in SELECT order. Drives the replace-vs-drop - * decision: Postgres permits `CREATE OR REPLACE VIEW` only when the existing - * columns are a PREFIX of the new ones (same names, same types, same order, - * additions at the end only). Undefined = unknown → fail safe to drop+create. - */ - columns?: readonly ViewColumnDescriptor[]; /** * Relations that depend on this view (transitively). Populated on the ACTUAL * side by introspection. A `DROP VIEW` with dependents needs CASCADE, which @@ -363,6 +350,14 @@ export type Change = * migration can recover. Blocked pending `allow.adoptView`. */ unmanagedActual?: boolean; + /** + * Postgres: the plan-time probe showed the live view's columns are a prefix of this + * view's, so `CREATE OR REPLACE VIEW` is legal (see view-replace.ts). `restorable` is + * true when the two column lists are the same, so the down migration can replace in + * place as well. The Postgres emitter writes `CREATE OR REPLACE VIEW` only for a + * change that carries this; any other replace is written as DROP + CREATE. + */ + proven?: { restorable: boolean }; }; export type ChangeKind = Change["kind"]; @@ -414,6 +409,8 @@ export type ViewChangeReason = export interface ChangeStatus { state: "allowed" | "blocked"; blockedReason?: string; + /** The `AllowOptions` field whose absence blocked the change; granting it lifts THIS block. */ + blockedBy?: keyof AllowOptions; } // --------------------------------------------------------------------------- diff --git a/server/typescript/packages/migrate-ts/src/view-column-types.ts b/server/typescript/packages/migrate-ts/src/view-column-types.ts deleted file mode 100644 index e56653621..000000000 --- a/server/typescript/packages/migrate-ts/src/view-column-types.ts +++ /dev/null @@ -1,110 +0,0 @@ -// view-column-types.ts — resolve a view's output columns to SqlTypes. -// -// Needed to decide whether a view change is non-destructively REPLACEABLE. Postgres -// permits `CREATE OR REPLACE VIEW` only when the existing output columns are a prefix -// of the new ones on (name, type, position); getting the types wrong would make the -// diff propose an OR REPLACE that Postgres rejects at APPLY time -// ("cannot change data type of view column ..."), aborting the migration mid-flight -// with no plan-time warning. So every resolution failure here degrades to `undefined` -// — "unknown" — which the diff treats as not-replaceable and routes through a gated, -// loud drop+create. Wrong-but-confident is the only outcome we must never produce. - -import type { SqlType } from "./sql-type.js"; -import { sqlTypeEquals } from "./sql-type.js"; -import type { TableDescriptor, ViewColumnDescriptor } from "./types.js"; - -/** The codegen-side description of a view column: physical, but untyped. */ -export type ExpectedViewColumnInput = - | { kind: "passthrough"; name: string; sourceTable: string; sourceColumn: string } - | { kind: "aggregate"; name: string; sourceTable: string; sourceColumn: string; agg: string }; - -/** - * Can `existing` be turned into `target` with a `CREATE OR REPLACE VIEW`? - * - * Postgres permits it only when the new query produces the same columns as the existing - * view — same names, same types, same order — with additions allowed at the END. - * Formally: `existing` must be a PREFIX of `target`. - * https://www.postgresql.org/docs/current/sql-createview.html - * - * Used in BOTH directions. Forward (target = expected, existing = live) it decides - * replace-vs-drop. Backward, in a down migration (target = the old view, existing = the - * one we just created), it decides the same thing — and usually says NO, because undoing - * an append means REMOVING a column, which OR REPLACE cannot do. - * - * Fails SAFE: unknown columns on either side → false → the caller uses drop+create. - * A wrong "yes" is not a failed check, it is a statement Postgres rejects at APPLY time, - * aborting the migration with no plan-time warning. - */ -export function viewReplaceIsLegal( - target: readonly ViewColumnDescriptor[] | undefined, - existing: readonly ViewColumnDescriptor[] | undefined, -): boolean { - if (target === undefined || existing === undefined) return false; - if (existing.length > target.length) return false; - return existing.every((ec, i) => { - const tc = target[i]!; - return ec.name === tc.name && sqlTypeEquals(ec.sqlType, tc.sqlType); - }); -} - -/** - * Postgres aggregate result types. Not the argument type — `count(int)` is bigint and - * `avg(int)` is numeric — so a naive "same as input" rule would mis-type the column and - * mis-plan the replace. - * - * https://www.postgresql.org/docs/current/functions-aggregate.html - */ -function aggregateResultType(agg: string, arg: SqlType | undefined): SqlType | undefined { - switch (agg) { - // count() is bigint regardless of its argument — it need not even resolve. - case "count": - return { kind: "integer", bits: 64 }; - case "min": - case "max": - return arg; // same as the argument type - case "sum": - if (arg === undefined) return undefined; - if (arg.kind === "integer") { - // smallint/integer → bigint; bigint → numeric (sum can overflow). - return arg.bits === 64 ? { kind: "numeric" } : { kind: "integer", bits: 64 }; - } - // numeric → numeric, real → real, double → double. - if (arg.kind === "numeric" || arg.kind === "real" || arg.kind === "real4") return arg; - return undefined; - case "avg": - if (arg === undefined) return undefined; - if (arg.kind === "integer" || arg.kind === "numeric") return { kind: "numeric" }; - if (arg.kind === "real" || arg.kind === "real4") return { kind: "real" }; - return undefined; - default: - return undefined; - } -} - -/** - * Resolve every view column against the expected TABLE descriptors. - * - * Returns undefined if ANY column cannot be resolved — a partial list is worse than - * none, because the prefix comparison would silently compare the wrong positions. - */ -export function resolveViewColumns( - inputs: readonly ExpectedViewColumnInput[] | undefined, - tables: readonly TableDescriptor[], -): ViewColumnDescriptor[] | undefined { - if (inputs === undefined || inputs.length === 0) return undefined; - - const byTable = new Map(tables.map((t) => [t.name, t] as const)); - const columnType = (table: string, column: string): SqlType | undefined => - byTable.get(table)?.columns.find((c) => c.name === column)?.sqlType; - - const out: ViewColumnDescriptor[] = []; - for (const input of inputs) { - const arg = columnType(input.sourceTable, input.sourceColumn); - const sqlType = input.kind === "aggregate" - ? aggregateResultType(input.agg, arg) - : arg; - if (sqlType === undefined) return undefined; - out.push({ name: input.name, sqlType }); - } - return out; -} diff --git a/server/typescript/packages/migrate-ts/src/view-replace-probe.ts b/server/typescript/packages/migrate-ts/src/view-replace-probe.ts new file mode 100644 index 000000000..d2a16d83f --- /dev/null +++ b/server/typescript/packages/migrate-ts/src/view-replace-probe.ts @@ -0,0 +1,70 @@ +// view-replace-probe.ts — the Postgres implementation of the view-replace probe (see +// view-replace.ts for why the planner asks the database instead of predicting). + +import type { Kysely } from "kysely"; +import { sql } from "kysely"; +import { DEFAULT_DB_SCHEMA_POSTGRES } from "@metaobjectsdev/metadata"; +import { readPgViewColumns } from "./introspect/postgres.js"; +import type { ViewReplaceProbe } from "./view-replace.js"; + +const PROBE_NAME = "mo_view_replace_probe"; +const ROLLBACK = Symbol("rollback"); + +/** + * Run `work` and undo everything it did. On a connection that is already inside a + * transaction the work is bracketed by a SAVEPOINT, so the caller's transaction is left + * exactly as it was, still open and still usable even when `work` fails. Otherwise it + * runs in a transaction of its own, which is rolled back. + */ +async function rolledBack( + db: Kysely>, + work: (q: Kysely>) => Promise, +): Promise { + if (db.isTransaction) { + await sql.raw(`SAVEPOINT ${PROBE_NAME}`).execute(db); + try { + return await work(db); + } finally { + await sql.raw(`ROLLBACK TO SAVEPOINT ${PROBE_NAME}`).execute(db); + await sql.raw(`RELEASE SAVEPOINT ${PROBE_NAME}`).execute(db); + } + } + let result: { value: T } | undefined; + try { + await db.transaction().execute(async (trx) => { + result = { value: await work(trx) }; + throw ROLLBACK; + }); + } catch (err) { + if (err !== ROLLBACK) throw err; + } + return result!.value; +} + +const quote = (ident: string): string => `"${ident.replace(/"/g, '""')}"`; + +/** + * The probe for a live Postgres database. It creates the expected definition as a + * TEMPORARY view, reads that view's columns and the live view's, and rolls back. The + * temporary view exists only inside the rolled-back work, so nothing persists. + */ +export function postgresViewReplaceProbe(db: Kysely>): ViewReplaceProbe { + return async (expected, live) => { + const body = expected.sql; + if (body === undefined) return undefined; + try { + return await rolledBack(db, async (q) => { + await sql.raw(`CREATE TEMPORARY VIEW ${quote(PROBE_NAME)} AS\n${body}`).execute(q); + return { + expected: await readPgViewColumns(q, `pg_temp.${quote(PROBE_NAME)}`), + live: await readPgViewColumns( + q, + `${quote(live.schema ?? DEFAULT_DB_SCHEMA_POSTGRES)}.${quote(live.name)}`, + ), + }; + }); + } catch { + return undefined; + } + }; +} diff --git a/server/typescript/packages/migrate-ts/src/view-replace.ts b/server/typescript/packages/migrate-ts/src/view-replace.ts new file mode 100644 index 000000000..65dbaf06a --- /dev/null +++ b/server/typescript/packages/migrate-ts/src/view-replace.ts @@ -0,0 +1,65 @@ +// view-replace.ts — can a live Postgres view be changed with `CREATE OR REPLACE VIEW`? +// +// Postgres permits it only when the new query keeps every existing column — same name, +// same type, same collation, same position — and adds columns only at the END. A replace +// that breaks the rule is refused at APPLY time ("cannot change data type of view column +// ..."), aborting the migration with no plan-time warning. +// +// The tool does not predict the answer. The types a query produces are Postgres's to +// decide (an aggregate drops its argument's length, `COALESCE(varchar(40), 'x')` is a bare +// `character varying`, `json` is not `jsonb`), and a prediction that is nearly right plans +// exactly the statement Postgres refuses. So the planner ASKS: it creates the expected +// definition as a temporary view inside a transaction it always rolls back, reads the +// columns Postgres gave it, and compares them with the live view's. A replace is planned +// only on that evidence. With no evidence (no database, no permission, a definition that +// does not compile against the schema as it stands) the plan is DROP + CREATE. +// https://www.postgresql.org/docs/current/sql-createview.html + +import type { ViewDescriptor } from "./types.js"; + +/** One output column of a view exactly as Postgres holds it. */ +export interface ExactViewColumn { + name: string; + /** `format_type(atttypid, atttypmod)`: `character varying(40)`, `jsonb`, `numeric(9,2)`, `text[]`. */ + type: string; + /** The column's collation OID; `0` for a type that has none. */ + collation: string; +} + +/** What Postgres reports for the two views a replace would involve. */ +export interface ViewReplaceEvidence { + /** The columns the expected definition produces. */ + expected: readonly ExactViewColumn[]; + /** The columns the live view has. */ + live: readonly ExactViewColumn[]; +} + +/** + * Ask the database for the columns of `expected` (as it would be created) and of the `live` + * view it would replace. Resolves `undefined` when the question could not be answered; it + * never throws and never leaves anything behind. + */ +export type ViewReplaceProbe = ( + expected: ViewDescriptor, + live: ViewDescriptor, +) => Promise; + +/** + * Can a view whose columns are `existing` be turned into one whose columns are `target` + * with `CREATE OR REPLACE VIEW`? `existing` must be a PREFIX of `target`. + * + * Used in both directions. Forward (target = expected, existing = live) it decides + * replace-vs-drop. Backward (target = the old view, existing = the one the migration + * created) it decides the same for the down migration, and usually says NO: undoing an + * appended column means removing one, which a replace cannot do. + */ +export function viewReplaceIsLegal( + target: readonly ExactViewColumn[], + existing: readonly ExactViewColumn[], +): boolean { + if (existing.length === 0 || existing.length > target.length) return false; + return existing.every((ec, i) => { + const tc = target[i]!; + return ec.name === tc.name && ec.type === tc.type && ec.collation === tc.collation; + }); +} diff --git a/server/typescript/packages/migrate-ts/src/write-migration-flyway.ts b/server/typescript/packages/migrate-ts/src/write-migration-flyway.ts index eaa4b2c11..7d830c21b 100644 --- a/server/typescript/packages/migrate-ts/src/write-migration-flyway.ts +++ b/server/typescript/packages/migrate-ts/src/write-migration-flyway.ts @@ -1,6 +1,6 @@ import { mkdir, readdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import type { EmitResult } from "./types.js"; +import type { EmitResult, SchemaSnapshot } from "./types.js"; export interface WriteMigrationFlywayOptions { /** Flyway migrations dir (e.g. "src/main/resources/db/migration"). Created if missing. */ @@ -96,3 +96,15 @@ function sanitizeSlug(raw: string): string { function ensureTrailingNewline(s: string): string { return s.endsWith("\n") ? s : s + "\n"; } + +/** Flyway's own ledger. The runner creates and owns it, so it is no part of the schema under migration. */ +export const FLYWAY_HISTORY_TABLE = "flyway_schema_history"; + +/** + * `snapshot` without Flyway's history table. Introspecting a Flyway-managed database reports + * that table as a live table the metadata never declares, and the diff answers with + * `DROP TABLE flyway_schema_history` — a hint that, followed, deletes the runner's ledger. + */ +export function withoutFlywayHistory(snapshot: S): S { + return { ...snapshot, tables: snapshot.tables.filter((t) => t.name !== FLYWAY_HISTORY_TABLE) }; +} diff --git a/server/typescript/packages/migrate-ts/test/check/serialize-upgrade.test.ts b/server/typescript/packages/migrate-ts/test/check/serialize-upgrade.test.ts index 7b883df86..466320a53 100644 --- a/server/typescript/packages/migrate-ts/test/check/serialize-upgrade.test.ts +++ b/server/typescript/packages/migrate-ts/test/check/serialize-upgrade.test.ts @@ -12,7 +12,7 @@ const snap = (): SchemaSnapshot => ({ }); describe("serialize with checks + snapshot format version", () => { - // v3 added ViewDescriptor.fingerprint/columns/dependents. An older reader that does + // v3 added ViewDescriptor.fingerprint/dependents. An older reader that does // not understand them would silently fall back to comparing view TEXT against // Postgres's deparsed output — which never matches — and re-propose every view on // every migrate. So the version must be bumped, to hard-fail instead of misread. @@ -34,4 +34,25 @@ describe("serialize with checks + snapshot format version", () => { const parsed = parseSnapshot(v1); expect(parsed.tables[0]?.checks).toEqual([]); }); + + // Releases before 1.1.0 wrote each view's guessed column types into a v3 snapshot. That + // file must keep loading, and the retired field must not be written back. + test("a v3 snapshot that records a view's columns still loads, without them", () => { + const older = JSON.stringify({ + formatVersion: 3, + snapshot: { + tables: [], + views: [{ + name: "v_orders", sql: "SELECT 1 AS n", fingerprint: "a".repeat(64), + columns: [{ name: "n", sqlType: { kind: "integer", bits: 32 } }], + }], + }, + }); + const parsed = parseSnapshot(older); + expect(parsed.views).toEqual([{ name: "v_orders", sql: "SELECT 1 AS n", fingerprint: "a".repeat(64) }]); + expect(JSON.parse(serializeSnapshot(parsed))).toEqual({ + formatVersion: 3, + snapshot: { tables: [], views: [{ name: "v_orders", sql: "SELECT 1 AS n", fingerprint: "a".repeat(64) }] }, + }); + }); }); diff --git a/server/typescript/packages/migrate-ts/test/emit-drop-if-exists.test.ts b/server/typescript/packages/migrate-ts/test/emit-drop-if-exists.test.ts index a51223830..22bff55f7 100644 --- a/server/typescript/packages/migrate-ts/test/emit-drop-if-exists.test.ts +++ b/server/typescript/packages/migrate-ts/test/emit-drop-if-exists.test.ts @@ -102,7 +102,6 @@ describe("down statements stay bare — a rollback must fail loudly", () => { view: { name: "v", sql: "SELECT 1 AS one", - columns: [{ name: "one", sqlType: { kind: "integer", bits: 32 } }], }, }, ]); diff --git a/server/typescript/packages/migrate-ts/test/emit-postgres-create-schema.test.ts b/server/typescript/packages/migrate-ts/test/emit-postgres-create-schema.test.ts index 7bd1cf6d8..a9b8f35d2 100644 --- a/server/typescript/packages/migrate-ts/test/emit-postgres-create-schema.test.ts +++ b/server/typescript/packages/migrate-ts/test/emit-postgres-create-schema.test.ts @@ -21,7 +21,6 @@ const v = (name: string, schema?: string): ViewDescriptor => ({ name, ...(schema !== undefined ? { schema } : {}), sql: "SELECT 1 AS one", - columns: [{ name: "one", sqlType: { kind: "integer", bits: 32 } }], }); describe("a chain that creates an object in a non-default schema creates the schema first", () => { diff --git a/server/typescript/packages/migrate-ts/test/expected-schema-enum-intvaluemap.test.ts b/server/typescript/packages/migrate-ts/test/expected-schema-enum-intvaluemap.test.ts index b950f3de5..83607c597 100644 --- a/server/typescript/packages/migrate-ts/test/expected-schema-enum-intvaluemap.test.ts +++ b/server/typescript/packages/migrate-ts/test/expected-schema-enum-intvaluemap.test.ts @@ -163,7 +163,7 @@ describe("buildExpectedSchema — int-backed field.enum (@intValueMap)", () => { // @intValueMap on a field that already has a column is a text<->integer // change-column-type, and `isWidening` returns false for ANY cross-kind pair // (sql-type.ts: `if (from.kind !== to.kind) return false`), so -// `blockedReasonFor`'s change-column-type branch blocks it unless allow.typeChange +// `blockFor`'s change-column-type branch blocks it unless allow.typeChange // is passed. This proves that end-to-end through the real metadata path rather // than asserting it about hand-built snapshots. describe("int-backed enum — backing-mode change is blocked by the existing guard", () => { diff --git a/server/typescript/packages/migrate-ts/test/integration/pg-adopt-view-239.test.ts b/server/typescript/packages/migrate-ts/test/integration/pg-adopt-view-239.test.ts index 4aef367dd..a9e9c4882 100644 --- a/server/typescript/packages/migrate-ts/test/integration/pg-adopt-view-239.test.ts +++ b/server/typescript/packages/migrate-ts/test/integration/pg-adopt-view-239.test.ts @@ -64,10 +64,6 @@ describe("PG adopt-view structural change round-trip (#239)", () => { name: "v_foo", sql: body, fingerprint: viewFingerprint(body), - columns: [ - { name: "a", sqlType: { kind: "text" as const } }, - { name: "c", sqlType: { kind: "text" as const } }, - ], dependsOn: ["t_239"], }; const expected: SchemaSnapshot = { tables: actual.tables, views: [expectedView] }; diff --git a/server/typescript/packages/migrate-ts/test/unit/diff-view-change-reason.test.ts b/server/typescript/packages/migrate-ts/test/unit/diff-view-change-reason.test.ts index b4b4ba716..289becd8f 100644 --- a/server/typescript/packages/migrate-ts/test/unit/diff-view-change-reason.test.ts +++ b/server/typescript/packages/migrate-ts/test/unit/diff-view-change-reason.test.ts @@ -20,8 +20,12 @@ import type { const BODY = "SELECT id, label FROM t"; -function col(name: string, nullable: boolean): ColumnDescriptor { - return { name, sqlType: name === "id" ? { kind: "integer", bits: 64 } : { kind: "text" }, nullable }; +function col(name: string, nullable: boolean, maxLength?: number): ColumnDescriptor { + return { + name, + sqlType: name === "id" ? { kind: "integer", bits: 64 } : { kind: "text", ...(maxLength !== undefined ? { maxLength } : {}) }, + nullable, + }; } const PARENT_FK: FkDescriptor = { @@ -30,7 +34,7 @@ const PARENT_FK: FkDescriptor = { /** Table `t` (read by the view) plus table `p` (which `t`'s FK points at). */ function snap( - opts: { labelNullable?: boolean; fk?: boolean; views: ViewDescriptor[] }, + opts: { labelNullable?: boolean; labelMaxLength?: number; fk?: boolean; views: ViewDescriptor[] }, ): SchemaSnapshot { return { tables: [ @@ -39,7 +43,7 @@ function snap( }, { name: "t", - columns: [col("id", false), col("label", opts.labelNullable ?? true), col("parentId", true)], + columns: [col("id", false), col("label", opts.labelNullable ?? true, opts.labelMaxLength), col("parentId", true)], indexes: [], foreignKeys: opts.fk === true ? [PARENT_FK] : [], primaryKey: ["id"], @@ -71,14 +75,29 @@ function viewChanges(changes: readonly Change[]): Change[] { } describe("D3 — a view recreated only around a table change is marked unchanged", () => { + // Postgres alters nullability in place, so it plans no view change for it (D4); a type + // change to a column the view reads is what strands a view there. + test("postgres: NOT NULL change on a table the view reads plans no view change", async () => { + const r = await diff({ + expected: snap({ labelNullable: false, views: [expectedView()] }), + actual: snap({ labelNullable: true, views: [actualView("postgres")] }), + dialect: "postgres", + }); + expect(r.changes.some((c) => c.kind === "change-column-nullable")).toBe(true); + expect(viewChanges(r.changes)).toEqual([]); + }); + for (const dialect of ["sqlite", "d1", "postgres"] as const) { - test(`${dialect}: NOT NULL change on a table the view reads`, async () => { + test(`${dialect}: ${dialect === "postgres" ? "type" : "NOT NULL"} change on a table the view reads`, async () => { const r = await diff({ - expected: snap({ labelNullable: false, views: [expectedView()] }), + expected: snap(dialect === "postgres" + ? { labelNullable: true, labelMaxLength: 255, views: [expectedView()] } + : { labelNullable: false, views: [expectedView()] }), actual: snap({ labelNullable: true, views: [actualView(dialect)] }), dialect, + allow: { typeChange: true }, }); - expect(r.changes.some((c) => c.kind === "change-column-nullable")).toBe(true); + expect(r.changes.some((c) => c.kind === (dialect === "postgres" ? "change-column-type" : "change-column-nullable"))).toBe(true); // The pair is still planned: the migration SQL needs it. const views = viewChanges(r.changes); @@ -214,7 +233,7 @@ describe("D3 — an adoption Pass 2c supersedes keeps its gate", () => { test("postgres: an unstamped view over an altered table still needs allow.adoptView", async () => { const unstamped: ViewDescriptor = { name: "v", sql: " SELECT t.id, t.label FROM t;" }; const args = { - expected: snap({ labelNullable: false, views: [expectedView()] }), + expected: snap({ labelNullable: true, labelMaxLength: 255, views: [expectedView()] }), actual: snap({ labelNullable: true, views: [unstamped] }), dialect: "postgres" as const, }; @@ -222,11 +241,34 @@ describe("D3 — an adoption Pass 2c supersedes keeps its gate", () => { const drop = r.changes.find((c) => c.kind === "drop-view"); expect(drop).toMatchObject({ unmanagedActual: true, - reason: { kind: "unfingerprinted", tables: ["t"] }, + reason: { kind: "unfingerprinted" }, status: { state: "blocked" }, }); - const allowed = await diff({ ...args, allow: { adoptView: true, nullableToNotNull: true } }); + const allowed = await diff({ ...args, allow: { adoptView: true, typeChange: true } }); expect(allowed.blocked).toEqual([]); }); + + // The database proved the replace legal, but a replace cannot run around the ALTER of a + // column the view reads. Pass 2c turns it into the pair, which names the altered table + // and keeps the adoption gate. + test("postgres: a proven replace of an unstamped view over an altered table becomes the pair, still gated", async () => { + const unstamped: ViewDescriptor = { name: "v", sql: " SELECT t.id, t.label FROM t;" }; + const columns = [ + { name: "id", type: "bigint", collation: "0" }, + { name: "label", type: "text", collation: "100" }, + ]; + const r = await diff({ + expected: snap({ labelNullable: true, labelMaxLength: 255, views: [expectedView()] }), + actual: snap({ labelNullable: true, views: [unstamped] }), + dialect: "postgres", + viewReplaceProbe: async () => ({ expected: columns, live: columns }), + }); + expect(viewChanges(r.changes).map((c) => c.kind).sort()).toEqual(["create-view", "drop-view"]); + expect(r.changes.find((c) => c.kind === "drop-view")).toMatchObject({ + unmanagedActual: true, + reason: { kind: "unfingerprinted", tables: ["t"] }, + status: { state: "blocked" }, + }); + }); }); diff --git a/server/typescript/packages/migrate-ts/test/unit/diff-view-recreate-columns.test.ts b/server/typescript/packages/migrate-ts/test/unit/diff-view-recreate-columns.test.ts new file mode 100644 index 000000000..1d6fe9891 --- /dev/null +++ b/server/typescript/packages/migrate-ts/test/unit/diff-view-recreate-columns.test.ts @@ -0,0 +1,120 @@ +/** + * Views and column changes (adopter report D4). + * + * Postgres only blocks ALTER on a column a view READS, and only for a type change, a drop + * or a rename. A default or nullability change, or a change to a column the view does not + * read, must not drop and recreate the view. (D2, when a view may be replaced in place, is + * in view-replace.test.ts.) + */ +import { test, expect, describe } from "bun:test"; +import { diff } from "../../src/diff/index.js"; +import { viewFingerprint } from "../../src/view-fingerprint.js"; +import type { ColumnDescriptor, Dialect, SchemaSnapshot, TableDescriptor, ViewDescriptor } from "../../src/types.js"; + +const BODY = "SELECT g.id AS id, g.label AS label FROM games g"; + +function table(cols: ColumnDescriptor[]): TableDescriptor { + return { name: "games", columns: cols, indexes: [], foreignKeys: [], primaryKey: ["id"], checks: [] }; +} +const ID: ColumnDescriptor = { name: "id", sqlType: { kind: "integer", bits: 64 }, nullable: false }; +const LABEL: ColumnDescriptor = { name: "label", sqlType: { kind: "text" }, nullable: false }; +const NOTE: ColumnDescriptor = { name: "note", sqlType: { kind: "text" }, nullable: false }; + +function view(over: Partial = {}): ViewDescriptor { + return { name: "v_games", sql: BODY, fingerprint: viewFingerprint(BODY), dependsOn: ["games"], ...over }; +} + +function snap(t: TableDescriptor, v: ViewDescriptor): SchemaSnapshot { + return { tables: [t], views: [v] }; +} + +const kinds = (r: { changes: { kind: string }[] }) => r.changes.map((c) => c.kind); + +describe("D4 — a view is recreated only when the migration alters a column it reads", () => { + const dialect: Dialect = "postgres"; + const run = (expected: TableDescriptor, actual: TableDescriptor) => + diff(snap(expected, view()), snap(actual, view()), { dialect, allow: { typeChange: true } }); + + test("a type change to a column the view reads recreates it", async () => { + const r = await run(table([ID, { ...LABEL, sqlType: { kind: "text", maxLength: 255 } }]), table([ID, LABEL])); + expect(kinds(r)).toEqual(expect.arrayContaining(["change-column-type", "drop-view", "create-view"])); + }); + + test("a type change to a column the view does NOT read leaves it alone", async () => { + const r = await run( + table([ID, LABEL, { ...NOTE, sqlType: { kind: "text", maxLength: 255 } }]), + table([ID, LABEL, NOTE]), + ); + expect(kinds(r)).toEqual(["change-column-type"]); + }); + + test("a nullability change does not recreate a view over it on Postgres", async () => { + const r = await run(table([ID, { ...LABEL, nullable: true }]), table([ID, LABEL])); + expect(kinds(r)).toEqual(["change-column-nullable"]); + }); + + test("a default-only change does not recreate a view over it on Postgres", async () => { + const r = await run( + table([ID, { ...LABEL, default: { kind: "literal", value: "x" } }]), + table([ID, LABEL]), + ); + expect(kinds(r)).toEqual(["change-column-default"]); + }); + + // Postgres folds an unquoted identifier to lower case, so `g.LABEL` reads column `label`. + test("a hand-written body that spells the column in another case still reads it", async () => { + const upper = "SELECT g.ID, g.LABEL FROM games g"; + const v = view({ sql: upper, fingerprint: viewFingerprint(upper) }); + const r = await diff( + snap(table([ID, { ...LABEL, sqlType: { kind: "text", maxLength: 255 } }]), v), + snap(table([ID, LABEL]), v), + { dialect, allow: { typeChange: true } }, + ); + expect(kinds(r)).toEqual(expect.arrayContaining(["change-column-type", "drop-view", "create-view"])); + }); + + test("a view that selects * is recreated by any type change", async () => { + const star = "SELECT * FROM games"; + const v = view({ sql: star, fingerprint: viewFingerprint(star) }); + const r = await diff( + snap(table([ID, LABEL, { ...NOTE, sqlType: { kind: "text", maxLength: 255 } }]), v), + snap(table([ID, LABEL, NOTE]), v), + { dialect, allow: { typeChange: true } }, + ); + expect(kinds(r)).toEqual(expect.arrayContaining(["drop-view", "create-view"])); + }); + + // Postgres: NATURAL JOIN joins on every column name the two tables share, so the view + // depends on `note` although its body never spells it, and the ALTER is refused while the + // view stands ("cannot alter type of a column used by a view or rule"). + test("a view that joins NATURAL is recreated by any type change", async () => { + const natural = "SELECT g.id FROM games g NATURAL JOIN archive"; + const v = view({ sql: natural, fingerprint: viewFingerprint(natural) }); + const r = await diff( + snap(table([ID, LABEL, { ...NOTE, sqlType: { kind: "text", maxLength: 255 } }]), v), + snap(table([ID, LABEL, NOTE]), v), + { dialect, allow: { typeChange: true } }, + ); + expect(kinds(r)).toEqual(expect.arrayContaining(["change-column-type", "drop-view", "create-view"])); + }); + + test("SQLite still recreates every view over a rebuilt table", async () => { + const r = await diff( + snap(table([ID, LABEL, { ...NOTE, nullable: true }]), view()), + snap(table([ID, LABEL, NOTE]), view()), + { dialect: "sqlite" }, + ); + expect(kinds(r)).toEqual(expect.arrayContaining(["drop-view", "create-view"])); + }); + + test("a view with a pending definition change is recreated even if the column is not in its new body", async () => { + const changed = view({ sql: "SELECT g.id AS id FROM games g", fingerprint: viewFingerprint("SELECT g.id AS id FROM games g") }); + const r = await diff( + snap(table([ID, { ...NOTE, sqlType: { kind: "text", maxLength: 255 } }]), changed), + snap(table([ID, NOTE]), view({ fingerprint: "f".repeat(64) })), + { dialect, allow: { typeChange: true } }, + ); + expect(kinds(r)).toEqual(expect.arrayContaining(["drop-view", "create-view"])); + expect(kinds(r)).not.toContain("replace-view"); + }); +}); diff --git a/server/typescript/packages/migrate-ts/test/unit/diff-view-recreate.test.ts b/server/typescript/packages/migrate-ts/test/unit/diff-view-recreate.test.ts index 60eba12ef..5191ced1f 100644 --- a/server/typescript/packages/migrate-ts/test/unit/diff-view-recreate.test.ts +++ b/server/typescript/packages/migrate-ts/test/unit/diff-view-recreate.test.ts @@ -58,6 +58,43 @@ describe("diff — Pass 2c view dependency-recreate", () => { expect(k).not.toContain("drop-view"); }); + // No dialect: the caller may emit for SQLite, which rebuilds the table for any of these + // (the README quick start diffs without one, then emits `{ dialect: "sqlite" }`). + describe("with no dialect", () => { + const tableOf = (columns: ColumnDescriptor[]): SchemaSnapshot => ({ + tables: [{ name: "t", columns, indexes: [], foreignKeys: [], primaryKey: [], checks: [] }], + views: [dependentView()], + }); + + test("a nullability change recreates the view over the table", async () => { + const r = await diff(tableOf([{ ...col("c", "text"), nullable: false }]), tableOf([col("c", "text")])); + const k = kinds(r.changes); + expect(k).toContain("change-column-nullable"); + expect(k.filter((x) => x === "drop-view").length).toBe(1); + expect(k.filter((x) => x === "create-view").length).toBe(1); + }); + + test("a default change recreates the view over the table", async () => { + const withDefault: ColumnDescriptor = { ...col("c", "text"), default: { kind: "literal", value: "x" } }; + const r = await diff(tableOf([withDefault]), tableOf([col("c", "text")])); + const k = kinds(r.changes); + expect(k).toContain("change-column-default"); + expect(k.filter((x) => x === "drop-view").length).toBe(1); + expect(k.filter((x) => x === "create-view").length).toBe(1); + }); + + test("a type change on a column the view does not name recreates the view", async () => { + const r = await diff( + tableOf([col("c", "text"), col("d", "integer")]), + tableOf([col("c", "text"), col("d", "text")]), + ); + const k = kinds(r.changes); + expect(k).toContain("change-column-type"); + expect(k.filter((x) => x === "drop-view").length).toBe(1); + expect(k.filter((x) => x === "create-view").length).toBe(1); + }); + }); + test("column change on a table the view does NOT depend on → no view change", async () => { const independent: ViewDescriptor = { name: "v", sql: BODY, dependsOn: ["other"] }; const r = await diff(snap("integer", [independent]), snap("text", [independent])); diff --git a/server/typescript/packages/migrate-ts/test/unit/diff.test.ts b/server/typescript/packages/migrate-ts/test/unit/diff.test.ts index b1f7a27cd..7517d917b 100644 --- a/server/typescript/packages/migrate-ts/test/unit/diff.test.ts +++ b/server/typescript/packages/migrate-ts/test/unit/diff.test.ts @@ -2,6 +2,7 @@ import { test, expect, describe } from "bun:test"; import { diff } from "../../src/diff/index.js"; import { emit } from "../../src/emit/index.js"; import type { SchemaSnapshot, ColumnDescriptor } from "../../src/types.js"; +import type { ViewReplaceProbe } from "../../src/view-replace.js"; const empty: SchemaSnapshot = { tables: [], views: [] }; @@ -142,27 +143,36 @@ describe("diff — view-body drift", () => { // --------------------------------------------------------------------------- // #239 — the Postgres adopt-view branch (DB view carries no fingerprint) must // make the SAME legal/illegal OR-REPLACE decision the managed path makes. -// A structural change (rename / reorder / mid-insert) is NOT a legal +// A structural change (rename / reorder / mid-insert / retype) is NOT a legal // CREATE OR REPLACE, so it must be emitted as drop-view + create-view — and the // adoption of an unmanaged view still requires allow.adoptView (the recreate-pair // auto-allow must not wave through clobbering hand-written SQL). +// +// Whether the replace is legal is asked of the database (view-replace.ts). Here the +// probe is a stub that reports the columns Postgres would: the planner's decision is +// the unit under test. // --------------------------------------------------------------------------- describe("diff — Postgres adopt-view legality (#239)", () => { - const T = { kind: "text" as const }; // A view metadata (expected) side always carries a fingerprint. const expView = (cols: string[]) => ({ name: "v_foo", sql: `SELECT ${cols.map((c) => `t.${c} AS ${c}`).join(", ")} FROM t`, fingerprint: "a".repeat(64), // raw sha256-hex; renderFingerprintMarker adds the prefix - columns: cols.map((name) => ({ name, sqlType: T })), }); // The DB (actual) side has NO fingerprint — pre-fingerprint or hand-written. const dbView = (cols: string[]) => ({ name: "v_foo", sql: `SELECT ${cols.map((c) => `t.${c} AS ${c}`).join(", ")} FROM t`, - columns: cols.map((name) => ({ name, sqlType: T })), }); - const opts = (adoptView = false) => ({ dialect: "postgres" as const, allow: { adoptView } }); + const text = (names: string[]) => names.map((name) => ({ name, type: "text", collation: "100" })); + /** What Postgres reports: the expected definition's columns and the live view's. */ + const probing = (expected: string[], live: string[]): ViewReplaceProbe => + async () => ({ expected: text(expected), live: text(live) }); + const opts = (viewReplaceProbe: ViewReplaceProbe | undefined, adoptView = true) => ({ + dialect: "postgres" as const, + allow: { adoptView }, + ...(viewReplaceProbe !== undefined ? { viewReplaceProbe } : {}), + }); test("STRUCTURAL change (renamed output column) → drop-view + create-view, NOT replace-view", async () => { // DB view outputs (a, b); metadata renames b→c → columns (a, c). Postgres cannot @@ -170,7 +180,7 @@ describe("diff — Postgres adopt-view legality (#239)", () => { const r = await diff( { tables: [], views: [expView(["a", "c"])] }, { tables: [], views: [dbView(["a", "b"])] }, - opts(true), + opts(probing(["a", "c"], ["a", "b"])), ); expect(r.changes.find((c) => c.kind === "replace-view")).toBeUndefined(); expect(r.changes.find((c) => c.kind === "drop-view")).toMatchObject({ kind: "drop-view", view: "v_foo" }); @@ -182,10 +192,11 @@ describe("diff — Postgres adopt-view legality (#239)", () => { test("adopting an unmanaged view via drop+create STILL requires allow.adoptView", async () => { // Without allow.adoptView the drop must be BLOCKED — the recreate-pair auto-allow // must not silently clobber the (possibly hand-written) unmanaged view. + const probe = probing(["a", "c"], ["a", "b"]); const r = await diff( { tables: [], views: [expView(["a", "c"])] }, { tables: [], views: [dbView(["a", "b"])] }, - opts(false), + opts(probe, false), ); const dropv = r.changes.find((c) => c.kind === "drop-view"); expect(dropv?.status.state).toBe("blocked"); @@ -193,61 +204,67 @@ describe("diff — Postgres adopt-view legality (#239)", () => { const r2 = await diff( { tables: [], views: [expView(["a", "c"])] }, { tables: [], views: [dbView(["a", "b"])] }, - opts(true), + opts(probe), ); expect(r2.changes.find((c) => c.kind === "drop-view")?.status.state).toBe("allowed"); }); - test("PURE APPEND (columns are a prefix) is still a legal replace-view (unmanagedActual)", async () => { - // DB (a, b); metadata (a, b, c) — c appended. Legal OR-REPLACE → replace-view, - // still gated on adoptView. + test("PURE APPEND (live columns are a prefix) is a legal replace-view (unmanagedActual), gated on adoptView", async () => { + // DB (a, b); metadata (a, b, c) — c appended. Undoing it removes a column, which a + // replace cannot do, so the change is not restorable in place. + const probe = probing(["a", "b", "c"], ["a", "b"]); const r = await diff( { tables: [], views: [expView(["a", "b", "c"])] }, { tables: [], views: [dbView(["a", "b"])] }, - opts(true), + opts(probe), ); - expect(r.changes.find((c) => c.kind === "replace-view")).toMatchObject({ - kind: "replace-view", view: { name: "v_foo" }, unmanagedActual: true, + expect(r.changes.map((c) => c.kind)).toEqual(["replace-view"]); + expect(r.changes[0]).toMatchObject({ + view: { name: "v_foo" }, unmanagedActual: true, proven: { restorable: false }, }); - expect(r.changes.find((c) => c.kind === "drop-view")).toBeUndefined(); + const blocked = await diff( + { tables: [], views: [expView(["a", "b", "c"])] }, + { tables: [], views: [dbView(["a", "b"])] }, + opts(probe, false), + ); + expect(blocked.blocked.map((c) => c.kind)).toEqual(["replace-view"]); }); - test("#240 offline adopt: expected columns KNOWN but actual (old snapshot) has NO columns → drop+create", async () => { - // A pre-fingerprint snapshot records no view columns (actual.columns undefined), but - // a projection's desired shape IS known. The decision keys on EXPECTED knowledge, so - // a structural change still drop+creates — never an illegal CREATE OR REPLACE. - const actualNoCols = { name: "v_foo", sql: "SELECT t.a AS a, t.b AS b FROM t" }; // no columns, no fingerprint + test("#240 offline adopt: no database to ask → drop+create, still gated on adoptView", async () => { + // An offline plan diffs against the committed snapshot. Nothing can prove the replace, + // so it is never planned, even for columns that look like a pure append. const r = await diff( - { tables: [], views: [expView(["a", "c"])] }, // expected knows [a, c] (b renamed to c) - { tables: [], views: [actualNoCols] }, - opts(true), + { tables: [], views: [expView(["a", "b", "c"])] }, + { tables: [], views: [dbView(["a", "b"])] }, + opts(undefined), ); - expect(r.changes.find((c) => c.kind === "replace-view")).toBeUndefined(); - expect(r.changes.find((c) => c.kind === "drop-view")).toMatchObject({ kind: "drop-view", view: "v_foo" }); - expect(r.changes.find((c) => c.kind === "create-view")).toBeDefined(); - // Still gated on adoptView (unmanagedActual on the drop half). + expect(r.changes.map((c) => c.kind)).toEqual(["drop-view", "create-view"]); const rBlocked = await diff( - { tables: [], views: [expView(["a", "c"])] }, - { tables: [], views: [actualNoCols] }, - opts(false), + { tables: [], views: [expView(["a", "b", "c"])] }, + { tables: [], views: [dbView(["a", "b"])] }, + opts(undefined, false), ); expect(rBlocked.changes.find((c) => c.kind === "drop-view")?.status.state).toBe("blocked"); }); - test("OPAQUE @sql body (columns unknown) adopts via non-destructive replace-view, not drop+create", async () => { - // A hand-written @sql view has no parsed columns, so replace legality is unprovable. - // The common case is re-stamping an identical pre-fingerprint view (#208), which a - // legal CREATE OR REPLACE handles without cascading — so adoption keeps replace-view. + test("OPAQUE @sql body: replaced in place when the database proves it, dropped and created when nothing can", async () => { + // A hand-written @sql body is never parsed, but Postgres can still say what columns + // it produces. Re-stamping an identical pre-fingerprint view (#208) is the common case. const opaque = { name: "v_foo", sql: "SELECT t.a AS a, t.b AS b FROM t", fingerprint: "b".repeat(64) }; - const r = await diff( + const proven = await diff( { tables: [], views: [opaque] }, { tables: [], views: [dbView(["a", "b"])] }, - opts(true), + opts(probing(["a", "b"], ["a", "b"])), ); - expect(r.changes.find((c) => c.kind === "replace-view")).toMatchObject({ - kind: "replace-view", view: { name: "v_foo" }, unmanagedActual: true, - }); - expect(r.changes.find((c) => c.kind === "drop-view")).toBeUndefined(); + expect(proven.changes.map((c) => c.kind)).toEqual(["replace-view"]); + expect(proven.changes[0]).toMatchObject({ unmanagedActual: true, proven: { restorable: true } }); + + const unproven = await diff( + { tables: [], views: [opaque] }, + { tables: [], views: [dbView(["a", "b"])] }, + opts(undefined), + ); + expect(unproven.changes.map((c) => c.kind)).toEqual(["drop-view", "create-view"]); }); test("emitted up.sql for a structural adoption is DROP+CREATE (no illegal CREATE OR REPLACE)", async () => { @@ -257,7 +274,7 @@ describe("diff — Postgres adopt-view legality (#239)", () => { const r = await diff( { tables: [], views: [expView(["a", "c"])] }, { tables: [], views: [dbView(["a", "b"])] }, - opts(true), + opts(probing(["a", "c"], ["a", "b"])), ); const up = emit(r.changes, { dialect: "postgres" }).up; expect(up).not.toMatch(/CREATE OR REPLACE VIEW/i); diff --git a/server/typescript/packages/migrate-ts/test/unit/emit-views.test.ts b/server/typescript/packages/migrate-ts/test/unit/emit-views.test.ts index 45f286bd6..1efea0ea2 100644 --- a/server/typescript/packages/migrate-ts/test/unit/emit-views.test.ts +++ b/server/typescript/packages/migrate-ts/test/unit/emit-views.test.ts @@ -18,7 +18,37 @@ const replaceV: Change[] = [ test("postgres emits all three view kinds", () => { expect(emit(createV, { dialect: "postgres" }).up).toMatch(/CREATE VIEW/i); expect(emit(dropV, { dialect: "postgres" }).up).toMatch(/DROP VIEW/i); - expect(emit(replaceV, { dialect: "postgres" }).up).toMatch(/CREATE OR REPLACE VIEW/i); + const proven: Change[] = [{ ...replaceV[0]!, kind: "replace-view", proven: { restorable: true } } as Change]; + expect(emit(proven, { dialect: "postgres" }).up).toMatch(/CREATE OR REPLACE VIEW "v" AS/); +}); + +// Postgres refuses a replace that changes an existing column, and it refuses at apply. A +// replace nothing proved (a diff with no dialect, whose changes can be emitted for any +// database) is written the way SQLite writes every replace. +test("postgres writes an unproven replace as DROP + CREATE", () => { + const { up, down } = emit( + [{ kind: "replace-view", view: { name: "v", sql: "SELECT 1 AS x" }, restore: { name: "v", sql: "SELECT 0 AS x" }, status: ALLOWED }], + { dialect: "postgres" }, + ); + expect(up).not.toMatch(/OR REPLACE/i); + expect(up.indexOf('DROP VIEW IF EXISTS "v";')).toBeGreaterThanOrEqual(0); + expect(up.indexOf('DROP VIEW IF EXISTS "v";')).toBeLessThan(up.indexOf('CREATE VIEW "v" AS')); + expect(down).not.toMatch(/OR REPLACE/i); + expect(down.indexOf('DROP VIEW "v";')).toBeLessThan(down.indexOf('CREATE VIEW "v" AS\nSELECT 0 AS x;')); +}); + +test("postgres restores a proven replace in place only when the columns are the same", () => { + const replace = (restorable: boolean): Change[] => [{ + kind: "replace-view", view: { name: "v", sql: "SELECT 1 AS x" }, restore: { name: "v", sql: "SELECT 0 AS x" }, + proven: { restorable }, status: ALLOWED, + }]; + const inPlace = emit(replace(true), { dialect: "postgres" }).down; + expect(inPlace).toContain('CREATE OR REPLACE VIEW "v" AS\nSELECT 0 AS x;'); + expect(inPlace).not.toContain("DROP VIEW"); + // An appended column: undoing it removes one, which a replace cannot do. + const dropped = emit(replace(false), { dialect: "postgres" }).down; + expect(dropped).not.toMatch(/OR REPLACE/i); + expect(dropped.indexOf('DROP VIEW "v";')).toBeLessThan(dropped.indexOf('CREATE VIEW "v" AS\nSELECT 0 AS x;')); }); test("sqlite renders view DDL (DROP+CREATE for replace, no OR REPLACE)", () => { diff --git a/server/typescript/packages/migrate-ts/test/unit/view-column-types.test.ts b/server/typescript/packages/migrate-ts/test/unit/view-column-types.test.ts deleted file mode 100644 index c58295db5..000000000 --- a/server/typescript/packages/migrate-ts/test/unit/view-column-types.test.ts +++ /dev/null @@ -1,115 +0,0 @@ -/** - * View column typing + the CREATE-OR-REPLACE legality rule. - * - * These decide whether a view change is applied NON-DESTRUCTIVELY. Get them wrong in - * the optimistic direction and the migration emits a statement Postgres rejects at - * APPLY time ("cannot change data type of view column ..."), aborting mid-flight with - * no plan-time warning. So the fail-safe direction is asserted explicitly below: every - * unknown resolves to "not replaceable", never to "replaceable". - * - * The aggregate result types are not guesses — they are checked against real Postgres - * by test/../integration-tests view-lifecycle-pg (min/max keep the argument type, - * count/sum over int widen to bigint, avg goes numeric). - */ - -import { describe, test, expect } from "bun:test"; -import { resolveViewColumns, viewReplaceIsLegal } from "../../src/view-column-types.js"; -import type { TableDescriptor, ViewColumnDescriptor } from "../../src/types.js"; - -const TABLES: TableDescriptor[] = [ - { - name: "programs", - columns: [ - { name: "id", sqlType: { kind: "integer", bits: 64 }, nullable: false }, - { name: "title", sqlType: { kind: "text", maxLength: 200 }, nullable: false }, - { name: "minutes", sqlType: { kind: "integer", bits: 32 }, nullable: true }, - { name: "price", sqlType: { kind: "numeric", precision: 9, scale: 2 }, nullable: true }, - ], - indexes: [], foreignKeys: [], checks: [], primaryKey: ["id"], - }, -]; - -const col = (name: string, sqlType: ViewColumnDescriptor["sqlType"]): ViewColumnDescriptor => - ({ name, sqlType }); - -describe("resolveViewColumns", () => { - test("a passthrough column takes its source column's type", () => { - expect(resolveViewColumns( - [{ kind: "passthrough", name: "title", sourceTable: "programs", sourceColumn: "title" }], - TABLES, - )).toEqual([col("title", { kind: "text", maxLength: 200 })]); - }); - - test("aggregates use POSTGRES result types, not the argument type", () => { - const of = (agg: string, sourceColumn: string) => - resolveViewColumns([{ kind: "aggregate", name: "x", sourceTable: "programs", sourceColumn, agg }], TABLES)?.[0]?.sqlType; - - // count(anything) is bigint — NOT the argument's int32. - expect(of("count", "minutes")).toEqual({ kind: "integer", bits: 64 }); - // sum(int32) widens to bigint; sum(bigint) goes numeric (it can overflow). - expect(of("sum", "minutes")).toEqual({ kind: "integer", bits: 64 }); - expect(of("sum", "id")).toEqual({ kind: "numeric" }); - // avg of any exact type is numeric. - expect(of("avg", "minutes")).toEqual({ kind: "numeric" }); - // min/max preserve the argument type. - expect(of("min", "minutes")).toEqual({ kind: "integer", bits: 32 }); - expect(of("max", "price")).toEqual({ kind: "numeric", precision: 9, scale: 2 }); - }); - - test("an unresolvable column drops the WHOLE list — a partial list would misalign the prefix compare", () => { - expect(resolveViewColumns( - [ - { kind: "passthrough", name: "title", sourceTable: "programs", sourceColumn: "title" }, - { kind: "passthrough", name: "gone", sourceTable: "programs", sourceColumn: "nonexistent" }, - ], - TABLES, - )).toBeUndefined(); - - expect(resolveViewColumns( - [{ kind: "aggregate", name: "x", sourceTable: "programs", sourceColumn: "minutes", agg: "bogus" }], - TABLES, - )).toBeUndefined(); - }); -}); - -describe("viewReplaceIsLegal — Postgres's prefix rule", () => { - const a = col("a", { kind: "text" }); - const b = col("b", { kind: "integer", bits: 64 }); - const c = col("c", { kind: "boolean" }); - - test("identical column lists are replaceable", () => { - expect(viewReplaceIsLegal([a, b], [a, b])).toBe(true); - }); - - test("APPENDING a column is replaceable — this is what keeps a projection change non-destructive", () => { - expect(viewReplaceIsLegal([a, b, c], [a, b])).toBe(true); - }); - - test("REMOVING a column is NOT replaceable (Postgres: cannot drop columns from view)", () => { - expect(viewReplaceIsLegal([a], [a, b])).toBe(false); - }); - - test("REORDERING is NOT replaceable — the old list is no longer a prefix", () => { - expect(viewReplaceIsLegal([b, a], [a, b])).toBe(false); - }); - - test("INSERTING mid-list is NOT replaceable", () => { - expect(viewReplaceIsLegal([a, c, b], [a, b])).toBe(false); - }); - - test("RENAMING a column is NOT replaceable", () => { - expect(viewReplaceIsLegal([col("renamed", { kind: "text" }), b], [a, b])).toBe(false); - }); - - test("RETYPING a column in place is NOT replaceable", () => { - expect(viewReplaceIsLegal([col("a", { kind: "integer", bits: 32 }), b], [a, b])).toBe(false); - }); - - test("FAIL SAFE: unknown columns on either side are NOT replaceable", () => { - // A wrong "true" here is not a failed assertion — it is a statement Postgres rejects - // at apply time, aborting a live migration. Unknown must always mean "drop+create". - expect(viewReplaceIsLegal(undefined, [a])).toBe(false); - expect(viewReplaceIsLegal([a], undefined)).toBe(false); - expect(viewReplaceIsLegal(undefined, undefined)).toBe(false); - }); -}); diff --git a/server/typescript/packages/migrate-ts/test/unit/view-replace.test.ts b/server/typescript/packages/migrate-ts/test/unit/view-replace.test.ts new file mode 100644 index 000000000..157fe5c5c --- /dev/null +++ b/server/typescript/packages/migrate-ts/test/unit/view-replace.test.ts @@ -0,0 +1,186 @@ +/** + * When may a live Postgres view be changed with `CREATE OR REPLACE VIEW`? (adopter report D2) + * + * Postgres permits it only when every existing column keeps its name, type, collation and + * position, with new columns at the end, and it refuses anything else at APPLY time. The + * planner does not predict the answer: it asks the database through a probe and plans a + * replace only on that evidence. These tests pin the rule itself, taken from the Postgres + * documentation (https://www.postgresql.org/docs/current/sql-createview.html), and the + * planner's use of it with a stubbed probe. The real probe runs against a live Postgres in + * integration-tests view-replace-probe-pg. + */ + +import { describe, test, expect } from "bun:test"; +import { diff } from "../../src/diff/index.js"; +import { viewFingerprint } from "../../src/view-fingerprint.js"; +import { viewReplaceIsLegal, type ExactViewColumn, type ViewReplaceProbe } from "../../src/view-replace.js"; +import type { SchemaSnapshot, ViewDescriptor } from "../../src/types.js"; + +const col = (name: string, type: string, collation = "0"): ExactViewColumn => ({ name, type, collation }); + +describe("viewReplaceIsLegal — Postgres's prefix rule", () => { + const a = col("a", "text", "100"); + const b = col("b", "bigint"); + const c = col("c", "boolean"); + + test("identical column lists are replaceable", () => { + expect(viewReplaceIsLegal([a, b], [a, b])).toBe(true); + }); + + test("APPENDING a column is replaceable; undoing the append is not", () => { + expect(viewReplaceIsLegal([a, b, c], [a, b])).toBe(true); + expect(viewReplaceIsLegal([a, b], [a, b, c])).toBe(false); + }); + + test("REORDERING, INSERTING mid-list and RENAMING are not replaceable", () => { + expect(viewReplaceIsLegal([b, a], [a, b])).toBe(false); + expect(viewReplaceIsLegal([a, c, b], [a, b])).toBe(false); + expect(viewReplaceIsLegal([col("renamed", "text", "100"), b], [a, b])).toBe(false); + }); + + // Each pair is one type under a coarser model and two types to Postgres, which refuses + // the replace with "cannot change data type of view column". + test.each([ + ["json", "jsonb"], + ["smallint", "integer"], + ["character varying", "text"], + ["character varying(40)", "character varying"], + ["character varying[]", "text[]"], + ["numeric(9,2)", "numeric"], + ["integer", "bigint"], + ])("RETYPING %s to %s is not replaceable", (from, to) => { + expect(viewReplaceIsLegal([col("x", to)], [col("x", from)])).toBe(false); + expect(viewReplaceIsLegal([col("x", from)], [col("x", from)])).toBe(true); + }); + + test("a changed collation is not replaceable", () => { + expect(viewReplaceIsLegal([col("x", "text", "950")], [col("x", "text", "100")])).toBe(false); + }); + + test("a view with no columns proves nothing", () => { + expect(viewReplaceIsLegal([a], [])).toBe(false); + expect(viewReplaceIsLegal([], [])).toBe(false); + }); +}); + +describe("the planner replaces a view only on the probe's evidence", () => { + const OLD_BODY = "SELECT g.id AS id, COALESCE(g.nickname, 'x') AS label FROM games g"; + const NEW_BODY = "SELECT g.id AS id, COALESCE(g.notes, 'x') AS label FROM games g"; + const view = (sql: string, stamped = true): ViewDescriptor => ({ + name: "v_games", sql, dependsOn: ["games"], + ...(stamped ? { fingerprint: viewFingerprint(sql) } : {}), + }); + const snap = (views: ViewDescriptor[]): SchemaSnapshot => ({ + tables: [{ + name: "games", + columns: [{ name: "id", sqlType: { kind: "integer", bits: 64 }, nullable: false }], + indexes: [], foreignKeys: [], primaryKey: ["id"], checks: [], + }], + views, + }); + const kinds = (r: { changes: { kind: string }[] }) => r.changes.map((c) => c.kind); + const ID = col("id", "bigint"); + + /** A probe that answers with `evidence` and records what it was asked. */ + function stub(evidence: Awaited>) { + const asked: { expected: string; live: string }[] = []; + const probe: ViewReplaceProbe = async (expected, live) => { + asked.push({ expected: expected.sql ?? "", live: live.sql ?? "" }); + return evidence; + }; + return { probe, asked }; + } + + const changed = (probe?: ViewReplaceProbe) => + diff(snap([view(NEW_BODY)]), snap([view(OLD_BODY)]), { + dialect: "postgres", + ...(probe !== undefined ? { viewReplaceProbe: probe } : {}), + }); + + test("a managed change the database proves legal is a replace, restorable when the columns are the same", async () => { + const label = col("label", "text", "100"); + const { probe, asked } = stub({ expected: [ID, label], live: [ID, label] }); + const r = await changed(probe); + expect(kinds(r)).toEqual(["replace-view"]); + expect(r.changes[0]).toMatchObject({ proven: { restorable: true } }); + expect(r.blocked).toEqual([]); + // Asked once, about the expected definition and the live view it would replace. + expect(asked).toEqual([{ expected: NEW_BODY, live: OLD_BODY }]); + }); + + // The regression: a coalesce that moves from a varchar(40) field to an unbounded one keeps + // its name and its string-ness, and changes type. Postgres refuses the replace. + test("a managed change that retypes a column is a drop and a create", async () => { + const { probe } = stub({ + expected: [ID, col("label", "text", "100")], + live: [ID, col("label", "character varying", "100")], + }); + const r = await changed(probe); + expect(kinds(r)).toEqual(["drop-view", "create-view"]); + expect(r.blocked).toEqual([]); + }); + + test("a probe that could not answer plans a drop and a create", async () => { + const { probe, asked } = stub(undefined); + expect(kinds(await changed(probe))).toEqual(["drop-view", "create-view"]); + expect(asked).toHaveLength(1); + }); + + test("no probe (an offline plan) plans a drop and a create", async () => { + expect(kinds(await changed())).toEqual(["drop-view", "create-view"]); + }); + + test("the probe is not asked about a view that is unchanged, new, or undeclared", async () => { + const { probe, asked } = stub({ expected: [ID], live: [ID] }); + const opts = { dialect: "postgres" as const, viewReplaceProbe: probe }; + expect(kinds(await diff(snap([view(OLD_BODY)]), snap([view(OLD_BODY)]), opts))).toEqual([]); + expect(kinds(await diff(snap([view(OLD_BODY)]), snap([]), opts))).toEqual(["create-view"]); + expect(kinds(await diff(snap([]), snap([view(OLD_BODY)]), { ...opts, allow: { dropView: true } }))).toEqual(["drop-view"]); + expect(asked).toEqual([]); + }); + + // The probe sees the schema before the plan's table changes; the replace runs after them. + // `SELECT *` re-expands then, so a column the plan adds lands in the middle of the view's + // column list and Postgres refuses the replace ("cannot change name of view column"). + describe("a body that reads every column, in a plan that adds a column", () => { + const TAGLINE = { name: "tagline", sqlType: { kind: "text" as const }, nullable: true }; + const withTagline = (views: ViewDescriptor[]): SchemaSnapshot => { + const s = snap(views); + return { ...s, tables: s.tables.map((t) => ({ ...t, columns: [...t.columns, TAGLINE] })) }; + }; + const same = { expected: [ID, col("extra", "integer")], live: [ID, col("extra", "integer")] }; + const STAR_OLD = "SELECT g.*, 1 AS extra FROM games g"; + const STAR_NEW = "SELECT g.*, 2 AS extra FROM games g"; + + test("is dropped and created, and the probe is not asked", async () => { + const { probe, asked } = stub(same); + const r = await diff(withTagline([view(STAR_NEW)]), snap([view(STAR_OLD)]), { dialect: "postgres", viewReplaceProbe: probe }); + expect(kinds(r)).toEqual(["add-column", "drop-view", "create-view"]); + expect(asked).toEqual([]); + }); + + test("is still replaced on the probe's evidence when the plan adds no column", async () => { + const { probe, asked } = stub(same); + const r = await diff(snap([view(STAR_NEW)]), snap([view(STAR_OLD)]), { dialect: "postgres", viewReplaceProbe: probe }); + expect(kinds(r)).toEqual(["replace-view"]); + expect(asked).toHaveLength(1); + }); + + test("a body that names its columns is still replaced beside an added column", async () => { + const { probe, asked } = stub(same); + const r = await diff(withTagline([view(NEW_BODY)]), snap([view(OLD_BODY)]), { dialect: "postgres", viewReplaceProbe: probe }); + expect(kinds(r)).toEqual(["add-column", "replace-view"]); + expect(asked).toHaveLength(1); + }); + }); + + test("SQLite and D1 never ask: a replace there is written as DROP + CREATE", async () => { + for (const dialect of ["sqlite", "d1"] as const) { + const { probe, asked } = stub({ expected: [ID], live: [ID] }); + const live: ViewDescriptor = { name: "v_games", sql: `CREATE VIEW "v_games" AS ${OLD_BODY}`, dependsOn: ["games"] }; + const r = await diff(snap([view(NEW_BODY)]), snap([live]), { dialect, viewReplaceProbe: probe }); + expect(kinds(r)).toEqual(["replace-view"]); + expect(asked).toEqual([]); + } + }); +}); diff --git a/tools/prerelease/prerelease-link.sh b/tools/prerelease/prerelease-link.sh index fea784e71..e24f04e02 100755 --- a/tools/prerelease/prerelease-link.sh +++ b/tools/prerelease/prerelease-link.sh @@ -83,9 +83,20 @@ need_registry() { HOSTPORT="${BASE#*://}" # ── version normalization — the same table as scripts/prerelease.mjs ────────────────── -# canonical -rc. -> npm/nuget verbatim · PEP 440 rc · Maven 7..-rc. +# canonical -rc. -> npm/nuget verbatim · PEP 440 rc · Maven ..-rc. py_version() { echo "$1" | sed -E 's/-rc\.([0-9]+)$/rc\1/'; } -maven_version() { echo "$1" | sed -E 's/^[0-9]+\./7./'; } +# The JVM major runs npm major + 7 (ADR-0035): 0.x -> 7.x, 1.x -> 8.x. Derived here in shell +# rather than through scripts/maven-coordinate.mjs because this script is handed to +# collaborators on its own, and a JVM-only machine has no Node. test-prerelease-link.sh holds +# it to that module's mavenVersion() so the two cannot drift. +maven_version() { + local v="$1" + if [[ ! "$v" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)(-.+)?$ ]]; then + printf '"%s" is not a three-part version\n' "$v" >&2 + return 1 + fi + echo "$((10#${BASH_REMATCH[1]} + 7)).${BASH_REMATCH[2]}.${BASH_REMATCH[3]}${BASH_REMATCH[4]}" +} # ── ecosystem detection ─────────────────────────────────────────────────────────────── # Detection walks the project, it does not test the root. A Maven- or pip-rooted monorepo @@ -129,11 +140,31 @@ strip_block() { # strip_block # Keep an untracked managed file out of commits without touching the shared .gitignore. local_exclude() { # local_exclude - local rel="$1" gi - gi="$(git -C "$PROJECT" rev-parse --git-dir 2>/dev/null)" || return 0 - [ -d "$PROJECT/$gi/info" ] || mkdir -p "$PROJECT/$gi/info" - local ex="$PROJECT/$gi/info/exclude" - grep -qxF "$rel" "$ex" 2>/dev/null || echo "$rel" >> "$ex" + local rel="$1" ex prefix + # --git-path, not --git-dir: in a linked worktree the git dir is a private directory, and the + # exclude file git READS is the shared one in the common dir. The answer is relative to the + # directory git ran in, or absolute when the git dir is elsewhere; joining an absolute one onto + # $PROJECT is what once wrote a stray `home/.../info/exclude` tree into the work tree. + ex="$(git -C "$PROJECT" rev-parse --git-path info/exclude 2>/dev/null)" || return 0 + case "$ex" in /*) ;; *) ex="$PROJECT/$ex" ;; esac + # Exclude patterns are relative to the repository root, and $PROJECT may be a subdirectory. + prefix="$(git -C "$PROJECT" rev-parse --show-prefix 2>/dev/null || true)" + mkdir -p "$(dirname "$ex")" + grep -qxF "$prefix$rel" "$ex" 2>/dev/null || echo "$prefix$rel" >> "$ex" +} + +# Refuse to report success while a file this script wrote could still be committed. The managed +# .npmrc can hold a registry credential, and `git add -A` takes anything untracked and not ignored. +# A tracked file is the project's own (the caller has already warned); outside a repository there +# is nothing to commit. +assert_ignored() { # assert_ignored ... + git -C "$PROJECT" rev-parse --git-dir >/dev/null 2>&1 || return 0 + local rel bad=() + for rel in "$@"; do + tracked "$rel" && continue + git -C "$PROJECT" check-ignore -q -- "$rel" || bad+=("$rel") + done + [ ${#bad[@]} -eq 0 ] || die "git does not ignore: ${bad[*]} — they hold the registry address (and a token when one was needed), and 'git add -A' would commit them. Add them to .git/info/exclude or .gitignore, then run link again." } tracked() { git -C "$PROJECT" ls-files --error-unmatch "$1" >/dev/null 2>&1; } @@ -433,7 +464,7 @@ link_npm() { # pre-release pin and no registry to resolve it from — `link` reported success and the # breakage surfaced later as `notarget` inside that sub-project. `check` already walked # nested roots; that asymmetry was the bug. - local wrote=0 root rel f + local wrote=0 root rel f written=() while IFS= read -r root; do f="$root/.npmrc" strip_block "$f" "$BEGIN" "$END" @@ -447,8 +478,10 @@ link_npm() { else local_exclude "$rel" fi + written+=("$rel") wrote=$(( wrote + 1 )) done < <(npm_install_roots) + assert_ignored ${written+"${written[@]}"} say "wrote $wrote .npmrc file(s) (scope $NPM_SCOPE only) + git local excludes" [ -n "$auth" ] && warn "the registry required a token, so .npmrc now holds a credential — do not commit it" repin_npm "$VERSION" diff --git a/tools/prerelease/test-prerelease-link.sh b/tools/prerelease/test-prerelease-link.sh new file mode 100755 index 000000000..7a926cb60 --- /dev/null +++ b/tools/prerelease/test-prerelease-link.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Tests for tools/prerelease/prerelease-link.sh — the two defects an adopter hit linking a project +# to a pre-release candidate. +# +# tools/prerelease/test-prerelease-link.sh +# +# D9 the Maven pin is npm major + 7 (1.1.0-rc.N is 8.1.0-rc.N), not a hardcoded 7. +# D11 in a linked git worktree the exclude file is the shared one in the common dir, nothing +# stray is written into the work tree, and `link` refuses to report success while a written +# .npmrc (which can hold a registry credential) is not ignored. +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +WORK="$(mktemp -d)" +# The script under test runs from a copy. It sources a `registry.env` beside itself, and a +# maintainer's checkout has one holding the real registry address and token, which would send +# these runs to that registry instead of the local one below. +mkdir -p "$WORK/tool" +cp "$HERE/prerelease-link.sh" "$HERE/detect-prerelease-pins.sh" "$WORK/tool/" +LINK="$WORK/tool/prerelease-link.sh" +SERVER_PID="" +cleanup() { [ -z "$SERVER_PID" ] || kill "$SERVER_PID" 2>/dev/null || true; rm -rf "$WORK"; } +trap cleanup EXIT + +fails=0 +pass() { printf 'ok: %s\n' "$1"; } +fail() { printf 'FAIL: %s\n' "$1" >&2; fails=$((fails + 1)); } +check() { if eval "$2"; then pass "$1"; else fail "$1"; fi; } + +# ── D9: the Maven version, through the function itself ──────────────────────────────── +maven_version_fn="$(awk '/^maven_version\(\) \{/,/^\}/' "$LINK")" +eval "$maven_version_fn" +# The shell function carries its own copy of the rule so the script works without Node or a +# checkout; scripts/maven-coordinate.mjs is where the rule is defined, so each input is also +# compared against mavenVersion() there. +reference_maven_version() { + node --input-type=module -e ' + import { pathToFileURL } from "node:url"; + const { mavenVersion } = await import(pathToFileURL(process.argv[1]).href); + process.stdout.write(mavenVersion(process.argv[2])); + ' "$HERE/../../scripts/maven-coordinate.mjs" "$1" +} +for pair in "0.25.0:7.25.0" "0.24.0-rc.3:7.24.0-rc.3" "1.0.0:8.0.0" "1.1.0-rc.2:8.1.0-rc.2" "2.0.0:9.0.0" "10.4.12-rc.11:17.4.12-rc.11"; do + in="${pair%%:*}"; want="${pair##*:}"; got="$(maven_version "$in")" + [ "$got" = "$want" ] && pass "maven_version $in -> $want" || fail "maven_version $in -> $got, want $want" + ref="$(reference_maven_version "$in")" + [ "$got" = "$ref" ] && pass "maven_version $in agrees with maven-coordinate.mjs" \ + || fail "maven_version $in -> $got, maven-coordinate.mjs says $ref" +done +if maven_version "1.1" >/dev/null 2>&1; then fail "maven_version accepted a two-part version"; else pass "maven_version refuses a two-part version"; fi +# The function runs with no Node on PATH: a collaborator's JVM-only machine. +got="$(PATH=/nonexistent maven_version "1.1.0-rc.2")" +[ "$got" = "8.1.0-rc.2" ] && pass "maven_version needs nothing on PATH" || fail "maven_version without PATH -> $got" + +# ── D11: link inside a linked worktree ──────────────────────────────────────────────── +# A registry that answers GET / is all `link` needs from the network. +# It also answers the anonymous-read probe (`/api/packages//npm/@metaobjectsdev%2Fcli`), so +# no token is needed. +mkdir -p "$WORK/registry/api/packages/owner/npm/@metaobjectsdev" +echo '{}' > "$WORK/registry/api/packages/owner/npm/@metaobjectsdev/cli" +PORT="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1])')" +(cd "$WORK/registry" && exec python3 -m http.server "$PORT" --bind 127.0.0.1 >/dev/null 2>&1) & +SERVER_PID=$! +for _ in $(seq 1 50); do curl -fsS -o /dev/null "http://127.0.0.1:$PORT/" 2>/dev/null && break; sleep 0.1; done + +export GIT_AUTHOR_NAME=t GIT_AUTHOR_EMAIL=t@example.invalid GIT_COMMITTER_NAME=t GIT_COMMITTER_EMAIL=t@example.invalid +export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 +git init -q "$WORK/main" +cat > "$WORK/main/package.json" <<'JSON' +{ "name": "consumer", "version": "1.0.0", "dependencies": { "@metaobjectsdev/metadata": "^1.0.0" } } +JSON +git -C "$WORK/main" add -A && git -C "$WORK/main" commit -q -m init +git -C "$WORK/main" worktree add -q "$WORK/wt" -b linked + +link() { # link -> runs link, output in $WORK/out + MO_REGISTRY_BASE="http://127.0.0.1:$PORT" MO_REGISTRY_OWNER=owner MO_REGISTRY_TOKEN="" \ + bash "$LINK" link --project "$1" --version 1.1.0-rc.9 >"$WORK/out" 2>&1 +} + +if link "$WORK/wt"; then pass "link succeeds in a linked worktree"; else fail "link failed: $(tail -5 "$WORK/out")"; fi +check "the .npmrc is written" '[ -f "$WORK/wt/.npmrc" ]' +check "the .npmrc is ignored by git" 'git -C "$WORK/wt" check-ignore -q -- .npmrc' +check "git status shows no .npmrc" '! git -C "$WORK/wt" status --porcelain | grep -q "\.npmrc"' +# The shared exclude file is the one git reads; no path component of the absolute git dir may +# appear inside the work tree. +common="$(git -C "$WORK/wt" rev-parse --git-common-dir)" +case "$common" in /*) ;; *) common="$WORK/wt/$common" ;; esac +check "the exclude entry is in the shared exclude file" 'grep -qxF .npmrc "$common/info/exclude"' +leading="$(printf '%s' "$WORK" | cut -d/ -f2)" +check "no stray exclude tree in the work tree" '[ ! -e "$WORK/wt/$leading" ]' +check "the dependency is pinned to the candidate" 'grep -q "1.1.0-rc.9" "$WORK/wt/package.json"' + +# The assertion: an .npmrc git does not ignore makes link fail. +git -C "$WORK/main" worktree add -q "$WORK/wt2" -b linked2 +# A global excludes file is not what we control, so defeat the ignore by negating it in .gitignore. +printf '!.npmrc\n' > "$WORK/wt2/.gitignore" +git -C "$WORK/wt2" add .gitignore && git -C "$WORK/wt2" commit -q -m "track a rule that un-ignores .npmrc" +# local_exclude appends to the exclude file, but .gitignore (higher precedence) re-includes it. +if link "$WORK/wt2"; then fail "link reported success with a non-ignored .npmrc"; else + grep -q "git does not ignore: .npmrc" "$WORK/out" && pass "link refuses when .npmrc is not ignored" \ + || fail "link failed, but not with the ignore message: $(tail -3 "$WORK/out")" +fi + +[ "$fails" -eq 0 ] || { echo "$fails failure(s)" >&2; exit 1; } +echo "all prerelease-link tests passed" From 2f97209f66d4b8c9cd22822fa92cc32225d242fd Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Sun, 11 Oct 2026 13:23:52 -0400 Subject: [PATCH 2/5] no-mistakes(review): fix(migrate-ts): skip view-replace probe when untracked table changes --- CHANGELOG.md | 7 +- docs/features/migrations-and-drift.md | 7 +- .../test/view-replace-probe-pg.test.ts | 84 ++++++++++++++++++- .../packages/migrate-ts/src/diff/index.ts | 46 ++++++++-- .../migrate-ts/test/unit/view-replace.test.ts | 73 ++++++++++++++++ 5 files changed, 203 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 128c8717d..3601ecf3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1001,9 +1001,12 @@ until you regenerate. - **A replace is never planned without that proof.** When the probe cannot run, the plan is a drop and a create: an offline `meta migrate` (the default, which diffs the committed snapshot and has no database to ask), a role with no `TEMPORARY` privilege, a new - definition that does not compile yet because it reads a column the same migration adds, or + definition that does not compile yet because it reads a column the same migration adds, an `@sql` body with `SELECT *` or `NATURAL JOIN` in a migration that also adds a column (its - column list changes once the column exists, so an answer taken before would not hold). + column list changes once the column exists, so an answer taken before would not hold), or + an `@sql` body that joins a table the view does not track in a migration that also retypes, + drops or renames a column of that table the body reads, or drops that table (Postgres + refuses those changes while the old view stands). Before, an offline plan replaced a view in place when its recorded columns looked compatible. - **A drop and a create discards the privileges granted on the view.** `DROP VIEW` removes the view's grants with it, and `meta migrate` writes no `GRANT`, so a role that had `SELECT` on a diff --git a/docs/features/migrations-and-drift.md b/docs/features/migrations-and-drift.md index 29bc49521..7dea8533d 100644 --- a/docs/features/migrations-and-drift.md +++ b/docs/features/migrations-and-drift.md @@ -438,7 +438,12 @@ and a `CREATE VIEW`: column the same migration adds; - an `@sql` body that reads columns it does not name (`SELECT *`, `NATURAL JOIN`) in a migration that also adds a column. The probe sees the schema before the column exists, and - the body's column list changes once it does, so the answer would not hold. + the body's column list changes once it does, so the answer would not hold; +- an `@sql` body that joins a table the view does not track (any table other than its own + table, its `extends` anchors or a report's `@from` table), in a migration that also changes + the type of, drops or renames a column of that table that the body reads, or drops that + table. A replace would leave the old view standing while the table changes, and Postgres + refuses to change what a view reads. A drop and a create of the same view needs no `--allow`. It has two costs a replace does not: diff --git a/server/typescript/packages/integration-tests/test/view-replace-probe-pg.test.ts b/server/typescript/packages/integration-tests/test/view-replace-probe-pg.test.ts index 6b8ddac1e..7f1ebf87c 100644 --- a/server/typescript/packages/integration-tests/test/view-replace-probe-pg.test.ts +++ b/server/typescript/packages/integration-tests/test/view-replace-probe-pg.test.ts @@ -222,9 +222,9 @@ const CASES: readonly Case[] = [ ] as const).map(([kind, column, type]): Case => ({ kind, host: REPORT, view: "v_week_stats", column, retyped: cast(type) })), ]; -async function load(host: unknown, programFields: unknown[] = []): Promise { +async function load(host: unknown, programFields: unknown[] = [], others: unknown[] = []): Promise { const json = JSON.stringify({ - "metadata.root": { package: "acme", children: [program(programFields), WEEK, WEEK_BRIEF, host] }, + "metadata.root": { package: "acme", children: [program(programFields), WEEK, WEEK_BRIEF, ...others, host] }, }); const { root, errors } = await new MetaDataLoader().load([new InMemoryStringSource(json)]); expect(errors).toEqual([]); @@ -240,8 +240,8 @@ async function applyRaw(text: string, db: Db = k): Promise { interface Model { expected: SchemaSnapshot; view: ViewDescriptor } /** The expected schema of a model, and the one view under test. */ -async function model(host: unknown, viewName: string, programFields: unknown[] = []): Promise { - const root = await load(host, programFields); +async function model(host: unknown, viewName: string, programFields: unknown[] = [], others: unknown[] = []): Promise { + const root = await load(host, programFields, others); const expected = buildExpectedSchema(root, { columnNamingStrategy: STRATEGY, views: buildProjectionViews(root, { dialect: "postgres", columnNamingStrategy: STRATEGY }), @@ -509,6 +509,82 @@ describe("an @sql body that reads every column (postgres)", () => { }, 60_000); }); +// An `@sql` view's tracked tables are its anchors, not the tables its body joins. The replace +// runs after the plan's table changes, with the live view standing while they run, and +// Postgres refuses to retype or drop what a standing view reads. +describe("an @sql body that joins a table the view does not track (postgres)", () => { + const badge = (points?: "int" | "long"): unknown => ({ + "object.entity": { name: "Badge", children: [ + { "source.rdb": { "@table": "badges" } }, + { "field.long": { name: "id" } }, + { "field.long": { name: "programId", "@required": true } }, + ...(points !== undefined ? [{ [`field.${points}`]: { name: "points", "@required": true } }] : []), + { "identity.primary": { name: "id", "@fields": "id", "@generation": "increment" } }, + ] }, + }); + const scoredView = (scored: string): unknown => ({ + "object.projection": { name: "ProgramSummary", children: [ + { "source.rdb": { "@kind": "view", "@view": SUMMARY_VIEW, "@sql": `SELECT p."id" AS "id", ${scored} AS "scored" FROM "programs" p` } }, + { "identity.primary": { name: "id", extends: "Program.id", "@fields": "id" } }, + { "field.long": { name: "id", extends: "Program.id" } }, + ] }, + }); + const overPoints = (threshold: number): string => + `(SELECT count(*) FROM "badges" b WHERE b."programId" = p."id" AND b."points" > ${threshold})`; + const OVER_BADGES = `(SELECT count(*) FROM "badges" b WHERE b."programId" = p."id")`; + const BEFORE = { scored: overPoints(0), others: [badge("int")] }; + + test.each([ + { + change: "retypes a column the body reads", scored: overPoints(1), others: [badge("long")], + statement: `ALTER TABLE "badges" ALTER COLUMN "points" TYPE BIGINT`, + refusal: /cannot alter type of a column used by a view or rule/, planned: "change-column-type", + }, + { + change: "drops a column the live body reads", scored: OVER_BADGES, others: [badge()], + statement: `ALTER TABLE "badges" DROP COLUMN "points"`, + refusal: /cannot drop column points of table badges because other objects depend on it/, planned: "drop-column", + }, + { + change: "drops the table the live body reads", scored: "0::bigint", others: [], + statement: `DROP TABLE "badges"`, + refusal: /cannot drop table badges because other objects depend on it/, planned: "drop-table", + }, + ])("a body change in a plan that $change is dropped and created, and applies", async (c) => { + await migrate((await model(scoredView(BEFORE.scored), SUMMARY_VIEW, [], BEFORE.others)).expected); + const after = await model(scoredView(c.scored), SUMMARY_VIEW, [], c.others); + expect(after.view.dependsOn).toEqual(["programs"]); + + // Postgres accepts the replace today, and refuses the table change while the view stands. + const rollback = new Error("rollback"); + await expect(k.transaction().execute(async (trx) => { + await sql.raw(`CREATE OR REPLACE VIEW "${SUMMARY_VIEW}" AS ${after.view.sql}`).execute(trx); + throw rollback; + })).rejects.toBe(rollback); + await expect(sql.raw(c.statement).execute(k)).rejects.toThrow(c.refusal); + + const changed = await plan(after.expected, { dropColumn: true, dropTable: true }); + expect(changed.blocked).toEqual([]); + expect(kindsOf(changed.changes)).toEqual([c.planned, "create-view", "drop-view"].sort()); + await apply(changed.changes); + + expect(await columnTypes(SUMMARY_VIEW)).toEqual({ id: "bigint", scored: "bigint" }); + expect((await plan(after.expected)).changes).toEqual([]); + }, 60_000); + + test("a body change beside a change to a table it does not read is replaced in place", async () => { + await migrate((await model(scoredView("0::bigint"), SUMMARY_VIEW, [], BEFORE.others)).expected); + const oid = await viewOid(SUMMARY_VIEW); + const after = await model(scoredView("1::bigint"), SUMMARY_VIEW, [], [badge("long")]); + + const changed = await plan(after.expected); + expect(kindsOf(changed.changes)).toEqual(["change-column-type", "replace-view"]); + await apply(changed.changes); + expect(await viewOid(SUMMARY_VIEW)).toBe(oid); + expect((await plan(after.expected)).changes).toEqual([]); + }, 60_000); +}); + describe("when the probe cannot answer, the plan is a drop and a create (postgres)", () => { const compatible = async (): Promise => { const m = await model(summary(COALESCE_NICKNAME), SUMMARY_VIEW); diff --git a/server/typescript/packages/migrate-ts/src/diff/index.ts b/server/typescript/packages/migrate-ts/src/diff/index.ts index 92676d8b5..32e2dbabf 100644 --- a/server/typescript/packages/migrate-ts/src/diff/index.ts +++ b/server/typescript/packages/migrate-ts/src/diff/index.ts @@ -967,10 +967,8 @@ async function diffViews( * and for a derived body and an opaque `@sql` one alike. * * The probe answers for the schema as it stands, and the replace runs after this plan's - * table changes. A body that reads columns it does not name (`SELECT *`) has a different - * column list once the plan adds a column, so the answer would not hold: it is not asked. - * Every other table change that alters a view's columns is a type change, a drop or a - * rename, which Pass 2c turns into the pair. + * table changes. When one of those changes would undo the answer, or be refused while the + * live view stands ({@link tableChangesPrecludeReplace}), the probe is not asked. */ async function pushViewUpdate( expected: ViewDescriptor, actual: ViewDescriptor, changes: Change[], @@ -986,9 +984,9 @@ async function pushViewUpdate( const reason: ViewChangeReason = unmanaged ? { kind: "unfingerprinted" } : { kind: "definition", compared: "fingerprint" }; - const columnsMayGrow = expected.sql !== undefined && viewReadsEveryColumn(expected.sql) - && changes.some((c) => c.kind === "add-column"); - const evidence = probe === undefined || columnsMayGrow ? undefined : await probe(expected, actual); + const evidence = probe === undefined || tableChangesPrecludeReplace(expected, actual, changes) + ? undefined + : await probe(expected, actual); if (evidence !== undefined && viewReplaceIsLegal(evidence.expected, evidence.live)) { changes.push({ kind: "replace-view", view: expected, ...sx, restore: actual, ...adopt, reason, @@ -1136,6 +1134,40 @@ function viewReadsAnyColumn(sql: string, cols: Iterable, readsEverything return false; } +/** + * Does a table change already in this plan rule out replacing the view in place? A replace + * runs after the plan's table changes and leaves the live view standing while they run. + * Pass 2 has run by the time a view is planned, so `changes` holds them. + * + * - A body that reads columns it does not name has a different column list once the plan + * adds a column, so what the probe saw would not hold. + * - Postgres refuses a type change, a drop or a rename of a column a standing view reads, + * and a drop of a table it reads. For a table in `dependsOn`, Pass 2c turns the replace + * into the pair. An `@sql` body's `dependsOn` leaves out the tables the body joins + * (ADR-0043), and a dropped table is in no expected view's, so for a table outside + * `dependsOn` the two bodies are the evidence: each names the tables and columns it + * reads, the live one as Postgres deparsed it. + */ +function tableChangesPrecludeReplace( + expected: ViewDescriptor, actual: ViewDescriptor, changes: readonly Change[], +): boolean { + const bodies = [expected.sql, actual.sql] + .filter((sql): sql is string => sql !== undefined) + .map((sql) => ({ sql, readsEverything: viewReadsEveryColumn(sql) })); + const columnsMayGrow = expected.sql !== undefined && viewReadsEveryColumn(expected.sql); + const tracked = expected.dependsOn ?? []; + return changes.some((c) => { + if (c.kind === "add-column") return columnsMayGrow; + if (c.kind !== "drop-table" && !VIEW_RECREATE_TRIGGERS_PG.has(c.kind)) return false; + const table = (c as { table?: string }).table; + if (typeof table !== "string" || tracked.includes(table)) return false; + const cols = changedColumns(c); + return bodies.some(({ sql, readsEverything }) => + columnIdentifierRe(table).test(sql) + && (cols === undefined || viewReadsAnyColumn(sql, cols, readsEverything))); + }); +} + function recreateViewsDependingOnChangedTables( expectedViews: ViewDescriptor[], actualViews: ViewDescriptor[], diff --git a/server/typescript/packages/migrate-ts/test/unit/view-replace.test.ts b/server/typescript/packages/migrate-ts/test/unit/view-replace.test.ts index 157fe5c5c..5412d36c8 100644 --- a/server/typescript/packages/migrate-ts/test/unit/view-replace.test.ts +++ b/server/typescript/packages/migrate-ts/test/unit/view-replace.test.ts @@ -174,6 +174,79 @@ describe("the planner replaces a view only on the probe's evidence", () => { }); }); + // The replace runs after the plan's table changes and leaves the live view standing while + // they run. Postgres refuses to retype or drop a column, or drop a table, that a standing + // view reads (https://www.postgresql.org/docs/current/sql-altertable.html, DROP COLUMN and + // SET DATA TYPE; sql-droptable.html, RESTRICT). Pass 2c covers a table in `dependsOn`; an + // `@sql` body joins tables its `dependsOn` does not list. + describe("a body that reads a table outside dependsOn, in a plan that changes that table", () => { + type Columns = SchemaSnapshot["tables"][number]["columns"]; + const POINTS: Columns[number] = { name: "points", sqlType: { kind: "integer", bits: 32 }, nullable: false }; + const NOTE: Columns[number] = { name: "note", sqlType: { kind: "text" }, nullable: true }; + const withScores = (views: ViewDescriptor[], columns?: Columns): SchemaSnapshot => { + const s = snap(views); + if (columns === undefined) return s; + return { ...s, tables: [...s.tables, { name: "scores", columns, indexes: [], foreignKeys: [], primaryKey: [], checks: [] }] }; + }; + const joined = (threshold: number): string => + `SELECT g.id AS id, (SELECT count(*) FROM scores s WHERE s.points > ${threshold}) AS scored FROM games g`; + const UNJOINED = "SELECT g.id AS id, 0::bigint AS scored FROM games g"; + const same = { expected: [ID, col("scored", "bigint")], live: [ID, col("scored", "bigint")] }; + const WIDE_POINTS: Columns[number] = { ...POINTS, sqlType: { kind: "integer", bits: 64 } }; + const opts = (probe: ViewReplaceProbe) => + ({ dialect: "postgres" as const, viewReplaceProbe: probe, allow: { dropColumn: true, dropTable: true } }); + + test.each([ + ["retypes a column the body reads", joined(1), [WIDE_POINTS, NOTE], "change-column-type"], + ["drops a column the live body reads", UNJOINED, [NOTE], "drop-column"], + ["drops the table the live body reads", UNJOINED, undefined, "drop-table"], + ] as const)("a plan that %s drops and creates the view, and the probe is not asked", async (_, body, columns, change) => { + const { probe, asked } = stub(same); + const r = await diff( + withScores([view(body)], columns === undefined ? undefined : [...columns]), + withScores([view(joined(0))], [POINTS, NOTE]), + opts(probe), + ); + expect(kinds(r)).toEqual([change, "drop-view", "create-view"]); + expect(r.blocked).toEqual([]); + expect(asked).toEqual([]); + }); + + test("a changed column the bodies do not read leaves the replace to the probe", async () => { + const { probe, asked } = stub(same); + const r = await diff( + withScores([view(joined(1))], [POINTS, { ...NOTE, sqlType: { kind: "integer", bits: 32 } }]), + withScores([view(joined(0))], [POINTS, NOTE]), + opts(probe), + ); + expect(kinds(r)).toEqual(["change-column-type", "replace-view"]); + expect(asked).toHaveLength(1); + }); + + test("a changed table the bodies do not read leaves the replace to the probe", async () => { + const { probe, asked } = stub(same); + const r = await diff( + withScores([view(NEW_BODY)], [WIDE_POINTS, NOTE]), + withScores([view(OLD_BODY)], [POINTS, NOTE]), + opts(probe), + ); + expect(kinds(r)).toEqual(["change-column-type", "replace-view"]); + expect(asked).toHaveLength(1); + }); + + // Postgres changes nullability in place under a view. + test("a nullability change on a column the body reads leaves the replace to the probe", async () => { + const { probe, asked } = stub(same); + const r = await diff( + withScores([view(joined(1))], [{ ...POINTS, nullable: true }, NOTE]), + withScores([view(joined(0))], [POINTS, NOTE]), + opts(probe), + ); + expect(kinds(r)).toEqual(["change-column-nullable", "replace-view"]); + expect(asked).toHaveLength(1); + }); + }); + test("SQLite and D1 never ask: a replace there is written as DROP + CREATE", async () => { for (const dialect of ["sqlite", "d1"] as const) { const { probe, asked } = stub({ expected: [ID], live: [ID] }); From 76869997884faceecc4da4880ff811d2ce88a253 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Sun, 11 Oct 2026 13:38:56 -0400 Subject: [PATCH 3/5] no-mistakes(review): fix(migrate-ts): one rule for when view-replace probe is trusted --- CHANGELOG.md | 11 +- docs/features/migrations-and-drift.md | 17 +-- .../src/projection/build-projection-views.ts | 6 + .../projection/build-projection-views.test.ts | 2 +- .../test/view-replace-probe-pg.test.ts | 74 +++++++++-- .../packages/migrate-ts/src/diff/index.ts | 64 +++++----- .../migrate-ts/src/expected-schema.ts | 3 + .../migrate-ts/src/snapshot/serialize.ts | 11 +- .../packages/migrate-ts/src/types.ts | 6 + .../test/check/serialize-upgrade.test.ts | 9 ++ .../migrate-ts/test/unit/view-replace.test.ts | 117 ++++++++++++------ 11 files changed, 224 insertions(+), 96 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3601ecf3d..608295243 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1002,11 +1002,12 @@ until you regenerate. drop and a create: an offline `meta migrate` (the default, which diffs the committed snapshot and has no database to ask), a role with no `TEMPORARY` privilege, a new definition that does not compile yet because it reads a column the same migration adds, - an `@sql` body with `SELECT *` or `NATURAL JOIN` in a migration that also adds a column (its - column list changes once the column exists, so an answer taken before would not hold), or - an `@sql` body that joins a table the view does not track in a migration that also retypes, - drops or renames a column of that table the body reads, or drops that table (Postgres - refuses those changes while the old view stands). + an `@sql` body in a migration that also changes any table's columns (the body is not + parsed, so the tables it joins are unknown, and Postgres refuses to retype or drop what a + standing view reads), or a body that reads columns it does not name (`SELECT *`, + `NATURAL JOIN`) in a migration that also alters any table or any other view (its column + list comes from what it reads, so an answer taken before would not hold). A derived + projection or report view lists every table it reads and is not affected. Before, an offline plan replaced a view in place when its recorded columns looked compatible. - **A drop and a create discards the privileges granted on the view.** `DROP VIEW` removes the view's grants with it, and `meta migrate` writes no `GRANT`, so a role that had `SELECT` on a diff --git a/docs/features/migrations-and-drift.md b/docs/features/migrations-and-drift.md index 7dea8533d..8ca4d8570 100644 --- a/docs/features/migrations-and-drift.md +++ b/docs/features/migrations-and-drift.md @@ -436,14 +436,15 @@ and a `CREATE VIEW`: - a role that may not create a temporary view (no `TEMPORARY` privilege on the database); - a new definition that does not compile against the schema as it stands, because it reads a column the same migration adds; -- an `@sql` body that reads columns it does not name (`SELECT *`, `NATURAL JOIN`) in a - migration that also adds a column. The probe sees the schema before the column exists, and - the body's column list changes once it does, so the answer would not hold; -- an `@sql` body that joins a table the view does not track (any table other than its own - table, its `extends` anchors or a report's `@from` table), in a migration that also changes - the type of, drops or renames a column of that table that the body reads, or drops that - table. A replace would leave the old view standing while the table changes, and Postgres - refuses to change what a view reads. +- an `@sql` body, in a migration that also changes any table's columns (adds, drops, renames + or retypes a column, changes its nullability or default, or drops a table). `meta migrate` + does not parse the body, so it cannot tell which tables the body joins. The probe sees the + schema before those changes, the replace runs after them, and Postgres refuses to retype or + drop what a standing view reads. A derived projection or report view is not affected: it + lists every table it reads; +- a body that reads columns it does not name (`SELECT *`, `NATURAL JOIN`), in a migration + that also alters any table or any other view. Its column list comes from what it reads, so + an answer taken before the change would not hold. A drop and a create of the same view needs no `--allow`. It has two costs a replace does not: diff --git a/server/typescript/packages/codegen-ts/src/projection/build-projection-views.ts b/server/typescript/packages/codegen-ts/src/projection/build-projection-views.ts index c58986bb8..45dba94dc 100644 --- a/server/typescript/packages/codegen-ts/src/projection/build-projection-views.ts +++ b/server/typescript/packages/codegen-ts/src/projection/build-projection-views.ts @@ -56,6 +56,11 @@ export interface ExpectedView { * so the view must be dropped before and recreated after. */ dependsOn: string[]; + /** + * The body is the author's `@sql`, verbatim. It may read tables `dependsOn` does not + * list (ADR-0043), so migrate-ts does not replace it in place beside a table change. + */ + opaque?: true; } export interface BuildProjectionViewsOptions { @@ -308,6 +313,7 @@ function emitSqlView( name: source.physicalName, // FR-016 four-step physical name sql: source.sqlBody!, // verbatim — never parsed, never re-wrapped dependsOn, + opaque: true, fqn: host.resolutionKey(), ...(schema !== undefined ? { schema } : {}), }); diff --git a/server/typescript/packages/codegen-ts/test/projection/build-projection-views.test.ts b/server/typescript/packages/codegen-ts/test/projection/build-projection-views.test.ts index be346a002..473bbb50b 100644 --- a/server/typescript/packages/codegen-ts/test/projection/build-projection-views.test.ts +++ b/server/typescript/packages/codegen-ts/test/projection/build-projection-views.test.ts @@ -454,7 +454,7 @@ describe("buildReportViews — view-backed reports (FR-044 Plan 2, Table A)", () }); const body = "SELECT COUNT(id) AS users FROM users WHERE kind = 'ADMIN'"; const sql = buildReportViews(await loadModel(tph({ "@kind": "view", "@view": "v_admins", "@sql": body })), PG); - expect(sql.map((v) => [v.name, v.sql, v.dependsOn])).toEqual([["v_admins", body, ["users"]]]); + expect(sql.map((v) => [v.name, v.sql, v.dependsOn, v.opaque])).toEqual([["v_admins", body, ["users"], true]]); // An @unmanaged view is likewise the author's: nothing is created and nothing is refused. expect(buildReportViews(await loadModel(tph({ "@kind": "view", "@view": "v_admins", "@unmanaged": true })), PG)).toEqual([]); // The derived path is the one that refuses. diff --git a/server/typescript/packages/integration-tests/test/view-replace-probe-pg.test.ts b/server/typescript/packages/integration-tests/test/view-replace-probe-pg.test.ts index 7f1ebf87c..5713c3b3b 100644 --- a/server/typescript/packages/integration-tests/test/view-replace-probe-pg.test.ts +++ b/server/typescript/packages/integration-tests/test/view-replace-probe-pg.test.ts @@ -509,10 +509,11 @@ describe("an @sql body that reads every column (postgres)", () => { }, 60_000); }); -// An `@sql` view's tracked tables are its anchors, not the tables its body joins. The replace -// runs after the plan's table changes, with the live view standing while they run, and -// Postgres refuses to retype or drop what a standing view reads. -describe("an @sql body that joins a table the view does not track (postgres)", () => { +// An `@sql` view's tracked tables are its anchors, not the tables its body joins, and the +// body is not parsed. The replace runs after the plan's table changes, with the live view +// standing while they run, and Postgres refuses to retype or drop what a standing view reads. +// So an `@sql` body is not replaced in place beside a change to any table's columns. +describe("an @sql body in a plan that changes a table's columns (postgres)", () => { const badge = (points?: "int" | "long"): unknown => ({ "object.entity": { name: "Badge", children: [ { "source.rdb": { "@table": "badges" } }, @@ -572,10 +573,12 @@ describe("an @sql body that joins a table the view does not track (postgres)", ( expect((await plan(after.expected)).changes).toEqual([]); }, 60_000); - test("a body change beside a change to a table it does not read is replaced in place", async () => { - await migrate((await model(scoredView("0::bigint"), SUMMARY_VIEW, [], BEFORE.others)).expected); + // A derived body lists every table it reads, so a change to another table cannot reach it. + test("a derived view that changes beside the same table change is replaced in place", async () => { + const flag = (field: string) => summary(computed("flag", "boolean", { op: "isNull", arg: { field } })); + await migrate((await model(flag("nickname"), SUMMARY_VIEW, [], BEFORE.others)).expected); const oid = await viewOid(SUMMARY_VIEW); - const after = await model(scoredView("1::bigint"), SUMMARY_VIEW, [], [badge("long")]); + const after = await model(flag("notes"), SUMMARY_VIEW, [], [badge("long")]); const changed = await plan(after.expected); expect(kindsOf(changed.changes)).toEqual(["change-column-type", "replace-view"]); @@ -585,6 +588,63 @@ describe("an @sql body that joins a table the view does not track (postgres)", ( }, 60_000); }); +// `SELECT *` over another view is re-expanded when the view is replaced. The probe sees that +// other view as it stands; once the same plan has appended a column to it, the new column +// lands in the middle of this view's column list. +describe("an @sql body that reads every column of another view (postgres)", () => { + const TOP_VIEW = "v_program_top"; + const top = (extra: number): unknown => ({ + "object.projection": { name: "ProgramTop", children: [ + { "source.rdb": { "@kind": "view", "@view": TOP_VIEW, "@sql": `SELECT b.*, ${extra} AS "extra" FROM "${SUMMARY_VIEW}" b` } }, + { "identity.primary": { name: "id", extends: "Program.id", "@fields": "id" } }, + { "field.long": { name: "id", extends: "Program.id" } }, + ] }, + }); + const IS_NAMED = computed("isNamed", "boolean", { op: "isNotNull", arg: { field: "nickname" } }); + const replace = (view: ViewDescriptor, db: Db) => + sql.raw(`CREATE OR REPLACE VIEW "${view.name}" AS ${view.sql}`).execute(db); + + test("a body change beside a column appended to that view is dropped and created, and applies", async () => { + await migrate((await model(top(1), TOP_VIEW, [], [summary(COALESCE_NICKNAME)])).expected); + const baseOid = await viewOid(SUMMARY_VIEW); + const after = await model(top(2), TOP_VIEW, [], [summary(COALESCE_NICKNAME, IS_NAMED)]); + const base = after.expected.views.find((v) => v.name === SUMMARY_VIEW)!; + + // Postgres accepts the replace today, and refuses it once the other view has its column. + const rollback = new Error("rollback"); + await expect(k.transaction().execute(async (trx) => { + await replace(after.view, trx); + throw rollback; + })).rejects.toBe(rollback); + await expect(k.transaction().execute(async (trx) => { + await replace(base, trx); + await replace(after.view, trx); + })).rejects.toThrow(/cannot change name of view column "extra" to "isNamed"/); + + const changed = await plan(after.expected); + expect(changed.blocked).toEqual([]); + expect(kindsOf(changed.changes)).toEqual(["create-view", "drop-view", "replace-view"]); + await apply(changed.changes); + + expect(await viewOid(SUMMARY_VIEW)).toBe(baseOid); + expect(Object.keys(await columnTypes(TOP_VIEW))).toEqual(["id", "displayName", "isNamed", "extra"]); + expect((await plan(after.expected)).changes).toEqual([]); + }, 60_000); + + test("a body change with no other view changing is replaced in place", async () => { + const before = await model(top(1), TOP_VIEW, [], [summary(COALESCE_NICKNAME)]); + await migrate(before.expected); + const oid = await viewOid(TOP_VIEW); + const after = await model(top(2), TOP_VIEW, [], [summary(COALESCE_NICKNAME)]); + + const changed = await plan(after.expected); + expect(kindsOf(changed.changes)).toEqual(["replace-view"]); + await apply(changed.changes); + expect(await viewOid(TOP_VIEW)).toBe(oid); + expect((await plan(after.expected)).changes).toEqual([]); + }, 60_000); +}); + describe("when the probe cannot answer, the plan is a drop and a create (postgres)", () => { const compatible = async (): Promise => { const m = await model(summary(COALESCE_NICKNAME), SUMMARY_VIEW); diff --git a/server/typescript/packages/migrate-ts/src/diff/index.ts b/server/typescript/packages/migrate-ts/src/diff/index.ts index 32e2dbabf..07e302333 100644 --- a/server/typescript/packages/migrate-ts/src/diff/index.ts +++ b/server/typescript/packages/migrate-ts/src/diff/index.ts @@ -966,9 +966,9 @@ async function diffViews( * about what depends on the view. This holds for a managed change and for an adoption, * and for a derived body and an opaque `@sql` one alike. * - * The probe answers for the schema as it stands, and the replace runs after this plan's - * table changes. When one of those changes would undo the answer, or be refused while the - * live view stands ({@link tableChangesPrecludeReplace}), the probe is not asked. + * The probe answers for the schema as it stands, and the replace runs after the rest of + * this plan. When the plan may change what the body reads + * ({@link planMayChangeWhatBodyReads}), the answer would not hold and the probe is not asked. */ async function pushViewUpdate( expected: ViewDescriptor, actual: ViewDescriptor, changes: Change[], @@ -984,7 +984,7 @@ async function pushViewUpdate( const reason: ViewChangeReason = unmanaged ? { kind: "unfingerprinted" } : { kind: "definition", compared: "fingerprint" }; - const evidence = probe === undefined || tableChangesPrecludeReplace(expected, actual, changes) + const evidence = probe === undefined || planMayChangeWhatBodyReads(expected, changes) ? undefined : await probe(expected, actual); if (evidence !== undefined && viewReplaceIsLegal(evidence.expected, evidence.live)) { @@ -1135,37 +1135,35 @@ function viewReadsAnyColumn(sql: string, cols: Iterable, readsEverything } /** - * Does a table change already in this plan rule out replacing the view in place? A replace - * runs after the plan's table changes and leaves the live view standing while they run. - * Pass 2 has run by the time a view is planned, so `changes` holds them. + * Postgres: every table change that can alter a column some view body reads. Only Postgres + * has a probe, so the kinds that rebuild a table on SQLite and D1 are not listed. + */ +const TABLE_COLUMN_CHANGES = new Set([ + ...VIEW_RECREATE_TRIGGERS_UNKNOWN_DIALECT, "add-column", "drop-table", +]); +const VIEW_CHANGES = new Set(["create-view", "replace-view", "drop-view"]); + +/** + * May the rest of this plan change what the view's body reads? * - * - A body that reads columns it does not name has a different column list once the plan - * adds a column, so what the probe saw would not hold. - * - Postgres refuses a type change, a drop or a rename of a column a standing view reads, - * and a drop of a table it reads. For a table in `dependsOn`, Pass 2c turns the replace - * into the pair. An `@sql` body's `dependsOn` leaves out the tables the body joins - * (ADR-0043), and a dropped table is in no expected view's, so for a table outside - * `dependsOn` the two bodies are the evidence: each names the tables and columns it - * reads, the live one as Postgres deparsed it. + * The invariant: a probe's answer is trusted only when the plan cannot change what the + * probed body reads. The probe sees the schema as it stands. The replace runs after the + * plan's table changes and after the view changes planned ahead of it, and the live view + * stands while they run. `changes` holds both by the time a view is planned. + * + * - A derived body names every column it reads and lists every table it reads in + * `dependsOn`. Pass 2c turns its replace into the pair when one of those tables + * changes, and nothing else in the plan reaches it: the probe decides. + * - An opaque `@sql` body may read a table `dependsOn` does not list (ADR-0043), so a + * change to any table's columns may be a change to what it reads. + * - A body that reads columns it does not name (`SELECT *`, `NATURAL JOIN`) takes its + * column list from what it reads, so a change to any table's columns or to any other + * view may change that list. */ -function tableChangesPrecludeReplace( - expected: ViewDescriptor, actual: ViewDescriptor, changes: readonly Change[], -): boolean { - const bodies = [expected.sql, actual.sql] - .filter((sql): sql is string => sql !== undefined) - .map((sql) => ({ sql, readsEverything: viewReadsEveryColumn(sql) })); - const columnsMayGrow = expected.sql !== undefined && viewReadsEveryColumn(expected.sql); - const tracked = expected.dependsOn ?? []; - return changes.some((c) => { - if (c.kind === "add-column") return columnsMayGrow; - if (c.kind !== "drop-table" && !VIEW_RECREATE_TRIGGERS_PG.has(c.kind)) return false; - const table = (c as { table?: string }).table; - if (typeof table !== "string" || tracked.includes(table)) return false; - const cols = changedColumns(c); - return bodies.some(({ sql, readsEverything }) => - columnIdentifierRe(table).test(sql) - && (cols === undefined || viewReadsAnyColumn(sql, cols, readsEverything))); - }); +function planMayChangeWhatBodyReads(expected: ViewDescriptor, changes: readonly Change[]): boolean { + const unnamedColumns = expected.sql !== undefined && viewReadsEveryColumn(expected.sql); + if (!unnamedColumns && expected.opaque !== true) return false; + return changes.some((c) => TABLE_COLUMN_CHANGES.has(c.kind) || (unnamedColumns && VIEW_CHANGES.has(c.kind))); } function recreateViewsDependingOnChangedTables( diff --git a/server/typescript/packages/migrate-ts/src/expected-schema.ts b/server/typescript/packages/migrate-ts/src/expected-schema.ts index c960b1aac..f7446f4f0 100644 --- a/server/typescript/packages/migrate-ts/src/expected-schema.ts +++ b/server/typescript/packages/migrate-ts/src/expected-schema.ts @@ -121,6 +121,8 @@ export interface ExpectedViewInput { schema?: string; sql?: string; dependsOn?: readonly string[]; + /** The body is author-written SQL (`@sql`). See `ViewDescriptor.opaque`. */ + opaque?: true; /** * `resolutionKey()` of the object that declared this view — its PROVENANCE. * Recorded in the provenance map and deliberately NEVER copied onto the @@ -379,6 +381,7 @@ export function buildExpectedSchemaWithProvenance( ...(v.schema !== undefined ? { schema: v.schema } : {}), ...(v.sql !== undefined ? { sql: v.sql, fingerprint: viewFingerprint(v.sql) } : {}), ...(v.dependsOn !== undefined ? { dependsOn: v.dependsOn } : {}), + ...(v.opaque === true ? { opaque: true as const } : {}), }; // The declaring FQN goes to the provenance map ONLY — never onto the descriptor, // which is what the committed snapshot serializes (see ExpectedViewInput.fqn). diff --git a/server/typescript/packages/migrate-ts/src/snapshot/serialize.ts b/server/typescript/packages/migrate-ts/src/snapshot/serialize.ts index 4849395ee..1b94b2a12 100644 --- a/server/typescript/packages/migrate-ts/src/snapshot/serialize.ts +++ b/server/typescript/packages/migrate-ts/src/snapshot/serialize.ts @@ -32,7 +32,10 @@ function sortByName(arr: readonly T[]): T[] { return [...arr].sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); } -/** Sort arrays by name so serialization is order-independent. */ +/** + * Sort arrays by name so serialization is order-independent, and leave out a view's + * `opaque` marker, which is not part of the on-disk format. + */ function canonicalize(s: SchemaSnapshot): SchemaSnapshot { return { tables: sortByName(s.tables).map((t) => ({ @@ -42,7 +45,11 @@ function canonicalize(s: SchemaSnapshot): SchemaSnapshot { foreignKeys: sortByName(t.foreignKeys), checks: sortByName(t.checks ?? []), })), - views: sortByName(s.views), + views: sortByName(s.views).map((v) => { + const written = { ...v }; + delete written.opaque; + return written; + }), ...(s.meta ? { meta: s.meta } : {}), }; } diff --git a/server/typescript/packages/migrate-ts/src/types.ts b/server/typescript/packages/migrate-ts/src/types.ts index 3693a3b63..098074af1 100644 --- a/server/typescript/packages/migrate-ts/src/types.ts +++ b/server/typescript/packages/migrate-ts/src/types.ts @@ -198,6 +198,12 @@ export interface ViewDescriptor { * a column a view depends on, and sqlite's recreate-and-copy rebuilds the table. */ dependsOn?: readonly string[]; + /** + * The body is author-written SQL (`@sql`), not derived from the model, so it may read + * tables `dependsOn` does not list (ADR-0043). EXPECTED side only, and read only while a + * plan is made against a live database: `serializeSnapshot` never writes it. + */ + opaque?: true; } // --------------------------------------------------------------------------- diff --git a/server/typescript/packages/migrate-ts/test/check/serialize-upgrade.test.ts b/server/typescript/packages/migrate-ts/test/check/serialize-upgrade.test.ts index 466320a53..84c12dd6e 100644 --- a/server/typescript/packages/migrate-ts/test/check/serialize-upgrade.test.ts +++ b/server/typescript/packages/migrate-ts/test/check/serialize-upgrade.test.ts @@ -35,6 +35,15 @@ describe("serialize with checks + snapshot format version", () => { expect(parsed.tables[0]?.checks).toEqual([]); }); + // The committed snapshot is a byte contract: `opaque` marks an `@sql` body for a live plan + // only, and a field written to the file would owe a format-version bump. + test("a view's opaque marker is not written", () => { + const view = { name: "v_orders", sql: "SELECT id FROM orders", dependsOn: ["orders"] }; + const withMarker: SchemaSnapshot = { ...snap(), views: [{ ...view, opaque: true }] }; + expect(serializeSnapshot(withMarker)).toBe(serializeSnapshot({ ...snap(), views: [view] })); + expect(withMarker.views[0]?.opaque).toBe(true); + }); + // Releases before 1.1.0 wrote each view's guessed column types into a v3 snapshot. That // file must keep loading, and the retired field must not be written back. test("a v3 snapshot that records a view's columns still loads, without them", () => { diff --git a/server/typescript/packages/migrate-ts/test/unit/view-replace.test.ts b/server/typescript/packages/migrate-ts/test/unit/view-replace.test.ts index 5412d36c8..387fbe2be 100644 --- a/server/typescript/packages/migrate-ts/test/unit/view-replace.test.ts +++ b/server/typescript/packages/migrate-ts/test/unit/view-replace.test.ts @@ -174,77 +174,114 @@ describe("the planner replaces a view only on the probe's evidence", () => { }); }); - // The replace runs after the plan's table changes and leaves the live view standing while - // they run. Postgres refuses to retype or drop a column, or drop a table, that a standing - // view reads (https://www.postgresql.org/docs/current/sql-altertable.html, DROP COLUMN and - // SET DATA TYPE; sql-droptable.html, RESTRICT). Pass 2c covers a table in `dependsOn`; an - // `@sql` body joins tables its `dependsOn` does not list. - describe("a body that reads a table outside dependsOn, in a plan that changes that table", () => { + // The probe sees the schema before the plan runs; the replace runs after the plan's table + // changes. Postgres refuses to retype or drop a column, or drop a table, that a standing + // view reads (https://www.postgresql.org/docs/current/sql-altertable.html, SET DATA TYPE + // and DROP COLUMN; sql-droptable.html, RESTRICT). An `@sql` body is not parsed and may join + // a table its `dependsOn` does not list, so any table's column change may be one it reads. + describe("an opaque @sql body, in a plan that changes a table's columns", () => { type Columns = SchemaSnapshot["tables"][number]["columns"]; const POINTS: Columns[number] = { name: "points", sqlType: { kind: "integer", bits: 32 }, nullable: false }; + const WIDE_POINTS: Columns[number] = { ...POINTS, sqlType: { kind: "integer", bits: 64 } }; const NOTE: Columns[number] = { name: "note", sqlType: { kind: "text" }, nullable: true }; - const withScores = (views: ViewDescriptor[], columns?: Columns): SchemaSnapshot => { + const withScores = (views: ViewDescriptor[], columns?: readonly Columns[number][]): SchemaSnapshot => { const s = snap(views); if (columns === undefined) return s; - return { ...s, tables: [...s.tables, { name: "scores", columns, indexes: [], foreignKeys: [], primaryKey: [], checks: [] }] }; + return { ...s, tables: [...s.tables, { name: "scores", columns: [...columns], indexes: [], foreignKeys: [], primaryKey: [], checks: [] }] }; }; + const opaque = (sql: string): ViewDescriptor => ({ ...view(sql), opaque: true }); const joined = (threshold: number): string => `SELECT g.id AS id, (SELECT count(*) FROM scores s WHERE s.points > ${threshold}) AS scored FROM games g`; const UNJOINED = "SELECT g.id AS id, 0::bigint AS scored FROM games g"; const same = { expected: [ID, col("scored", "bigint")], live: [ID, col("scored", "bigint")] }; - const WIDE_POINTS: Columns[number] = { ...POINTS, sqlType: { kind: "integer", bits: 64 } }; const opts = (probe: ViewReplaceProbe) => ({ dialect: "postgres" as const, viewReplaceProbe: probe, allow: { dropColumn: true, dropTable: true } }); test.each([ - ["retypes a column the body reads", joined(1), [WIDE_POINTS, NOTE], "change-column-type"], - ["drops a column the live body reads", UNJOINED, [NOTE], "drop-column"], - ["drops the table the live body reads", UNJOINED, undefined, "drop-table"], + ["retypes a column of a table outside dependsOn", joined(1), [WIDE_POINTS, NOTE], "change-column-type"], + ["drops a column of a table outside dependsOn", UNJOINED, [NOTE], "drop-column"], + ["drops a table outside dependsOn", UNJOINED, undefined, "drop-table"], + ["adds a column to a table outside dependsOn", joined(1), [POINTS, NOTE, { ...NOTE, name: "memo" }], "add-column"], ] as const)("a plan that %s drops and creates the view, and the probe is not asked", async (_, body, columns, change) => { const { probe, asked } = stub(same); - const r = await diff( - withScores([view(body)], columns === undefined ? undefined : [...columns]), - withScores([view(joined(0))], [POINTS, NOTE]), - opts(probe), - ); + const r = await diff(withScores([opaque(body)], columns), withScores([view(joined(0))], [POINTS, NOTE]), opts(probe)); expect(kinds(r)).toEqual([change, "drop-view", "create-view"]); expect(r.blocked).toEqual([]); expect(asked).toEqual([]); }); - test("a changed column the bodies do not read leaves the replace to the probe", async () => { + test("a plan that changes no table still replaces it on the probe's evidence", async () => { const { probe, asked } = stub(same); - const r = await diff( - withScores([view(joined(1))], [POINTS, { ...NOTE, sqlType: { kind: "integer", bits: 32 } }]), - withScores([view(joined(0))], [POINTS, NOTE]), - opts(probe), - ); - expect(kinds(r)).toEqual(["change-column-type", "replace-view"]); + const r = await diff(withScores([opaque(joined(1))], [POINTS, NOTE]), withScores([view(joined(0))], [POINTS, NOTE]), opts(probe)); + expect(kinds(r)).toEqual(["replace-view"]); expect(asked).toHaveLength(1); }); - test("a changed table the bodies do not read leaves the replace to the probe", async () => { + // A derived body lists every table it reads in dependsOn, so a change elsewhere cannot reach it. + test("a derived body beside the same table change is still replaced on the probe's evidence", async () => { const { probe, asked } = stub(same); - const r = await diff( - withScores([view(NEW_BODY)], [WIDE_POINTS, NOTE]), - withScores([view(OLD_BODY)], [POINTS, NOTE]), - opts(probe), - ); + const r = await diff(withScores([view(NEW_BODY)], [WIDE_POINTS, NOTE]), withScores([view(OLD_BODY)], [POINTS, NOTE]), opts(probe)); expect(kinds(r)).toEqual(["change-column-type", "replace-view"]); expect(asked).toHaveLength(1); }); + }); - // Postgres changes nullability in place under a view. - test("a nullability change on a column the body reads leaves the replace to the probe", async () => { - const { probe, asked } = stub(same); - const r = await diff( - withScores([view(joined(1))], [{ ...POINTS, nullable: true }, NOTE]), - withScores([view(joined(0))], [POINTS, NOTE]), - opts(probe), - ); - expect(kinds(r)).toEqual(["change-column-nullable", "replace-view"]); - expect(asked).toHaveLength(1); + // `SELECT *` over another view is re-expanded when the view is replaced. Once the plan has + // given that other view a new column, the column lands mid-list and Postgres refuses the + // replace ("cannot change name of view column", sql-createview.html). + describe("a body that reads every column of another view, in a plan that changes that view", () => { + const named = (name: string, sql: string, over: Partial = {}): ViewDescriptor => + ({ name, sql, dependsOn: ["games"], fingerprint: viewFingerprint(sql), ...over }); + const base = (sql: string) => named("v_base", sql); + const top = (extra: number) => named("v_top", `SELECT b.*, ${extra} AS extra FROM v_base b`, { opaque: true }); + const BASE = "SELECT g.id AS id FROM games g"; + const BASE_APPENDED = "SELECT g.id AS id, g.id AS twice FROM games g"; + const TWICE = col("twice", "bigint"); + const EXTRA = col("extra", "integer"); + + /** Answers for each view as Postgres would before the plan runs, and records who was asked. */ + function probeBeforeThePlan() { + const asked: string[] = []; + const probe: ViewReplaceProbe = async (expected) => { + asked.push(expected.name); + return expected.name === "v_base" + ? { expected: [ID, TWICE], live: [ID] } + : { expected: [ID, EXTRA], live: [ID, EXTRA] }; + }; + return { probe, asked }; + } + + test("the view it reads is replaced in place, and it is dropped and created", async () => { + const { probe, asked } = probeBeforeThePlan(); + const r = await diff(snap([base(BASE_APPENDED), top(2)]), snap([base(BASE), top(1)]), { dialect: "postgres", viewReplaceProbe: probe }); + expect(r.changes.map((c) => [c.kind, c.kind === "drop-view" ? c.view : (c as { view: ViewDescriptor }).view.name])).toEqual([ + ["replace-view", "v_base"], ["drop-view", "v_top"], ["create-view", "v_top"], + ]); + expect(r.blocked).toEqual([]); + expect(asked).toEqual(["v_base"]); }); + + test("with no other view changing, it is replaced on the probe's evidence", async () => { + const { probe, asked } = probeBeforeThePlan(); + const r = await diff(snap([base(BASE), top(2)]), snap([base(BASE), top(1)]), { dialect: "postgres", viewReplaceProbe: probe }); + expect(kinds(r)).toEqual(["replace-view"]); + expect(asked).toEqual(["v_top"]); + }); + }); + + test.each([ + ["a derived body", {}], + ["an opaque @sql body", { opaque: true as const }], + ])("adopting an unstamped view with %s the database proves compatible is a replace in place", async (_, marker) => { + const label = col("label", "text", "100"); + const { probe, asked } = stub({ expected: [ID, label], live: [ID, label] }); + const r = await diff(snap([{ ...view(NEW_BODY), ...marker }]), snap([view(OLD_BODY, false)]), { + dialect: "postgres", viewReplaceProbe: probe, allow: { adoptView: true }, + }); + expect(kinds(r)).toEqual(["replace-view"]); + expect(r.changes[0]).toMatchObject({ unmanagedActual: true }); + expect(r.blocked).toEqual([]); + expect(asked).toHaveLength(1); }); test("SQLite and D1 never ask: a replace there is written as DROP + CREATE", async () => { From 9964716ae5774f38ce1266fff88ac8cb22c42eb8 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Sun, 11 Oct 2026 14:01:20 -0400 Subject: [PATCH 4/5] no-mistakes(review): fix(migrate-ts): recreate views whose live body reads changed tables Postgres: a view is dropped and created, not replaced, when the plan retypes, drops or renames a column of, or drops, a table outside dependsOn that the live body names (X8 exception to X7; a renamed table is matched by its old name). Sibling checks: - rename-table: live Postgres test; no refusal, the proven replace-view applies and the view keeps its OID. No change. - rename-column: covered by the same exception; live Postgres test, drop + create applies. - mysql: migrate-ts Dialect is postgres | sqlite | d1; no MySQL dialect. No change. - sqlite: real bun:sqlite, statements applied one at a time. DROP COLUMN and a table rebuild both fail while a view reading the table stands. Fixed in Pass 2c for every table the stored view body names; executed test added. - d1: same engine, in one transaction with foreign keys on. Native path and the foreign-key cascade both failed; both fixed, including a view the migration drops; executed test added. Not covered: an @unmanaged live view reading a cascade-rebuilt table is left alone --- CHANGELOG.md | 14 +- docs/features/migrations-and-drift.md | 19 +- .../test/view-replace-probe-pg.test.ts | 107 +++++++++ .../packages/migrate-ts/src/diff/index.ts | 93 +++++--- .../migrate-ts/src/emit/d1-cascade.ts | 18 +- .../packages/migrate-ts/src/view-reads.ts | 34 +++ ...te-live-view-reads-untracked-table.test.ts | 225 ++++++++++++++++++ .../test/unit/diff-view-recreate.test.ts | 2 +- .../migrate-ts/test/unit/view-replace.test.ts | 89 +++++-- 9 files changed, 540 insertions(+), 61 deletions(-) create mode 100644 server/typescript/packages/migrate-ts/src/view-reads.ts create mode 100644 server/typescript/packages/migrate-ts/test/integration/sqlite-live-view-reads-untracked-table.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 608295243..914a7adce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1007,8 +1007,20 @@ until you regenerate. standing view reads), or a body that reads columns it does not name (`SELECT *`, `NATURAL JOIN`) in a migration that also alters any table or any other view (its column list comes from what it reads, so an answer taken before would not hold). A derived - projection or report view lists every table it reads and is not affected. + projection or report view lists every table its new definition reads, so a change to some + other table does not do this to it, with one exception that holds for every view: when + the definition in the database reads a table the new one no longer reads, and the same + migration retypes, drops or renames a column of that table or drops it, the plan is a drop + and a create. The probe proves the new definition only, and Postgres refuses those changes + while the old view stands. The table is found by name in the old view's stored + definition. Renaming the table alone keeps the replace: a view follows a renamed table. Before, an offline plan replaced a view in place when its recorded columns looked compatible. + - **SQLite and D1 drop a view before changing any table its stored definition reads.** They + refuse a `DROP COLUMN`, and the rebuild of a table, while a view that reads the table + stands. A view was already dropped before and created after such a change for the tables + its model lists. It now is for every table the definition in the database names as well: + one the new definition no longer reads, and one an `@sql` body joins. The D1 foreign-key + cascade does the same for every table it rebuilds, including a view the migration drops. - **A drop and a create discards the privileges granted on the view.** `DROP VIEW` removes the view's grants with it, and `meta migrate` writes no `GRANT`, so a role that had `SELECT` on a projection's view loses it, with nothing reported. It needs no `--allow` and applies cleanly. diff --git a/docs/features/migrations-and-drift.md b/docs/features/migrations-and-drift.md index 8ca4d8570..2175cdc45 100644 --- a/docs/features/migrations-and-drift.md +++ b/docs/features/migrations-and-drift.md @@ -440,11 +440,18 @@ and a `CREATE VIEW`: or retypes a column, changes its nullability or default, or drops a table). `meta migrate` does not parse the body, so it cannot tell which tables the body joins. The probe sees the schema before those changes, the replace runs after them, and Postgres refuses to retype or - drop what a standing view reads. A derived projection or report view is not affected: it - lists every table it reads; + drop what a standing view reads. A derived projection or report view lists every table its + new definition reads, so a change to some other table does not do this to it; - a body that reads columns it does not name (`SELECT *`, `NATURAL JOIN`), in a migration that also alters any table or any other view. Its column list comes from what it reads, so - an answer taken before the change would not hold. + an answer taken before the change would not hold; +- any view, derived ones included, whose definition **in the database** reads a table the new + definition no longer reads, in a migration that also retypes, drops or renames a column of + that table, or drops the table. The probe proves the new definition only. The old view + stands while the table changes, and Postgres refuses to retype or drop what it reads. + `meta migrate` finds the table by name in the old view's stored definition, so a view that + merely names something spelled like that table is dropped and created too. Renaming the + table alone does not do this: a view follows a renamed table. A drop and a create of the same view needs no `--allow`. It has two costs a replace does not: @@ -458,7 +465,11 @@ Plan with `--from-db` to get the in-place replace when it is legal, which keeps grant the privileges again after the migration is applied. SQLite and D1 have no `CREATE OR REPLACE VIEW`: every replaced view is written as `DROP VIEW` -and `CREATE VIEW` there, so there is no refusal to avoid and no probe. +and `CREATE VIEW` there, so there is no refusal to avoid and no probe. They refuse something +else: a `DROP COLUMN`, or the rebuild of a table, while a view that reads the table stands. So +a view is dropped before such a change and created after it, for every table its definition +in the database names, including one the new definition no longer reads and one an `@sql` +body joins. The D1 foreign-key cascade does the same for every table it rebuilds. **A cascading drop is blocked, loudly.** If dropping a view would destroy dependent objects — another application's view, a materialized view — `meta migrate` blocks and diff --git a/server/typescript/packages/integration-tests/test/view-replace-probe-pg.test.ts b/server/typescript/packages/integration-tests/test/view-replace-probe-pg.test.ts index 5713c3b3b..0f6ded67c 100644 --- a/server/typescript/packages/integration-tests/test/view-replace-probe-pg.test.ts +++ b/server/typescript/packages/integration-tests/test/view-replace-probe-pg.test.ts @@ -588,6 +588,113 @@ describe("an @sql body in a plan that changes a table's columns (postgres)", () }, 60_000); }); +// The probe proves the NEW body. The LIVE view stands while the plan's table changes run, and +// it may read a table the new body no longer reads: that table is not among the tables the +// new view lists, so nothing but the live body's own text says the view reads it. +describe("a derived view whose live body reads a table the new body does not (postgres)", () => { + const BADGES = [{ "relationship.aggregation": { name: "badges", "@cardinality": "many", "@objectRef": "Badge" } }]; + const ownedBadge = (points: Record | undefined, table = "badges"): unknown => ({ + "object.entity": { name: "Badge", children: [ + { "source.rdb": { "@table": table } }, + { "field.long": { name: "id" } }, + { "field.long": { name: "programId", "@required": true } }, + ...(points !== undefined ? [points] : []), + { "identity.primary": { name: "id", "@fields": "id", "@generation": "increment" } }, + { "identity.reference": { name: "ref_program", "@fields": "programId", "@references": "Program" } }, + ] }, + }); + const POINTS = { "field.int": { name: "points", "@required": true } }; + const total = (origin: Record): unknown => + summary({ "field.long": { name: "total", children: [{ "origin.aggregate": { "@agg": "sum", ...origin } }] } }); + const OVER_BADGES = total({ "@of": "Badge.points", "@via": "Program.badges" }); + const OVER_WEEKS = total({ "@of": "Week.durationMinutes", ...VIA }); + + /** The view sums badge points; the model then sums week minutes into the same column. */ + async function repointed(programFields: unknown[], others: unknown[]): Promise { + await migrate((await model(OVER_BADGES, SUMMARY_VIEW, BADGES, [ownedBadge(POINTS)])).expected); + expect(await columnTypes(SUMMARY_VIEW)).toEqual({ id: "bigint", total: "bigint" }); + const after = await model(OVER_WEEKS, SUMMARY_VIEW, programFields, others); + expect(after.view.dependsOn).not.toContain("badges"); + return after; + } + const planWith = async (expected: SchemaSnapshot, renames: Parameters[0]["renames"] = []) => + diff({ + expected, actual: await introspectPostgres(k), dialect: "postgres", renames, + allow: { dropColumn: true, dropTable: true }, viewReplaceProbe: postgresViewReplaceProbe(k), + }); + const converged = async (expected: SchemaSnapshot) => { + expect(await columnTypes(SUMMARY_VIEW)).toEqual({ id: "bigint", total: "bigint" }); + expect((await plan(expected)).changes).toEqual([]); + }; + + test.each([ + { + change: "retypes a column the live body reads", programFields: BADGES, + others: [ownedBadge({ "field.long": { name: "points", "@required": true } })], + statement: `ALTER TABLE "badges" ALTER COLUMN "points" TYPE BIGINT`, + refusal: /cannot alter type of a column used by a view or rule/, planned: "change-column-type", + }, + { + change: "drops a column the live body reads", programFields: BADGES, others: [ownedBadge(undefined)], + statement: `ALTER TABLE "badges" DROP COLUMN "points"`, + refusal: /cannot drop column points of table badges because other objects depend on it/, planned: "drop-column", + }, + { + change: "drops the table the live body reads", programFields: [], others: [], + statement: `DROP TABLE "badges"`, + refusal: /cannot drop table badges because other objects depend on it/, planned: "drop-table", + }, + ])("a plan that $change drops and creates the view, and applies", async (c) => { + const after = await repointed(c.programFields, c.others); + + // Postgres accepts the replace today, and refuses the table change while the view stands. + const rollback = new Error("rollback"); + await expect(k.transaction().execute(async (trx) => { + await sql.raw(`CREATE OR REPLACE VIEW "${SUMMARY_VIEW}" AS ${after.view.sql}`).execute(trx); + throw rollback; + })).rejects.toBe(rollback); + await expect(sql.raw(c.statement).execute(k)).rejects.toThrow(c.refusal); + + const changed = await planWith(after.expected); + expect(changed.blocked).toEqual([]); + const kinds = kindsOf(changed.changes); + expect(kinds).toContain(c.planned); + expect(kinds).toContain("drop-view"); + expect(kinds).toContain("create-view"); + expect(kinds).not.toContain("replace-view"); + await apply(changed.changes); + await converged(after.expected); + }, 60_000); + + // Postgres lets a column be renamed under a view, so this one is not a refusal. The view + // is recreated all the same: the planner sees the rename as a drop until it is resolved. + test("a plan that renames a column the live body reads drops and creates the view, and applies", async () => { + const after = await repointed(BADGES, [ownedBadge({ "field.int": { name: "pts", "@required": true } })]); + + const changed = await planWith(after.expected, [{ kind: "column", table: "badges", from: "points", to: "pts" }]); + expect(changed.blocked).toEqual([]); + expect(kindsOf(changed.changes)).toEqual(["create-view", "drop-view", "rename-column"]); + await apply(changed.changes); + await converged(after.expected); + }, 60_000); + + // A view holds the OID of each table it reads, so it follows a rename: nothing is refused, + // and the view is replaced in place. + test("a plan that renames the table the live body reads replaces the view in place, and applies", async () => { + const after = await repointed(BADGES, [ownedBadge(POINTS, "awards")]); + const oid = await viewOid(SUMMARY_VIEW); + + const changed = await planWith(after.expected, [{ kind: "table", from: "badges", to: "awards" }]); + expect(changed.blocked).toEqual([]); + expect(kindsOf(changed.changes)).toEqual(["rename-table", "replace-view"]); + await apply(changed.changes); + + expect(await viewOid(SUMMARY_VIEW)).toBe(oid); + expect(await scalar(`SELECT to_regclass('public.awards') IS NOT NULL AND to_regclass('public.badges') IS NULL`)).toBe(true); + await converged(after.expected); + }, 60_000); +}); + // `SELECT *` over another view is re-expanded when the view is replaced. The probe sees that // other view as it stands; once the same plan has appended a column to it, the new column // lands in the middle of this view's column list. diff --git a/server/typescript/packages/migrate-ts/src/diff/index.ts b/server/typescript/packages/migrate-ts/src/diff/index.ts index 07e302333..3ad6ee69f 100644 --- a/server/typescript/packages/migrate-ts/src/diff/index.ts +++ b/server/typescript/packages/migrate-ts/src/diff/index.ts @@ -12,6 +12,7 @@ import { PrimaryKeyChangeError } from "../errors.js"; import { detectColumnRenames, detectTableRenames } from "./rename-heuristic.js"; import { firstViewSqlDifference, viewSqlEquals } from "../view-sql-compare.js"; import { viewReplaceIsLegal, type ViewReplaceProbe } from "../view-replace.js"; +import { liveViewReadsTable, sqlNamesIdentifier } from "../view-reads.js"; import { checkExprEquals, checkExprRespelledNullSafe, normalizeCheckExpr, renameExprIdentifiers } from "../check-expr-compare.js"; import { isPgAutoSequenceDefault } from "../pg-identity-default.js"; import { DEFAULT_DB_SCHEMA_POSTGRES } from "@metaobjectsdev/metadata"; @@ -967,8 +968,8 @@ async function diffViews( * and for a derived body and an opaque `@sql` one alike. * * The probe answers for the schema as it stands, and the replace runs after the rest of - * this plan. When the plan may change what the body reads - * ({@link planMayChangeWhatBodyReads}), the answer would not hold and the probe is not asked. + * this plan, with the live view standing meanwhile. When the plan may change what either + * body reads ({@link planMayChangeWhatBodyReads}), the probe is not asked. */ async function pushViewUpdate( expected: ViewDescriptor, actual: ViewDescriptor, changes: Change[], @@ -984,7 +985,7 @@ async function pushViewUpdate( const reason: ViewChangeReason = unmanaged ? { kind: "unfingerprinted" } : { kind: "definition", compared: "fingerprint" }; - const evidence = probe === undefined || planMayChangeWhatBodyReads(expected, changes) + const evidence = probe === undefined || planMayChangeWhatBodyReads(expected, actual, changes) ? undefined : await probe(expected, actual); if (evidence !== undefined && viewReplaceIsLegal(evidence.expected, evidence.live)) { @@ -1104,36 +1105,30 @@ function viewReadsEveryColumn(sql: string): boolean { } /** - * Per-column identifier matches, compiled once per column name: a wide schema diff revisits - * the same column names across tables and views, and recompiling a RegExp per test would - * rebuild identical state over and over. - */ -const columnIdentifierRes = new Map(); -function columnIdentifierRe(column: string): RegExp { - let re = columnIdentifierRes.get(column); - if (re === undefined) { - const escaped = column.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); - re = new RegExp(`(?, readsEverything: boolean): boolean { if (readsEverything) return true; for (const col of cols) { - if (columnIdentifierRe(col).test(sql)) return true; + if (sqlNamesIdentifier(sql, col)) return true; } return false; } +/** + * The name each table has in the database before this plan runs. Pass 1c turns a renamed + * table's drop and create into a `rename-table`, and every other change on that table is + * keyed by its NEW name, which a live view body cannot hold yet. + */ +function liveTableNames(changes: readonly Change[]): (table: string) => string { + const renamedFrom = new Map(); + for (const c of changes) if (c.kind === "rename-table") renamedFrom.set(c.to, c.from); + return (table) => renamedFrom.get(table) ?? table; +} + /** * Postgres: every table change that can alter a column some view body reads. Only Postgres * has a probe, so the kinds that rebuild a table on SQLite and D1 are not listed. @@ -1142,28 +1137,45 @@ const TABLE_COLUMN_CHANGES = new Set([ ...VIEW_RECREATE_TRIGGERS_UNKNOWN_DIALECT, "add-column", "drop-table", ]); const VIEW_CHANGES = new Set(["create-view", "replace-view", "drop-view"]); +/** Postgres refuses each of these on a table while a view that reads it stands. */ +const TABLE_CHANGES_A_VIEW_BLOCKS = new Set([...VIEW_RECREATE_TRIGGERS_PG, "drop-table"]); /** - * May the rest of this plan change what the view's body reads? + * May the rest of this plan change what the view reads, in its new body or its live one? * * The invariant: a probe's answer is trusted only when the plan cannot change what the - * probed body reads. The probe sees the schema as it stands. The replace runs after the - * plan's table changes and after the view changes planned ahead of it, and the live view - * stands while they run. `changes` holds both by the time a view is planned. + * probed body reads, and the live view may stand only while the plan changes nothing it + * reads. The probe sees the schema as it stands and proves the NEW body. The replace runs + * after the plan's table changes and after the view changes planned ahead of it, and the + * LIVE view stands while they run. `changes` holds both by the time a view is planned. * * - A derived body names every column it reads and lists every table it reads in * `dependsOn`. Pass 2c turns its replace into the pair when one of those tables - * changes, and nothing else in the plan reaches it: the probe decides. + * changes, so for the new body the probe decides. * - An opaque `@sql` body may read a table `dependsOn` does not list (ADR-0043), so a * change to any table's columns may be a change to what it reads. * - A body that reads columns it does not name (`SELECT *`, `NATURAL JOIN`) takes its * column list from what it reads, so a change to any table's columns or to any other * view may change that list. + * - The live body, of any view, may read a table the new body no longer reads. That + * table is outside `dependsOn`, so Pass 2c does not see it, and the probe never saw the + * live body. Postgres refuses to retype, drop or rename a column of it, or to drop it, + * while the live view stands. The live body's own text says which tables it reads. */ -function planMayChangeWhatBodyReads(expected: ViewDescriptor, changes: readonly Change[]): boolean { +function planMayChangeWhatBodyReads( + expected: ViewDescriptor, actual: ViewDescriptor, changes: readonly Change[], +): boolean { const unnamedColumns = expected.sql !== undefined && viewReadsEveryColumn(expected.sql); - if (!unnamedColumns && expected.opaque !== true) return false; - return changes.some((c) => TABLE_COLUMN_CHANGES.has(c.kind) || (unnamedColumns && VIEW_CHANGES.has(c.kind))); + const untracked = unnamedColumns || expected.opaque === true; + const tracked = expected.dependsOn ?? []; + const liveName = liveTableNames(changes); + return changes.some((c) => { + if (untracked && TABLE_COLUMN_CHANGES.has(c.kind)) return true; + if (unnamedColumns && VIEW_CHANGES.has(c.kind)) return true; + if (!TABLE_CHANGES_A_VIEW_BLOCKS.has(c.kind)) return false; + const table = (c as { table?: string }).table; + return typeof table === "string" && !tracked.includes(table) && liveViewReadsTable(actual, liveName(table)); + }); } function recreateViewsDependingOnChangedTables( @@ -1208,12 +1220,14 @@ function recreateViewsDependingOnChangedTables( if (c.kind === "create-view") createQueued.add(queuedId); } + const liveName = liveTableNames(changes); for (const v of expectedViews) { const id = viewIdentity(v); const viewChanges = definitionQueued.has(id); const sql = v.sql; const readsEverything = sql !== undefined && viewReadsEveryColumn(sql); - const tables = (v.dependsOn ?? []).filter((t) => { + const tracked = v.dependsOn ?? []; + const tables = tracked.filter((t) => { if (!alteredTables.has(t)) return false; // A view whose own definition is unchanged is stranded only if it READS an altered // column. One already queued for a replace (its definition differs) is recreated @@ -1222,6 +1236,16 @@ function recreateViewsDependingOnChangedTables( if (cols === null || viewChanges || sql === undefined) return true; return viewReadsAnyColumn(sql, cols, readsEverything); }); + const prior = actualById.get(id); + // Where a triggering change rebuilds the table, the view in the database is stranded by + // every table ITS body reads, and that body may read one `dependsOn` leaves out: a + // table the new body no longer reads, or one an `@sql` body joins (ADR-0043). On + // Postgres the same case is settled when the view is planned (pushViewUpdate). + if (!columnScoped && prior !== undefined) { + for (const t of alteredTables.keys()) { + if (!tracked.includes(t) && liveViewReadsTable(prior, liveName(t))) tables.push(t); + } + } if (tables.length === 0) continue; // Brand-new view (create-view already queued): the DB has no prior view to @@ -1239,7 +1263,6 @@ function recreateViewsDependingOnChangedTables( changes.splice(i, 1); } } - const prior = actualById.get(id); const reason = recreateReason(v, prior, superseded, tables, dialect); changes.push({ kind: "drop-view", view: v.name, ...schemaSpread(v.schema), diff --git a/server/typescript/packages/migrate-ts/src/emit/d1-cascade.ts b/server/typescript/packages/migrate-ts/src/emit/d1-cascade.ts index 37832f9e5..e217af258 100644 --- a/server/typescript/packages/migrate-ts/src/emit/d1-cascade.ts +++ b/server/typescript/packages/migrate-ts/src/emit/d1-cascade.ts @@ -8,6 +8,7 @@ import { quote, } from "./sqlite.js"; import { buildFkEdges, unionEdges, affectedSet, topoOrder } from "./fk-graph.js"; +import { liveViewReadsTable } from "../view-reads.js"; /** * Prefix for the transient rebuild tables the cascade recipe creates @@ -81,14 +82,27 @@ export function emitD1Cascade( // Drop them before the rebuild and recreate them after every table is back at its // final name. The dispatcher filters any diff-injected view change for these from the // native "rest" so they are not emitted twice / out of order. + // + // A view in the database is stranded by every affected table ITS body reads, and that + // body may read one the expected view's `dependsOn` leaves out: a table the new body no + // longer reads, or one an `@sql` body joins (ADR-0043). Such a view is the cascade's too + // when the model still declares it (recreated below) or this migration drops it. + const liveReaders = new Set( + actualSchema.views + .filter((v) => [...affected].some((t) => liveViewReadsTable(v, t))) + .map((v) => v.name), + ); const affectedViews = expectedSchema.views.filter((v) => - (v.dependsOn ?? []).some((t) => affected.has(t)), + liveReaders.has(v.name) || (v.dependsOn ?? []).some((t) => affected.has(t)), ); const handledViews = new Set(affectedViews.map((v) => v.name)); + for (const c of changes) { + if (c.kind === "drop-view" && liveReaders.has(c.view)) handledViews.add(c.view); + } const stmts: string[] = []; - for (const v of affectedViews) stmts.push(`DROP VIEW IF EXISTS ${quote(v.name)};`); + for (const view of handledViews) stmts.push(`DROP VIEW IF EXISTS ${quote(view)};`); // Defer FK enforcement to commit — the D1-legal alternative to the (no-op) OFF bracket. stmts.push("PRAGMA defer_foreign_keys = ON;"); diff --git a/server/typescript/packages/migrate-ts/src/view-reads.ts b/server/typescript/packages/migrate-ts/src/view-reads.ts new file mode 100644 index 000000000..9e11c9497 --- /dev/null +++ b/server/typescript/packages/migrate-ts/src/view-reads.ts @@ -0,0 +1,34 @@ +// src/view-reads.ts +import type { ViewDescriptor } from "./types.js"; + +/** + * Identifier matches, compiled once per name: a wide schema diff revisits the same table + * and column names across views, and recompiling a RegExp per test would rebuild identical + * state over and over. + */ +const identifierRes = new Map(); + +/** + * Does `sql` name `identifier`, quoted or bare? A textual match. It can over-report (a + * column name that is also an alias, a table name that is also a column), which is the safe + * direction: a wrongly recreated view costs a drop and a create, a missed one fails the + * table change at apply. + */ +export function sqlNamesIdentifier(sql: string, identifier: string): boolean { + let re = identifierRes.get(identifier); + if (re === undefined) { + const escaped = identifier.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + re = new RegExp(`(? after drop column`; + * - the recreate-and-copy rebuild fails at its `ALTER TABLE … RENAME TO`, because the view + * names a table that does not exist at that moment (`error in view : no such table`). + * + * So the view has to be dropped before the table change and created after it. SQLite is the + * oracle: each test first shows the refusal on the database as it stands, then applies the + * emitted migration one statement at a time and reads the view back. D1 is SQLite at the SQL + * level; its cases run inside one transaction with foreign keys on, as remote D1 does, and + * cover both the native path and the foreign-key cascade rebuild. + */ + +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { Database } from "bun:sqlite"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Kysely } from "kysely"; +import { LibsqlDialect } from "@libsql/kysely-libsql"; +import { MetaDataLoader, InMemoryStringSource } from "@metaobjectsdev/metadata"; +import { buildExpectedSchema, type ExpectedViewInput } from "../../src/expected-schema.js"; +import { introspectSqlite } from "../../src/introspect/sqlite.js"; +import { diff } from "../../src/diff/index.js"; +import { emit } from "../../src/emit/index.js"; +import type { SchemaSnapshot } from "../../src/types.js"; + +type SqliteDialect = "sqlite" | "d1"; + +let tmpDir: string; +let dbPath: string; + +beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), "live-view-reads-")); + dbPath = join(tmpDir, "t.db"); +}); +afterEach(() => { + rmSync(tmpDir, { recursive: true, force: true }); +}); + +const VIEW = "v_games"; +const READS_SCORES = "SELECT g.id AS id, (SELECT count(*) FROM scores s WHERE s.points > 0) AS scored FROM games g"; +const READS_GAMES_ONLY = "SELECT g.id AS id, 0 AS scored FROM games g"; +/** The view as the model declares it: `games` is the one table it is known to read. */ +const view = (sql: string): ExpectedViewInput => ({ name: VIEW, sql, dependsOn: ["games"] }); + +const entity = (name: string, table: string, children: unknown[]): unknown => ({ + "object.entity": { name, children: [ + { "source.rdb": { "@table": table } }, + { "field.long": { name: "id" } }, + ...children, + { "identity.primary": { name: "pk", "@fields": ["id"], "@generation": "increment" } }, + ] }, +}); +const GAME = entity("Game", "games", []); +const points = (required: boolean): unknown => ({ "field.int": { name: "points", ...(required ? { "@required": true } : {}) } }); +const score = (...fields: unknown[]): unknown => entity("Score", "scores", fields); +/** A table that references `scores`, which makes a rebuild of `scores` a D1 cascade. */ +const TALLY = entity("Tally", "tallies", [ + { "field.long": { name: "scoreId", "@required": true } }, + { "identity.reference": { name: "ref_score", "@fields": ["scoreId"], "@references": "Score" } }, +]); + +async function build(dialect: SqliteDialect, entities: unknown[], views: ExpectedViewInput[]): Promise { + const json = JSON.stringify({ "metadata.root": { package: "acme", children: entities } }); + const { root, errors } = await new MetaDataLoader().load([new InMemoryStringSource(json)]); + expect(errors).toEqual([]); + return buildExpectedSchema(root, { dialect, columnNamingStrategy: "snake_case", views }); +} + +async function introspect(): Promise { + const k = new Kysely>({ dialect: new LibsqlDialect({ url: `file:${dbPath}` }) }); + try { + return await introspectSqlite(k); + } finally { + await k.destroy(); + } +} + +function withDb(use: (db: Database) => T): T { + const db = new Database(dbPath); + try { + return use(db); + } finally { + db.close(); + } +} + +/** The statements of an emitted migration, without comment-only lines. */ +function statements(up: string): string[] { + return up.split(";") + .map((s) => s.split("\n").filter((line) => !line.trimStart().startsWith("--")).join("\n").trim()) + .filter(Boolean); +} + +/** Apply one statement at a time. D1 runs a file in one transaction with foreign keys on. */ +function apply(dialect: SqliteDialect, up: string): void { + withDb((db) => { + if (dialect === "sqlite") { + for (const s of statements(up)) db.run(s); + return; + } + db.run("PRAGMA foreign_keys = ON"); + db.run("BEGIN"); + try { + for (const s of statements(up)) db.run(s); + db.run("COMMIT"); + } catch (e) { + db.run("ROLLBACK"); + throw e; + } + }); +} + +async function plan(dialect: SqliteDialect, expected: SchemaSnapshot) { + const actual = await introspect(); + const planned = await diff({ expected, actual, dialect, allow: { dropColumn: true, dropView: true } }); + expect(planned.blocked).toEqual([]); + const { up } = emit(planned.changes, { + dialect, expectedSchema: expected, actualSchema: actual, + ...(actual.meta !== undefined ? { actualMeta: actual.meta } : {}), + }); + return { changes: planned.changes, up }; +} + +async function migrate(dialect: SqliteDialect, expected: SchemaSnapshot): Promise { + const { changes, up } = await plan(dialect, expected); + apply(dialect, up); + expect((await plan(dialect, expected)).changes).toEqual([]); + return changes.map((c) => c.kind); +} + +/** SQLite refuses `sqlText` on the database as it stands; nothing of the attempt is kept. */ +function refuses(sqlText: string[], refusal: RegExp): void { + withDb((db) => { + db.run("BEGIN"); + try { + expect(() => { for (const s of sqlText) db.run(s); }).toThrow(refusal); + } finally { + db.run("ROLLBACK"); + } + }); +} +const DROP_POINTS = [`ALTER TABLE "scores" DROP COLUMN "points"`]; +const REBUILD_SCORES = [ + `CREATE TABLE "__rebuilt_scores" AS SELECT * FROM "scores"`, + `DROP TABLE "scores"`, + `ALTER TABLE "__rebuilt_scores" RENAME TO "scores"`, +]; +const AFTER_DROP_COLUMN = /error in view v_games after drop column/; +const NO_SUCH_TABLE = /error in view v_games: no such table: main\.scores/; + +const viewRows = (): unknown[] => withDb((db) => db.query(`SELECT id, scored FROM "${VIEW}"`).all()); +const viewExists = (): boolean => + withDb((db) => db.query(`SELECT count(*) AS n FROM sqlite_master WHERE type = 'view' AND name = '${VIEW}'`).get() as { n: number }).n === 1; +const scoresColumns = (): string[] => + withDb((db) => (db.query(`PRAGMA table_info("scores")`).all() as { name: string }[]).map((c) => c.name)); +const seedGame = (): void => withDb((db) => { db.run(`INSERT INTO "games" ("id") VALUES (7)`); }); + +describe.each(["sqlite", "d1"] as const)("a live view that reads a table outside dependsOn (%s)", (dialect) => { + test("the new body no longer reads the table, and the plan drops a column the live body reads", async () => { + await migrate(dialect, await build(dialect, [GAME, score(points(false))], [view(READS_SCORES)])); + seedGame(); + refuses(DROP_POINTS, AFTER_DROP_COLUMN); + + const kinds = await migrate(dialect, await build(dialect, [GAME, score()], [view(READS_GAMES_ONLY)])); + expect(kinds).toContain("drop-column"); + expect(scoresColumns()).toEqual(["id"]); + expect(viewRows()).toEqual([{ id: 7, scored: 0 }]); + }); + + test("the new body no longer reads the table, and the plan rebuilds the table", async () => { + await migrate(dialect, await build(dialect, [GAME, score(points(true))], [view(READS_SCORES)])); + seedGame(); + refuses(REBUILD_SCORES, NO_SUCH_TABLE); + + const kinds = await migrate(dialect, await build(dialect, [GAME, score(points(false))], [view(READS_GAMES_ONLY)])); + expect(kinds).toContain("change-column-nullable"); + expect(viewRows()).toEqual([{ id: 7, scored: 0 }]); + }); + + test("an unchanged body that joins the table, and the plan rebuilds the table", async () => { + await migrate(dialect, await build(dialect, [GAME, score(points(true))], [view(READS_SCORES)])); + seedGame(); + refuses(REBUILD_SCORES, NO_SUCH_TABLE); + + const kinds = await migrate(dialect, await build(dialect, [GAME, score(points(false))], [view(READS_SCORES)])); + expect(kinds).toContain("change-column-nullable"); + expect(viewRows()).toEqual([{ id: 7, scored: 0 }]); + }); +}); + +// A table another table references is rebuilt by the D1 foreign-key cascade, which drops and +// renames every table in the affected set inside the one transaction. +describe("a live view that reads a table the D1 cascade rebuilds", () => { + const before = () => build("d1", [GAME, score(points(true)), TALLY], [view(READS_SCORES)]); + const rebuilt = (views: ExpectedViewInput[]) => build("d1", [GAME, score(points(false)), TALLY], views); + + test.each([ + ["the new body no longer reads the table", READS_GAMES_ONLY], + ["the body is unchanged", READS_SCORES], + ])("%s: the view is dropped before the cascade and created after it", async (_, body) => { + await migrate("d1", await before()); + seedGame(); + refuses(REBUILD_SCORES, NO_SUCH_TABLE); + + await migrate("d1", await rebuilt([view(body)])); + expect(viewRows()).toEqual([{ id: 7, scored: 0 }]); + }); + + test("the model no longer declares the view: it is dropped before the cascade", async () => { + await migrate("d1", await before()); + refuses(REBUILD_SCORES, NO_SUCH_TABLE); + + const kinds = await migrate("d1", await rebuilt([])); + expect(kinds).toContain("drop-view"); + expect(viewExists()).toBe(false); + }); +}); diff --git a/server/typescript/packages/migrate-ts/test/unit/diff-view-recreate.test.ts b/server/typescript/packages/migrate-ts/test/unit/diff-view-recreate.test.ts index 5191ced1f..4dc50d024 100644 --- a/server/typescript/packages/migrate-ts/test/unit/diff-view-recreate.test.ts +++ b/server/typescript/packages/migrate-ts/test/unit/diff-view-recreate.test.ts @@ -96,7 +96,7 @@ describe("diff — Pass 2c view dependency-recreate", () => { }); test("column change on a table the view does NOT depend on → no view change", async () => { - const independent: ViewDescriptor = { name: "v", sql: BODY, dependsOn: ["other"] }; + const independent: ViewDescriptor = { name: "v", sql: "SELECT c FROM other", dependsOn: ["other"] }; const r = await diff(snap("integer", [independent]), snap("text", [independent])); const k = kinds(r.changes); expect(k).toContain("change-column-type"); diff --git a/server/typescript/packages/migrate-ts/test/unit/view-replace.test.ts b/server/typescript/packages/migrate-ts/test/unit/view-replace.test.ts index 387fbe2be..b0e3c972f 100644 --- a/server/typescript/packages/migrate-ts/test/unit/view-replace.test.ts +++ b/server/typescript/packages/migrate-ts/test/unit/view-replace.test.ts @@ -14,7 +14,7 @@ import { describe, test, expect } from "bun:test"; import { diff } from "../../src/diff/index.js"; import { viewFingerprint } from "../../src/view-fingerprint.js"; import { viewReplaceIsLegal, type ExactViewColumn, type ViewReplaceProbe } from "../../src/view-replace.js"; -import type { SchemaSnapshot, ViewDescriptor } from "../../src/types.js"; +import type { DeclaredRename, SchemaSnapshot, ViewDescriptor } from "../../src/types.js"; const col = (name: string, type: string, collation = "0"): ExactViewColumn => ({ name, type, collation }); @@ -174,29 +174,30 @@ describe("the planner replaces a view only on the probe's evidence", () => { }); }); + // A second table, `scores`, that a view body may read without listing it in dependsOn. + type Columns = SchemaSnapshot["tables"][number]["columns"]; + const POINTS: Columns[number] = { name: "points", sqlType: { kind: "integer", bits: 32 }, nullable: false }; + const WIDE_POINTS: Columns[number] = { ...POINTS, sqlType: { kind: "integer", bits: 64 } }; + const NOTE: Columns[number] = { name: "note", sqlType: { kind: "text" }, nullable: true }; + const withScores = (views: ViewDescriptor[], columns?: readonly Columns[number][], table = "scores"): SchemaSnapshot => { + const s = snap(views); + if (columns === undefined) return s; + return { ...s, tables: [...s.tables, { name: table, columns: [...columns], indexes: [], foreignKeys: [], primaryKey: [], checks: [] }] }; + }; + const opaque = (sql: string): ViewDescriptor => ({ ...view(sql), opaque: true }); + const joined = (threshold: number): string => + `SELECT g.id AS id, (SELECT count(*) FROM scores s WHERE s.points > ${threshold}) AS scored FROM games g`; + const UNJOINED = "SELECT g.id AS id, 0::bigint AS scored FROM games g"; + const same = { expected: [ID, col("scored", "bigint")], live: [ID, col("scored", "bigint")] }; + const opts = (probe: ViewReplaceProbe, renames: readonly DeclaredRename[] = []) => + ({ dialect: "postgres" as const, viewReplaceProbe: probe, allow: { dropColumn: true, dropTable: true }, renames }); + // The probe sees the schema before the plan runs; the replace runs after the plan's table // changes. Postgres refuses to retype or drop a column, or drop a table, that a standing // view reads (https://www.postgresql.org/docs/current/sql-altertable.html, SET DATA TYPE // and DROP COLUMN; sql-droptable.html, RESTRICT). An `@sql` body is not parsed and may join // a table its `dependsOn` does not list, so any table's column change may be one it reads. describe("an opaque @sql body, in a plan that changes a table's columns", () => { - type Columns = SchemaSnapshot["tables"][number]["columns"]; - const POINTS: Columns[number] = { name: "points", sqlType: { kind: "integer", bits: 32 }, nullable: false }; - const WIDE_POINTS: Columns[number] = { ...POINTS, sqlType: { kind: "integer", bits: 64 } }; - const NOTE: Columns[number] = { name: "note", sqlType: { kind: "text" }, nullable: true }; - const withScores = (views: ViewDescriptor[], columns?: readonly Columns[number][]): SchemaSnapshot => { - const s = snap(views); - if (columns === undefined) return s; - return { ...s, tables: [...s.tables, { name: "scores", columns: [...columns], indexes: [], foreignKeys: [], primaryKey: [], checks: [] }] }; - }; - const opaque = (sql: string): ViewDescriptor => ({ ...view(sql), opaque: true }); - const joined = (threshold: number): string => - `SELECT g.id AS id, (SELECT count(*) FROM scores s WHERE s.points > ${threshold}) AS scored FROM games g`; - const UNJOINED = "SELECT g.id AS id, 0::bigint AS scored FROM games g"; - const same = { expected: [ID, col("scored", "bigint")], live: [ID, col("scored", "bigint")] }; - const opts = (probe: ViewReplaceProbe) => - ({ dialect: "postgres" as const, viewReplaceProbe: probe, allow: { dropColumn: true, dropTable: true } }); - test.each([ ["retypes a column of a table outside dependsOn", joined(1), [WIDE_POINTS, NOTE], "change-column-type"], ["drops a column of a table outside dependsOn", UNJOINED, [NOTE], "drop-column"], @@ -226,6 +227,58 @@ describe("the planner replaces a view only on the probe's evidence", () => { }); }); + // The probe proves the NEW body. The LIVE view stands while the plan's table changes run, + // and Postgres refuses to retype or drop a column, or drop a table, that it reads + // (sql-altertable.html; sql-droptable.html). A table the new body no longer reads is + // outside dependsOn, so only the live body's own text says the view reads it. + describe("a derived body whose live view reads a table the new body does not", () => { + const live = () => withScores([view(joined(0))], [POINTS, NOTE]); + const PTS: Columns[number] = { ...POINTS, name: "pts" }; + const kindsSorted = (r: { changes: { kind: string }[] }) => kinds(r).sort(); + + test.each([ + ["retypes a column of that table", [WIDE_POINTS, NOTE], "change-column-type"], + ["drops a column of that table", [NOTE], "drop-column"], + ["drops that table", undefined, "drop-table"], + ] as const)("a plan that %s drops and creates the view, and the probe is not asked", async (_, columns, change) => { + const { probe, asked } = stub(same); + const r = await diff(withScores([view(UNJOINED)], columns), live(), opts(probe)); + expect(kinds(r)).toEqual([change, "drop-view", "create-view"]); + expect(r.blocked).toEqual([]); + expect(asked).toEqual([]); + }); + + // Postgres lets a column be renamed under a view; recreating the view anyway is safe. + test("a plan that renames a column of that table drops and creates the view", async () => { + const { probe, asked } = stub(same); + const renames = [{ kind: "column", table: "scores", from: "points", to: "pts" }] as const; + const r = await diff(withScores([view(UNJOINED)], [PTS, NOTE]), live(), opts(probe, renames)); + expect(kindsSorted(r)).toEqual(["create-view", "drop-view", "rename-column"]); + expect(asked).toEqual([]); + }); + + // A change on a renamed table is keyed by its new name; the live body holds the old one. + test("a plan that renames that table and drops a column of it drops and creates the view", async () => { + const { probe, asked } = stub(same); + const renames = [{ kind: "table", from: "scores", to: "tallies" }] as const; + const r = await diff(withScores([view(UNJOINED)], [NOTE], "tallies"), live(), opts(probe, renames)); + expect(kindsSorted(r)).toEqual(["create-view", "drop-column", "drop-view", "rename-table"]); + expect(asked).toEqual([]); + }); + + // A view follows a renamed table (it holds the table's OID), and nullability changes + // in place under a view, so neither is refused: the probe decides. + test.each([ + ["only renames that table", [POINTS, NOTE], "tallies", [{ kind: "table", from: "scores", to: "tallies" }], "rename-table"], + ["changes a column's nullability on that table", [{ ...POINTS, nullable: true }, NOTE], "scores", [], "change-column-nullable"], + ] as const)("a plan that %s still replaces the view on the probe's evidence", async (_, columns, table, renames, change) => { + const { probe, asked } = stub(same); + const r = await diff(withScores([view(UNJOINED)], columns, table), live(), opts(probe, renames)); + expect(kinds(r)).toEqual([change, "replace-view"]); + expect(asked).toHaveLength(1); + }); + }); + // `SELECT *` over another view is re-expanded when the view is replaced. Once the plan has // given that other view a new column, the column lands mid-list and Postgres refuses the // replace ("cannot change name of view column", sql-createview.html). From 0a8d679fcd3741eff4028ef8fae12a7a6c3c8fd0 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Sun, 11 Oct 2026 14:20:48 -0400 Subject: [PATCH 5/5] no-mistakes(review): fix(migrate-ts): recreate unchanged Postgres views over retyped untracked tables Pass 2c now covers Postgres: an unchanged view is dropped before and created after a table change when its stored body names an altered table outside dependsOn and the altered column (column-scoped). Unit case and an executed live Postgres case added; both fail without the fix. F2: the CHANGELOG bullet "SQLite and D1 are unchanged" now says the replace rule does not apply to SQLite and D1. MySQL: migrate-ts Dialect is postgres | sqlite | d1; no MySQL path exists, no change --- CHANGELOG.md | 12 ++++--- docs/features/migrations-and-drift.md | 7 ++++ .../test/view-replace-probe-pg.test.ts | 35 ++++++++++++++++++ .../packages/migrate-ts/src/diff/index.ts | 19 ++++++---- .../unit/diff-view-recreate-columns.test.ts | 36 +++++++++++++++++++ 5 files changed, 98 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 914a7adce..4bf80bd7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1039,9 +1039,9 @@ until you regenerate. - **The snapshot no longer records a view's columns.** Nothing reads them. The format version stays `3`, and an older snapshot that carries them still loads; they are dropped on read. `ExpectedView.columns` is removed from `@metaobjectsdev/codegen-ts` for the same reason. - - SQLite and D1 are unchanged. Neither has `CREATE OR REPLACE VIEW`; a replaced view is - written as `DROP VIEW` and `CREATE VIEW` there, as it always was, so there is nothing to - refuse. + - The replace rule does not apply to SQLite and D1. Neither has `CREATE OR REPLACE VIEW`; a + replaced view is written as `DROP VIEW` and `CREATE VIEW` there, as it always was, so there + is nothing to refuse. - **The `--allow` hint for a blocked view change names a token the parser accepts.** It printed `--allow replace-view`, which `meta migrate` rejects (`invalid --allow token`). It names `adopt-view` for an unfingerprinted view and `drop-view-cascade` when the drop would cascade. @@ -1051,7 +1051,11 @@ until you regenerate. - **A column change recreates only the views that read the column.** On Postgres a new default or a changed nullability no longer drops and rebuilds the views over the table (neither blocks a view), and a type change recreates only the views whose SQL names the column (a view that - selects `*` or joins `NATURAL` reads columns it does not name, and is always recreated). SQLite and D1 still + selects `*` or joins `NATURAL` reads columns it does not name, and is always recreated). That + now holds for a table the model does not list for the view as well: an unchanged `@sql` view + that joins such a table is dropped before and created after a retype of a column it reads, + where the migration failed at apply before (`cannot alter type of a column used by a view or + rule`). The table is found by name in the view's stored definition. SQLite and D1 still recreate every view over a rebuilt table, and so does a `diff()` called with no `dialect`, whose changes may be emitted for SQLite. `verify --db` and `meta migrate` now print the full column type in a type change (`text(255) -> text`, `timestamp -> timestamptz`), so a length or diff --git a/docs/features/migrations-and-drift.md b/docs/features/migrations-and-drift.md index 2175cdc45..df7318411 100644 --- a/docs/features/migrations-and-drift.md +++ b/docs/features/migrations-and-drift.md @@ -464,6 +464,13 @@ A drop and a create of the same view needs no `--allow`. It has two costs a repl Plan with `--from-db` to get the in-place replace when it is legal, which keeps both. Otherwise grant the privileges again after the migration is applied. +A view whose definition does **not** change is dropped and created too, when the migration +retypes, drops or renames a column the view reads: Postgres refuses to retype or drop it +while the view stands. The table may be one the model does not list for the view, which is +the case of an `@sql` body that joins it. `meta migrate` finds it by name in the view's +stored definition, and recreates the view only when that definition also names the changed +column. The same two costs apply. + SQLite and D1 have no `CREATE OR REPLACE VIEW`: every replaced view is written as `DROP VIEW` and `CREATE VIEW` there, so there is no refusal to avoid and no probe. They refuse something else: a `DROP COLUMN`, or the rebuild of a table, while a view that reads the table stands. So diff --git a/server/typescript/packages/integration-tests/test/view-replace-probe-pg.test.ts b/server/typescript/packages/integration-tests/test/view-replace-probe-pg.test.ts index 0f6ded67c..673478feb 100644 --- a/server/typescript/packages/integration-tests/test/view-replace-probe-pg.test.ts +++ b/server/typescript/packages/integration-tests/test/view-replace-probe-pg.test.ts @@ -573,6 +573,41 @@ describe("an @sql body in a plan that changes a table's columns (postgres)", () expect((await plan(after.expected)).changes).toEqual([]); }, 60_000); + // The view's definition does not change here, so nothing replaces it: it only has to be + // out of the way while the column it reads is retyped. + test("an unchanged body in a plan that retypes a column it reads is dropped and created, and applies", async () => { + const before = await model(scoredView(BEFORE.scored), SUMMARY_VIEW, [], BEFORE.others); + await migrate(before.expected); + const after = await model(scoredView(BEFORE.scored), SUMMARY_VIEW, [], [badge("long")]); + expect(after.view.fingerprint).toBe(before.view.fingerprint); + expect(after.view.dependsOn).toEqual(["programs"]); + await expect(sql.raw(`ALTER TABLE "badges" ALTER COLUMN "points" TYPE BIGINT`).execute(k)) + .rejects.toThrow(/cannot alter type of a column used by a view or rule/); + + const changed = await plan(after.expected); + expect(changed.blocked).toEqual([]); + expect(kindsOf(changed.changes)).toEqual(["change-column-type", "create-view", "drop-view"]); + await apply(changed.changes); + + expect(await scalar(`SELECT format_type(atttypid, atttypmod) FROM pg_attribute + WHERE attrelid = 'public.badges'::regclass AND attname = 'points'`)).toBe("bigint"); + expect(await columnTypes(SUMMARY_VIEW)).toEqual({ id: "bigint", scored: "bigint" }); + expect((await plan(after.expected)).changes).toEqual([]); + }, 60_000); + + // Postgres retypes a column no view reads with the view standing, so the view stays. + test("an unchanged body in a plan that retypes a column it does not read is left standing", async () => { + await migrate((await model(scoredView(OVER_BADGES), SUMMARY_VIEW, [], BEFORE.others)).expected); + const oid = await viewOid(SUMMARY_VIEW); + const after = await model(scoredView(OVER_BADGES), SUMMARY_VIEW, [], [badge("long")]); + + const changed = await plan(after.expected); + expect(kindsOf(changed.changes)).toEqual(["change-column-type"]); + await apply(changed.changes); + expect(await viewOid(SUMMARY_VIEW)).toBe(oid); + expect((await plan(after.expected)).changes).toEqual([]); + }, 60_000); + // A derived body lists every table it reads, so a change to another table cannot reach it. test("a derived view that changes beside the same table change is replaced in place", async () => { const flag = (field: string) => summary(computed("flag", "boolean", { op: "isNull", arg: { field } })); diff --git a/server/typescript/packages/migrate-ts/src/diff/index.ts b/server/typescript/packages/migrate-ts/src/diff/index.ts index 3ad6ee69f..db8bae9a4 100644 --- a/server/typescript/packages/migrate-ts/src/diff/index.ts +++ b/server/typescript/packages/migrate-ts/src/diff/index.ts @@ -1237,13 +1237,18 @@ function recreateViewsDependingOnChangedTables( return viewReadsAnyColumn(sql, cols, readsEverything); }); const prior = actualById.get(id); - // Where a triggering change rebuilds the table, the view in the database is stranded by - // every table ITS body reads, and that body may read one `dependsOn` leaves out: a - // table the new body no longer reads, or one an `@sql` body joins (ADR-0043). On - // Postgres the same case is settled when the view is planned (pushViewUpdate). - if (!columnScoped && prior !== undefined) { - for (const t of alteredTables.keys()) { - if (!tracked.includes(t) && liveViewReadsTable(prior, liveName(t))) tables.push(t); + // The view in the database is stranded by every table ITS body reads, and that body + // may read one `dependsOn` leaves out: a table the new body no longer reads, or one an + // `@sql` body joins (ADR-0043). Its own text says which. On Postgres it is stranded + // only when it also reads an altered column, as above. (A Postgres view whose + // definition changes never gets this far as a replace: pushViewUpdate plans the pair.) + if (prior !== undefined) { + const priorReadsEverything = prior.sql !== undefined && viewReadsEveryColumn(prior.sql); + for (const [t, cols] of alteredTables) { + if (tracked.includes(t) || !liveViewReadsTable(prior, liveName(t))) continue; + if (cols === null || prior.sql === undefined || viewReadsAnyColumn(prior.sql, cols, priorReadsEverything)) { + tables.push(t); + } } } if (tables.length === 0) continue; diff --git a/server/typescript/packages/migrate-ts/test/unit/diff-view-recreate-columns.test.ts b/server/typescript/packages/migrate-ts/test/unit/diff-view-recreate-columns.test.ts index 1d6fe9891..7a07ce1ac 100644 --- a/server/typescript/packages/migrate-ts/test/unit/diff-view-recreate-columns.test.ts +++ b/server/typescript/packages/migrate-ts/test/unit/diff-view-recreate-columns.test.ts @@ -118,3 +118,39 @@ describe("D4 — a view is recreated only when the migration alters a column it expect(kinds(r)).not.toContain("replace-view"); }); }); + +// A view's `dependsOn` lists the tables the model knows it reads. An `@sql` body may join +// another one (ADR-0043), and Postgres refuses `ALTER COLUMN … TYPE` on a column that a +// standing view reads, whichever table holds it +// (https://www.postgresql.org/docs/current/sql-altertable.html, SET DATA TYPE). The view's +// stored body is what names that table. +describe("an unchanged view whose stored body reads a table outside dependsOn (postgres)", () => { + const POINTS: ColumnDescriptor = { name: "points", sqlType: { kind: "integer", bits: 32 }, nullable: false }; + const BONUS: ColumnDescriptor = { name: "bonus", sqlType: { kind: "integer", bits: 32 }, nullable: false }; + const wide = (c: ColumnDescriptor): ColumnDescriptor => ({ ...c, sqlType: { kind: "integer", bits: 64 } }); + const scores = (cols: ColumnDescriptor[]): TableDescriptor => + ({ name: "scores", columns: [ID, ...cols], indexes: [], foreignKeys: [], primaryKey: ["id"], checks: [] }); + const JOINED = "SELECT g.id AS id, (SELECT count(*) FROM scores s WHERE s.points > 0) AS scored FROM games g"; + const joined = view({ sql: JOINED, fingerprint: viewFingerprint(JOINED) }); + const run = (v: ViewDescriptor, expected: ColumnDescriptor[]) => + diff( + { tables: [table([ID, LABEL]), scores(expected)], views: [v] }, + { tables: [table([ID, LABEL]), scores([POINTS, BONUS])], views: [v] }, + { dialect: "postgres", allow: { typeChange: true } }, + ); + + test("a type change to a column the body reads recreates it, and names the table", async () => { + const r = await run(joined, [wide(POINTS), BONUS]); + expect(kinds(r)).toEqual(["change-column-type", "drop-view", "create-view"]); + expect(r.changes[1]).toMatchObject({ view: "v_games", reason: { kind: "unchanged", tables: ["scores"] } }); + expect(r.blocked).toEqual([]); + }); + + test("a type change to a column of that table the body does not read leaves it alone", async () => { + expect(kinds(await run(joined, [POINTS, wide(BONUS)]))).toEqual(["change-column-type"]); + }); + + test("a view whose body does not name that table is left alone", async () => { + expect(kinds(await run(view(), [wide(POINTS), BONUS]))).toEqual(["change-column-type"]); + }); +});