From 4d6fe8166d5375a8f90f5f6201d99149b5f4ed22 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sun, 1 Nov 2026 16:18:48 +1100 Subject: [PATCH 1/3] test(conformance): enroll value and query fixtures --- conformance/stack-encrypt/README.md | 20 ++- conformance/stack-encrypt/coverage.md | 18 ++- conformance/stack-encrypt/fixture-contract.md | 100 +++++++++++++++ .../stack-encrypt/v1/fixture-profiles.json | 83 ++++++++++++ .../2026-10-31-stack-encrypt-conformance.md | 32 ++++- .../encrypt/fixture_conformance_test.go | 63 ++++++++++ languages/golang/encrypt/query_test.go | 6 +- languages/golang/encrypt/value_kinds_test.go | 4 +- packages/stack-encrypt/tests/common/mod.rs | 11 ++ .../stack-encrypt/tests/query_operators.rs | 3 +- packages/stack-encrypt/tests/value_kinds.rs | 3 +- .../__tests__/stack-conformance-ci.test.mjs | 118 +++++++++++++++++- scripts/stack-conformance.sh | 6 +- 13 files changed, 446 insertions(+), 21 deletions(-) create mode 100644 conformance/stack-encrypt/fixture-contract.md create mode 100644 conformance/stack-encrypt/v1/fixture-profiles.json create mode 100644 languages/golang/encrypt/fixture_conformance_test.go diff --git a/conformance/stack-encrypt/README.md b/conformance/stack-encrypt/README.md index abd176bd8..11d4cd96b 100644 --- a/conformance/stack-encrypt/README.md +++ b/conformance/stack-encrypt/README.md @@ -21,13 +21,31 @@ native Rust against the ciphertexts Go has just produced. It never regenerates expectations. CI also runs the Go cases on its other supported host platforms. The same task runs the shared declaration acceptance/rejection corpus in both languages, with native observations of normalization. It also runs the shared -runtime and read cases described below. +runtime and read cases, and enrolls the existing value-kind and query-operator +fixtures described below. Set no fixture-update variables when checking conformance. The Rust runner is `packages/stack-encrypt/tests/conformance.rs`; the Go runner is `languages/golang/encrypt/conformance_test.go`. Go calls the record interface used by generated code. Testing public SDK declarations is a separate layer. Python and TypeScript runners are pending; their absence is not a passing result. +## Existing value-kind and query-operator fixtures + +The [fixture contract](fixture-contract.md) defines `record-value-kinds-v1` and +`record-query-operators-v1`. Their [enrollment index](v1/fixture-profiles.json) +maps stable requirement IDs to the existing Rust and Go tests. The fixtures +remain under `packages/stack-encrypt/tests/fixtures`; no vectors are copied. + +These profiles cover the represented narrow/128-bit integers, dates, +timestamps, decimals, equality/order terms, query operand metadata, match +verification and fetch limits. They preserve typed Rust and generated Go API +checks. The task runs the existing tests directly, and a missing guest fails +when conformance is required. The ordinary CI suites already execute them. + +This is representative fixture coverage, not an exhaustive type/operator +matrix. Wider fresh artifact exchange, shared query refusals and EQL/JSON +operations remain separate work. + ## `record-reads-v1` contract The [read contract](read-contract.md) defines cases in `v1/reads.json` for diff --git a/conformance/stack-encrypt/coverage.md b/conformance/stack-encrypt/coverage.md index 6b54fdc02..bad3e2a5d 100644 --- a/conformance/stack-encrypt/coverage.md +++ b/conformance/stack-encrypt/coverage.md @@ -6,11 +6,11 @@ Paths are relative to the repository root. New profile cases run from `v1/`. | Requirement | Existing evidence | Remaining work | |---|---|---| -| Scalar value widths and lossless transport | `packages/stack-encrypt/tests/value_kinds.rs`; `languages/golang/encrypt/value_kinds_test.go`; initial shared scalar cases | Expand shared cases to decimals, dates, timestamps, floats, bytes and all widths; Python/TS mappings | +| Scalar value widths and lossless transport | Initial scalar cases and [value-kind requirements](fixture-contract.md#value-kind-requirements), executed by the existing Rust and generated Go runners | Complete per-kind boundary matrices, bytes and broader floats; Python/TS mappings | | Owned opaque ciphertext and storage envelope | `packages/stack-encrypt/tests/scalar.rs`, `frozen_bytes.rs`; initial cross-opening cases | Blob profile; encoded targets; supported version matrix | | Record declaration and lowering | [Declaration grammar](declaration-grammar.md); `v1/declarations.json` shared by native Rust and Go; native normalization observations; existing record lowering suites | EQL-resolver profile; SDK-to-declaration mappings; runtime source/read shapes | | Context binding and composition | `packages/stack-encrypt/tests/target.rs`, `descriptor.rs`, `record_lowering.rs`; `languages/golang/encrypt/context_field_test.go`; initial wrong-context cases | Shared composition vectors; expected-context and required-input matrix | -| Query terms and diagnostics | `packages/stack-encrypt/tests/query_operators.rs`; `languages/golang/encrypt/query_test.go`; initial equality/query-error cases | Common rejection corpus for every operator, capability and input shape | +| Query terms and diagnostics | [Query-operator requirements](fixture-contract.md#query-operator-requirements), existing Rust/Go runners and initial equality/query-error cases | Common rejection corpus for every operator, capability and input shape; wider kind/index combinations | | Missing, SQL NULL and sealed null | `packages/stack-encrypt/src/dynamic/record.rs` null tests; `languages/golang/encrypt/null_test.go` | Explicit language mappings and common missing/null fixtures | | Fresh cross-language storage | `.github/workflows/tests-golang.yml` exports fresh Go EQL rows for Rust; initial scalar artifact exchange | Every producer/consumer/backend pair and supported released versions | | JSON queries, extraction and tampering | `packages/stack-encrypt/tests/json_tree_vectors.rs`; `packages/eql/tests/encryption/tests/json_tree.rs`; `languages/golang/encrypt/json_tree_test.go`, `json_postgres_test.go` | Formal operation profile; JSON vectors consumed by all runners | @@ -59,17 +59,25 @@ raw-byte profile must cover that boundary explicitly. ## Consolidation policy +The value-kind/query-operator enrollment adds named requirements for the +existing fixtures through [the enrollment index](v1/fixture-profiles.json). +The conformance command selects their original runners; CI guards the mapping +and selection. Typed Rust comparisons, generated Go methods, verification and +fetch-limit checks stay in those runners. No fixture is copied and no existing +assertion is removed. Shared negative cases and fresh exchange of the wider +value kinds remain open. + Shared semantic expectations should be maintained once and executed by each language runner. Conformance can replace duplicate case tables and fixture maintenance. It does not eliminate the need to execute the Rust and Go paths. -No existing test has been deleted in these first four slices. +No existing test has been deleted by conformance enrollment. | Existing evidence | Relationship to conformance | Consolidation action | |---|---|---| | Rust `dynamic/record.rs` tests `a_malformed_plan`, `a_null_is_refused_as_its_key`, `a_missing_or_null_index_payload_is_the_default` | Many inputs and diagnostic outcomes are now shared declaration cases | Move the remaining cases and retain any unique display/low-level assertions before removing duplicate rows | | Rust kind/index admission tests | The corpus has the complete 21-kind/five-index acceptance matrix | Share case data; retain direct tests that compare declaration admission with actual term execution | -| Rust `value_kinds.rs`, Go `value_kinds_test.go` | Already share `value_kinds.json`; cover more types and generated SDK code than the initial scalar profile | Enroll that existing fixture under named requirements instead of copying it; keep generated-code assertions | -| Rust `query_operators.rs`, Go `query_test.go` | Already share operator vectors, including verification and target limits | Reuse their fixture and cases when adding an operator profile; preserve host/generated query mapping tests | +| Rust `value_kinds.rs`, Go `value_kinds_test.go` | Enrolled as `record-value-kinds-v1`, retaining the shared fixture and generated SDK calls | Extend the existing fixture/requirements for missing boundaries; retain typed and generated-code assertions | +| Rust `query_operators.rs`, Go `query_test.go` | Enrolled as `record-query-operators-v1`, including verification and target limits | Add shared refusal cases separately; retain host/generated query mapping tests | | Go checker and host validation tests in `guest_test.go` | Exercise host rejection and diagnostic forwarding through public checker methods | Keep adapter-specific assertions; representable declaration cases also use `Checker.Check`, while raw frames observe the guest boundary | | Native scalar/record round trips and cross-opening | Some behavior overlaps, but existing tests cover more types and API paths | Replace only exact duplicates after matching requirements and execution paths | diff --git a/conformance/stack-encrypt/fixture-contract.md b/conformance/stack-encrypt/fixture-contract.md new file mode 100644 index 000000000..54e418038 --- /dev/null +++ b/conformance/stack-encrypt/fixture-contract.md @@ -0,0 +1,100 @@ +# Value-kind and query-operator profiles + +`record-value-kinds-v1` and `record-query-operators-v1` enroll the existing +Rust/Go fixtures in the conformance contract. They are experimental profiles +of the cases those fixtures contain, not exhaustive type/operator certification. +They use the plain record resolver (`NoTargets`), with public deterministic +keys and no database or live credentials. + +The fixtures remain at their existing paths: + +- [`value_kinds.json`](../../packages/stack-encrypt/tests/fixtures/value_kinds.json) +- [`query_operators.json`](../../packages/stack-encrypt/tests/fixtures/query_operators.json) + +Their [existing schema documentation](../../packages/stack-encrypt/tests/fixtures/README.md) +continues to own the formats. No expected frames, ciphertexts, terms, verified +pages or fetch limits are copied into this directory. + +[`v1/fixture-profiles.json`](v1/fixture-profiles.json) maps stable requirement IDs +to the existing Rust test functions and Go tests that provide their evidence. +It is an enrollment index, not another input corpus or a runtime capability +negotiation format. The CI wiring test checks that its fixtures/tests exist, +that each requirement is documented here, and that the conformance command +selects every listed test. Both language implementations are mandatory for +these profiles; Python and TypeScript remain pending. + +## Value-kind requirements + +The fixture represents int8, int16, uint8, uint16, int128, uint128, date, +timestamp and decimal values. It includes the signed 128-bit minimum and +unsigned 128-bit maximum. These are representative cases, not every boundary +of every type. Values use the fixture's `type` and `text` with a frozen `frame`; +128-bit integers must never pass through a floating-point JSON number. + +| Requirement ID | Required observation | Fixture evidence | +|---|---|---| +| `values.typed-frames` | The actual host value encodes to the exact frozen typed frame, preserving integer width, decimal scale and timestamp nanoseconds. | Every `values` entry | +| `values.frozen-open` | The committed Rust record opens to those values and types, retaining decimal scale and timestamp precision. | `record` and `values` | +| `values.stored-terms` | Fresh records round-trip and store the frozen equality/order terms. Queries for their values derive the corresponding stored terms. Randomized ciphertext bytes are not compared. | Every non-`c` output in `record`, under `plan` | +| `values.normalized-queries` | Decimal queries differing only in trailing zeros and timestamp queries truncated to microseconds derive the stored equality and order terms, while the opened values retain their original precision. | Every `queries` entry and its field's order output | + +Rust additionally compares dynamic execution with typed term derivation and +retains its wrong-width write refusal test. Go runs the real generated +`testusers.ValueKinds` encrypt/decrypt and field-query methods, including its +field-context substitution refusal. These assertions stay in their owning +tests. The separate Go invalid-date/decimal/timestamp tests remain in the full +Go suite; they are not claimed as shared fixture requirements here. + +## Query-operator requirements + +The fixture covers equality on text and uint32, scalar/range ordering on uint32 +and float64, and probabilistic text matching. Its operators are `eq`, `lt`, +`lte`, `gt`, `gte`, `between` and `matches`, in the combinations declared by +the fixture. It does not claim every operator for every kind or index. + +| Requirement ID | Required observation | Fixture evidence | +|---|---|---| +| `queries.term-contract` | Derive the frozen operand bytes and metadata: operator, fidelity, index, domain, and scalar versus ordered low/high range shape. | Every `queries` entry | +| `queries.match-verification` | Verify the complete page in input order, including matches, nonmatches and null; results agree with the frozen boolean vector. | `verify` | +| `queries.fetch-limits` | Given the frozen match term, requested count and statistics, return the recorded bounded row count or explicit unbounded result. | Every `target_limit` entry | + +Rust compares the dynamic path with typed query chains, `MatchQuery::verify` +and typed fetch sizing, and checks that the fixture contains both verification +outcomes and both bounded/unbounded limits. Go exercises the record interface +and generated query methods. Generated string-page verification omits null, +which the record-interface test covers. Its existing SQL parameter conversion, +empty-page, zero-query, inverted-range and floating-point ordering assertions +remain in those selected tests. Wider operator refusal and secret-diagnostic +tests remain in the full suites; this slice does not turn them into shared +negative cases. + +## Running and maintaining the profiles + +`mise run test:conformance` selects the existing `value_kinds` and +`query_operators` Rust binaries and the Go tests listed in the enrollment index, +alongside the earlier profiles. Each existing test runs once per invocation; +no wrapper test invokes another test a second time. The ordinary Rust and Go +CI suites already run these tests, so enrollment adds no duplicate CI suite. + +When `STACK_CONFORMANCE_REQUIRED=1`, a missing deterministic guest fails the +enrolled Go tests instead of skipping them. The aggregate command rejects both +`STACK_CONFORMANCE_UPDATE` and `STACK_ENCRYPT_UPDATE_FIXTURES`, even when set to +an empty value. The enrolled Rust fixture writers independently reject updates +in required mode. Deliberate fixture maintenance remains available outside +required mode through the existing documented regeneration commands. + +Keep requirement IDs stable. Add a new profile version for incompatible +expectations, and review fixture regeneration explicitly. The enrollment +index's `version` is its metadata format, not a production storage version. + +This slice consolidates the coverage inventory and execution selection, not +the underlying test assertions. No test or frozen artifact is removed. Fresh +cross-language artifact exchange remains the scalar profile's guarantee; +these wider fixtures prove frozen Rust-to-Go opening, local round trips and +term agreement, not fresh Go-to-Rust opening of every represented kind. + +Python and TypeScript should consume these same fixture paths and implement +the listed observations through their own host values and public APIs. Their +integer/decimal/time mappings and actual backends must be specified by those +implementations. EQL domains, JSON operations, all-type boundary matrices, +shared query refusals and supported-release matrices remain separate work. diff --git a/conformance/stack-encrypt/v1/fixture-profiles.json b/conformance/stack-encrypt/v1/fixture-profiles.json new file mode 100644 index 000000000..050c76be8 --- /dev/null +++ b/conformance/stack-encrypt/v1/fixture-profiles.json @@ -0,0 +1,83 @@ +{ + "version": 1, + "profiles": [ + { + "id": "record-value-kinds-v1", + "fixture": "packages/stack-encrypt/tests/fixtures/value_kinds.json", + "rust_binary": "value_kinds", + "go_source": "languages/golang/encrypt/value_kinds_test.go", + "requirements": [ + { + "id": "values.typed-frames", + "rust_tests": [ + "the_committed_fixture_opens_and_its_terms_are_todays" + ], + "go_tests": ["TestGeneratedCodeOpensTheRustValueKindsFixture"] + }, + { + "id": "values.frozen-open", + "rust_tests": [ + "the_committed_fixture_opens_and_its_terms_are_todays" + ], + "go_tests": ["TestGeneratedCodeOpensTheRustValueKindsFixture"] + }, + { + "id": "values.stored-terms", + "rust_tests": [ + "every_new_kind_seals_opens_and_queries_as_the_typed_path", + "the_committed_fixture_opens_and_its_terms_are_todays" + ], + "go_tests": ["TestGeneratedCodeOpensTheRustValueKindsFixture"] + }, + { + "id": "values.normalized-queries", + "rust_tests": [ + "every_new_kind_seals_opens_and_queries_as_the_typed_path", + "the_committed_fixture_opens_and_its_terms_are_todays" + ], + "go_tests": ["TestGeneratedCodeOpensTheRustValueKindsFixture"] + } + ] + }, + { + "id": "record-query-operators-v1", + "fixture": "packages/stack-encrypt/tests/fixtures/query_operators.json", + "rust_binary": "query_operators", + "go_source": "languages/golang/encrypt/query_test.go", + "requirements": [ + { + "id": "queries.term-contract", + "rust_tests": [ + "the_committed_query_fixture_is_what_the_engine_derives" + ], + "go_tests": [ + "TestQueryTermsAreTheRustFixtures", + "TestGeneratedQueriesAreTheRustFixtures" + ] + }, + { + "id": "queries.match-verification", + "rust_tests": [ + "the_committed_query_fixture_is_what_the_engine_derives", + "the_fixture_pins_both_outcomes" + ], + "go_tests": [ + "TestVerifyIsTheRustFixtures", + "TestAMatchesQueryVerifiesAndSizesAsTheRustFixtures" + ] + }, + { + "id": "queries.fetch-limits", + "rust_tests": [ + "the_committed_query_fixture_is_what_the_engine_derives", + "the_fixture_pins_both_outcomes" + ], + "go_tests": [ + "TestTargetLimitIsTheRustFixtures", + "TestAMatchesQueryVerifiesAndSizesAsTheRustFixtures" + ] + } + ] + } + ] +} diff --git a/docs/plans/2026-10-31-stack-encrypt-conformance.md b/docs/plans/2026-10-31-stack-encrypt-conformance.md index df11cd397..af6cae7ad 100644 --- a/docs/plans/2026-10-31-stack-encrypt-conformance.md +++ b/docs/plans/2026-10-31-stack-encrypt-conformance.md @@ -1,6 +1,6 @@ # Stack Encrypt cross-language conformance -Date: 2026-10-31. Status: scalar, declaration, initial runtime and read slices implemented; the complete contract is not frozen. +Date: 2026-10-31. Status: scalar, declaration, initial runtime and read slices implemented; existing value-kind and query-operator fixtures enrolled. The complete contract is not frozen. This work is stacked on [PR #1404](https://github.com/cipherstash/stack/pull/1404), at `b684a217e2da629c762efa70fac958ec7e21646a`. It follows the accepted @@ -145,11 +145,37 @@ Local validation passes for all four profiles, fresh scalar artifact exchange, Rust clippy/formatting, Go lint, Biome and the CI wiring checks. The deterministic guest is reused from the unchanged parent engine. -Next, enroll the existing wider value-kind and operator fixtures under named -requirements, then extend context composition and EQL/JSON operation profiles. +The fixture checkpoint below enrolls the existing wider value-kind and operator +fixtures. Next, extend context composition and EQL/JSON operation profiles. Repeated-key and malformed-leaf cases need an explicit raw-byte transport path. Python and TypeScript consumers remain pending. +## Value-kind and query-operator checkpoint + +The next slice is on `feat/stack-encrypt-value-query-conformance`, stacked on +[PR #1462](https://github.com/cipherstash/stack/pull/1462). The +[fixture contract](../../conformance/stack-encrypt/fixture-contract.md) and +[enrollment index](../../conformance/stack-encrypt/v1/fixture-profiles.json) +assign stable requirement IDs to the existing `value_kinds.json` and +`query_operators.json` evidence. The conformance task directly runs their +existing Rust and Go tests. Typed Rust comparisons, generated Go methods, +match verification and fetch sizing remain intact. + +The fixture files stay in place with no duplicated expectations. CI guards +the requirement-to-test mapping and test selection. Required Go runs fail if +the guest is missing; required checks cannot regenerate frozen fixtures. +This is consolidation of inventory and execution, not removal of assertions. + +Local validation passes for the expanded conformance command, fresh scalar +artifact exchange, missing-guest regressions, Rust clippy/formatting, Go lint, +Biome and CI wiring checks. Both enrolled Rust writers refuse regeneration in +required mode, and the frozen fixture files remain unchanged. + +The coverage is representative, not a complete kind/operator boundary matrix. +Shared operator refusals, wider fresh artifact exchange, Python/TypeScript +consumers and version matrices remain open. The next implementation slice is +shared context-composition vectors, followed by EQL/JSON operation profiles. + ## Follow-up slices - Extend the declaration grammar to EQL resolver capabilities and the operation diff --git a/languages/golang/encrypt/fixture_conformance_test.go b/languages/golang/encrypt/fixture_conformance_test.go new file mode 100644 index 000000000..3d7724f76 --- /dev/null +++ b/languages/golang/encrypt/fixture_conformance_test.go @@ -0,0 +1,63 @@ +package encrypt_test + +import ( + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// Exercise the real fixture entry points in an isolated tree with their frozen +// inputs but no guest. A missing guest must never make required conformance green. +func TestFixtureConformanceRequiresGuest(t *testing.T) { + binary, err := os.Executable() + if err != nil { + t.Fatal(err) + } + root := t.TempDir() + workdir := filepath.Join(root, "languages", "golang", "encrypt") + fixtures := filepath.Join("packages", "stack-encrypt", "tests", "fixtures") + for _, dir := range []string{workdir, filepath.Join(root, fixtures)} { + if err := os.MkdirAll(dir, 0o750); err != nil { + t.Fatal(err) + } + } + for _, name := range []string{"value_kinds.json", "query_operators.json"} { + data, err := os.ReadFile(filepath.Join("..", "..", "..", fixtures, name)) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, fixtures, name), data, 0o600); err != nil { + t.Fatal(err) + } + } + tests := []string{"TestGeneratedCodeOpensTheRustValueKindsFixture", "TestQueryTermsAreTheRustFixtures"} + for _, required := range []string{"0", "1"} { + t.Run("required="+required, func(t *testing.T) { + t.Setenv("STACK_CONFORMANCE_REQUIRED", required) + cmd := exec.CommandContext(t.Context(), binary, "-test.v", "-test.timeout=30s", "-test.run=^("+strings.Join(tests, "|")+")$") + cmd.Dir = workdir + out, err := cmd.CombinedOutput() + marker := "--- SKIP: " + if required == "1" { + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != 1 { + t.Fatalf("required fixtures must fail: %v\n%s", err, out) + } + marker = "--- FAIL: " + } else if err != nil { + t.Fatalf("optional fixtures must skip: %v\n%s", err, out) + } + for _, test := range tests { + if !strings.Contains(string(out), marker+test) { + t.Errorf("missing %q result for %s:\n%s", marker, test, out) + } + } + if !strings.Contains(string(out), "deterministic guest not built") { + t.Errorf("expected missing-guest diagnostic:\n%s", out) + } + }) + } +} diff --git a/languages/golang/encrypt/query_test.go b/languages/golang/encrypt/query_test.go index 2e7c9e3ec..d4ef243c6 100644 --- a/languages/golang/encrypt/query_test.go +++ b/languages/golang/encrypt/query_test.go @@ -28,8 +28,8 @@ import ( // declaration, the term each query by operator derives (each equal to the // typed Rust query chain's), a page of verified values and a few fetch // limits. These tests derive, verify and size the same queries through the -// deterministic test build of the guest and compare. Skipped when that build -// is absent. +// deterministic test build of the guest and compare. A missing build skips +// ordinary tests, but fails when STACK_CONFORMANCE_REQUIRED=1. type queryFixture struct { KeySource struct { @@ -129,7 +129,7 @@ func queryFixtureClient(t *testing.T, f queryFixture) *encrypt.Client { t.Fatalf("seed: %v (%d bytes)", err, len(seed)) } c, err := encrypt.NewDeterministicClient(context.Background(), [32]byte(seed)) - if errors.Is(err, encrypt.ErrDeterministicGuestNotBuilt) || errors.Is(err, encrypt.ErrGuestNotBuilt) { + if (errors.Is(err, encrypt.ErrDeterministicGuestNotBuilt) || errors.Is(err, encrypt.ErrGuestNotBuilt)) && os.Getenv("STACK_CONFORMANCE_REQUIRED") != "1" { t.Skip(err) } if err != nil { diff --git a/languages/golang/encrypt/value_kinds_test.go b/languages/golang/encrypt/value_kinds_test.go index cc25a4703..32ae018df 100644 --- a/languages/golang/encrypt/value_kinds_test.go +++ b/languages/golang/encrypt/value_kinds_test.go @@ -27,7 +27,7 @@ import ( // testusers.ValueKinds — the same context, identities and indexes as tags — // writes each value as the same frame, opens the Rust record, and derives // the same term bytes through the guest built over the same source. -// Skipped when the deterministic test build of the guest is absent. +// Skipped when the test guest is absent, unless conformance is required. type valueKindsFixture struct { KeySource struct { @@ -152,7 +152,7 @@ func TestGeneratedCodeOpensTheRustValueKindsFixture(t *testing.T) { t.Fatalf("seed: %v (%d bytes)", err, len(seedBytes)) } c, err := encrypt.NewDeterministicClient(context.Background(), [32]byte(seedBytes)) - if errors.Is(err, encrypt.ErrDeterministicGuestNotBuilt) || errors.Is(err, encrypt.ErrGuestNotBuilt) { + if (errors.Is(err, encrypt.ErrDeterministicGuestNotBuilt) || errors.Is(err, encrypt.ErrGuestNotBuilt)) && os.Getenv("STACK_CONFORMANCE_REQUIRED") != "1" { t.Skip(err) } if err != nil { diff --git a/packages/stack-encrypt/tests/common/mod.rs b/packages/stack-encrypt/tests/common/mod.rs index 9e4fb1b8b..0224668d2 100644 --- a/packages/stack-encrypt/tests/common/mod.rs +++ b/packages/stack-encrypt/tests/common/mod.rs @@ -19,6 +19,17 @@ use stack_kms::{ }; use uuid::Uuid; +/// Fixture regeneration is an explicit maintenance operation, never evidence +/// for a required conformance run. Check before deriving or writing any fixture. +pub fn fixture_updates_requested() -> bool { + let update = std::env::var_os("STACK_ENCRYPT_UPDATE_FIXTURES").is_some(); + assert!( + !update || std::env::var("STACK_CONFORMANCE_REQUIRED").as_deref() != Ok("1"), + "conformance checks must not update frozen fixtures" + ); + update +} + /// A cipher over the deterministic fake source. The fake index key is /// deterministic per keyset, so two separately built ciphers stand in for the /// write path and a query path in another process. diff --git a/packages/stack-encrypt/tests/query_operators.rs b/packages/stack-encrypt/tests/query_operators.rs index 11cf45f6a..cf50643a2 100644 --- a/packages/stack-encrypt/tests/query_operators.rs +++ b/packages/stack-encrypt/tests/query_operators.rs @@ -365,9 +365,10 @@ fn fixture_path() -> PathBuf { /// `STACK_ENCRYPT_UPDATE_FIXTURES=1` it is written first. #[tokio::test] async fn the_committed_query_fixture_is_what_the_engine_derives() { + let update = common::fixture_updates_requested(); let now = derived().await; let path = fixture_path(); - if std::env::var_os("STACK_ENCRYPT_UPDATE_FIXTURES").is_some() { + if update { let mut text = serde_json::to_string_pretty(&now).expect("serialise"); text.push('\n'); std::fs::write(&path, text).expect("write the fixture"); diff --git a/packages/stack-encrypt/tests/value_kinds.rs b/packages/stack-encrypt/tests/value_kinds.rs index 52905fc57..b019d4965 100644 --- a/packages/stack-encrypt/tests/value_kinds.rs +++ b/packages/stack-encrypt/tests/value_kinds.rs @@ -448,6 +448,7 @@ async fn a_value_of_another_width_is_refused_when_sealed() { /// and the file written. #[tokio::test] async fn the_committed_fixture_opens_and_its_terms_are_todays() { + let update = common::fixture_updates_requested(); let cipher = deterministic_cipher(SEED).await; let keyset = cipher.default_keyset(); let plan = data_plan(); @@ -466,7 +467,7 @@ async fn the_committed_fixture_opens_and_its_terms_are_todays() { .collect() }; - if std::env::var_os("STACK_ENCRYPT_UPDATE_FIXTURES").is_some() { + if update { let record = json_of_record(seal(&cipher, &plan).await); let mut queries = Vec::new(); for (field, text, input) in folded_queries() { diff --git a/scripts/__tests__/stack-conformance-ci.test.mjs b/scripts/__tests__/stack-conformance-ci.test.mjs index e3006df7b..7b4528e89 100644 --- a/scripts/__tests__/stack-conformance-ci.test.mjs +++ b/scripts/__tests__/stack-conformance-ci.test.mjs @@ -1,9 +1,23 @@ +import { spawnSync } from 'node:child_process' import { readFileSync } from 'node:fs' import { join } from 'node:path' import { describe, expect, it } from 'vitest' import { REPO_ROOT } from './lib/repo-root.mjs' import { readWorkflow } from './lib/workflows.mjs' +const readRepo = (path) => readFileSync(join(REPO_ROOT, path), 'utf8') +const conformanceScript = () => + readRepo('scripts/stack-conformance.sh').replace(/^\s*#.*$/gm, '') + +function goSelection(script) { + const match = /go test -count=1 -v -run '([^']+)' \.\/encrypt/.exec(script) + expect( + match, + 'the Go conformance command must execute uncached', + ).not.toBeNull() + return new RegExp(match[1]) +} + describe('shared Stack Encrypt conformance is required in CI', () => { it('the conformance task runs all profiles in both languages', () => { const script = readFileSync( @@ -21,17 +35,117 @@ describe('shared Stack Encrypt conformance is required in CI', () => { expect(script).toMatch(/cargo nextest run[^\n]*--test declarations/) expect(script).toMatch(/cargo nextest run[^\n]*--test runtime_conformance/) expect(script).toMatch(/cargo nextest run[^\n]*--test read_conformance/) - expect(script).toContain( - "go test -count=1 -v -run '^(TestStackConformance|TestDeclarationConformance|TestRuntimeConformance|TestReadConformance)$' ./encrypt", + const selected = goSelection(script) + for (const test of [ + 'TestStackConformance', + 'TestDeclarationConformance', + 'TestRuntimeConformance', + 'TestReadConformance', + 'TestFixtureConformanceRequiresGuest', + ]) { + expect(selected.test(test), test).toBe(true) + } + }) + + it('enrolls existing fixture evidence without copying or dropping tests', () => { + const index = JSON.parse( + readRepo('conformance/stack-encrypt/v1/fixture-profiles.json'), ) + expect(Object.keys(index).sort()).toEqual(['profiles', 'version']) + expect(index.version).toBe(1) + expect(index.profiles.map((profile) => profile.id).sort()).toEqual([ + 'record-query-operators-v1', + 'record-value-kinds-v1', + ]) + const script = conformanceScript() + const selected = goSelection(script) + const contract = readRepo('conformance/stack-encrypt/fixture-contract.md') + const requirements = new Set() + for (const profile of index.profiles) { + expect(Object.keys(profile).sort()).toEqual([ + 'fixture', + 'go_source', + 'id', + 'requirements', + 'rust_binary', + ]) + expect(profile.rust_binary).toMatch(/^[a-z_]+$/) + expect(profile.fixture).toBe( + `packages/stack-encrypt/tests/fixtures/${profile.rust_binary}.json`, + ) + expect(JSON.parse(readRepo(profile.fixture))).toHaveProperty('plan') + expect(profile.go_source).toMatch( + /^languages\/golang\/encrypt\/\w+_test\.go$/, + ) + const rust = readRepo( + `packages/stack-encrypt/tests/${profile.rust_binary}.rs`, + ) + const go = readRepo(profile.go_source) + expect(rust).toContain(`${profile.rust_binary}.json`) + expect(go).toContain(`${profile.rust_binary}.json`) + expect(script).toMatch( + new RegExp( + `cargo nextest run[^\\n]*--test ${profile.rust_binary}(?: |$)`, + 'm', + ), + ) + expect(profile.requirements.length).toBeGreaterThan(0) + expect(contract).toContain(profile.id) + for (const requirement of profile.requirements) { + expect(Object.keys(requirement).sort()).toEqual([ + 'go_tests', + 'id', + 'rust_tests', + ]) + expect(requirement.id).toMatch(/^(values|queries)\.[a-z-]+$/) + expect(requirements.has(requirement.id), requirement.id).toBe(false) + requirements.add(requirement.id) + expect(contract).toContain(`\`${requirement.id}\``) + expect(requirement.rust_tests.length).toBeGreaterThan(0) + expect(requirement.go_tests.length).toBeGreaterThan(0) + for (const test of requirement.rust_tests) { + expect(test).toMatch(/^[a-z_]+$/) + expect(rust).toMatch( + new RegExp(`#\\[(?:tokio::)?test\\]\\s+(?:async )?fn ${test}\\(`), + ) + } + for (const test of requirement.go_tests) { + expect(test).toMatch(/^Test\w+$/) + expect(go).toContain(`func ${test}(t *testing.T)`) + expect(selected.test(test), `${requirement.id}: ${test}`).toBe(true) + } + } + } + const documented = [ + ...contract.matchAll(/\| `((?:values|queries)\.[a-z-]+)` \|/g), + ].map((match) => match[1]) + expect([...requirements].sort()).toEqual(documented.sort()) }) + it.each(['STACK_CONFORMANCE_UPDATE', 'STACK_ENCRYPT_UPDATE_FIXTURES'])( + 'refuses %s before invoking either language or modifying fixtures', + (variable) => { + // Empty is still set: Rust's fixture writers use var_os().is_some(). + const result = spawnSync('bash', ['scripts/stack-conformance.sh'], { + cwd: REPO_ROOT, + env: { ...process.env, [variable]: '' }, + encoding: 'utf8', + }) + expect(result.status).toBe(1) + expect(result.stderr).toContain( + 'Conformance checks must not update frozen artifacts.', + ) + expect(result.stdout).toBe('') + }, + ) + it('runs both consumers when the corpus changes', () => { for (const file of ['tests-crates.yml', 'tests-golang.yml']) { const workflow = readWorkflow(`.github/workflows/${file}`) const triggers = workflow.on ?? workflow[true] for (const event of ['pull_request', 'push']) { expect(triggers[event].paths).toContain('conformance/stack-encrypt/**') + expect(triggers[event].paths).toContain('packages/stack-encrypt/**') } } }) diff --git a/scripts/stack-conformance.sh b/scripts/stack-conformance.sh index 670b9085c..dd765cafc 100644 --- a/scripts/stack-conformance.sh +++ b/scripts/stack-conformance.sh @@ -3,7 +3,7 @@ set -euo pipefail cd "$(dirname "$0")/.." -if [[ -n "${STACK_CONFORMANCE_UPDATE:-}" ]]; then +if [[ -n "${STACK_CONFORMANCE_UPDATE+x}" || -n "${STACK_ENCRYPT_UPDATE_FIXTURES+x}" ]]; then echo 'Conformance checks must not update frozen artifacts.' >&2 exit 1 fi @@ -18,11 +18,11 @@ fi artifacts=$(mktemp -d) trap 'rm -rf "$artifacts"' EXIT -mise x --env test -- cargo nextest run --locked -p stack-encrypt --all-features --test conformance --test declarations --test runtime_conformance --test read_conformance +mise x --env test -- cargo nextest run --locked -p stack-encrypt --all-features --test conformance --test declarations --test runtime_conformance --test read_conformance --test value_kinds --test query_operators ( cd languages/golang STACK_CONFORMANCE_OUTPUT="$artifacts/go.json" CGO_ENABLED=0 \ - go test -count=1 -v -run '^(TestStackConformance|TestDeclarationConformance|TestRuntimeConformance|TestReadConformance)$' ./encrypt + go test -count=1 -v -run '^(TestStackConformance|TestDeclarationConformance|TestRuntimeConformance|TestReadConformance|TestFixtureConformanceRequiresGuest|TestGeneratedCodeOpensTheRustValueKindsFixture|TestQueryTermsAreTheRustFixtures|TestVerifyIsTheRustFixtures|TestTargetLimitIsTheRustFixtures|TestGeneratedQueriesAreTheRustFixtures|TestAMatchesQueryVerifiesAndSizesAsTheRustFixtures)$' ./encrypt ) test -s "$artifacts/go.json" STACK_CONFORMANCE_INPUT="$artifacts/go.json" mise x --env test -- \ From 52d96d990ca976a93f00ece270ec37d2ccfd0701 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sun, 1 Nov 2026 17:32:31 +1100 Subject: [PATCH 2/3] test(conformance): enforce fixture enrollment --- .github/workflows/tests-golang.yml | 2 + conformance/stack-encrypt/fixture-contract.md | 28 ++++++--- .../2026-10-31-stack-encrypt-conformance.md | 8 +-- languages/golang/encrypt/conformance_test.go | 4 +- .../encrypt/declaration_conformance_test.go | 5 +- .../encrypt/fixture_conformance_test.go | 63 +++++++++++++++++-- languages/golang/encrypt/query_test.go | 4 +- .../golang/encrypt/read_conformance_test.go | 9 +-- .../encrypt/runtime_conformance_test.go | 6 +- languages/golang/encrypt/value_kinds_test.go | 4 +- packages/stack-encrypt/tests/common/mod.rs | 46 +++++++++++++- .../stack-encrypt/tests/fixture_safety.rs | 44 +++++++++++++ .../stack-encrypt/tests/query_operators.rs | 10 ++- .../stack-encrypt/tests/record_lowering.rs | 11 ++-- packages/stack-encrypt/tests/value_kinds.rs | 9 ++- .../__tests__/stack-conformance-ci.test.mjs | 57 ++++++++++++++--- scripts/stack-conformance-selection.mjs | 41 ++++++++++++ scripts/stack-conformance.sh | 5 +- 18 files changed, 294 insertions(+), 62 deletions(-) create mode 100644 packages/stack-encrypt/tests/fixture_safety.rs create mode 100644 scripts/stack-conformance-selection.mjs diff --git a/.github/workflows/tests-golang.yml b/.github/workflows/tests-golang.yml index 6cc483abf..0fa4a6967 100644 --- a/.github/workflows/tests-golang.yml +++ b/.github/workflows/tests-golang.yml @@ -59,6 +59,7 @@ on: - scripts/check-wasm-imports.py - scripts/go-binding-test.sh - scripts/stack-conformance.sh + - scripts/stack-conformance-selection.mjs - Cargo.toml - Cargo.lock - .cargo/** @@ -97,6 +98,7 @@ on: - scripts/check-wasm-imports.py - scripts/go-binding-test.sh - scripts/stack-conformance.sh + - scripts/stack-conformance-selection.mjs - Cargo.toml - Cargo.lock - .cargo/** diff --git a/conformance/stack-encrypt/fixture-contract.md b/conformance/stack-encrypt/fixture-contract.md index 54e418038..efb589960 100644 --- a/conformance/stack-encrypt/fixture-contract.md +++ b/conformance/stack-encrypt/fixture-contract.md @@ -18,10 +18,12 @@ pages or fetch limits are copied into this directory. [`v1/fixture-profiles.json`](v1/fixture-profiles.json) maps stable requirement IDs to the existing Rust test functions and Go tests that provide their evidence. It is an enrollment index, not another input corpus or a runtime capability -negotiation format. The CI wiring test checks that its fixtures/tests exist, -that each requirement is documented here, and that the conformance command -selects every listed test. Both language implementations are mandatory for -these profiles; Python and TypeScript remain pending. +negotiation format. The CI wiring test checks fixture paths, source declarations, +requirement documentation and exact command selection. Rust also checks each +binary’s compiled test inventory and rejects missing or ignored enrolled tests. +The Go missing-guest regression checks each indexed test’s actual failure/skip +outcome, so a test excluded from compilation cannot silently pass enrollment. +Both language implementations are mandatory for these profiles; Python and TypeScript remain pending. ## Value-kind requirements @@ -66,21 +68,29 @@ which the record-interface test covers. Its existing SQL parameter conversion, empty-page, zero-query, inverted-range and floating-point ordering assertions remain in those selected tests. Wider operator refusal and secret-diagnostic tests remain in the full suites; this slice does not turn them into shared -negative cases. +negative cases. In Go, `TestAQueryTheFieldDoesNotAnswerIsRefused` and +`TestAMatchesQueryNeverPrintsItsText` share the fixture client, so they also +fail on a missing guest in required mode. They remain outside the aggregate +selection and the shared requirements; the missing-guest regression covers them. ## Running and maintaining the profiles `mise run test:conformance` selects the existing `value_kinds` and `query_operators` Rust binaries and the Go tests listed in the enrollment index, -alongside the earlier profiles. Each existing test runs once per invocation; -no wrapper test invokes another test a second time. The ordinary Rust and Go +alongside the earlier profiles. The Go selection is generated from the index; +adding an enrolled Go test does not require editing a shell regular expression. +Fixture assertions stay in their owning tests; the separate missing-guest probes +stop at client creation. The ordinary Rust and Go CI suites already run these tests, so enrollment adds no duplicate CI suite. When `STACK_CONFORMANCE_REQUIRED=1`, a missing deterministic guest fails the enrolled Go tests instead of skipping them. The aggregate command rejects both `STACK_CONFORMANCE_UPDATE` and `STACK_ENCRYPT_UPDATE_FIXTURES`, even when set to -an empty value. The enrolled Rust fixture writers independently reject updates -in required mode. Deliberate fixture maintenance remains available outside +an empty value. All four Rust fixture writers, including the two record-lowering +writers, +independently reject updates in required mode. A subprocess regression exercises +the guard with absent, empty and nonempty update flags without writing fixtures. +Deliberate fixture maintenance remains available outside required mode through the existing documented regeneration commands. Keep requirement IDs stable. Add a new profile version for incompatible diff --git a/docs/plans/2026-10-31-stack-encrypt-conformance.md b/docs/plans/2026-10-31-stack-encrypt-conformance.md index af6cae7ad..5cde8cea2 100644 --- a/docs/plans/2026-10-31-stack-encrypt-conformance.md +++ b/docs/plans/2026-10-31-stack-encrypt-conformance.md @@ -152,8 +152,7 @@ Python and TypeScript consumers remain pending. ## Value-kind and query-operator checkpoint -The next slice is on `feat/stack-encrypt-value-query-conformance`, stacked on -[PR #1462](https://github.com/cipherstash/stack/pull/1462). The +Building on the read-compatibility profile, the [fixture contract](../../conformance/stack-encrypt/fixture-contract.md) and [enrollment index](../../conformance/stack-encrypt/v1/fixture-profiles.json) assign stable requirement IDs to the existing `value_kinds.json` and @@ -162,13 +161,14 @@ existing Rust and Go tests. Typed Rust comparisons, generated Go methods, match verification and fetch sizing remain intact. The fixture files stay in place with no duplicated expectations. CI guards -the requirement-to-test mapping and test selection. Required Go runs fail if +the requirement-to-test mapping, compiled test availability and exact test +selection. The Go selection is generated from the enrollment index. Required Go runs fail if the guest is missing; required checks cannot regenerate frozen fixtures. This is consolidation of inventory and execution, not removal of assertions. Local validation passes for the expanded conformance command, fresh scalar artifact exchange, missing-guest regressions, Rust clippy/formatting, Go lint, -Biome and CI wiring checks. Both enrolled Rust writers refuse regeneration in +Biome and CI wiring checks. All four Rust fixture writers refuse regeneration in required mode, and the frozen fixture files remain unchanged. The coverage is representative, not a complete kind/operator boundary matrix. diff --git a/languages/golang/encrypt/conformance_test.go b/languages/golang/encrypt/conformance_test.go index 5fa883e7d..88039d9eb 100644 --- a/languages/golang/encrypt/conformance_test.go +++ b/languages/golang/encrypt/conformance_test.go @@ -160,9 +160,7 @@ func TestStackConformance(t *testing.T) { } ctx := context.Background() client, err := encrypt.NewDeterministicClient(ctx, [32]byte(seed)) - if errors.Is(err, encrypt.ErrDeterministicGuestNotBuilt) && os.Getenv("STACK_CONFORMANCE_REQUIRED") != "1" { - t.Skip(err) - } + skipMissingConformanceGuest(t, err) if err != nil { t.Fatal(err) } diff --git a/languages/golang/encrypt/declaration_conformance_test.go b/languages/golang/encrypt/declaration_conformance_test.go index 35e19e28b..cd06c4382 100644 --- a/languages/golang/encrypt/declaration_conformance_test.go +++ b/languages/golang/encrypt/declaration_conformance_test.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "encoding/json" "errors" - "os" "path/filepath" "sort" "strconv" @@ -87,9 +86,7 @@ func TestDeclarationConformance(t *testing.T) { for name, load := range map[string]func() ([]byte, error){"release": encrypt.PlainGuest, "deterministic": encrypt.DeterministicGuest} { t.Run(name, func(t *testing.T) { wasm, err := load() - if (errors.Is(err, encrypt.ErrGuestNotBuilt) || errors.Is(err, encrypt.ErrDeterministicGuestNotBuilt)) && os.Getenv("STACK_CONFORMANCE_REQUIRED") != "1" { - t.Skip(err) - } + skipMissingConformanceGuest(t, err) if err != nil { t.Fatal(err) } diff --git a/languages/golang/encrypt/fixture_conformance_test.go b/languages/golang/encrypt/fixture_conformance_test.go index 3d7724f76..de62ed107 100644 --- a/languages/golang/encrypt/fixture_conformance_test.go +++ b/languages/golang/encrypt/fixture_conformance_test.go @@ -1,16 +1,31 @@ package encrypt_test import ( + "encoding/json" "errors" "os" "os/exec" "path/filepath" + "regexp" + "slices" "strings" "testing" + + "github.com/cipherstash/stack/languages/golang/encrypt" ) +// All conformance runners use the same missing-guest policy. Other errors +// remain failures in the caller, including errors wrapping neither sentinel. +func skipMissingConformanceGuest(t *testing.T, err error) { + t.Helper() + if (errors.Is(err, encrypt.ErrGuestNotBuilt) || errors.Is(err, encrypt.ErrDeterministicGuestNotBuilt)) && os.Getenv("STACK_CONFORMANCE_REQUIRED") != "1" { + t.Skip(err) + } +} + // Exercise the real fixture entry points in an isolated tree with their frozen -// inputs but no guest. A missing guest must never make required conformance green. +// inputs but no guest. Check every indexed test's outcome, so a missing compiled +// test also fails this guard. A missing guest must never make conformance green. func TestFixtureConformanceRequiresGuest(t *testing.T) { binary, err := os.Executable() if err != nil { @@ -24,20 +39,56 @@ func TestFixtureConformanceRequiresGuest(t *testing.T) { t.Fatal(err) } } - for _, name := range []string{"value_kinds.json", "query_operators.json"} { - data, err := os.ReadFile(filepath.Join("..", "..", "..", fixtures, name)) + var index struct { + Profiles []struct { + Fixture string `json:"fixture"` + Requirements []struct { + GoTests []string `json:"go_tests"` + } `json:"requirements"` + } `json:"profiles"` + } + data, err := os.ReadFile(filepath.Join("..", "..", "..", "conformance", "stack-encrypt", "v1", "fixture-profiles.json")) + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(data, &index); err != nil { + t.Fatal(err) + } + names := make(map[string]bool) + for _, profile := range index.Profiles { + data, err := os.ReadFile(filepath.Join("..", "..", "..", profile.Fixture)) if err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(root, fixtures, name), data, 0o600); err != nil { + if err := os.WriteFile(filepath.Join(root, profile.Fixture), data, 0o600); err != nil { t.Fatal(err) } + for _, requirement := range profile.Requirements { + for _, name := range requirement.GoTests { + names[name] = true + } + } + } + if len(names) == 0 { + t.Fatal("fixture index selects no Go tests") + } + // These full-suite tests also share queryFixtureClient's required-mode policy; + // they remain outside the shared fixture requirements and aggregate selection. + names["TestAQueryTheFieldDoesNotAnswerIsRefused"] = true + names["TestAMatchesQueryNeverPrintsItsText"] = true + tests := make([]string, 0, len(names)) + for name := range names { + tests = append(tests, name) + } + slices.Sort(tests) + patterns := make([]string, len(tests)) + for i, name := range tests { + patterns[i] = regexp.QuoteMeta(name) } - tests := []string{"TestGeneratedCodeOpensTheRustValueKindsFixture", "TestQueryTermsAreTheRustFixtures"} for _, required := range []string{"0", "1"} { t.Run("required="+required, func(t *testing.T) { t.Setenv("STACK_CONFORMANCE_REQUIRED", required) - cmd := exec.CommandContext(t.Context(), binary, "-test.v", "-test.timeout=30s", "-test.run=^("+strings.Join(tests, "|")+")$") + cmd := exec.CommandContext(t.Context(), binary, "-test.v", "-test.timeout=30s", "-test.run=^("+strings.Join(patterns, "|")+")$") cmd.Dir = workdir out, err := cmd.CombinedOutput() marker := "--- SKIP: " diff --git a/languages/golang/encrypt/query_test.go b/languages/golang/encrypt/query_test.go index d4ef243c6..d4cbc7baf 100644 --- a/languages/golang/encrypt/query_test.go +++ b/languages/golang/encrypt/query_test.go @@ -129,9 +129,7 @@ func queryFixtureClient(t *testing.T, f queryFixture) *encrypt.Client { t.Fatalf("seed: %v (%d bytes)", err, len(seed)) } c, err := encrypt.NewDeterministicClient(context.Background(), [32]byte(seed)) - if (errors.Is(err, encrypt.ErrDeterministicGuestNotBuilt) || errors.Is(err, encrypt.ErrGuestNotBuilt)) && os.Getenv("STACK_CONFORMANCE_REQUIRED") != "1" { - t.Skip(err) - } + skipMissingConformanceGuest(t, err) if err != nil { t.Fatal(err) } diff --git a/languages/golang/encrypt/read_conformance_test.go b/languages/golang/encrypt/read_conformance_test.go index 894bc8958..60a225916 100644 --- a/languages/golang/encrypt/read_conformance_test.go +++ b/languages/golang/encrypt/read_conformance_test.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "os" "path/filepath" "reflect" "testing" @@ -149,9 +148,7 @@ func TestReadConformance(t *testing.T) { } ctx := context.Background() client, err := encrypt.NewDeterministicClient(ctx, [32]byte{}) - if errors.Is(err, encrypt.ErrDeterministicGuestNotBuilt) && os.Getenv("STACK_CONFORMANCE_REQUIRED") != "1" { - t.Skip(err) - } + skipMissingConformanceGuest(t, err) if err != nil { t.Fatal(err) } @@ -416,9 +413,7 @@ func TestReadCorpusSchema(t *testing.T) { func TestReadTreePreservesCipherScope(t *testing.T) { ctx := context.Background() client, err := encrypt.NewDeterministicClient(ctx, [32]byte{}) - if errors.Is(err, encrypt.ErrDeterministicGuestNotBuilt) && os.Getenv("STACK_CONFORMANCE_REQUIRED") != "1" { - t.Skip(err) - } + skipMissingConformanceGuest(t, err) if err != nil { t.Fatal(err) } diff --git a/languages/golang/encrypt/runtime_conformance_test.go b/languages/golang/encrypt/runtime_conformance_test.go index 5dfcfddee..0a34db566 100644 --- a/languages/golang/encrypt/runtime_conformance_test.go +++ b/languages/golang/encrypt/runtime_conformance_test.go @@ -3,8 +3,6 @@ package encrypt_test import ( "bytes" "context" - "errors" - "os" "path/filepath" "reflect" "testing" @@ -59,9 +57,7 @@ func TestRuntimeConformance(t *testing.T) { } ctx := context.Background() client, err := encrypt.NewDeterministicClient(ctx, [32]byte{}) - if errors.Is(err, encrypt.ErrDeterministicGuestNotBuilt) && os.Getenv("STACK_CONFORMANCE_REQUIRED") != "1" { - t.Skip(err) - } + skipMissingConformanceGuest(t, err) if err != nil { t.Fatal(err) } diff --git a/languages/golang/encrypt/value_kinds_test.go b/languages/golang/encrypt/value_kinds_test.go index 32ae018df..1b734daba 100644 --- a/languages/golang/encrypt/value_kinds_test.go +++ b/languages/golang/encrypt/value_kinds_test.go @@ -152,9 +152,7 @@ func TestGeneratedCodeOpensTheRustValueKindsFixture(t *testing.T) { t.Fatalf("seed: %v (%d bytes)", err, len(seedBytes)) } c, err := encrypt.NewDeterministicClient(context.Background(), [32]byte(seedBytes)) - if (errors.Is(err, encrypt.ErrDeterministicGuestNotBuilt) || errors.Is(err, encrypt.ErrGuestNotBuilt)) && os.Getenv("STACK_CONFORMANCE_REQUIRED") != "1" { - t.Skip(err) - } + skipMissingConformanceGuest(t, err) if err != nil { t.Fatal(err) } diff --git a/packages/stack-encrypt/tests/common/mod.rs b/packages/stack-encrypt/tests/common/mod.rs index 0224668d2..0bcd7faa3 100644 --- a/packages/stack-encrypt/tests/common/mod.rs +++ b/packages/stack-encrypt/tests/common/mod.rs @@ -21,7 +21,10 @@ use uuid::Uuid; /// Fixture regeneration is an explicit maintenance operation, never evidence /// for a required conformance run. Check before deriving or writing any fixture. -pub fn fixture_updates_requested() -> bool { +/// +/// # Panics +/// Panics if updates are requested while `STACK_CONFORMANCE_REQUIRED=1`. +pub fn fixture_updates_requested_or_panic() -> bool { let update = std::env::var_os("STACK_ENCRYPT_UPDATE_FIXTURES").is_some(); assert!( !update || std::env::var("STACK_CONFORMANCE_REQUIRED").as_deref() != Ok("1"), @@ -30,6 +33,47 @@ pub fn fixture_updates_requested() -> bool { update } +/// Check the compiled libtest inventory, including ignored tests, rather than +/// trusting source declarations that may be disabled by `cfg`. +pub fn assert_fixture_tests_registered(binary: &str) { + let index: serde_json::Value = serde_json::from_str(include_str!( + "../../../../conformance/stack-encrypt/v1/fixture-profiles.json" + )) + .expect("fixture enrollment index"); + let profile = index["profiles"] + .as_array() + .expect("profiles") + .iter() + .find(|profile| profile["rust_binary"] == binary) + .expect("binary is enrolled"); + let inventory = |ignored: bool| { + let mut command = std::process::Command::new(std::env::current_exe().expect("test binary")); + command.arg("--list").arg("--format=terse"); + if ignored { + command.arg("--ignored"); + } + let output = command.output().expect("list compiled tests"); + assert!(output.status.success(), "listing failed: {output:?}"); + String::from_utf8(output.stdout) + .expect("UTF-8 inventory") + .lines() + .filter_map(|line| line.strip_suffix(": test").map(str::to_owned)) + .collect::>() + }; + let registered = inventory(false); + let ignored = inventory(true); + for requirement in profile["requirements"].as_array().expect("requirements") { + for test in requirement["rust_tests"].as_array().expect("Rust tests") { + let name = test.as_str().expect("test name"); + assert!( + registered.contains(name), + "{binary}::{name} is not compiled" + ); + assert!(!ignored.contains(name), "{binary}::{name} is ignored"); + } + } +} + /// A cipher over the deterministic fake source. The fake index key is /// deterministic per keyset, so two separately built ciphers stand in for the /// write path and a query path in another process. diff --git a/packages/stack-encrypt/tests/fixture_safety.rs b/packages/stack-encrypt/tests/fixture_safety.rs new file mode 100644 index 000000000..f054e9c07 --- /dev/null +++ b/packages/stack-encrypt/tests/fixture_safety.rs @@ -0,0 +1,44 @@ +//! Exercise the regeneration guard without ever reaching a fixture writer. +mod common; + +#[test] +fn fixture_update_probe() { + if let Ok(expected) = std::env::var("STACK_FIXTURE_GUARD_PROBE") { + assert_eq!( + common::fixture_updates_requested_or_panic(), + expected == "true" + ); + } +} + +#[test] +fn required_conformance_refuses_fixture_updates() { + for required in [None, Some("0"), Some("1")] { + for update in [None, Some(""), Some("1")] { + let mut command = + std::process::Command::new(std::env::current_exe().expect("test binary")); + command + .args(["--exact", "fixture_update_probe", "--nocapture"]) + .env_remove("STACK_CONFORMANCE_REQUIRED") + .env_remove("STACK_ENCRYPT_UPDATE_FIXTURES") + .env("STACK_FIXTURE_GUARD_PROBE", update.is_some().to_string()); + if let Some(value) = required { + command.env("STACK_CONFORMANCE_REQUIRED", value); + } + if let Some(value) = update { + command.env("STACK_ENCRYPT_UPDATE_FIXTURES", value); + } + let output = command.output().expect("run fixture guard probe"); + let refused = required == Some("1") && update.is_some(); + assert_eq!( + output.status.success(), + !refused, + "required={required:?}, update={update:?}: {output:?}" + ); + if refused { + assert!(String::from_utf8_lossy(&output.stderr) + .contains("conformance checks must not update frozen fixtures")); + } + } + } +} diff --git a/packages/stack-encrypt/tests/query_operators.rs b/packages/stack-encrypt/tests/query_operators.rs index cf50643a2..9103a2674 100644 --- a/packages/stack-encrypt/tests/query_operators.rs +++ b/packages/stack-encrypt/tests/query_operators.rs @@ -362,10 +362,11 @@ fn fixture_path() -> PathBuf { } /// The committed fixture is what this run derives. With -/// `STACK_ENCRYPT_UPDATE_FIXTURES=1` it is written first. +/// `STACK_ENCRYPT_UPDATE_FIXTURES=1` it is written first. Updates panic when +/// `STACK_CONFORMANCE_REQUIRED=1`. #[tokio::test] async fn the_committed_query_fixture_is_what_the_engine_derives() { - let update = common::fixture_updates_requested(); + let update = common::fixture_updates_requested_or_panic(); let now = derived().await; let path = fixture_path(); if update { @@ -395,3 +396,8 @@ async fn the_fixture_pins_both_outcomes() { assert!(limits.iter().any(|limit| limit.is_u64())); assert!(limits.iter().any(|limit| limit.is_null())); } + +#[test] +fn enrolled_fixture_tests_are_compiled_and_enabled() { + common::assert_fixture_tests_registered("query_operators"); +} diff --git a/packages/stack-encrypt/tests/record_lowering.rs b/packages/stack-encrypt/tests/record_lowering.rs index 16dbcbdd4..ce5e7f22a 100644 --- a/packages/stack-encrypt/tests/record_lowering.rs +++ b/packages/stack-encrypt/tests/record_lowering.rs @@ -421,14 +421,16 @@ async fn the_chain_and_the_lowering_are_one_engine_today() { /// The committed fixture: records each author sealed in an earlier run, /// opened by the other now, their terms the ones derived now. With /// `STACK_ENCRYPT_UPDATE_FIXTURES=1` the records are sealed afresh and -/// written; the terms and the plaintext never change. +/// written; the terms and the plaintext never change. Updates panic when +/// `STACK_CONFORMANCE_REQUIRED=1`. #[tokio::test] async fn the_committed_fixture_opens_both_ways() { + let update = common::fixture_updates_requested_or_panic(); let cipher = deterministic_cipher(SEED).await; let plan = record::plan(&frame_of_json(&data_plan_json())).expect("the data plan parses"); let path = fixture_path(); - if std::env::var_os("STACK_ENCRYPT_UPDATE_FIXTURES").is_some() { + if update { let mut fixture = committed_fixture(&path); let entries = fixture.as_object_mut().expect("an object"); let _ = entries.insert("_comment".into(), json!("Records sealed by the typed Rust chain and by the data-plan lowering under one declaration (ADR-0007). Read by tests/record_lowering.rs; the schema is in README.md beside this file. Regenerate with STACK_ENCRYPT_UPDATE_FIXTURES=1; do not edit by hand.")); @@ -736,14 +738,15 @@ async fn the_chain_and_the_lowering_agree_on_a_context_field_today() { /// The committed context-field case, under the fixture's `context_field` /// key: sealed by each author in an earlier run, opened by the other now. /// With `STACK_ENCRYPT_UPDATE_FIXTURES=1` this test rewrites that key and -/// nothing else in the file. +/// nothing else in the file. Updates panic when `STACK_CONFORMANCE_REQUIRED=1`. #[tokio::test] async fn the_committed_context_field_case_opens_both_ways() { + let update = common::fixture_updates_requested_or_panic(); let cipher = deterministic_cipher(SEED).await; let plan = record::plan(&frame_of_json(¬e_plan_json())).expect("the data plan parses"); let path = fixture_path(); - if std::env::var_os("STACK_ENCRYPT_UPDATE_FIXTURES").is_some() { + if update { let mut fixture = committed_fixture(&path); let _ = fixture.as_object_mut().expect("an object").insert( "context_field".into(), diff --git a/packages/stack-encrypt/tests/value_kinds.rs b/packages/stack-encrypt/tests/value_kinds.rs index b019d4965..cb55e3290 100644 --- a/packages/stack-encrypt/tests/value_kinds.rs +++ b/packages/stack-encrypt/tests/value_kinds.rs @@ -445,10 +445,10 @@ async fn a_value_of_another_width_is_refused_when_sealed() { /// The committed fixture: the record sealed in an earlier run opens now, its /// terms and frames are today's, and each folded query lands on its stored /// term. With `STACK_ENCRYPT_UPDATE_FIXTURES=1` the record is sealed afresh -/// and the file written. +/// and the file written. Updates panic when `STACK_CONFORMANCE_REQUIRED=1`. #[tokio::test] async fn the_committed_fixture_opens_and_its_terms_are_todays() { - let update = common::fixture_updates_requested(); + let update = common::fixture_updates_requested_or_panic(); let cipher = deterministic_cipher(SEED).await; let keyset = cipher.default_keyset(); let plan = data_plan(); @@ -525,3 +525,8 @@ async fn the_committed_fixture_opens_and_its_terms_are_todays() { assert_eq!(json!(hex(&term)), entry["term"], "{field}: today's"); } } + +#[test] +fn enrolled_fixture_tests_are_compiled_and_enabled() { + common::assert_fixture_tests_registered("value_kinds"); +} diff --git a/scripts/__tests__/stack-conformance-ci.test.mjs b/scripts/__tests__/stack-conformance-ci.test.mjs index 7b4528e89..25501e67d 100644 --- a/scripts/__tests__/stack-conformance-ci.test.mjs +++ b/scripts/__tests__/stack-conformance-ci.test.mjs @@ -2,6 +2,7 @@ import { spawnSync } from 'node:child_process' import { readFileSync } from 'node:fs' import { join } from 'node:path' import { describe, expect, it } from 'vitest' +import { goSelection as selectGoTests } from '../stack-conformance-selection.mjs' import { REPO_ROOT } from './lib/repo-root.mjs' import { readWorkflow } from './lib/workflows.mjs' @@ -10,12 +11,20 @@ const conformanceScript = () => readRepo('scripts/stack-conformance.sh').replace(/^\s*#.*$/gm, '') function goSelection(script) { - const match = /go test -count=1 -v -run '([^']+)' \.\/encrypt/.exec(script) - expect( - match, - 'the Go conformance command must execute uncached', - ).not.toBeNull() - return new RegExp(match[1]) + expect(script).toContain( + 'go_tests=$(node scripts/stack-conformance-selection.mjs)', + ) + expect(script).toContain('go test -count=1 -v -run "$go_tests" ./encrypt') + const result = spawnSync( + process.execPath, + ['scripts/stack-conformance-selection.mjs'], + { + cwd: REPO_ROOT, + encoding: 'utf8', + }, + ) + expect(result.status, result.stderr).toBe(0) + return new RegExp(result.stdout.trim()) } describe('shared Stack Encrypt conformance is required in CI', () => { @@ -26,7 +35,7 @@ describe('shared Stack Encrypt conformance is required in CI', () => { ).replace(/^\s*#.*$/gm, '') const commands = script.replace(/\\\r?\n\s*/g, ' ') expect(commands).toMatch( - /STACK_CONFORMANCE_OUTPUT=[^\n]+go test[^\n]+TestStackConformance/, + /STACK_CONFORMANCE_OUTPUT=[^\n]+go test[^\n]+"\$go_tests"/, ) expect(commands).toMatch( /STACK_CONFORMANCE_INPUT=[^\n]+cargo nextest run[^\n]+--test conformance/, @@ -35,6 +44,7 @@ describe('shared Stack Encrypt conformance is required in CI', () => { expect(script).toMatch(/cargo nextest run[^\n]*--test declarations/) expect(script).toMatch(/cargo nextest run[^\n]*--test runtime_conformance/) expect(script).toMatch(/cargo nextest run[^\n]*--test read_conformance/) + expect(script).toMatch(/cargo nextest run[^\n]*--test fixture_safety/) const selected = goSelection(script) for (const test of [ 'TestStackConformance', @@ -47,6 +57,39 @@ describe('shared Stack Encrypt conformance is required in CI', () => { } }) + it('selects exactly the core runners and indexed fixture tests', () => { + const index = JSON.parse( + readRepo('conformance/stack-encrypt/v1/fixture-profiles.json'), + ) + const expected = [ + ...new Set([ + 'TestStackConformance', + 'TestDeclarationConformance', + 'TestRuntimeConformance', + 'TestReadConformance', + 'TestFixtureConformanceRequiresGuest', + ...index.profiles.flatMap((p) => + p.requirements.flatMap((r) => r.go_tests), + ), + ]), + ].sort() + const selected = goSelection(conformanceScript()) + expect(selected.source).toBe(`^(${expected.join('|')})$`) + for (const name of expected) { + expect(selected.test(`${name}Extra`)).toBe(false) + expect(selected.test(`Prefix${name}`)).toBe(false) + } + const extended = structuredClone(index) + extended.profiles[0].requirements[0].go_tests.push( + 'TestNewFixtureRequirement', + ) + expect( + new RegExp(selectGoTests(extended)).test('TestNewFixtureRequirement'), + ).toBe(true) + extended.profiles[0].requirements[0].go_tests.push('Test.*') + expect(() => selectGoTests(extended)).toThrow('Invalid Go test name') + }) + it('enrolls existing fixture evidence without copying or dropping tests', () => { const index = JSON.parse( readRepo('conformance/stack-encrypt/v1/fixture-profiles.json'), diff --git a/scripts/stack-conformance-selection.mjs b/scripts/stack-conformance-selection.mjs new file mode 100644 index 000000000..2c4c64193 --- /dev/null +++ b/scripts/stack-conformance-selection.mjs @@ -0,0 +1,41 @@ +#!/usr/bin/env node +// The fixture index owns enrollment; the command adds the earlier runners. +import { readFileSync } from 'node:fs' +import { pathToFileURL } from 'node:url' + +export function goSelection(index) { + const tests = new Set([ + 'TestStackConformance', + 'TestDeclarationConformance', + 'TestRuntimeConformance', + 'TestReadConformance', + 'TestFixtureConformanceRequiresGuest', + ]) + for (const profile of index.profiles) { + for (const requirement of profile.requirements) { + for (const name of requirement.go_tests) { + if (!/^Test[A-Za-z0-9_]+$/.test(name)) { + throw new Error(`Invalid Go test name: ${name}`) + } + tests.add(name) + } + } + } + return `^(${[...tests].sort().join('|')})$` +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + const index = JSON.parse( + readFileSync( + new URL( + '../conformance/stack-encrypt/v1/fixture-profiles.json', + import.meta.url, + ), + 'utf8', + ), + ) + process.stdout.write(`${goSelection(index)}\n`) +} diff --git a/scripts/stack-conformance.sh b/scripts/stack-conformance.sh index dd765cafc..025c834a4 100644 --- a/scripts/stack-conformance.sh +++ b/scripts/stack-conformance.sh @@ -15,14 +15,15 @@ if [[ ! -s languages/golang/encrypt/testdata/stack_encrypt_guest_deterministic.w exit 1 fi +go_tests=$(node scripts/stack-conformance-selection.mjs) artifacts=$(mktemp -d) trap 'rm -rf "$artifacts"' EXIT -mise x --env test -- cargo nextest run --locked -p stack-encrypt --all-features --test conformance --test declarations --test runtime_conformance --test read_conformance --test value_kinds --test query_operators +mise x --env test -- cargo nextest run --locked -p stack-encrypt --all-features --test conformance --test declarations --test runtime_conformance --test read_conformance --test value_kinds --test query_operators --test fixture_safety ( cd languages/golang STACK_CONFORMANCE_OUTPUT="$artifacts/go.json" CGO_ENABLED=0 \ - go test -count=1 -v -run '^(TestStackConformance|TestDeclarationConformance|TestRuntimeConformance|TestReadConformance|TestFixtureConformanceRequiresGuest|TestGeneratedCodeOpensTheRustValueKindsFixture|TestQueryTermsAreTheRustFixtures|TestVerifyIsTheRustFixtures|TestTargetLimitIsTheRustFixtures|TestGeneratedQueriesAreTheRustFixtures|TestAMatchesQueryVerifiesAndSizesAsTheRustFixtures)$' ./encrypt + go test -count=1 -v -run "$go_tests" ./encrypt ) test -s "$artifacts/go.json" STACK_CONFORMANCE_INPUT="$artifacts/go.json" mise x --env test -- \ From 5d52bc661754212e41ac3f7ea42d188ad7a0cf0f Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sun, 1 Nov 2026 18:37:20 +1100 Subject: [PATCH 3/3] fix(ci): use the standard script entry guard --- scripts/stack-conformance-selection.mjs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scripts/stack-conformance-selection.mjs b/scripts/stack-conformance-selection.mjs index 2c4c64193..d3ca1e787 100644 --- a/scripts/stack-conformance-selection.mjs +++ b/scripts/stack-conformance-selection.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node // The fixture index owns enrollment; the command adds the earlier runners. import { readFileSync } from 'node:fs' -import { pathToFileURL } from 'node:url' +import { fileURLToPath } from 'node:url' export function goSelection(index) { const tests = new Set([ @@ -24,10 +24,7 @@ export function goSelection(index) { return `^(${[...tests].sort().join('|')})$` } -if ( - process.argv[1] && - import.meta.url === pathToFileURL(process.argv[1]).href -) { +if (process.argv[1] === fileURLToPath(import.meta.url)) { const index = JSON.parse( readFileSync( new URL(