From 72520ad920c450f09eb404ddc948127a0662032d Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 28 Oct 2026 19:22:46 +0900 Subject: [PATCH 1/2] feat(python): generate the EQL type markers from the catalog (#1385) A model declares each field with a marker named after its EQL type, such as TextEq. The spec (#1341) wants the markers generated from the EQL catalog, beside the Go package, so Python and Go cannot disagree about a type. eql-codegen gains a py-eql emitter over the same rows as go-eql. It writes stackencrypt/_eql.py: one Marker subclass per row with the ten catalog attributes, a TYPES tuple, and an __all__ of the producible markers that the package re-exports. types:generate runs it and types:check diffs it. Two parity tests hold it: the committed file must match the render, and every marker must match its Go Types row. The file is not added to the EQL workflow path filters, the same as eql_gen.go. A path outside packages/eql/ would need a new exception in the filter guard, and the catalog changes that matter already start the EQL workflow. Refs #1385 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/tests-python.yml | 12 + .../python/python/stackencrypt/__init__.py | 14 +- languages/python/python/stackencrypt/_eql.py | 1034 +++++++++++++++++ .../python/python/stackencrypt/_markers.py | 29 + languages/python/tests/test_markers.py | 93 ++ packages/eql/crates/eql-codegen/src/lib.rs | 1 + packages/eql/crates/eql-codegen/src/main.rs | 18 + packages/eql/crates/eql-codegen/src/py_eql.rs | 199 ++++ .../crates/eql-codegen/tests/py_eql_parity.rs | 201 ++++ packages/eql/mise.toml | 9 +- 10 files changed, 1603 insertions(+), 7 deletions(-) create mode 100644 languages/python/python/stackencrypt/_eql.py create mode 100644 languages/python/python/stackencrypt/_markers.py create mode 100644 languages/python/tests/test_markers.py create mode 100644 packages/eql/crates/eql-codegen/src/py_eql.rs create mode 100644 packages/eql/crates/eql-codegen/tests/py_eql_parity.rs diff --git a/.github/workflows/tests-python.yml b/.github/workflows/tests-python.yml index 9c88ddafc..1a91ff6bf 100644 --- a/.github/workflows/tests-python.yml +++ b/.github/workflows/tests-python.yml @@ -160,6 +160,18 @@ jobs: - name: pytest run: mise run test + # The committed stackencrypt/_eql.py is generated from the EQL catalog + # by eql-codegen (#1385). `mise run types:check` in test-eql.yml diffs + # it, but that workflow does not run on a change under languages/python, + # so a hand edit to the file reached no check. This runs the one test + # that renders the catalog and compares it with the committed file, + # plus the Go/Python attribute parity in the same file. It compiles + # eql-codegen and its catalog only, in the EQL workspace, under the + # root Rust toolchain; tests-golang.yml compiles that workspace the + # same way. The target directory is not cached: the build is seconds. + - name: stackencrypt._eql matches the EQL catalog + run: cargo test --locked --manifest-path ../../packages/eql/Cargo.toml -p eql-codegen --test py_eql_parity + # One abi3-py39 wheel per platform the FFI ships for, by the same Rust # triples. python-ci.test.mjs holds the list to the protect-ffi platform # packages. The wheels are uploaded as artifacts. Nothing publishes them: diff --git a/languages/python/python/stackencrypt/__init__.py b/languages/python/python/stackencrypt/__init__.py index 77bd367be..aa7d47136 100644 --- a/languages/python/python/stackencrypt/__init__.py +++ b/languages/python/python/stackencrypt/__init__.py @@ -1,14 +1,20 @@ """Searchable encryption for Python, over the CipherStash Stack engine. -Walking skeleton. The package loads its native extension, ``_engine``, and -exposes two facts about the build. Nothing encrypts yet. +Walking skeleton. The package loads its native extension, ``_engine``, exposes +two facts about the build, and exports the markers a model's fields are +declared with. Nothing encrypts yet. """ -from stackencrypt import _engine +from stackencrypt import _engine, _eql +# The markers for the EQL types the engine produces today. `_eql.__all__` +# lists them; a marker the engine cannot produce yet stays in `_eql` only. +from stackencrypt._eql import * # noqa: F403 +from stackencrypt._markers import Ignore, Passthrough # The distribution's version, and the commit of the engine it was built from. __version__: str __engine__: str __version__, __engine__ = _engine.build_info() -__all__ = ["__engine__", "__version__"] +__all__ = ["Ignore", "Passthrough", "__engine__", "__version__"] +__all__ += _eql.__all__ diff --git a/languages/python/python/stackencrypt/_eql.py b/languages/python/python/stackencrypt/_eql.py new file mode 100644 index 000000000..8844e2fc6 --- /dev/null +++ b/languages/python/python/stackencrypt/_eql.py @@ -0,0 +1,1034 @@ +# Code generated by eql-codegen from the eql-domains catalog. DO NOT EDIT. +"""The EQL types as stash markers: one class per catalog row, in catalog order. + +Generated from the same catalog rows as the Go package encrypt/eql. ``__all__`` +lists the markers the engine produces today, which the package re-exports. +""" + +from stackencrypt._markers import Marker + +__all__ = [ + "Integer", + "IntegerEq", + "IntegerOrdOre", + "IntegerOrd", + "IntegerOrdOpe", + "Smallint", + "SmallintEq", + "SmallintOrdOre", + "SmallintOrd", + "SmallintOrdOpe", + "Bigint", + "BigintEq", + "BigintOrdOre", + "BigintOrd", + "BigintOrdOpe", + "Date", + "DateEq", + "DateOrdOre", + "DateOrd", + "DateOrdOpe", + "Timestamp", + "TimestampEq", + "TimestampOrdOre", + "TimestampOrd", + "TimestampOrdOpe", + "Numeric", + "NumericEq", + "NumericOrdOre", + "NumericOrd", + "NumericOrdOpe", + "Text", + "TextEq", + "TextMatch", + "TextOrdOre", + "TextOrd", + "TextOrdOpe", + "TextSearchOre", + "TextSearch", + "Boolean", + "Real", + "RealEq", + "RealOrdOre", + "RealOrd", + "RealOrdOpe", + "Double", + "DoubleEq", + "DoubleOrdOre", + "DoubleOrd", + "DoubleOrdOpe", + "JsonSearch", +] + + +class Integer(Marker): + """The EQL type stored in public.eql_v4_integer: the integer family with no index. + + The engine produces it from an int32. + """ + + name = "Integer" + family = "integer" + suffix = "" + plaintext = "int32" + sql_domain = "public.eql_v4_integer" + indexes = () + query = None + query_sql_domain = None + producible = True + reason = None + + +class IntegerEq(Marker): + """The EQL type stored in public.eql_v4_integer_eq: the integer family with the eq index. + + The engine produces it from an int32. + """ + + name = "IntegerEq" + family = "integer" + suffix = "Eq" + plaintext = "int32" + sql_domain = "public.eql_v4_integer_eq" + indexes = ("eq",) + query = "IntegerEqQuery" + query_sql_domain = "eql_v4.query_integer_eq" + producible = True + reason = None + + +class IntegerOrdOre(Marker): + """The EQL type stored in public.eql_v4_integer_ord_ore: the integer family with the ore_block index. + + The engine produces it from an int32. + """ + + name = "IntegerOrdOre" + family = "integer" + suffix = "OrdOre" + plaintext = "int32" + sql_domain = "public.eql_v4_integer_ord_ore" + indexes = ("ore_block",) + query = "IntegerOrdOreQuery" + query_sql_domain = "eql_v4.query_integer_ord_ore" + producible = True + reason = None + + +class IntegerOrd(Marker): + """The EQL type stored in public.eql_v4_integer_ord: the integer family with the ope index. + + The engine produces it from an int32. + """ + + name = "IntegerOrd" + family = "integer" + suffix = "Ord" + plaintext = "int32" + sql_domain = "public.eql_v4_integer_ord" + indexes = ("ope",) + query = "IntegerOrdQuery" + query_sql_domain = "eql_v4.query_integer_ord" + producible = True + reason = None + + +class IntegerOrdOpe(Marker): + """The EQL type stored in public.eql_v4_integer_ord_ope: the integer family with the ope index. + + The engine produces it from an int32. + """ + + name = "IntegerOrdOpe" + family = "integer" + suffix = "OrdOpe" + plaintext = "int32" + sql_domain = "public.eql_v4_integer_ord_ope" + indexes = ("ope",) + query = "IntegerOrdOpeQuery" + query_sql_domain = "eql_v4.query_integer_ord_ope" + producible = True + reason = None + + +class Smallint(Marker): + """The EQL type stored in public.eql_v4_smallint: the smallint family with no index. + + The engine produces it from an int16. + """ + + name = "Smallint" + family = "smallint" + suffix = "" + plaintext = "int16" + sql_domain = "public.eql_v4_smallint" + indexes = () + query = None + query_sql_domain = None + producible = True + reason = None + + +class SmallintEq(Marker): + """The EQL type stored in public.eql_v4_smallint_eq: the smallint family with the eq index. + + The engine produces it from an int16. + """ + + name = "SmallintEq" + family = "smallint" + suffix = "Eq" + plaintext = "int16" + sql_domain = "public.eql_v4_smallint_eq" + indexes = ("eq",) + query = "SmallintEqQuery" + query_sql_domain = "eql_v4.query_smallint_eq" + producible = True + reason = None + + +class SmallintOrdOre(Marker): + """The EQL type stored in public.eql_v4_smallint_ord_ore: the smallint family with the ore_block index. + + The engine produces it from an int16. + """ + + name = "SmallintOrdOre" + family = "smallint" + suffix = "OrdOre" + plaintext = "int16" + sql_domain = "public.eql_v4_smallint_ord_ore" + indexes = ("ore_block",) + query = "SmallintOrdOreQuery" + query_sql_domain = "eql_v4.query_smallint_ord_ore" + producible = True + reason = None + + +class SmallintOrd(Marker): + """The EQL type stored in public.eql_v4_smallint_ord: the smallint family with the ope index. + + The engine produces it from an int16. + """ + + name = "SmallintOrd" + family = "smallint" + suffix = "Ord" + plaintext = "int16" + sql_domain = "public.eql_v4_smallint_ord" + indexes = ("ope",) + query = "SmallintOrdQuery" + query_sql_domain = "eql_v4.query_smallint_ord" + producible = True + reason = None + + +class SmallintOrdOpe(Marker): + """The EQL type stored in public.eql_v4_smallint_ord_ope: the smallint family with the ope index. + + The engine produces it from an int16. + """ + + name = "SmallintOrdOpe" + family = "smallint" + suffix = "OrdOpe" + plaintext = "int16" + sql_domain = "public.eql_v4_smallint_ord_ope" + indexes = ("ope",) + query = "SmallintOrdOpeQuery" + query_sql_domain = "eql_v4.query_smallint_ord_ope" + producible = True + reason = None + + +class Bigint(Marker): + """The EQL type stored in public.eql_v4_bigint: the bigint family with no index. + + The engine produces it from an int64. + """ + + name = "Bigint" + family = "bigint" + suffix = "" + plaintext = "int64" + sql_domain = "public.eql_v4_bigint" + indexes = () + query = None + query_sql_domain = None + producible = True + reason = None + + +class BigintEq(Marker): + """The EQL type stored in public.eql_v4_bigint_eq: the bigint family with the eq index. + + The engine produces it from an int64. + """ + + name = "BigintEq" + family = "bigint" + suffix = "Eq" + plaintext = "int64" + sql_domain = "public.eql_v4_bigint_eq" + indexes = ("eq",) + query = "BigintEqQuery" + query_sql_domain = "eql_v4.query_bigint_eq" + producible = True + reason = None + + +class BigintOrdOre(Marker): + """The EQL type stored in public.eql_v4_bigint_ord_ore: the bigint family with the ore_block index. + + The engine produces it from an int64. + """ + + name = "BigintOrdOre" + family = "bigint" + suffix = "OrdOre" + plaintext = "int64" + sql_domain = "public.eql_v4_bigint_ord_ore" + indexes = ("ore_block",) + query = "BigintOrdOreQuery" + query_sql_domain = "eql_v4.query_bigint_ord_ore" + producible = True + reason = None + + +class BigintOrd(Marker): + """The EQL type stored in public.eql_v4_bigint_ord: the bigint family with the ope index. + + The engine produces it from an int64. + """ + + name = "BigintOrd" + family = "bigint" + suffix = "Ord" + plaintext = "int64" + sql_domain = "public.eql_v4_bigint_ord" + indexes = ("ope",) + query = "BigintOrdQuery" + query_sql_domain = "eql_v4.query_bigint_ord" + producible = True + reason = None + + +class BigintOrdOpe(Marker): + """The EQL type stored in public.eql_v4_bigint_ord_ope: the bigint family with the ope index. + + The engine produces it from an int64. + """ + + name = "BigintOrdOpe" + family = "bigint" + suffix = "OrdOpe" + plaintext = "int64" + sql_domain = "public.eql_v4_bigint_ord_ope" + indexes = ("ope",) + query = "BigintOrdOpeQuery" + query_sql_domain = "eql_v4.query_bigint_ord_ope" + producible = True + reason = None + + +class Date(Marker): + """The EQL type stored in public.eql_v4_date: the date family with no index. + + The engine produces it from a date. + """ + + name = "Date" + family = "date" + suffix = "" + plaintext = "date" + sql_domain = "public.eql_v4_date" + indexes = () + query = None + query_sql_domain = None + producible = True + reason = None + + +class DateEq(Marker): + """The EQL type stored in public.eql_v4_date_eq: the date family with the eq index. + + The engine produces it from a date. + """ + + name = "DateEq" + family = "date" + suffix = "Eq" + plaintext = "date" + sql_domain = "public.eql_v4_date_eq" + indexes = ("eq",) + query = "DateEqQuery" + query_sql_domain = "eql_v4.query_date_eq" + producible = True + reason = None + + +class DateOrdOre(Marker): + """The EQL type stored in public.eql_v4_date_ord_ore: the date family with the ore_block index. + + The engine produces it from a date. + """ + + name = "DateOrdOre" + family = "date" + suffix = "OrdOre" + plaintext = "date" + sql_domain = "public.eql_v4_date_ord_ore" + indexes = ("ore_block",) + query = "DateOrdOreQuery" + query_sql_domain = "eql_v4.query_date_ord_ore" + producible = True + reason = None + + +class DateOrd(Marker): + """The EQL type stored in public.eql_v4_date_ord: the date family with the ope index. + + The engine produces it from a date. + """ + + name = "DateOrd" + family = "date" + suffix = "Ord" + plaintext = "date" + sql_domain = "public.eql_v4_date_ord" + indexes = ("ope",) + query = "DateOrdQuery" + query_sql_domain = "eql_v4.query_date_ord" + producible = True + reason = None + + +class DateOrdOpe(Marker): + """The EQL type stored in public.eql_v4_date_ord_ope: the date family with the ope index. + + The engine produces it from a date. + """ + + name = "DateOrdOpe" + family = "date" + suffix = "OrdOpe" + plaintext = "date" + sql_domain = "public.eql_v4_date_ord_ope" + indexes = ("ope",) + query = "DateOrdOpeQuery" + query_sql_domain = "eql_v4.query_date_ord_ope" + producible = True + reason = None + + +class Timestamp(Marker): + """The EQL type stored in public.eql_v4_timestamp: the timestamp family with no index. + + The engine produces it from a timestamp. + """ + + name = "Timestamp" + family = "timestamp" + suffix = "" + plaintext = "timestamp" + sql_domain = "public.eql_v4_timestamp" + indexes = () + query = None + query_sql_domain = None + producible = True + reason = None + + +class TimestampEq(Marker): + """The EQL type stored in public.eql_v4_timestamp_eq: the timestamp family with the eq index. + + The engine produces it from a timestamp. + """ + + name = "TimestampEq" + family = "timestamp" + suffix = "Eq" + plaintext = "timestamp" + sql_domain = "public.eql_v4_timestamp_eq" + indexes = ("eq",) + query = "TimestampEqQuery" + query_sql_domain = "eql_v4.query_timestamp_eq" + producible = True + reason = None + + +class TimestampOrdOre(Marker): + """The EQL type stored in public.eql_v4_timestamp_ord_ore: the timestamp family with the ore_block index. + + The engine produces it from a timestamp. + """ + + name = "TimestampOrdOre" + family = "timestamp" + suffix = "OrdOre" + plaintext = "timestamp" + sql_domain = "public.eql_v4_timestamp_ord_ore" + indexes = ("ore_block",) + query = "TimestampOrdOreQuery" + query_sql_domain = "eql_v4.query_timestamp_ord_ore" + producible = True + reason = None + + +class TimestampOrd(Marker): + """The EQL type stored in public.eql_v4_timestamp_ord: the timestamp family with the ope index. + + The engine produces it from a timestamp. + """ + + name = "TimestampOrd" + family = "timestamp" + suffix = "Ord" + plaintext = "timestamp" + sql_domain = "public.eql_v4_timestamp_ord" + indexes = ("ope",) + query = "TimestampOrdQuery" + query_sql_domain = "eql_v4.query_timestamp_ord" + producible = True + reason = None + + +class TimestampOrdOpe(Marker): + """The EQL type stored in public.eql_v4_timestamp_ord_ope: the timestamp family with the ope index. + + The engine produces it from a timestamp. + """ + + name = "TimestampOrdOpe" + family = "timestamp" + suffix = "OrdOpe" + plaintext = "timestamp" + sql_domain = "public.eql_v4_timestamp_ord_ope" + indexes = ("ope",) + query = "TimestampOrdOpeQuery" + query_sql_domain = "eql_v4.query_timestamp_ord_ope" + producible = True + reason = None + + +class Numeric(Marker): + """The EQL type stored in public.eql_v4_numeric: the numeric family with no index. + + The engine produces it from a decimal. + """ + + name = "Numeric" + family = "numeric" + suffix = "" + plaintext = "decimal" + sql_domain = "public.eql_v4_numeric" + indexes = () + query = None + query_sql_domain = None + producible = True + reason = None + + +class NumericEq(Marker): + """The EQL type stored in public.eql_v4_numeric_eq: the numeric family with the eq index. + + The engine produces it from a decimal. + """ + + name = "NumericEq" + family = "numeric" + suffix = "Eq" + plaintext = "decimal" + sql_domain = "public.eql_v4_numeric_eq" + indexes = ("eq",) + query = "NumericEqQuery" + query_sql_domain = "eql_v4.query_numeric_eq" + producible = True + reason = None + + +class NumericOrdOre(Marker): + """The EQL type stored in public.eql_v4_numeric_ord_ore: the numeric family with the ore_block index. + + The engine produces it from a decimal. + """ + + name = "NumericOrdOre" + family = "numeric" + suffix = "OrdOre" + plaintext = "decimal" + sql_domain = "public.eql_v4_numeric_ord_ore" + indexes = ("ore_block",) + query = "NumericOrdOreQuery" + query_sql_domain = "eql_v4.query_numeric_ord_ore" + producible = True + reason = None + + +class NumericOrd(Marker): + """The EQL type stored in public.eql_v4_numeric_ord: the numeric family with the ope index. + + The engine produces it from a decimal. + """ + + name = "NumericOrd" + family = "numeric" + suffix = "Ord" + plaintext = "decimal" + sql_domain = "public.eql_v4_numeric_ord" + indexes = ("ope",) + query = "NumericOrdQuery" + query_sql_domain = "eql_v4.query_numeric_ord" + producible = True + reason = None + + +class NumericOrdOpe(Marker): + """The EQL type stored in public.eql_v4_numeric_ord_ope: the numeric family with the ope index. + + The engine produces it from a decimal. + """ + + name = "NumericOrdOpe" + family = "numeric" + suffix = "OrdOpe" + plaintext = "decimal" + sql_domain = "public.eql_v4_numeric_ord_ope" + indexes = ("ope",) + query = "NumericOrdOpeQuery" + query_sql_domain = "eql_v4.query_numeric_ord_ope" + producible = True + reason = None + + +class Text(Marker): + """The EQL type stored in public.eql_v4_text: the text family with no index. + + The engine produces it from a string. + """ + + name = "Text" + family = "text" + suffix = "" + plaintext = "string" + sql_domain = "public.eql_v4_text" + indexes = () + query = None + query_sql_domain = None + producible = True + reason = None + + +class TextEq(Marker): + """The EQL type stored in public.eql_v4_text_eq: the text family with the eq index. + + The engine produces it from a string. + """ + + name = "TextEq" + family = "text" + suffix = "Eq" + plaintext = "string" + sql_domain = "public.eql_v4_text_eq" + indexes = ("eq",) + query = "TextEqQuery" + query_sql_domain = "eql_v4.query_text_eq" + producible = True + reason = None + + +class TextMatch(Marker): + """The EQL type stored in public.eql_v4_text_match: the text family with the match index. + + The engine produces it from a string. + """ + + name = "TextMatch" + family = "text" + suffix = "Match" + plaintext = "string" + sql_domain = "public.eql_v4_text_match" + indexes = ("match",) + query = "TextMatchQuery" + query_sql_domain = "eql_v4.query_text_match" + producible = True + reason = None + + +class TextOrdOre(Marker): + """The EQL type stored in public.eql_v4_text_ord_ore: the text family with the eq and ore_block indexes. + + The engine produces it from a string. + """ + + name = "TextOrdOre" + family = "text" + suffix = "OrdOre" + plaintext = "string" + sql_domain = "public.eql_v4_text_ord_ore" + indexes = ("eq", "ore_block") + query = "TextOrdOreQuery" + query_sql_domain = "eql_v4.query_text_ord_ore" + producible = True + reason = None + + +class TextOrd(Marker): + """The EQL type stored in public.eql_v4_text_ord: the text family with the eq and ope indexes. + + The engine produces it from a string. + """ + + name = "TextOrd" + family = "text" + suffix = "Ord" + plaintext = "string" + sql_domain = "public.eql_v4_text_ord" + indexes = ("eq", "ope") + query = "TextOrdQuery" + query_sql_domain = "eql_v4.query_text_ord" + producible = True + reason = None + + +class TextOrdOpe(Marker): + """The EQL type stored in public.eql_v4_text_ord_ope: the text family with the eq and ope indexes. + + The engine produces it from a string. + """ + + name = "TextOrdOpe" + family = "text" + suffix = "OrdOpe" + plaintext = "string" + sql_domain = "public.eql_v4_text_ord_ope" + indexes = ("eq", "ope") + query = "TextOrdOpeQuery" + query_sql_domain = "eql_v4.query_text_ord_ope" + producible = True + reason = None + + +class TextSearchOre(Marker): + """The EQL type stored in public.eql_v4_text_search_ore: the text family with the eq and ore_block and match indexes. + + The engine produces it from a string. + """ + + name = "TextSearchOre" + family = "text" + suffix = "SearchOre" + plaintext = "string" + sql_domain = "public.eql_v4_text_search_ore" + indexes = ("eq", "ore_block", "match") + query = "TextSearchOreQuery" + query_sql_domain = "eql_v4.query_text_search_ore" + producible = True + reason = None + + +class TextSearch(Marker): + """The EQL type stored in public.eql_v4_text_search: the text family with the eq and ope and match indexes. + + The engine produces it from a string. + """ + + name = "TextSearch" + family = "text" + suffix = "Search" + plaintext = "string" + sql_domain = "public.eql_v4_text_search" + indexes = ("eq", "ope", "match") + query = "TextSearchQuery" + query_sql_domain = "eql_v4.query_text_search" + producible = True + reason = None + + +class Boolean(Marker): + """The EQL type stored in public.eql_v4_boolean: the boolean family with no index. + + The engine produces it from a bool. + """ + + name = "Boolean" + family = "boolean" + suffix = "" + plaintext = "bool" + sql_domain = "public.eql_v4_boolean" + indexes = () + query = None + query_sql_domain = None + producible = True + reason = None + + +class Real(Marker): + """The EQL type stored in public.eql_v4_real: the real family with no index. + + The engine produces it from a float32. + """ + + name = "Real" + family = "real" + suffix = "" + plaintext = "float32" + sql_domain = "public.eql_v4_real" + indexes = () + query = None + query_sql_domain = None + producible = True + reason = None + + +class RealEq(Marker): + """The EQL type stored in public.eql_v4_real_eq: the real family with the eq index. + + The engine produces it from a float32. + """ + + name = "RealEq" + family = "real" + suffix = "Eq" + plaintext = "float32" + sql_domain = "public.eql_v4_real_eq" + indexes = ("eq",) + query = "RealEqQuery" + query_sql_domain = "eql_v4.query_real_eq" + producible = True + reason = None + + +class RealOrdOre(Marker): + """The EQL type stored in public.eql_v4_real_ord_ore: the real family with the ore_block index. + + The engine produces it from a float32. + """ + + name = "RealOrdOre" + family = "real" + suffix = "OrdOre" + plaintext = "float32" + sql_domain = "public.eql_v4_real_ord_ore" + indexes = ("ore_block",) + query = "RealOrdOreQuery" + query_sql_domain = "eql_v4.query_real_ord_ore" + producible = True + reason = None + + +class RealOrd(Marker): + """The EQL type stored in public.eql_v4_real_ord: the real family with the ope index. + + The engine produces it from a float32. + """ + + name = "RealOrd" + family = "real" + suffix = "Ord" + plaintext = "float32" + sql_domain = "public.eql_v4_real_ord" + indexes = ("ope",) + query = "RealOrdQuery" + query_sql_domain = "eql_v4.query_real_ord" + producible = True + reason = None + + +class RealOrdOpe(Marker): + """The EQL type stored in public.eql_v4_real_ord_ope: the real family with the ope index. + + The engine produces it from a float32. + """ + + name = "RealOrdOpe" + family = "real" + suffix = "OrdOpe" + plaintext = "float32" + sql_domain = "public.eql_v4_real_ord_ope" + indexes = ("ope",) + query = "RealOrdOpeQuery" + query_sql_domain = "eql_v4.query_real_ord_ope" + producible = True + reason = None + + +class Double(Marker): + """The EQL type stored in public.eql_v4_double: the double family with no index. + + The engine produces it from a float64. + """ + + name = "Double" + family = "double" + suffix = "" + plaintext = "float64" + sql_domain = "public.eql_v4_double" + indexes = () + query = None + query_sql_domain = None + producible = True + reason = None + + +class DoubleEq(Marker): + """The EQL type stored in public.eql_v4_double_eq: the double family with the eq index. + + The engine produces it from a float64. + """ + + name = "DoubleEq" + family = "double" + suffix = "Eq" + plaintext = "float64" + sql_domain = "public.eql_v4_double_eq" + indexes = ("eq",) + query = "DoubleEqQuery" + query_sql_domain = "eql_v4.query_double_eq" + producible = True + reason = None + + +class DoubleOrdOre(Marker): + """The EQL type stored in public.eql_v4_double_ord_ore: the double family with the ore_block index. + + The engine produces it from a float64. + """ + + name = "DoubleOrdOre" + family = "double" + suffix = "OrdOre" + plaintext = "float64" + sql_domain = "public.eql_v4_double_ord_ore" + indexes = ("ore_block",) + query = "DoubleOrdOreQuery" + query_sql_domain = "eql_v4.query_double_ord_ore" + producible = True + reason = None + + +class DoubleOrd(Marker): + """The EQL type stored in public.eql_v4_double_ord: the double family with the ope index. + + The engine produces it from a float64. + """ + + name = "DoubleOrd" + family = "double" + suffix = "Ord" + plaintext = "float64" + sql_domain = "public.eql_v4_double_ord" + indexes = ("ope",) + query = "DoubleOrdQuery" + query_sql_domain = "eql_v4.query_double_ord" + producible = True + reason = None + + +class DoubleOrdOpe(Marker): + """The EQL type stored in public.eql_v4_double_ord_ope: the double family with the ope index. + + The engine produces it from a float64. + """ + + name = "DoubleOrdOpe" + family = "double" + suffix = "OrdOpe" + plaintext = "float64" + sql_domain = "public.eql_v4_double_ord_ope" + indexes = ("ope",) + query = "DoubleOrdOpeQuery" + query_sql_domain = "eql_v4.query_double_ord_ope" + producible = True + reason = None + + +class Json(Marker): + """The EQL type stored in public.eql_v4_json: the json family with no index. + + The engine cannot produce it yet: a JSON value seals as a tree of leaves and EQL's `c` holds one; how a document becomes one sealed leaf is not decided. + """ + + name = "Json" + family = "json" + suffix = "" + plaintext = None + sql_domain = "public.eql_v4_json" + indexes = () + query = None + query_sql_domain = None + producible = False + reason = "a JSON value seals as a tree of leaves and EQL's `c` holds one; how a document becomes one sealed leaf is not decided" + + +class JsonSearch(Marker): + """The EQL type stored in public.eql_v4_json_search: the json_tree family with the json index. + + The engine produces it from a json. + """ + + name = "JsonSearch" + family = "json_tree" + suffix = "" + plaintext = "json" + sql_domain = "public.eql_v4_json_search" + indexes = ("json",) + query = "JsonSearchQuery" + query_sql_domain = "eql_v4.query_json" + producible = True + reason = None + + +TYPES = ( + Integer, + IntegerEq, + IntegerOrdOre, + IntegerOrd, + IntegerOrdOpe, + Smallint, + SmallintEq, + SmallintOrdOre, + SmallintOrd, + SmallintOrdOpe, + Bigint, + BigintEq, + BigintOrdOre, + BigintOrd, + BigintOrdOpe, + Date, + DateEq, + DateOrdOre, + DateOrd, + DateOrdOpe, + Timestamp, + TimestampEq, + TimestampOrdOre, + TimestampOrd, + TimestampOrdOpe, + Numeric, + NumericEq, + NumericOrdOre, + NumericOrd, + NumericOrdOpe, + Text, + TextEq, + TextMatch, + TextOrdOre, + TextOrd, + TextOrdOpe, + TextSearchOre, + TextSearch, + Boolean, + Real, + RealEq, + RealOrdOre, + RealOrd, + RealOrdOpe, + Double, + DoubleEq, + DoubleOrdOre, + DoubleOrd, + DoubleOrdOpe, + Json, + JsonSearch, +) diff --git a/languages/python/python/stackencrypt/_markers.py b/languages/python/python/stackencrypt/_markers.py new file mode 100644 index 000000000..5ae1d1855 --- /dev/null +++ b/languages/python/python/stackencrypt/_markers.py @@ -0,0 +1,29 @@ +"""The hand-written markers. + +``Marker`` is the base of every stash marker. ``Passthrough`` and ``Ignore`` +name no EQL type, so they are written here. The markers that name an EQL type +are generated from the EQL catalog into ``_eql.py``. + +A marker is used as a class, in ``Annotated`` metadata, and is found there by +identity: ``Annotated[str, TextEq]``. +""" + + +class Marker: + """The base of every stash marker.""" + + +class Passthrough(Marker): + """A field stored as it is, without encryption. + + The value goes through the engine with the rest of the record, and comes + back unchanged. + """ + + +class Ignore(Marker): + """An attribute that is not a column, such as a SQLAlchemy relationship. + + The SDK does not read, store, require, or show it. The marker exists so + that every field still carries a marker, and a field with none is an error. + """ diff --git a/languages/python/tests/test_markers.py b/languages/python/tests/test_markers.py new file mode 100644 index 000000000..76db131dd --- /dev/null +++ b/languages/python/tests/test_markers.py @@ -0,0 +1,93 @@ +"""The markers: the generated EQL-type markers and the two hand-written ones. + +The catalog itself is checked in Rust, where it lives: eql-codegen's +`py_eql_parity.rs` compares every marker with the Go `Types` table. These +tests check what a Python developer sees. +""" + +import pytest + +import stackencrypt +from stackencrypt import _eql +from stackencrypt._markers import Marker + +ATTRIBUTES = ( + "name", + "family", + "suffix", + "plaintext", + "sql_domain", + "indexes", + "query", + "query_sql_domain", + "producible", + "reason", +) + +UNPRODUCIBLE = [marker.name for marker in _eql.TYPES if not marker.producible] + + +def test_the_common_markers_import_from_the_package(): + from stackencrypt import Ignore, Passthrough, Text, TextEq + + assert issubclass(TextEq, Marker) + assert issubclass(Text, Marker) + assert issubclass(Passthrough, Marker) + assert issubclass(Ignore, Marker) + + +def test_text_eq_carries_the_catalog_row(): + from stackencrypt import TextEq + + assert {name: getattr(TextEq, name) for name in ATTRIBUTES} == { + "name": "TextEq", + "family": "text", + "suffix": "Eq", + "plaintext": "string", + "sql_domain": "public.eql_v4_text_eq", + "indexes": ("eq",), + "query": "TextEqQuery", + "query_sql_domain": "eql_v4.query_text_eq", + "producible": True, + "reason": None, + } + + +def test_a_marker_without_a_query_has_no_query_domain(): + from stackencrypt import Text + + assert Text.query is None + assert Text.query_sql_domain is None + + +@pytest.mark.parametrize("marker", _eql.TYPES, ids=lambda m: m.name) +def test_every_marker_carries_exactly_the_ten_attributes(marker): + generated = [name for name in vars(marker) if not name.startswith("__")] + assert tuple(generated) == ATTRIBUTES + assert marker.__name__ == marker.name + assert marker.producible is (marker.reason is None) + + +def test_types_lists_every_marker_class_in_the_module_once(): + classes = [ + value + for value in vars(_eql).values() + if isinstance(value, type) and issubclass(value, Marker) and value is not Marker + ] + assert len(_eql.TYPES) == len(set(_eql.TYPES)) + assert set(_eql.TYPES) == set(classes) + + +def test_the_package_exports_exactly_the_producible_markers(): + exported = set(_eql.__all__) + assert exported == {marker.name for marker in _eql.TYPES if marker.producible} + assert exported <= set(stackencrypt.__all__) + for name in exported: + assert getattr(stackencrypt, name) is getattr(_eql, name) + + +@pytest.mark.skipif(not UNPRODUCIBLE, reason="every catalog type is producible") +@pytest.mark.parametrize("name", UNPRODUCIBLE) +def test_an_unproducible_marker_does_not_import_from_the_package(name): + with pytest.raises(ImportError): + exec(f"from stackencrypt import {name}", {}) diff --git a/packages/eql/crates/eql-codegen/src/lib.rs b/packages/eql/crates/eql-codegen/src/lib.rs index 02137825b..083961ec4 100644 --- a/packages/eql/crates/eql-codegen/src/lib.rs +++ b/packages/eql/crates/eql-codegen/src/lib.rs @@ -15,6 +15,7 @@ pub mod generate; pub mod go_eql; pub mod operator_surface; pub mod ordering; +pub mod py_eql; pub mod range; pub mod render; pub mod targets; diff --git a/packages/eql/crates/eql-codegen/src/main.rs b/packages/eql/crates/eql-codegen/src/main.rs index 5c473d439..41ffe217a 100644 --- a/packages/eql/crates/eql-codegen/src/main.rs +++ b/packages/eql/crates/eql-codegen/src/main.rs @@ -104,6 +104,23 @@ fn main() -> ExitCode { } } + // `py-eql`: regenerate the committed Python module stackencrypt._eql in + // the Python workspace (languages/python) from the same catalog rows as + // `go-eql`: one marker class per EQL type, and the TYPES tuple. Wired + // into `mise run types:generate`; `types:check` diffs the file. + if args.len() == 2 && args[1] == "py-eql" { + match eql_codegen::py_eql::generate_py_eql(&out_root()) { + Ok(path) => { + println!("generated {}", path.display()); + return ExitCode::SUCCESS; + } + Err(e) => { + eprintln!("error: {e}"); + return ExitCode::FAILURE; + } + } + } + // `order`: print the install order of the whole src/v3 SQL surface, one // repo-relative path per line, dependency before dependent. Consumed by // tasks/build.sh (`> src/deps-ordered-v3.txt`), which concatenates the files @@ -259,5 +276,6 @@ fn main() -> ExitCode { eprintln!(" eql-codegen dump-catalog (print catalog surface as JSON)"); eprintln!(" eql-codegen bindings (regenerate eql-bindings Rust payload types)"); eprintln!(" eql-codegen go-eql (regenerate the Go package encrypt/eql)"); + eprintln!(" eql-codegen py-eql (regenerate the Python module stackencrypt._eql)"); ExitCode::from(2) } diff --git a/packages/eql/crates/eql-codegen/src/py_eql.rs b/packages/eql/crates/eql-codegen/src/py_eql.rs new file mode 100644 index 000000000..e05acec2d --- /dev/null +++ b/packages/eql/crates/eql-codegen/src/py_eql.rs @@ -0,0 +1,199 @@ +//! The Python `stackencrypt._eql` module emitter: renders `eql_domains::CATALOG` +//! to the committed `languages/python/python/stackencrypt/_eql.py` — one +//! marker class per EQL type, each carrying the catalog row's columns as +//! class attributes, and a `TYPES` tuple listing them in catalog order. The +//! spec (#1341, "Markers (generated)") fixes the ten attributes and puts the +//! emitter here, beside the Go one. +//! +//! The rows are [`crate::targets`]', the same the Go `Types` table and the +//! Rust `TARGETS` table are rendered from, so Python and Go cannot disagree +//! about a type; `tests/py_eql_parity.rs` compares the two outputs attribute +//! by attribute. A marker is named by the catalog name as it is (`TextEq`, +//! `Json`): Python has no initialism rule to apply. +//! +//! `__all__` lists the producible markers only. The package re-exports the +//! module with `from stackencrypt._eql import *`, so a marker the engine +//! cannot produce yet is in the module but not importable from the package. +//! +//! Generate-to-file like the Go module: `eql-codegen py-eql` writes the file, +//! `mise run types:check` diffs it, and `tests/py_eql_parity.rs` asserts the +//! committed file is what the catalog renders. + +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; + +use crate::targets::{rows, Row}; + +/// The generated file, relative to the EQL subtree root (`repo_root()`): +/// the Python workspace lives beside `packages/` in the monorepo. +pub const PY_EQL_PATH: &str = "../../languages/python/python/stackencrypt/_eql.py"; + +/// The first line of the generated file, in the form Go uses and editors +/// recognise. +pub const PY_GENERATED_MARKER: &str = + "# Code generated by eql-codegen from the eql-domains catalog. DO NOT EDIT."; + +/// A Python string literal. Catalog text is printable, so only the quote +/// and the backslash need escaping; anything else is refused rather than +/// written in an escape Python would read differently. +fn py_str(s: &str) -> String { + assert!( + !s.chars().any(char::is_control), + "catalog text with a control character: {s:?}" + ); + format!("\"{}\"", s.replace('\\', "\\\\").replace('"', "\\\"")) +} + +/// A Python string literal, or `None` for an absent value. +fn py_opt(s: Option<&str>) -> String { + s.map_or_else(|| "None".to_owned(), py_str) +} + +/// A Python tuple of strings: `()`, `("eq",)` or `("eq", "ore")`. +fn py_tuple(items: &[&str]) -> String { + match items { + [] => "()".to_owned(), + [one] => format!("({},)", py_str(one)), + _ => { + let quoted: Vec = items.iter().map(|s| py_str(s)).collect(); + format!("({})", quoted.join(", ")) + } + } +} + +/// The first docstring line and the status line, in the Go emitter's words. +fn docstring(row: &Row) -> String { + let indexes = if row.indexes.is_empty() { + "no index".to_owned() + } else { + format!( + "the {} index{}", + row.indexes.join(" and "), + if row.indexes.len() > 1 { "es" } else { "" } + ) + }; + let status = match (row.reason, row.plaintext) { + (None, Some(kind)) => { + let article = if kind.starts_with(['a', 'e', 'i', 'o', 'u']) { + "an" + } else { + "a" + }; + format!("The engine produces it from {article} {kind}.") + } + (None, None) => "The engine produces it.".to_owned(), + (Some(reason), _) => format!("The engine cannot produce it yet: {reason}."), + }; + let text = format!( + "The EQL type stored in {domain}: the {family} family with {indexes}.\n\n {status}\n ", + domain = row.sql_domain, + family = row.family, + ); + assert!(!text.contains("\"\"\""), "catalog text ends the docstring"); + format!("\"\"\"{}\"\"\"", text.replace('\\', "\\\\")) +} + +fn class(out: &mut String, row: &Row) { + let (query, query_sql_domain) = match &row.query { + Some((query, domain)) => (Some(query.as_str()), Some(domain.as_str())), + None => (None, None), + }; + let _ = write!( + out, + "\n\nclass {name}(Marker):\n {doc}\n\n \ + name = {name_lit}\n \ + family = {family}\n \ + suffix = {suffix}\n \ + plaintext = {plaintext}\n \ + sql_domain = {sql_domain}\n \ + indexes = {indexes}\n \ + query = {query}\n \ + query_sql_domain = {query_sql_domain}\n \ + producible = {producible}\n \ + reason = {reason}\n", + name = row.name, + doc = docstring(row), + name_lit = py_str(&row.name), + family = py_str(row.family), + suffix = py_str(&row.suffix), + plaintext = py_opt(row.plaintext), + sql_domain = py_str(&row.sql_domain), + indexes = py_tuple(&row.indexes), + query = py_opt(query), + query_sql_domain = py_opt(query_sql_domain), + producible = if row.reason.is_none() { + "True" + } else { + "False" + }, + reason = py_opt(row.reason), + ); +} + +/// Render the Python source of `_eql.py`. +pub fn render_py_eql() -> String { + let rows = rows(); + let mut out = String::new(); + let _ = writeln!( + out, + "{PY_GENERATED_MARKER}\n\ + \"\"\"The EQL types as stash markers: one class per catalog row, in catalog order.\n\ + \n\ + Generated from the same catalog rows as the Go package encrypt/eql. ``__all__``\n\ + lists the markers the engine produces today, which the package re-exports.\n\ + \"\"\"\n\ + \n\ + from stackencrypt._markers import Marker\n\ + \n\ + __all__ = [" + ); + for row in rows.iter().filter(|row| row.reason.is_none()) { + let _ = writeln!(out, " {},", py_str(&row.name)); + } + out.push_str("]\n"); + for row in &rows { + class(&mut out, row); + } + out.push_str("\n\nTYPES = (\n"); + for row in &rows { + let _ = writeln!(out, " {},", row.name); + } + out.push_str(")\n"); + out +} + +/// Write `_eql.py` under `out_root` (the EQL subtree root). +pub fn generate_py_eql(out_root: &Path) -> std::io::Result { + let path = out_root.join(PY_EQL_PATH); + if let Some(dir) = path.parent() { + std::fs::create_dir_all(dir)?; + } + std::fs::write(&path, render_py_eql())?; + Ok(path) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn strings_escape_the_quote_and_the_backslash() { + assert_eq!(py_str(r#"a "b" \c"#), r#""a \"b\" \\c""#); + } + + #[test] + fn tuples_have_python_syntax_for_every_length() { + assert_eq!(py_tuple(&[]), "()"); + assert_eq!(py_tuple(&["eq"]), r#"("eq",)"#); + assert_eq!(py_tuple(&["eq", "ore"]), r#"("eq", "ore")"#); + } + + #[test] + fn every_row_is_one_class_and_one_entry_in_types() { + let rendered = render_py_eql(); + let count = rows().len(); + assert_eq!(rendered.matches("(Marker):\n").count(), count); + let types = rendered.split("TYPES = (\n").nth(1).expect("a TYPES tuple"); + assert_eq!(types.lines().filter(|l| l.ends_with(',')).count(), count); + } +} diff --git a/packages/eql/crates/eql-codegen/tests/py_eql_parity.rs b/packages/eql/crates/eql-codegen/tests/py_eql_parity.rs new file mode 100644 index 000000000..3b4585f4f --- /dev/null +++ b/packages/eql/crates/eql-codegen/tests/py_eql_parity.rs @@ -0,0 +1,201 @@ +//! The Python `stackencrypt._eql` parity gates. +//! +//! The first is the twin of `go_eql_parity.rs`: the committed +//! `languages/python/python/stackencrypt/_eql.py` must be byte for byte what +//! the catalog renders. The second reads the two rendered outputs back and +//! compares every marker with the Go `Types` row of the same position, +//! attribute by attribute, so the two languages cannot disagree about a type +//! even if one emitter changes how it reads the catalog. + +use std::fs; + +use eql_codegen::go_eql::render_go_eql; +use eql_codegen::py_eql::{render_py_eql, PY_EQL_PATH}; +use eql_codegen::repo_root; + +#[test] +fn py_eql_matches_the_committed_file() { + let path = repo_root().join(PY_EQL_PATH); + let committed = fs::read_to_string(&path).unwrap_or_else(|e| { + panic!( + "committed Python module {} is missing or unreadable ({e}); run `mise run types:generate` and commit", + path.display() + ) + }); + assert_eq!( + render_py_eql(), + committed, + "{}: the committed Python module is stale or hand-edited — run `mise run types:generate` and commit the result", + path.display() + ); +} + +/// One attribute value, as both languages can express it. +#[derive(Debug, PartialEq)] +enum Value { + Str(Option), + List(Vec), + Bool(bool), +} + +/// The ten attributes, in the order both outputs write them, with the Go +/// field each one is. Go writes an absent value as `""`; Python as `None`. +const ATTRIBUTES: [(&str, &str); 10] = [ + ("name", "Name"), + ("family", "Family"), + ("suffix", "Suffix"), + ("plaintext", "Plaintext"), + ("sql_domain", "SQLDomain"), + ("indexes", "Indexes"), + ("query", "Query"), + ("query_sql_domain", "QuerySQLDomain"), + ("producible", "Producible"), + ("reason", "Reason"), +]; + +/// The attributes that are absent rather than empty when Go writes `""`. +const OPTIONAL: [&str; 4] = ["plaintext", "query", "query_sql_domain", "reason"]; + +/// Read one double-quoted string at the start of `s`, with the two escapes +/// both emitters write (`\"` and `\\`); return it and the rest of `s`. +fn quoted(s: &str) -> (String, &str) { + let mut chars = s + .strip_prefix('"') + .expect("a string literal") + .char_indices(); + let mut out = String::new(); + while let Some((at, c)) = chars.next() { + match c { + '"' => return (out, &s[at + 2..]), + '\\' => match chars.next() { + Some((_, e @ ('"' | '\\'))) => out.push(e), + other => panic!("an escape neither emitter writes: {other:?}"), + }, + c => out.push(c), + } + } + panic!("an unterminated string literal: {s}") +} + +/// The strings of a list literal whose opening bracket has been read, up to +/// `close`; return them and the rest after `close`. +fn strings(mut s: &str, close: char) -> (Vec, &str) { + let mut out = Vec::new(); + loop { + s = s.trim_start_matches([',', ' ']); + if let Some(rest) = s.strip_prefix(close) { + return (out, rest); + } + let (item, rest) = quoted(s); + out.push(item); + s = rest; + } +} + +/// Every row of the Go `Types` table, as (Go field, value) pairs. +fn go_rows() -> Vec> { + let go = render_go_eql(); + let table = go + .split("var Types = []Type{\n") + .nth(1) + .and_then(|rest| rest.split("\n}\n").next()) + .expect("the Go Types table"); + table + .lines() + .map(|line| { + let mut s = line + .trim() + .strip_prefix('{') + .and_then(|l| l.strip_suffix("},")) + .expect("one Go table row per line"); + let mut fields = Vec::new(); + while !s.is_empty() { + let (key, rest) = s.split_once(": ").expect("a Go field"); + let (value, rest) = if rest.starts_with('"') { + let (v, rest) = quoted(rest); + (Value::Str(Some(v)), rest) + } else if let Some(rest) = rest.strip_prefix("[]string{") { + let (v, rest) = strings(rest, '}'); + (Value::List(v), rest) + } else if let Some(rest) = rest.strip_prefix("nil") { + (Value::List(Vec::new()), rest) + } else if let Some(rest) = rest.strip_prefix("true") { + (Value::Bool(true), rest) + } else if let Some(rest) = rest.strip_prefix("false") { + (Value::Bool(false), rest) + } else { + panic!("a Go value neither emitter writes: {rest}") + }; + fields.push((key.to_owned(), value)); + s = rest.trim_start_matches(", "); + } + fields + }) + .collect() +} + +/// Every marker class of the Python module, as (attribute, value) pairs. +fn py_rows() -> Vec> { + let py = render_py_eql(); + py.split("(Marker):\n") + .skip(1) + .map(|class| { + class + .lines() + .filter_map(|line| line.strip_prefix(" ")) + .filter_map(|line| line.split_once(" = ")) + .map(|(key, raw)| { + let value = if raw.starts_with('"') { + Value::Str(Some(quoted(raw).0)) + } else if raw == "None" { + Value::Str(None) + } else if let Some(rest) = raw.strip_prefix('(') { + Value::List(strings(rest, ')').0) + } else if raw == "True" || raw == "False" { + Value::Bool(raw == "True") + } else { + panic!("a Python value the emitter does not write: {raw}") + }; + (key.to_owned(), value) + }) + .collect() + }) + .collect() +} + +#[test] +fn every_python_marker_matches_the_go_row_in_the_same_position() { + let go = go_rows(); + let py = py_rows(); + assert_eq!(py.len(), go.len(), "one marker per Go Types row"); + assert!(!py.is_empty(), "the catalog renders no rows"); + + for (py_row, go_row) in py.iter().zip(&go) { + let keys: Vec<&str> = py_row.iter().map(|(k, _)| k.as_str()).collect(); + let expected: Vec<&str> = ATTRIBUTES.iter().map(|(py, _)| *py).collect(); + assert_eq!( + keys, expected, + "a marker carries exactly the ten attributes" + ); + + for (py_key, go_key) in ATTRIBUTES { + let py_value = &py_row.iter().find(|(k, _)| k == py_key).unwrap().1; + let go_value = &go_row + .iter() + .find(|(k, _)| k == go_key) + .unwrap_or_else(|| panic!("a Go row without {go_key}")) + .1; + let go_value = match go_value { + Value::Str(Some(s)) if s.is_empty() && OPTIONAL.contains(&py_key) => { + &Value::Str(None) + } + other => other, + }; + assert_eq!( + py_value, go_value, + "{py_key} of {:?} differs from the Go row's {go_key}", + py_row[0].1 + ); + } + } +} diff --git a/packages/eql/mise.toml b/packages/eql/mise.toml index 79bea8ab0..d967687d6 100644 --- a/packages/eql/mise.toml +++ b/packages/eql/mise.toml @@ -411,6 +411,9 @@ cargo run -q -p eql-codegen -- bindings # The Go package encrypt/eql in the Go module, from the same catalog rows # as the Rust target table (crates/eql-codegen/src/go_eql.rs). cargo run -q -p eql-codegen -- go-eql +# The Python markers module stackencrypt._eql, from the same rows +# (crates/eql-codegen/src/py_eql.rs). +cargo run -q -p eql-codegen -- py-eql TS_RS_EXPORT_DIR="$tmp/bindings" EQL_TYPES_SCHEMA_DIR="$tmp/schema" cargo test -p eql-bindings rm -rf "$bindings" "$schema" mv "$tmp/bindings" "$bindings" @@ -429,12 +432,12 @@ set -euo pipefail # runs in contexts (test-eql.yml's Rust job, the pre-commit hook) that # provision mise tools but not pnpm/node_modules. node packages/eql/scripts/sync-generated.mjs -git diff --exit-code -- crates/eql-bindings/src/v3 crates/eql-bindings/src/v4 crates/eql-bindings/bindings crates/eql-bindings/schema packages/eql/src/generated ../../languages/golang/encrypt/eql/eql_gen.go || { - echo "eql-bindings generated Rust/TS/JSON, the Go package encrypt/eql or @cipherstash/eql package output is stale — run 'mise run types:generate' and 'mise run typescript:generate' and commit the result" >&2 +git diff --exit-code -- crates/eql-bindings/src/v3 crates/eql-bindings/src/v4 crates/eql-bindings/bindings crates/eql-bindings/schema packages/eql/src/generated ../../languages/golang/encrypt/eql/eql_gen.go ../../languages/python/python/stackencrypt/_eql.py || { + echo "eql-bindings generated Rust/TS/JSON, the Go package encrypt/eql, the Python module stackencrypt._eql or @cipherstash/eql package output is stale — run 'mise run types:generate' and 'mise run typescript:generate' and commit the result" >&2 exit 1 } # git diff is blind to brand-new files; untracked output is stale too. -untracked=$(git ls-files --others --exclude-standard -- crates/eql-bindings/src/v3 crates/eql-bindings/src/v4 crates/eql-bindings/bindings crates/eql-bindings/schema packages/eql/src/generated ../../languages/golang/encrypt/eql/eql_gen.go) +untracked=$(git ls-files --others --exclude-standard -- crates/eql-bindings/src/v3 crates/eql-bindings/src/v4 crates/eql-bindings/bindings crates/eql-bindings/schema packages/eql/src/generated ../../languages/golang/encrypt/eql/eql_gen.go ../../languages/python/python/stackencrypt/_eql.py) if [ -n "$untracked" ]; then echo "eql-bindings has uncommitted generated files:" >&2 echo "$untracked" >&2 From d777757243ba03afed7e6797e60dfea96421dbe9 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Wed, 28 Oct 2026 19:32:37 +0900 Subject: [PATCH 2/2] fix(python): address the review of the generated EQL markers (#1385) `go-eql` and `py-eql` write outside the EQL subtree, so an `EQL_CODEGEN_OUT_ROOT` override sent them outside the throwaway tree. Both now refuse the override with exit 2, and a CLI test covers it. `types:generate` now also snapshots `eql_gen.go` and `_eql.py`. A failed run restores them with the bindings, so the tree is never left mixed. `Marker` declares the ten attributes with types, so mypy and pyright accept them on a `type[Marker]`. The Go and Python emitters now take their shared wording from two methods on `Row` in `targets.rs`: `summary` and `status`. Both generated files are unchanged. The parity test skips each docstring before it reads attributes, so a catalog reason containing ` = ` cannot be read as one. The eql-bindings README and AGENTS.md now say that `_eql.py` is generated. Refs #1385 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- .../python/python/stackencrypt/_markers.py | 21 ++++++- packages/eql/crates/eql-bindings/README.md | 5 +- packages/eql/crates/eql-codegen/src/go_eql.rs | 34 +++-------- packages/eql/crates/eql-codegen/src/main.rs | 57 ++++++++++++------- packages/eql/crates/eql-codegen/src/py_eql.rs | 30 ++-------- .../eql/crates/eql-codegen/src/targets.rs | 46 ++++++++++++++- packages/eql/crates/eql-codegen/tests/cli.rs | 22 +++++++ .../crates/eql-codegen/tests/py_eql_parity.rs | 25 +++++++- packages/eql/mise.toml | 10 +++- 10 files changed, 171 insertions(+), 81 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 2b732895c..d307ec739 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,7 +93,7 @@ Every npm package except EQL lives under `languages/typescript/`: packages in `l - `packages/stack-auth`, `packages/stack-profile`, `packages/stack-kms`, `packages/stack-encrypt`, `packages/stack-encrypt-derive`, `packages/stack-guest-abi`: The Rust crates imported from `cipherstash/cipherstash-suite` with their history — `stack-auth` and `stack-profile` (published to crates.io, one version group), `stack-kms` (published to crates.io from 0.1.0, its own version group, re-exported by `stack-encrypt` as `stack_encrypt::kms`), `stack-encrypt` and `stack-encrypt-derive` (published to crates.io from 0.1.0, one version group; `eql-bindings`' `stack-encrypt` feature depends on them from the registry), and `stack-guest-abi` (`publish = false`). They are the members of the **root Cargo workspace**, with the three node binding crates below. See "Working on the Rust crates". - `languages/typescript/packages/auth`, `languages/typescript/packages/profile`, `languages/typescript/packages/stack-auth-wasm`: The node bindings of those crates. `@cipherstash/auth` (napi-rs v2) and its six `platforms/*` packages are published to npm from this repository by `release.yml` (`auth-artifacts`, `publish-auth`); a change to what it ships, the `stack-auth` crate included, needs an `@cipherstash/auth` changeset (`require-auth-npm-changeset.yml`). `@cipherstash/profile` and its platforms are private and never published; `@cipherstash/stack-auth-wasm` is private and builds the wasm that `@cipherstash/auth` ships. Their `build` and `test` scripts never invoke cargo; `build:native`, `build:debug` and `test:cargo` do. - `languages/golang`: The Go module — the SDK `encrypt` with its generated-code support `encrypt/gensupport` and the policy packages `encrypt/policy` and `encrypt/policy/protosource`; the credential package `auth`; the generator `stashgen` and its command `cmd/stashgen`; and `internal` (the shared guest plumbing and the `record` wire model). A wazero host with no cgo. Its two WASI guests (`encrypt/guest`, `auth/guest`) are detached Cargo workspaces built by `mise run wasm:guest:build` and `mise run wasm:auth-guest:build`; `mise run wasm:guest:build:deterministic` builds the seeded TEST build the hermetic Go tests use; the `.wasm` files they embed are gitignored. The stack-encrypt guest has a second build with the EQL types (`mise run wasm:guest:build:eql`, cargo feature `eql`, linking `eql-bindings` by path; `wasm:guest:build:eql:deterministic` is its test build), embedded by `encrypt/eql` and registered on import, so a program whose generated code names an EQL type runs it (ADR-0007, amended). `encrypt/eql`'s types and `Types` table are generated from the EQL catalog by `eql-codegen` (`eql_gen.go`, written by `mise run types:generate` in `packages/eql` and drift-gated by `types:check` and `cargo test -p eql-codegen`). Generated `*_stash.go` files are committed and CI fails when `go generate ./...` changes one. Releases are cut by hand with `.github/workflows/release-golang.yml` (`gh workflow run release-golang.yml -f version=X.Y.Z[-pre] --ref `): because the guests are gitignored on main and Go fetches a version as the files committed at its tag, the workflow builds them, commits them on a release-only commit no branch points at, tags it `languages/golang/vX.Y.Z` and pushes the tag alone, then attaches the EQL v4 SQL bundle to the GitHub release. The ref must carry the EQL v4 bundle, or the build stops at "Build the EQL SQL bundles". `scripts/go-release-version.mjs` refuses, before anything is built, every version Go would refuse (non-canonical semver, `+build` metadata, a v2+ major on a module path with no `/vN` suffix, a version shaped like a pseudo-version), a final version dispatched from anywhere but `main`, and a final version while `languages/golang/go.mod` pins a `github.com/cipherstash/vitaminc/…` module at a pseudo-version (vitaminc's Go bindings have no tags, and a TEMPORARY pin on an unreleased commit must be gone before the module is tagged); a pre-release may come from a branch and may carry such a pin. The build job also refuses a version the Go proxy already lists in `@v/list`, which catches a tag deleted after the proxy fetched it; it never asks the proxy for the version itself before the tag exists, because the proxy would cache "not found" for up to 30 minutes and `verify` would fail after the push. The build checks the `Cargo.lock` of the EQL workspace and both guest workspaces with `--locked` first, since none of those builds passes it. It runs in a job with no write token and hands the guests to a `publish` job that holds `contents: write` and runs only git and gh. `publish` fetches the commit with plain unauthenticated git rather than `actions/checkout`, whose `token` input defaults to the job's writable token, and refuses a release commit whose tree is not the one the build tested. `publish` takes the paths it commits from the workflow's fixed `GUESTS` list, never from the artifact, and refuses a `guests.sha256` that is not exactly one SHA-256 line for each of those three guests: the build job runs third-party code after writing that manifest, so a path read from it could put any file on the tag (`scripts/__tests__/go-release-artifact.test.mjs`). A version is immutable once the Go proxy has fetched it, so a tag is never moved or deleted: if `publish` or `verify` fails after the tag is pushed, use "Re-run failed jobs" on the same run, which keeps a tag that run pushed and creates or completes the release; a new dispatch of the same version is refused. Never tag a main commit by hand: `go get` would fetch a module with no guests. -- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. The `extension-module` feature is set only by maturin in `pyproject.toml`, never in a crate manifest, so `cargo test` links libpython. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so every crate it shares with the root workspace resolves to the version the root tests. After a root dependency moves, re-seed it the same way: copy the root `Cargo.lock` over it and run `cargo metadata --offline` there. That command prunes the copy to what the workspace uses. Until then, `tests-python.yml` fails on `--locked`. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. It then builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. There is no Python lockfile yet: the package has no runtime dependency, so only pytest could be locked. Design: Python ADR-0001 under `languages/python/docs/adr/`, the spec in #1341. +- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. The `extension-module` feature is set only by maturin in `pyproject.toml`, never in a crate manifest, so `cargo test` links libpython. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so every crate it shares with the root workspace resolves to the version the root tests. After a root dependency moves, re-seed it the same way: copy the root `Cargo.lock` over it and run `cargo metadata --offline` there. That command prunes the copy to what the workspace uses. Until then, `tests-python.yml` fails on `--locked`. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. It then builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. There is no Python lockfile yet: the package has no runtime dependency, so only pytest could be locked. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: Python ADR-0001 under `languages/python/docs/adr/`, the spec in #1341. - `e2e/*`: Cross-package end-to-end tests (package managers, supply chain, Prisma example README) - `languages/typescript/examples/*`: Working apps (basic, prisma, supabase-worker) - `docs/plans/*`: Internal design plans. User-facing documentation lives at https://cipherstash.com/docs (not in this repo). diff --git a/languages/python/python/stackencrypt/_markers.py b/languages/python/python/stackencrypt/_markers.py index 5ae1d1855..f62d404bc 100644 --- a/languages/python/python/stackencrypt/_markers.py +++ b/languages/python/python/stackencrypt/_markers.py @@ -8,9 +8,28 @@ identity: ``Annotated[str, TextEq]``. """ +from typing import ClassVar, Optional, Tuple + class Marker: - """The base of every stash marker.""" + """The base of every stash marker. + + The ten attributes are declared here, with their types, so that type + checkers accept them on a value typed ``type[Marker]``. The generated + markers set all ten. ``Passthrough`` and ``Ignore`` name no EQL type and + set none of them. + """ + + name: ClassVar[str] + family: ClassVar[str] + suffix: ClassVar[str] + plaintext: ClassVar[Optional[str]] + sql_domain: ClassVar[str] + indexes: ClassVar[Tuple[str, ...]] + query: ClassVar[Optional[str]] + query_sql_domain: ClassVar[Optional[str]] + producible: ClassVar[bool] + reason: ClassVar[Optional[str]] class Passthrough(Marker): diff --git a/packages/eql/crates/eql-bindings/README.md b/packages/eql/crates/eql-bindings/README.md index 8a35f0c1e..f8873cd91 100644 --- a/packages/eql/crates/eql-bindings/README.md +++ b/packages/eql/crates/eql-bindings/README.md @@ -380,7 +380,10 @@ The table and the dispatch are generated into `src/v4/targets.rs` beside the v4 family modules (`mise run types:generate`), gated to the `targets` feature, and drift-gated by the same parity tests. The same generator writes the Go package `languages/golang/encrypt/eql` from the same rows -(`eql-codegen go-eql`, also under `types:generate` / `types:check`). The +(`eql-codegen go-eql`, also under `types:generate` / `types:check`), and the +Python markers module `languages/python/python/stackencrypt/_eql.py` +(`eql-codegen py-eql`, likewise). Both are outside this subtree, so neither +command accepts an `EQL_CODEGEN_OUT_ROOT` override. The engine's `TargetResolver` (stack-encrypt's `dynamic` module) is implemented over this module by the Go guest's `eql` build, not here: a resolver in this crate would need stack-encrypt API newer than the crates.io release the diff --git a/packages/eql/crates/eql-codegen/src/go_eql.rs b/packages/eql/crates/eql-codegen/src/go_eql.rs index 2ecbf63d5..e0dec2c3c 100644 --- a/packages/eql/crates/eql-codegen/src/go_eql.rs +++ b/packages/eql/crates/eql-codegen/src/go_eql.rs @@ -130,33 +130,11 @@ pub fn render_go_eql() -> String { out.push_str("}\n"); for row in &rows { let go = go_name(&row.name); - let indexes = if row.indexes.is_empty() { - "no index".to_owned() - } else { - format!( - "the {} index{}", - row.indexes.join(" and "), - if row.indexes.len() > 1 { "es" } else { "" } - ) - }; - let status = match (row.reason, row.plaintext) { - (None, Some(kind)) => { - // The Go kinds are read as words: "an int32", "a float64". - let article = if kind.starts_with(['a', 'e', 'i', 'o', 'u']) { - "an" - } else { - "a" - }; - format!("The engine produces it from {article} {kind}.") - } - (None, None) => "The engine produces it.".to_owned(), - (Some(reason), _) => format!("The engine cannot produce it yet: {reason}."), - }; let _ = write!( out, - "\n// {go} is the EQL type stored in {domain}: the {family} family with {indexes}.\n// {status}\ntype {go} []byte\n", - domain = row.sql_domain, - family = row.family, + "\n// {go} is the EQL type {summary}.\n// {status}\ntype {go} []byte\n", + summary = row.summary(), + status = row.status(), ); methods(&mut out, &go, &row.sql_domain); if let Some((query, query_domain)) = &row.query { @@ -231,8 +209,10 @@ fn range_types(out: &mut String) { } } -/// Write `eql_gen.go` under `out_root` (the EQL subtree root, or the -/// override `EQL_CODEGEN_OUT_ROOT`). Returns the path written. +/// Write `eql_gen.go` under `out_root`, the EQL subtree root. The CLI +/// refuses an `EQL_CODEGEN_OUT_ROOT` override here: the path leaves the +/// subtree, so under an override it would land outside the throwaway tree. +/// Returns the path written. pub fn generate_go_eql(out_root: &Path) -> std::io::Result { let path = out_root.join(GO_EQL_PATH); if let Some(dir) = path.parent() { diff --git a/packages/eql/crates/eql-codegen/src/main.rs b/packages/eql/crates/eql-codegen/src/main.rs index 41ffe217a..a2065a896 100644 --- a/packages/eql/crates/eql-codegen/src/main.rs +++ b/packages/eql/crates/eql-codegen/src/main.rs @@ -14,6 +14,41 @@ fn out_root() -> PathBuf { } } +/// The root for `go-eql` and `py-eql`, whose output lives outside the EQL +/// subtree (`../../languages/...`). Those paths are relative to the subtree +/// root, so under an `EQL_CODEGEN_OUT_ROOT` override they would resolve +/// outside the throwaway tree, at its grandparent. The override is refused +/// for them rather than followed there. +fn cross_tree_root(command: &str) -> Result { + if std::env::var_os("EQL_CODEGEN_OUT_ROOT").is_some() { + return Err(format!( + "{command} writes outside the EQL subtree and cannot honour EQL_CODEGEN_OUT_ROOT; unset it" + )); + } + Ok(repo_root()) +} + +/// Run one cross-tree emitter at [`cross_tree_root`]. +fn cross_tree(command: &str, emit: fn(&std::path::Path) -> std::io::Result) -> ExitCode { + let root = match cross_tree_root(command) { + Ok(root) => root, + Err(e) => { + eprintln!("error: {e}"); + return ExitCode::from(2); + } + }; + match emit(&root) { + Ok(path) => { + println!("generated {}", path.display()); + ExitCode::SUCCESS + } + Err(e) => { + eprintln!("error: {e}"); + ExitCode::FAILURE + } + } +} + fn main() -> ExitCode { let args: Vec = std::env::args().collect(); @@ -92,16 +127,7 @@ fn main() -> ExitCode { // the query types, and the Types table. Wired into `mise run // types:generate` beside `bindings`; `types:check` diffs the file. if args.len() == 2 && args[1] == "go-eql" { - match eql_codegen::go_eql::generate_go_eql(&out_root()) { - Ok(path) => { - println!("generated {}", path.display()); - return ExitCode::SUCCESS; - } - Err(e) => { - eprintln!("error: {e}"); - return ExitCode::FAILURE; - } - } + return cross_tree("go-eql", eql_codegen::go_eql::generate_go_eql); } // `py-eql`: regenerate the committed Python module stackencrypt._eql in @@ -109,16 +135,7 @@ fn main() -> ExitCode { // `go-eql`: one marker class per EQL type, and the TYPES tuple. Wired // into `mise run types:generate`; `types:check` diffs the file. if args.len() == 2 && args[1] == "py-eql" { - match eql_codegen::py_eql::generate_py_eql(&out_root()) { - Ok(path) => { - println!("generated {}", path.display()); - return ExitCode::SUCCESS; - } - Err(e) => { - eprintln!("error: {e}"); - return ExitCode::FAILURE; - } - } + return cross_tree("py-eql", eql_codegen::py_eql::generate_py_eql); } // `order`: print the install order of the whole src/v3 SQL surface, one diff --git a/packages/eql/crates/eql-codegen/src/py_eql.rs b/packages/eql/crates/eql-codegen/src/py_eql.rs index e05acec2d..4a074273b 100644 --- a/packages/eql/crates/eql-codegen/src/py_eql.rs +++ b/packages/eql/crates/eql-codegen/src/py_eql.rs @@ -61,33 +61,13 @@ fn py_tuple(items: &[&str]) -> String { } } -/// The first docstring line and the status line, in the Go emitter's words. +/// The class docstring: the summary and the status line the Go emitter +/// writes too ([`Row::summary`], [`Row::status`]). fn docstring(row: &Row) -> String { - let indexes = if row.indexes.is_empty() { - "no index".to_owned() - } else { - format!( - "the {} index{}", - row.indexes.join(" and "), - if row.indexes.len() > 1 { "es" } else { "" } - ) - }; - let status = match (row.reason, row.plaintext) { - (None, Some(kind)) => { - let article = if kind.starts_with(['a', 'e', 'i', 'o', 'u']) { - "an" - } else { - "a" - }; - format!("The engine produces it from {article} {kind}.") - } - (None, None) => "The engine produces it.".to_owned(), - (Some(reason), _) => format!("The engine cannot produce it yet: {reason}."), - }; let text = format!( - "The EQL type stored in {domain}: the {family} family with {indexes}.\n\n {status}\n ", - domain = row.sql_domain, - family = row.family, + "The EQL type {summary}.\n\n {status}\n ", + summary = row.summary(), + status = row.status(), ); assert!(!text.contains("\"\"\""), "catalog text ends the docstring"); format!("\"\"\"{}\"\"\"", text.replace('\\', "\\\\")) diff --git a/packages/eql/crates/eql-codegen/src/targets.rs b/packages/eql/crates/eql-codegen/src/targets.rs index c42b532aa..204c74b2d 100644 --- a/packages/eql/crates/eql-codegen/src/targets.rs +++ b/packages/eql/crates/eql-codegen/src/targets.rs @@ -160,9 +160,10 @@ pub fn target_gap(family: &DomainFamily, domain: &Domain) -> Option<&'static str encryption_gap(TARGET_VERSION, family, domain) } -/// One row of the generated table, computed from the catalog: what both -/// the Rust `TARGETS` table and the Go `encrypt/eql` package -/// ([`crate::go_eql`]) are rendered from, so the two cannot disagree. +/// One row of the generated table, computed from the catalog: what the Rust +/// `TARGETS` table, the Go `encrypt/eql` package ([`crate::go_eql`]) and the +/// Python `stackencrypt._eql` module ([`crate::py_eql`]) are rendered from, +/// so the three cannot disagree. pub(crate) struct Row { pub(crate) name: String, pub(crate) family: &'static str, @@ -189,6 +190,45 @@ pub(crate) struct Row { pub(crate) document: bool, } +impl Row { + /// Where the type is stored and what it indexes, as the Go and Python + /// emitters both describe it: "stored in public.eql_v4_text_eq: the text + /// family with the eq index". One copy, so the two cannot word it apart. + pub(crate) fn summary(&self) -> String { + let indexes = if self.indexes.is_empty() { + "no index".to_owned() + } else { + format!( + "the {} index{}", + self.indexes.join(" and "), + if self.indexes.len() > 1 { "es" } else { "" } + ) + }; + format!( + "stored in {}: the {} family with {indexes}", + self.sql_domain, self.family + ) + } + + /// Whether the engine produces the type, as one sentence for the same + /// two emitters. + pub(crate) fn status(&self) -> String { + match (self.reason, self.plaintext) { + (None, Some(kind)) => { + // The kinds are read as words: "an int32", "a float64". + let article = if kind.starts_with(['a', 'e', 'i', 'o', 'u']) { + "an" + } else { + "a" + }; + format!("The engine produces it from {article} {kind}.") + } + (None, None) => "The engine produces it.".to_owned(), + (Some(reason), _) => format!("The engine cannot produce it yet: {reason}."), + } + } +} + /// The catalog domains that are targets, in catalog order: every stored /// scalar domain. The `json` family's SteVec document is not one: its SQL is /// EQL v3 only, and in v4 its domain name, `json_search`, is the JSON tree diff --git a/packages/eql/crates/eql-codegen/tests/cli.rs b/packages/eql/crates/eql-codegen/tests/cli.rs index c8b41ea8e..494cc8672 100644 --- a/packages/eql/crates/eql-codegen/tests/cli.rs +++ b/packages/eql/crates/eql-codegen/tests/cli.rs @@ -221,6 +221,28 @@ fn select_subcommand_drops_files_confined_to_the_other_version() { assert_eq!(out.status.code(), Some(2)); } +/// `go-eql` and `py-eql` write outside the EQL subtree, so an +/// `EQL_CODEGEN_OUT_ROOT` override would send them outside the throwaway tree. +/// Each refuses the override with exit 2, before writing anything. +#[test] +fn cross_tree_emitters_refuse_the_output_root_override() { + let out_root = tempdir(); + for command in ["go-eql", "py-eql"] { + let out = Command::new(bin()) + .arg(command) + .env("EQL_CODEGEN_OUT_ROOT", out_root.0.as_os_str()) + .output() + .unwrap_or_else(|e| panic!("run eql-codegen {command}: {e}")); + let stderr = String::from_utf8_lossy(&out.stderr); + assert_eq!(out.status.code(), Some(2), "{command}: {stderr}"); + assert!(out.stdout.is_empty(), "{command} printed a written path"); + assert!( + stderr.contains("EQL_CODEGEN_OUT_ROOT"), + "{command}: {stderr}" + ); + } +} + /// An unrecognised argument prints usage and exits 2 (the `ExitCode::from(2)` /// fall-through in `main.rs`). #[test] diff --git a/packages/eql/crates/eql-codegen/tests/py_eql_parity.rs b/packages/eql/crates/eql-codegen/tests/py_eql_parity.rs index 3b4585f4f..2f7c71844 100644 --- a/packages/eql/crates/eql-codegen/tests/py_eql_parity.rs +++ b/packages/eql/crates/eql-codegen/tests/py_eql_parity.rs @@ -1,4 +1,4 @@ -//! The Python `stackencrypt._eql` parity gates. +//! The Python `stackencrypt._eql` parity checks. //! //! The first is the twin of `go_eql_parity.rs`: the committed //! `languages/python/python/stackencrypt/_eql.py` must be byte for byte what @@ -134,13 +134,34 @@ fn go_rows() -> Vec> { .collect() } +/// A class body after its docstring. The docstring quotes catalog text, so +/// a reason containing ` = ` would otherwise read as an attribute. +fn after_docstring(class: &str) -> &str { + let body = class + .strip_prefix(" \"\"\"") + .expect("every marker class opens with a docstring"); + let end = body.find("\"\"\"").expect("a closed docstring"); + &body[end + 3..] +} + +#[test] +fn a_docstring_line_with_an_equals_sign_is_not_an_attribute() { + let class = " \"\"\"The EQL type ...\n\n The engine cannot produce it yet: a = b.\n \"\"\"\n\n name = \"X\"\n"; + let attributes: Vec<&str> = after_docstring(class) + .lines() + .filter_map(|line| line.strip_prefix(" ")) + .filter_map(|line| line.split_once(" = ").map(|(key, _)| key)) + .collect(); + assert_eq!(attributes, ["name"]); +} + /// Every marker class of the Python module, as (attribute, value) pairs. fn py_rows() -> Vec> { let py = render_py_eql(); py.split("(Marker):\n") .skip(1) .map(|class| { - class + after_docstring(class) .lines() .filter_map(|line| line.strip_prefix(" ")) .filter_map(|line| line.split_once(" = ")) diff --git a/packages/eql/mise.toml b/packages/eql/mise.toml index d967687d6..029ca33a6 100644 --- a/packages/eql/mise.toml +++ b/packages/eql/mise.toml @@ -388,15 +388,21 @@ tmp="$(mktemp -d)" # restores the snapshot so the working tree is left exactly as it was — never # fresh structs against stale committed TS/JSON that an automated commit could # capture. The snapshot is a filesystem copy (not `git checkout`) so a dev's -# legitimate uncommitted edits survive a failed run. +# legitimate uncommitted edits survive a failed run. The Go and Python files +# the same rows generate outside the subtree are snapshotted too, so a failed +# run cannot leave them regenerated against rolled-back bindings. v3="crates/eql-bindings/src/v3" v4="crates/eql-bindings/src/v4" bindings="crates/eql-bindings/bindings" schema="crates/eql-bindings/schema" +go_gen="../../languages/golang/encrypt/eql/eql_gen.go" +py_gen="../../languages/python/python/stackencrypt/_eql.py" cp -R "$v3" "$tmp/v3-snapshot" cp -R "$v4" "$tmp/v4-snapshot" cp -R "$bindings" "$tmp/bindings-snapshot" cp -R "$schema" "$tmp/schema-snapshot" +cp "$go_gen" "$tmp/go-gen-snapshot" +cp "$py_gen" "$tmp/py-gen-snapshot" ok=0 trap 'if [ "$ok" != 1 ]; then rm -rf "$v3" "$v4" "$bindings" "$schema" @@ -404,6 +410,8 @@ trap 'if [ "$ok" != 1 ]; then mv "$tmp/v4-snapshot" "$v4" mv "$tmp/bindings-snapshot" "$bindings" mv "$tmp/schema-snapshot" "$schema" + mv -f "$tmp/go-gen-snapshot" "$go_gen" + mv -f "$tmp/py-gen-snapshot" "$py_gen" fi rm -rf "$tmp"' EXIT # Default no-arg eql-codegen stays SQL-only; `bindings` is the Rust-only subcommand.