From 59755994d68c17ba28db7d2897ef2cdbc4358ca3 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 30 Oct 2026 00:18:01 +1100 Subject: [PATCH 1/3] feat(eql): test equality and ranges on one array element of a json_tree document An equality and a range at two paths under one array could only be asked as `@>` plus an ordering over `json_tree_path_query`, which two different elements can answer: a superset of the plaintext jsonpath `exists($.a[*] ? (@.x == $v && @.y > $w))`. `json_tree_subtree` already computed `k`, the place in `tokens` of the token that found each row, and dropped it. `json_tree_subtree_indexed` returns it, and `json_tree_subtree` is now its projection, so the two reads have one implementation. A client expands a wildcard in element order to one bound, so reads of two fields of one array agree on `k` for one element. `eql_v4.json_tree_element_match(col, p eql_v4.query_json_elements)` takes one operand: per field, its read tokens (probe included) and its bounds as `query_json_leaf` operands. It opens the document once, finds the node each token names by its self tag (the node the subtree read returns at `d = 0`, with nothing beneath it read), pairs the members' nodes by `k`, and compares with the existing entry/leaf operators. A bound is checked by the same validator as the `query_json_leaf` CHECK. A token that names a node at another selector than its member's bounds is the probe's hit on an array longer than the bound, and raises `invalid_parameter_value` instead of answering over part of the array. The domain joins the catalog as the json_tree family's ElementsQuery role, so the eql-bindings inventory and the Go `eql` package pick it up from the generator (`eql.JSONSearchElementsQuery`). Refs #1440 --- .changeset/eql-json-tree-element-match.md | 9 ++ languages/golang/encrypt/eql/eql_gen.go | 23 +++ packages/eql/AGENTS.md | 2 +- .../eql-bindings/src/encryption/targets.rs | 12 ++ .../crates/eql-bindings/src/v4/inventory.rs | 1 + .../crates/eql-bindings/src/v4/json_tree.rs | 75 ++++++++++ .../eql/crates/eql-codegen/src/bindings.rs | 3 +- packages/eql/crates/eql-codegen/src/go_eql.rs | 4 + packages/eql/crates/eql-domains/src/lib.rs | 16 ++- packages/eql/crates/eql-domains/src/tests.rs | 10 +- packages/eql/docs/reference/eql-functions.md | 47 ++++++ .../eql/docs/reference/json-tree-leakage.md | 1 + packages/eql/src/v3/json_tree/elements.sql | 120 ++++++++++++++++ packages/eql/src/v3/json_tree/functions.sql | 117 ++++++++++----- packages/eql/src/v3/json_tree/types.sql | 134 ++++++++++++++++-- 15 files changed, 521 insertions(+), 53 deletions(-) create mode 100644 .changeset/eql-json-tree-element-match.md create mode 100644 packages/eql/src/v3/json_tree/elements.sql diff --git a/.changeset/eql-json-tree-element-match.md b/.changeset/eql-json-tree-element-match.md new file mode 100644 index 000000000..3d41291e7 --- /dev/null +++ b/.changeset/eql-json-tree-element-match.md @@ -0,0 +1,9 @@ +--- +"@cipherstash/eql": minor +--- + +EQL v4 `json_tree`: a predicate on one array element, `eql_v4.json_tree_element_match(col, $1::eql_v4.query_json_elements)` (#1440). + +`exists($.contacts[*] ? (@.relationship == "Sibling" && @.calls > 5))` asks for one contact that is both. On a `public.eql_v4_json_search` column the nearest query, `@>` for the relationship and an ordering over `eql_v4.json_tree_path_query` for the calls, can be answered by two different contacts, so it returned more rows than the plaintext query. The new function answers the element-scoped question with one operand, the new domain `eql_v4.query_json_elements`, which Stack Encrypt's `element_match` query operator derives: for each field of the element, the tokens of a subtree read of its path and its bounds. Every field must be at a path the column orders, and equality is the ordering term's, so text compares case- and accent-folded and integers past 2^53 can tie. A row whose array is longer than the bound the operand was derived to raises `invalid_parameter_value` rather than answer over part of the array. + +`eql_v4.json_tree_subtree_indexed(col, tokens, budget)` is the subtree read with `k`, the place in `tokens` of the token whose node each row sits in, which pairs the fields of one element. `eql_v4.json_tree_subtree` now returns its rows without `k`, and is otherwise unchanged. diff --git a/languages/golang/encrypt/eql/eql_gen.go b/languages/golang/encrypt/eql/eql_gen.go index 0a1111301..5094e8087 100644 --- a/languages/golang/encrypt/eql/eql_gen.go +++ b/languages/golang/encrypt/eql/eql_gen.go @@ -2229,6 +2229,29 @@ func (v *JSONSearchLeafQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } +// JSONSearchElementsQuery is the json_tree family's element-scoped predicate, which eql_v4.json_tree_element_match tests: the value eql_v4.query_json_elements holds. +type JSONSearchElementsQuery []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v JSONSearchElementsQuery) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a eql_v4.query_json_elements column. +func (v *JSONSearchElementsQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "JSONSearchElementsQuery") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v JSONSearchElementsQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "JSONSearchElementsQuery") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *JSONSearchElementsQuery) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} + // JSONEntry is the json_tree family's entry: one node of a document, as `->` and the walks return it: the value public.eql_v4_json_entry holds. type JSONEntry []byte diff --git a/packages/eql/AGENTS.md b/packages/eql/AGENTS.md index 05e5bc03d..b742b9d9c 100644 --- a/packages/eql/AGENTS.md +++ b/packages/eql/AGENTS.md @@ -60,7 +60,7 @@ Where the two versions' SQL must differ, the source holds both forms in **versio #### Files confined to one EQL version -A source file belongs to every version unless it carries the line `-- ONLY EQL VERSION: N` (the payload version, not the schema name, which the source may not write). Three surfaces do today. The generated range surface: `src/v3/scalars//query__range_types.sql` and `query__range_functions.sql` hold the range query domains (`eql_v4.query__range_ore` / `_range_ope`), their `lo` / `hi` accessors and `eql_v4.between(column, range)` (ADR-0009 decision 7). And the hand-written JSON tree family under `src/v3/json_tree/` (`types.sql`, `functions.sql`, `operators.sql`, `blockers.sql`): `public.eql_v4_json_search`, `public.eql_v4_json_entry`, `eql_v4.query_json` and `eql_v4.query_json_leaf`, the document Stack Encrypt's JSON index writes, under the names v3 gives its searchable JSON document but not a port of the SteVec surface (design: `docs/plans/2026-10-17-json-tree-index.md` at the repository root; bindings: `crates/eql-bindings/src/v4/json_tree.rs`, hand-written like the SteVec structs; catalog: `eql_domains::DOCUMENT_FAMILIES`, whose stored type is the target `JsonSearch`). Both are `-- ONLY EQL VERSION: 4`. The third surface goes the other way: the SteVec SQL under `src/v3/json/` and the generated `json_entry_` cross files under `src/v3/scalars//` are `-- ONLY EQL VERSION: 3`, because in v4 their domain names belong to the JSON tree family. EQL v3 is what cipherstash-client writes, nothing there produces a range or a JSON tree document, and its SQL must not change: the v3 bundle differs from an unconfined build only by those directive comment lines. `tasks/build.sh` runs the install order through `eql-codegen select <3|4>` before rendering each version, `eql-codegen render` refuses to render a confined file as the other version, `verify_installer_complete.sh` filters its disk side the same way with its own `grep`, the symbol-order checker runs on the v4 order too (on a copy mapped back to `eql_v3` names), the Doxygen filter reads a confined file from its own version's rendered tree, and `docs:validate:documented-sql` runs the whole v4 install order (`build/render/eql_v4/deps-ordered.txt`) in one rolled-back transaction, so every statement of the confined files runs after what it requires, besides running each file of the v3 render on its own. `eql-codegen order` still orders every file, so `src/deps-ordered-v3.txt` lists the files confined to either version; the per-version lists are `build/render/eql_v/deps-ordered.txt`. +A source file belongs to every version unless it carries the line `-- ONLY EQL VERSION: N` (the payload version, not the schema name, which the source may not write). Three surfaces do today. The generated range surface: `src/v3/scalars//query__range_types.sql` and `query__range_functions.sql` hold the range query domains (`eql_v4.query__range_ore` / `_range_ope`), their `lo` / `hi` accessors and `eql_v4.between(column, range)` (ADR-0009 decision 7). And the hand-written JSON tree family under `src/v3/json_tree/` (`types.sql`, `functions.sql`, `operators.sql`, `elements.sql`, `blockers.sql`): `public.eql_v4_json_search`, `public.eql_v4_json_entry`, `eql_v4.query_json`, `eql_v4.query_json_leaf` and `eql_v4.query_json_elements`, the document Stack Encrypt's JSON index writes, under the names v3 gives its searchable JSON document but not a port of the SteVec surface (design: `docs/plans/2026-10-17-json-tree-index.md` at the repository root; bindings: `crates/eql-bindings/src/v4/json_tree.rs`, hand-written like the SteVec structs; catalog: `eql_domains::DOCUMENT_FAMILIES`, whose stored type is the target `JsonSearch`). Both are `-- ONLY EQL VERSION: 4`. The third surface goes the other way: the SteVec SQL under `src/v3/json/` and the generated `json_entry_` cross files under `src/v3/scalars//` are `-- ONLY EQL VERSION: 3`, because in v4 their domain names belong to the JSON tree family. EQL v3 is what cipherstash-client writes, nothing there produces a range or a JSON tree document, and its SQL must not change: the v3 bundle differs from an unconfined build only by those directive comment lines. `tasks/build.sh` runs the install order through `eql-codegen select <3|4>` before rendering each version, `eql-codegen render` refuses to render a confined file as the other version, `verify_installer_complete.sh` filters its disk side the same way with its own `grep`, the symbol-order checker runs on the v4 order too (on a copy mapped back to `eql_v3` names), the Doxygen filter reads a confined file from its own version's rendered tree, and `docs:validate:documented-sql` runs the whole v4 install order (`build/render/eql_v4/deps-ordered.txt`) in one rolled-back transaction, so every statement of the confined files runs after what it requires, besides running each file of the v3 render on its own. `eql-codegen order` still orders every file, so `src/deps-ordered-v3.txt` lists the files confined to either version; the per-version lists are `build/render/eql_v/deps-ordered.txt`. Anything that needs the source as real SQL reads the rendered tree, not `src/`: `docs:validate:documented-sql` runs psql over `build/render/eql_v3/src`, and the Doxygen input filter (`tasks/docs/doxygen-filter.sh`) reads each file's rendered copy from `build/render/eql_v3/`. A new tool that parses `src/v3` must do the same, and depend on `build`; do not substitute the placeholders a second way. diff --git a/packages/eql/crates/eql-bindings/src/encryption/targets.rs b/packages/eql/crates/eql-bindings/src/encryption/targets.rs index dc3edf2e1..aed81c9d9 100644 --- a/packages/eql/crates/eql-bindings/src/encryption/targets.rs +++ b/packages/eql/crates/eql-bindings/src/encryption/targets.rs @@ -152,6 +152,18 @@ impl Target { document(self.name).map(|_| crate::v4::json_tree::JsonTreeLeafQuery::sql_domain_static()) } + /// The PostgreSQL domain of the type's element-scoped predicate, for a + /// type that stores a JSON document (`JsonSearch`: + /// `eql_v4.query_json_elements`); `None` for every other type. Stack + /// Encrypt binds `element_match` as it + /// (`TargetDescriptor::with_elements_query_sql_domain`), and answers + /// `element_match` only on a type that names it. + pub fn elements_query_sql_domain(&self) -> Option<&'static str> { + use crate::v4::DomainType; + document(self.name) + .map(|_| crate::v4::json_tree::JsonTreeElementsQuery::sql_domain_static()) + } + /// The options this type's match index is built under, for a type that /// carries one (`TextMatch`): [`EqlMatch`](super::EqlMatch)'s, the one /// match configuration every EQL type's filter is built under. `None` diff --git a/packages/eql/crates/eql-bindings/src/v4/inventory.rs b/packages/eql/crates/eql-bindings/src/v4/inventory.rs index 0793fa22f..613a60377 100644 --- a/packages/eql/crates/eql-bindings/src/v4/inventory.rs +++ b/packages/eql/crates/eql-bindings/src/v4/inventory.rs @@ -106,6 +106,7 @@ pub fn all_query() -> Vec> { Box::new(PhantomData::), Box::new(PhantomData::), Box::new(PhantomData::), + Box::new(PhantomData::), ] } /// Every v4 RANGE query operand (`eql_v4.query__range_`), diff --git a/packages/eql/crates/eql-bindings/src/v4/json_tree.rs b/packages/eql/crates/eql-bindings/src/v4/json_tree.rs index ef827967f..1c2d56c2a 100644 --- a/packages/eql/crates/eql-bindings/src/v4/json_tree.rs +++ b/packages/eql/crates/eql-bindings/src/v4/json_tree.rs @@ -177,6 +177,48 @@ pub struct JsonTreeLeafQuery { pub op: JsonTreeOrderingTerm, } +/// A read token, in [`Base64Text`]: sixteen bytes a client derives from a +/// concrete path alone, which `eql_v4.json_tree_subtree` and its indexed +/// form take to find the node at that path in every row. +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, TS, JsonSchema)] +pub struct JsonTreeToken(pub Base64Text); + +/// One member of an element-scoped predicate: the tokens of a read of one +/// field's path through the array's wildcard, one per element below the +/// bound in element order and then the probe, and the field's bounds, +/// each an ordering operand. `gt` or `gte` is the lower bound, `lt` or +/// `lte` the upper; equality is `gte` and `lte` at one value. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct JsonTreeElementsMember { + pub t: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub gt: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub gte: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub lt: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub lte: Option, +} + +/// `eql_v4.query_json_elements`: an element-scoped predicate, `{"m": +/// [member, …]}`, true of a row where some one element of the array +/// satisfies every member (`eql_v4.json_tree_element_match`). What Stack +/// Encrypt's `element_match` operator derives. The domain's CHECK also +/// holds what this type does not: at least one member, each with at least +/// one bound and at most one of each side, all its bounds at one selector, +/// and every member's tokens as many as every other's. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct JsonTreeElementsQuery { + pub m: Vec, +} + /// The entry `col -> selector` returned, with the selector that asked for /// it: opens to the scalar the entry holds, or to the whole document for a /// sealed root asked for at `$`. The only constructor, @@ -287,6 +329,7 @@ json_tree_domain_type!(JsonTreeDocument, "public.eql_v4_json_search"); json_tree_domain_type!(JsonTreeEntry, "public.eql_v4_json_entry"); json_tree_domain_type!(JsonTreeQuery, "eql_v4.query_json"); json_tree_domain_type!(JsonTreeLeafQuery, "eql_v4.query_json_leaf"); +json_tree_domain_type!(JsonTreeElementsQuery, "eql_v4.query_json_elements"); // ============================================================================= // Conversions to and from the engine's types, through their one wire form @@ -547,6 +590,38 @@ mod tests { JsonTreeLeafQuery::sql_domain_static(), "eql_v4.query_json_leaf" ); + assert_eq!( + JsonTreeElementsQuery::sql_domain_static(), + "eql_v4.query_json_elements" + ); + } + + #[test] + fn an_elements_query_is_members_of_tokens_and_bounds() { + let leaf = serde_json::json!({"s": b64(0xaa), "op": "AQI="}); + let query = serde_json::json!({"m": [ + {"t": [b64(1), b64(2)], "gte": leaf, "lte": leaf}, + {"t": [b64(3), b64(4)], "gt": leaf}, + ]}); + let parsed: JsonTreeElementsQuery = serde_json::from_value(query.clone()).unwrap(); + assert_eq!(parsed.m.len(), 2, "one member per field"); + assert!(parsed.m[1].lt.is_none(), "an absent bound is none"); + assert_eq!( + serde_json::to_value(&parsed).unwrap(), + query, + "absent bounds are left out" + ); + for wrong in [ + serde_json::json!({"m": [{"t": [b64(1)], "eq": leaf}]}), + serde_json::json!({"m": [{"gt": leaf}]}), + serde_json::json!({"m": [], "x": 1}), + serde_json::json!({"m": [{"t": ["not base64"], "gt": leaf}]}), + ] { + assert!( + serde_json::from_value::(wrong.clone()).is_err(), + "{wrong} should not deserialize" + ); + } } #[test] diff --git a/packages/eql/crates/eql-codegen/src/bindings.rs b/packages/eql/crates/eql-codegen/src/bindings.rs index 32aa51d33..627be79f5 100644 --- a/packages/eql/crates/eql-codegen/src/bindings.rs +++ b/packages/eql/crates/eql-codegen/src/bindings.rs @@ -1061,7 +1061,7 @@ pub fn render_v4_inventory_rs() -> String { }) .collect(); // The document families' hand-written types: the stored document in - // `all()`, its two query operands in `all_query()`. The entry `->` + // `all()`, its query operands in `all_query()`. The entry `->` // returns is neither a column nor an operand, so it is in neither. let document = |roles: &'static [eql_domains::DocumentRole]| -> TokenStream { eql_domains::DOCUMENT_FAMILIES @@ -1083,6 +1083,7 @@ pub fn render_v4_inventory_rs() -> String { let documents = document(&[ eql_domains::DocumentRole::Query, eql_domains::DocumentRole::LeafQuery, + eql_domains::DocumentRole::ElementsQuery, ]); quote! { #query_entries #documents } }; diff --git a/packages/eql/crates/eql-codegen/src/go_eql.rs b/packages/eql/crates/eql-codegen/src/go_eql.rs index 2ecbf63d5..9dbc4636b 100644 --- a/packages/eql/crates/eql-codegen/src/go_eql.rs +++ b/packages/eql/crates/eql-codegen/src/go_eql.rs @@ -186,6 +186,9 @@ fn document_types(out: &mut String) { DocumentRole::LeafQuery => { "ordering operand at a path, compared with an entry `->` extracts" } + DocumentRole::ElementsQuery => { + "element-scoped predicate, which eql_v4.json_tree_element_match tests" + } DocumentRole::Entry => { "entry: one node of a document, as `->` and the walks return it" } @@ -271,6 +274,7 @@ mod tests { "JSONSearch", "JSONSearchQuery", "JSONSearchLeafQuery", + "JSONSearchElementsQuery", "JSONEntry", ] { assert_eq!( diff --git a/packages/eql/crates/eql-domains/src/lib.rs b/packages/eql/crates/eql-domains/src/lib.rs index 0c5b95e10..1e7738eaa 100644 --- a/packages/eql/crates/eql-domains/src/lib.rs +++ b/packages/eql/crates/eql-domains/src/lib.rs @@ -554,7 +554,7 @@ pub const CATALOG: &[DomainFamily] = &[ ]; /// The role a domain of a [`DocumentFamily`] plays: the column type, the -/// two query operands, or the entry the SQL returns from a document. +/// query operands, or the entry the SQL returns from a document. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DocumentRole { /// The stored document: the column type, and the target a data plan @@ -564,6 +564,9 @@ pub enum DocumentRole { Query, /// The ordering operand at a path. LeafQuery, + /// The element-scoped predicate: fields of one array element, each + /// with its read tokens and its bounds. + ElementsQuery, /// One node, as `->` and the walks return it. Entry, } @@ -621,8 +624,9 @@ impl DocumentFamily { /// gives its own (`public.eql_v3_json_search`, the SteVec document): /// `JsonSearch` over `public.eql_v4_json_search`, its needle /// `JsonSearchQuery` (`eql_v4.query_json`), its ordering operand -/// `JsonSearchLeafQuery` (`eql_v4.query_json_leaf`) and its entry -/// `JsonEntry` (`public.eql_v4_json_entry`). The storage-only `Json` +/// `JsonSearchLeafQuery` (`eql_v4.query_json_leaf`), its element-scoped +/// predicate `JsonSearchElementsQuery` (`eql_v4.query_json_elements`) and +/// its entry `JsonEntry` (`public.eql_v4_json_entry`). The storage-only `Json` /// (`public.eql_v4_json`) is the `json` family's. The Rust structs keep the /// implementation's name, as v3's `SteVecDocument` does. pub const JSON_TREE: DocumentFamily = DocumentFamily { @@ -648,6 +652,12 @@ pub const JSON_TREE: DocumentFamily = DocumentFamily { rust: "JsonTreeLeafQuery", sql_domain: "eql_v4.query_json_leaf", }, + DocumentDomain { + role: DocumentRole::ElementsQuery, + name: "JsonSearchElementsQuery", + rust: "JsonTreeElementsQuery", + sql_domain: "eql_v4.query_json_elements", + }, DocumentDomain { role: DocumentRole::Entry, name: "JsonEntry", diff --git a/packages/eql/crates/eql-domains/src/tests.rs b/packages/eql/crates/eql-domains/src/tests.rs index a59b855c7..103414893 100644 --- a/packages/eql/crates/eql-domains/src/tests.rs +++ b/packages/eql/crates/eql-domains/src/tests.rs @@ -1633,6 +1633,7 @@ mod document_families { DocumentRole::Stored, DocumentRole::Query, DocumentRole::LeafQuery, + DocumentRole::ElementsQuery, DocumentRole::Entry, ] { assert_eq!( @@ -1645,7 +1646,9 @@ mod document_families { for domain in family.domains { let schema = match domain.role { DocumentRole::Stored | DocumentRole::Entry => "public.eql_v4_", - DocumentRole::Query | DocumentRole::LeafQuery => "eql_v4.query_", + DocumentRole::Query | DocumentRole::LeafQuery | DocumentRole::ElementsQuery => { + "eql_v4.query_" + } }; assert!( domain.sql_domain.starts_with(schema), @@ -1677,6 +1680,11 @@ mod document_families { "JsonSearchLeafQuery", "eql_v4.query_json_leaf", ), + ( + DocumentRole::ElementsQuery, + "JsonSearchElementsQuery", + "eql_v4.query_json_elements", + ), (DocumentRole::Entry, "JsonEntry", "public.eql_v4_json_entry"), ] { let domain = JSON_TREE.domain(role).unwrap(); diff --git a/packages/eql/docs/reference/eql-functions.md b/packages/eql/docs/reference/eql-functions.md index 27e57d28d..7eaf74266 100644 --- a/packages/eql/docs/reference/eql-functions.md +++ b/packages/eql/docs/reference/eql-functions.md @@ -11,6 +11,7 @@ A reference for the functions and operators EQL exposes for querying encrypted d - [Index Term Extraction](#index-term-extraction) - [Encrypted JSON (`public.eql_v3_json_search`)](#encrypted-json-publiceql_v3_json_search) - [Range queries in EQL v4: `eql_v4.between`](#range-queries-in-eql-v4-eql_v4between) +- [One array element in EQL v4: `eql_v4.json_tree_element_match`](#one-array-element-in-eql-v4-eql_v4json_tree_element_match) - [Aggregate Functions](#aggregate-functions) - [Diagnostics](#diagnostics) @@ -172,6 +173,52 @@ SELECT * FROM users WHERE eql_v4.between(age, $1::eql_v4.query_integer_range_ope --- +## One array element in EQL v4: `eql_v4.json_tree_element_match` + +On an encrypted JSON document (`public.eql_v4_json_search`), equality at a path is a containment needle (`col @> $1::eql_v4.query_json`) and a range at a path is an ordering over every match (`eql_v4.json_tree_path_query(col, $sel)`). Each can be answered by a different element of an array. So the two together ask "some contact is a Sibling, and some contact, maybe another one, has more than five calls", which is more rows than the plaintext query asks for: + +``` +exists($.contacts[*] ? (@.relationship == "Sibling" && @.calls > 5)) +``` + +`eql_v4.json_tree_element_match` asks that question: **some one element** satisfies every part of the predicate. + +```sql +eql_v4.json_tree_element_match(e public.eql_v4_json_search, p eql_v4.query_json_elements) RETURNS boolean + +SELECT id FROM people + WHERE eql_v4.json_tree_element_match(profile, $1::eql_v4.query_json_elements); +``` + +The operand `eql_v4.query_json_elements` is one parameter, `{"m": [member, …]}`, with one member per field of the element the predicate tests. Stack Encrypt's `element_match` query operator derives it. Each member holds: + +- `t`: the tokens of a subtree read of the field's path through the array (`$.contacts[*].relationship`), one per element below a bound, in element order, then the **probe**, the token of the element at the bound; +- one or two bounds, each an ordering operand of the shape `eql_v4.query_json_leaf` holds: `gt` or `gte` for the lower bound, `lt` or `lte` for the upper. Equality is `gte` and `lte` at one value. + +The function opens the row's document once, finds the node each token of every member names (the node `eql_v4.json_tree_subtree_indexed` returns for that token at `d = 0`), and pairs the members' nodes by `k`, the token's place in its list. The members' tokens were expanded to one bound in one order, so one `k` is one element. It is true when some `k` has a node for every member, inside that member's bounds. Nothing beneath a node is read. + +- **Every field must be at an ordered path.** The comparison is on the ordering term, which a leaf carries only at a path the column's index orders (`ordered=`). Stack Encrypt refuses a member whose path is not ordered. +- **Equality is folded.** Equality here is the ordering term's equality: text compares after case and accents are folded (`"Sibling"`, `"sibling"` and `"Síbling"` are equal), and numbers compare as 64-bit floats, so two integers past 2⁵³ can be equal. That suits an enum-like field. Exact equality at a path is containment (`@>`), which this function does not provide per element. +- **An array longer than the bound raises.** The probe finds a node only when the array is longer than the bound. Then the function raises SQLSTATE `22023` (`invalid_parameter_value`) rather than answer over part of the array. Derive the operand with a bound past the longest array (`wildcard_bound`, 64 by default). The server tells the probe apart without any marker: every other token names the field, at the selector the member's bounds carry, and the probe names the element, at the array's selector. +- **No index serves it.** The function is PL/pgSQL, so the planner cannot see into it: it filters the rows it is given. Narrow them first with an indexed predicate on the same values, such as `col @> $1::eql_v4.query_json` for the equality half. +- **What it reveals** is what its parts reveal sent alone: a subtree read's tokens for each field, to the bound, with the probes, and each bound's ordering term. See [What a query shows](./json-tree-leakage.md#what-a-query-shows). + +`eql_v4.json_tree_subtree_indexed(e, tokens text[], budget integer DEFAULT 64) RETURNS TABLE (k integer, entry public.eql_v4_json_entry)` is the subtree read with `k` on every row: the 1-based place in `tokens` of the token whose node the row sits in. `eql_v4.json_tree_subtree` is the same read without `k`. Past the budget, the sealed root comes back alone with `k = 0`. The same pairing by `k` can be written by hand, one parameter per token list and bound: + +```sql +WHERE EXISTS ( + SELECT 1 + FROM eql_v4.json_tree_subtree_indexed(col, $1::text[], 0) AS r + JOIN eql_v4.json_tree_subtree_indexed(col, $2::text[], 0) AS c ON c.k = r.k + WHERE r.entry ->> 'd' = '0' AND c.entry ->> 'd' = '0' + AND r.entry >= $3::eql_v4.query_json_leaf AND r.entry <= $3::eql_v4.query_json_leaf + AND c.entry > $4::eql_v4.query_json_leaf) +``` + +That form does not check the probe: on a row whose array is longer than the bound it answers over the first elements only. Prefer the function. + +--- + ## Aggregate Functions ### `eql_v3.min()` / `eql_v3.max()` (per-domain) diff --git a/packages/eql/docs/reference/json-tree-leakage.md b/packages/eql/docs/reference/json-tree-leakage.md index 7070ff7db..9bf763e10 100644 --- a/packages/eql/docs/reference/json-tree-leakage.md +++ b/packages/eql/docs/reference/json-tree-leakage.md @@ -51,6 +51,7 @@ A query hands the database a term derived from the plaintext it asks about, and - **What a token is.** A capability over the structure of its node's subtree in *every row of the column*: whoever holds it can find that node in any row (through its self tag `x`), see how the nodes beneath it nest, link each one to its parent, and derive the tokens of the nodes beneath. It also learns whether each node beneath is an object member or an array element, and each element's index: nothing stored says which a node is, but to derive a node's token the database tests its self tag against the member rule and then the element rule, and the one that matches says which; an element's index is the ordinal it is found at. An object's members' ordinals are a random permutation drawn when the row was sealed: they say nothing about the members' keys, and a key's ordinal in one row says nothing about its ordinal in another. It opens no value. Tokens are column-wide and do not expire, so a token sent once is useful to the database for as long as the column lives. Send a token to the database in a query and nowhere else. - **The root.** A read of `$` sends the root's token, and a read of anything below `$` does not. With the sealed root on (the default) that token covers no document's structure: it finds the root entry, whose ciphertext is the document, and nothing beneath it, since the top-level fields chain from nothing. With it off, the root's token covers every row's whole document. - **`json_tree_children` and `json_tree_child_count`** take one token, of any node a path names, and send nothing else. Under the sealed root the root's token enumerates nothing (its count is NULL), for the same reason. +- **An element-scoped predicate (`json_tree_element_match`).** For each field it tests, the tokens of a wildcard read of that field's path, probes included, and the bounds' ordering terms, as a subtree read and an ordering send them. So the database learns what those reads and orderings learn, and one thing more, which the answer itself is: in which rows one element holds every field inside its bounds. It does not open a value. Equality there is the ordering term's, of folded values (see `op` above). ## Native jsonb functions read the envelope diff --git a/packages/eql/src/v3/json_tree/elements.sql b/packages/eql/src/v3/json_tree/elements.sql new file mode 100644 index 000000000..0b9856222 --- /dev/null +++ b/packages/eql/src/v3/json_tree/elements.sql @@ -0,0 +1,120 @@ +-- ONLY EQL VERSION: 4 +-- REQUIRE: src/v3/schema.sql +-- REQUIRE: src/v3/json_tree/types.sql +-- REQUIRE: src/v3/json_tree/functions.sql +-- REQUIRE: src/v3/json_tree/operators.sql + +--! @file v3/json_tree/elements.sql +--! @brief Element-scoped predicates on public.{{prefix}}_json_search: tests +--! that one element of an array satisfies them all. + +--! @brief Whether some one element of an array in the document satisfies +--! every member of an element-scoped predicate. +--! +--! The jsonpath `exists($.contacts[*] ? (@.relationship == $a && +--! @.calls > $b))`: one element where both hold, which `@>` and an +--! ordering over `{{prefix}}.json_tree_path_query` cannot say together, +--! since each can be answered by a different element. Each member of +--! `p` holds the tokens of a subtree read of one field's path through the +--! array (`$.contacts[*].relationship`), in element order to one bound, +--! and that field's bounds. The function opens the document once, finds +--! the node each token of every member names, as +--! {{prefix}}.json_tree_subtree_indexed finds a token's node at `d = 0` +--! (by its self tag), and pairs the members' nodes by `k`, the token's +--! place in its member's array: the members' tokens were expanded to the +--! same bound in the same order, so one `k` is one element. Nothing +--! beneath a node is read: a member compares its field's own entry. It is true when some `k` has a node for every +--! member and each node satisfies its member's bounds, compared by the +--! ordering operators an entry has against a {{prefix}}.query_json_leaf: +--! `gt` by `>`, `gte` by `>=`, `lt` by `<`, `lte` by `<=`. An element where +--! a member's field is absent, or holds no ordering term (a container, a +--! boolean, a null), satisfies no bound of that member. +--! +--! **The probe.** Each member's tokens end with a probe: the token of the +--! element at the bound itself, which a document holds only when its array +--! is longer than the bound. A read finds the probe's node like any other, +--! and it is told apart by its selector: the element's, where every other +--! token names a node at the member's field, whose selector is the one the +--! member's bounds carry. A predicate over part of an array would answer +--! false for a row where an element past the bound matches, so a probe that +--! names a node raises rather than answer: derive the predicate with a +--! bound past the longest array, as a subtree read is raised. +--! +--! Equality rides on the ordering term, as `gte` and `lte` at one value, so +--! it is what the term compares: text case- and accent-folded, and numbers +--! as float64, so integers past 2^53 can tie. Containment (`@>`) is exact. +--! +--! @param e public.{{prefix}}_json_search The stored document. +--! @param p {{prefix}}.query_json_elements The predicate. +--! @return boolean True when one element satisfies every member; NULL when +--! either argument is NULL. +--! @throws invalid_parameter_value when a member's probe names a node: the +--! array is longer than the bound the predicate was derived to. +--! @note plpgsql, to raise, and so opaque to the planner: the call filters +--! a scan, as a subtree read in a predicate does. Narrow the rows with +--! an indexed predicate first, such as `@>` on the same values. +CREATE FUNCTION {{prefix}}.json_tree_element_match(e public.{{prefix}}_json_search, p {{prefix}}.query_json_elements) + RETURNS boolean + LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE +AS $$ +DECLARE + truncated boolean; + matched boolean; +BEGIN + WITH + -- The document, decompressed and indexed once for every member. + raw AS MATERIALIZED (SELECT e::jsonb #> '{}'::text[] AS d), + doc AS MATERIALIZED ( + SELECT {{prefix}}_internal.json_tree_iv(raw.d ->> 'h') AS iv, + raw.d -> 'sv' AS sv, + idx.nodes + FROM raw + CROSS JOIN LATERAL {{prefix}}_internal.json_tree_index(raw.d) AS idx + ), + member AS MATERIALIZED ( + SELECT m.i, + m.v, + -- The selector of the member's field: every bound carries it. + COALESCE(m.v -> 'gt', m.v -> 'gte', m.v -> 'lt', m.v -> 'lte') ->> 's' AS s + FROM jsonb_array_elements(p::jsonb -> 'm') WITH ORDINALITY AS m(v, i) + ), + -- The node each member's `k`-th token names, by its self tag; a token + -- that names none yields no row. + node AS MATERIALIZED ( + SELECT member.i, member.v, member.s, t.k, doc.sv -> found.j AS entry + FROM member + CROSS JOIN doc + CROSS JOIN LATERAL jsonb_array_elements_text(member.v -> 't') WITH ORDINALITY AS t(token, k) + CROSS JOIN LATERAL ( + SELECT (doc.nodes ->> {{prefix}}_internal.json_tree_self_tag(doc.iv, decode(t.token, 'base64')))::integer AS j + ) AS found + WHERE found.j IS NOT NULL + ), + satisfied AS ( + SELECT node.k + FROM node + CROSS JOIN LATERAL (SELECT node.entry::public.{{prefix}}_json_entry AS entry) AS typed + WHERE node.entry ->> 's' = node.s + AND (NOT node.v ? 'gt' + OR {{prefix}}.gt(typed.entry, (node.v -> 'gt')::{{prefix}}.query_json_leaf)) + AND (NOT node.v ? 'gte' + OR {{prefix}}.gte(typed.entry, (node.v -> 'gte')::{{prefix}}.query_json_leaf)) + AND (NOT node.v ? 'lt' + OR {{prefix}}.lt(typed.entry, (node.v -> 'lt')::{{prefix}}.query_json_leaf)) + AND (NOT node.v ? 'lte' + OR {{prefix}}.lte(typed.entry, (node.v -> 'lte')::{{prefix}}.query_json_leaf)) + GROUP BY node.k + HAVING count(DISTINCT node.i) = (SELECT count(*) FROM member) + ) + SELECT EXISTS (SELECT 1 FROM node WHERE node.entry ->> 's' IS DISTINCT FROM node.s), + EXISTS (SELECT 1 FROM satisfied) + INTO truncated, matched; + + IF truncated THEN + RAISE EXCEPTION 'an array the element predicate reads is longer than the bound it was derived to, so the predicate would answer over part of it' + USING ERRCODE = 'invalid_parameter_value', + HINT = 'Derive the predicate (element_match) with a wildcard_bound past the longest array.'; + END IF; + RETURN matched; +END; +$$; diff --git a/packages/eql/src/v3/json_tree/functions.sql b/packages/eql/src/v3/json_tree/functions.sql index 15ed4b82d..013dfa221 100644 --- a/packages/eql/src/v3/json_tree/functions.sql +++ b/packages/eql/src/v3/json_tree/functions.sql @@ -631,51 +631,34 @@ AS $$ FROM doc $$; ---! @brief The nodes the tokens name, each with everything beneath it, as ---! rows: the subtree read, in core SQL. ---! ---! For each token in turn, the entry whose self tag is the token's, at ---! `d = 0`, then its descendants, each with `d` its depth below that entry, ---! found by probing the ordinals under each node in hand until the first ---! miss and recursing under each hit's own token. So a read returns the ---! node it names, and `{}`, `[]`, a scalar and an absent node are an empty ---! object's entry, an empty array's entry, the scalar's entry and no rows, ---! the four answers `jsonb #>` gives. Rows come out in no promised order: ---! the client places each one by its tags and checks it under its parent's ---! digest, so the order they arrive in carries nothing. A token that names ---! no node yields no row, and nothing in a row is linkable to anything without its token. ---! The client sends one token for a path with no wildcard, and for a ---! wildcard one per position below its bound and a probe at the bound. The ---! server treats a probe like any other token, but can pick it out: it is ---! the last token of its level, the token count gives the bound away, and ---! a hit on it carries the element's selector rather than the leaf's. ---! Wanting a whole array is a read of the array's path, which sends the ---! array's own token. +--! @brief The subtree read, each row with the place in `tokens` of the +--! token whose node it sits in. --! ---! **The budget.** A read returns about two hundred characters per entry, ---! where a sealed root holds the whole document in one ciphertext of about ---! the plaintext's size. So when the root is sealed and the read would ---! return more than `budget` rows, counted across all the tokens and the ---! nodes at `d = 0` included, it stops and returns the root entry alone, ---! with `d = 0`: the client opens it to the whole document and evaluates ---! the same paths over it, which is cheaper on both sides than the read ---! past that point. The default of 64 is past the crossover by bytes ---! (about a dozen entries) by a margin that keeps a small subtree arriving ---! as the entries it is. A `budget` of `0` never falls back. Tokens that ---! name no node yield no row under any budget, and a document whose root ---! is not sealed is always read in full, since its root holds no document ---! to return. +--! The read {{prefix}}.json_tree_subtree describes, returning `k` beside +--! each row: `1` for the first token of `tokens`, `2` for the second, as +--! `unnest … WITH ORDINALITY` numbers them, for the node a token names and +--! for every row beneath it. `k` is a place in the caller's array and +--! nothing more: what it means is what the caller's order means. A client +--! expands a wildcard (`$.items[*].price`) into one token per position in +--! element order, `$.items[0].price` first, then the probe for the element +--! at the bound, so for such a read `k` is the element's position plus +--! one, and two reads expanded to the same bound over two paths below one +--! array (`$.items[*].price`, `$.items[*].sku`) agree on `k` for the same +--! element: a join on `k` pairs the fields of one element. Past the budget, +--! the root entry comes back alone with `k = 0`, as its `j` is: no token +--! found it. --! --! @param e public.{{prefix}}_json_search The stored document. --! @param tokens text[] The tokens of the nodes to read, base64, each --! derived by the client from a concrete path alone. --! @param budget integer The most rows to return before falling back to --! the sealed root; `0` to always read in full. Default 64. ---! @return SETOF public.{{prefix}}_json_entry The rows, each with its ---! depth `d` below the node its token named; or the root entry ---! alone, with `j = 0` and `d = 0`, past the budget. -CREATE FUNCTION {{prefix}}.json_tree_subtree(e public.{{prefix}}_json_search, tokens text[], budget integer DEFAULT 64) - RETURNS SETOF public.{{prefix}}_json_entry +--! @return TABLE(k integer, entry public.{{prefix}}_json_entry) The rows, +--! ordered by `k`, each entry with its depth `d` below the node its +--! token named; or the root entry alone, with `k = 0`, `j = 0` and +--! `d = 0`, past the budget. +CREATE FUNCTION {{prefix}}.json_tree_subtree_indexed(e public.{{prefix}}_json_search, tokens text[], budget integer DEFAULT 64) + RETURNS TABLE(k integer, entry public.{{prefix}}_json_entry) LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ WITH RECURSIVE @@ -754,7 +737,7 @@ AS $$ over AS MATERIALIZED ( SELECT doc.capped AND (SELECT count(*) FROM found) > budget AS fallback FROM doc ) - SELECT rows.entry + SELECT rows.k::integer, rows.entry FROM ( SELECT 0::bigint AS k, 0 AS j, (doc.meta || (doc.sv -> 0) || '{"j": 0, "d": 0}'::jsonb)::public.{{prefix}}_json_entry AS entry @@ -769,6 +752,62 @@ AS $$ ORDER BY rows.k, rows.j $$; +--! @brief The nodes the tokens name, each with everything beneath it, as +--! rows: the subtree read, in core SQL. +--! +--! For each token in turn, the entry whose self tag is the token's, at +--! `d = 0`, then its descendants, each with `d` its depth below that entry, +--! found by probing the ordinals under each node in hand until the first +--! miss and recursing under each hit's own token. So a read returns the +--! node it names, and `{}`, `[]`, a scalar and an absent node are an empty +--! object's entry, an empty array's entry, the scalar's entry and no rows, +--! the four answers `jsonb #>` gives. Rows come out in no promised order: +--! the client places each one by its tags and checks it under its parent's +--! digest, so the order they arrive in carries nothing. A token that names +--! no node yields no row, and nothing in a row is linkable to anything without its token. +--! The client sends one token for a path with no wildcard, and for a +--! wildcard one per position below its bound and a probe at the bound. The +--! server treats a probe like any other token, but can pick it out: it is +--! the last token of its level, the token count gives the bound away, and +--! a hit on it carries the element's selector rather than the leaf's. +--! Wanting a whole array is a read of the array's path, which sends the +--! array's own token. +--! +--! **The budget.** A read returns about two hundred characters per entry, +--! where a sealed root holds the whole document in one ciphertext of about +--! the plaintext's size. So when the root is sealed and the read would +--! return more than `budget` rows, counted across all the tokens and the +--! nodes at `d = 0` included, it stops and returns the root entry alone, +--! with `d = 0`: the client opens it to the whole document and evaluates +--! the same paths over it, which is cheaper on both sides than the read +--! past that point. The default of 64 is past the crossover by bytes +--! (about a dozen entries) by a margin that keeps a small subtree arriving +--! as the entries it is. A `budget` of `0` never falls back. Tokens that +--! name no node yield no row under any budget, and a document whose root +--! is not sealed is always read in full, since its root holds no document +--! to return. +--! +--! The rows are {{prefix}}.json_tree_subtree_indexed's, without the place +--! of the token that found each: one implementation, so the two reads +--! cannot disagree. +--! +--! @param e public.{{prefix}}_json_search The stored document. +--! @param tokens text[] The tokens of the nodes to read, base64, each +--! derived by the client from a concrete path alone. +--! @param budget integer The most rows to return before falling back to +--! the sealed root; `0` to always read in full. Default 64. +--! @return SETOF public.{{prefix}}_json_entry The rows, each with its +--! depth `d` below the node its token named; or the root entry +--! alone, with `j = 0` and `d = 0`, past the budget. +--! @see {{prefix}}.json_tree_subtree_indexed +CREATE FUNCTION {{prefix}}.json_tree_subtree(e public.{{prefix}}_json_search, tokens text[], budget integer DEFAULT 64) + RETURNS SETOF public.{{prefix}}_json_entry + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT r.entry + FROM {{prefix}}.json_tree_subtree_indexed(e, tokens, budget) AS r +$$; + ------------------------------------------------------------------------------ -- Ordering at a path ------------------------------------------------------------------------------ diff --git a/packages/eql/src/v3/json_tree/types.sql b/packages/eql/src/v3/json_tree/types.sql index 7c874ebdc..e1cb324c0 100644 --- a/packages/eql/src/v3/json_tree/types.sql +++ b/packages/eql/src/v3/json_tree/types.sql @@ -8,7 +8,7 @@ --! --! This is a new domain family beside `public.{{prefix}}_json`, for documents --! Stack Encrypt's JSON index writes. It shares no wire shape and no SQL with ---! the SteVec surface. Four jsonb-backed domains (none over another domain, +--! the SteVec surface. Five jsonb-backed domains (none over another domain, --! so the native-jsonb firewall in blockers.sql attaches): --! - public.{{prefix}}_json_search — storage: an EQL envelope `{v, i, h, sv, q}`. --! - public.{{prefix}}_json_entry — one `sv` element (returned by `->` @@ -17,6 +17,9 @@ --! projection of the needle's selectors. --! - {{prefix}}.query_json_leaf — an ordering operand at a path: a --! selector and a leaf ordering term. +--! - {{prefix}}.query_json_elements — an element-scoped predicate: per +--! member of an array's elements, the +--! member's read tokens and its bounds. --! --! The design is `docs/plans/2026-10-17-json-tree-index.md` --! at the repository root. The document's per-entry fields: @@ -253,6 +256,25 @@ BEGIN END $$; +--! @brief Validate an ordering operand at a JSON path. +--! @internal +--! @param val jsonb Candidate operand. +--! @return boolean True when `val` is an object with string `s` and `op` +--! and no other key: the {{prefix}}.query_json_leaf CHECK, and each +--! bound of a {{prefix}}.query_json_elements member. +CREATE OR REPLACE FUNCTION {{prefix}}_internal.json_tree_is_valid_leaf_query_payload(val jsonb) + RETURNS boolean + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT COALESCE( + jsonb_typeof(val) = 'object' + AND jsonb_typeof(val -> 's') = 'string' + AND jsonb_typeof(val -> 'op') = 'string' + AND val - ARRAY['s', 'op']::text[] = '{}'::jsonb, + false + ) +$$; + --! @brief Domain type for an ordering operand at a JSON path. --! --! `{"s": "…", "op": "…"}`: the path's selector, which `->` takes, and the @@ -271,16 +293,112 @@ BEGIN CREATE DOMAIN {{prefix}}.query_json_leaf AS jsonb CHECK ( VALUE IS NULL - OR COALESCE( - jsonb_typeof(VALUE) = 'object' - AND jsonb_typeof(VALUE -> 's') = 'string' - AND jsonb_typeof(VALUE -> 'op') = 'string' - AND VALUE - ARRAY['s', 'op']::text[] = '{}'::jsonb, - false - ) + OR {{prefix}}_internal.json_tree_is_valid_leaf_query_payload(VALUE) ); END IF; COMMENT ON DOMAIN {{prefix}}.query_json_leaf IS 'EQL JSON tree query operand (ordering at a path)'; END $$; + +--! @brief Validate one member of an element-scoped predicate. +--! @internal +--! +--! A member is `{"t": ["…", …], "gt" | "gte": leaf, "lt" | "lte": leaf}`: +--! the tokens of a read of the member's path, one per element below the +--! wildcard bound in element order and then the probe, and one or two +--! bounds, each an ordering operand of the shape `{{prefix}}.query_json_leaf` +--! holds (`{"s", "op"}`). At most one lower bound (`gt` or `gte`) and at most +--! one upper (`lt` or `lte`), at least one in all, and every bound of the +--! member at the one selector: the member's path's. +--! +--! @param val jsonb Candidate member. +--! @return boolean True when `val` is a member. +--! @note plpgsql, for the subqueries over the tokens and bounds, which a +--! CHECK expression cannot hold. +CREATE OR REPLACE FUNCTION {{prefix}}_internal.json_tree_is_valid_elements_member(val jsonb) + RETURNS boolean + LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE +AS $$ +BEGIN + RETURN COALESCE( + jsonb_typeof(val) = 'object' + AND val - ARRAY['t', 'gt', 'gte', 'lt', 'lte']::text[] = '{}'::jsonb + AND jsonb_typeof(val -> 't') = 'array' + AND jsonb_array_length(val -> 't') >= 1 + AND NOT EXISTS ( + SELECT 1 FROM jsonb_array_elements(val -> 't') AS token + WHERE jsonb_typeof(token) <> 'string' + ) + AND val ?| ARRAY['gt', 'gte', 'lt', 'lte']::text[] + AND NOT (val ? 'gt' AND val ? 'gte') + AND NOT (val ? 'lt' AND val ? 'lte') + -- Each bound by the `{{prefix}}.query_json_leaf` CHECK's own rule. + AND NOT EXISTS ( + SELECT 1 FROM jsonb_each(val - 't') AS bound + WHERE NOT {{prefix}}_internal.json_tree_is_valid_leaf_query_payload(bound.value) + ) + AND (SELECT count(DISTINCT bound.value ->> 's') FROM jsonb_each(val - 't') AS bound) = 1, + false + ); +END; +$$; + +--! @brief Validate an element-scoped predicate. +--! @internal +--! @param val jsonb Candidate predicate. +--! @return boolean True when `val` is `{"m": [member, …]}`, at least one +--! member, each valid, every member's tokens as many as every +--! other's: one wildcard expansion, at one bound, under each. +--! @see {{prefix}}_internal.json_tree_is_valid_elements_member +CREATE OR REPLACE FUNCTION {{prefix}}_internal.json_tree_is_valid_elements_payload(val jsonb) + RETURNS boolean + LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE +AS $$ +BEGIN + RETURN COALESCE( + jsonb_typeof(val) = 'object' + AND val - 'm' = '{}'::jsonb + AND jsonb_typeof(val -> 'm') = 'array' + AND jsonb_array_length(val -> 'm') >= 1 + AND NOT EXISTS ( + SELECT 1 FROM jsonb_array_elements(val -> 'm') AS member + WHERE NOT {{prefix}}_internal.json_tree_is_valid_elements_member(member) + ) + AND (SELECT count(DISTINCT jsonb_array_length(member -> 't')) + FROM jsonb_array_elements(val -> 'm') AS member) = 1, + false + ); +END; +$$; + +--! @brief Domain type for an element-scoped predicate over an array. +--! +--! `{"m": [{"t": ["…", …], "gte": {"s", "op"}, "lte": {"s", "op"}}, …]}`: +--! one member per field of an element the predicate tests, each the +--! tokens of a subtree read of that field's path through the array's +--! wildcard and the field's bounds, as Stack Encrypt's `element_match` +--! operator derives them. {{prefix}}.json_tree_element_match reads it: some +--! one element of the array satisfies every member's bounds. A member +--! carries `gt` or `gte` for a lower bound and `lt` or `lte` for an upper +--! one, each the ordering operand {{prefix}}.query_json_leaf holds; an +--! equality is `gte` and `lte` at one value. +--! +--! @note Construct from inline JSON via the DOMAIN cast: +--! `'{"m":[{"t":["…"],"gt":{"s":"…","op":"…"}}]}'::{{prefix}}.query_json_elements`. +--! @see {{prefix}}.json_tree_element_match +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_type + WHERE typname = 'query_json_elements' AND typnamespace = '{{prefix}}'::regnamespace + ) THEN + CREATE DOMAIN {{prefix}}.query_json_elements AS jsonb + CHECK ( + {{prefix}}_internal.json_tree_is_valid_elements_payload(VALUE) + ); + END IF; + + COMMENT ON DOMAIN {{prefix}}.query_json_elements IS 'EQL JSON tree query operand (element-scoped predicate)'; +END +$$; From 26294a88abc4573631d4bbde4e94cceedb4753c2 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 30 Oct 2026 00:18:02 +1100 Subject: [PATCH 2/3] feat(stack-encrypt): derive element-scoped predicates for a JSON document field `element_match` takes `{"path", "wildcard_bound"?, "predicates"}`: an array's elements by a path with a wildcard, and predicates on fields of an element by a member path relative to it. Its operand is `eql_v4.query_json_elements`: per predicate, the `subtree` tokens of the member's full path at the bound, and the leaf operands the orderings derive for its values. `eq` is one leaf as both `gte` and `lte`, so its equality is the ordering term's (folded text, numbers as float64), and `between` is `between_at`'s two. Each member's tokens are the subtree key of its full path, so the k-th token of every member names a field of one element, and the last the probe. No new derivation. Refused before anything is derived: a path with no wildcard, a `wildcard_bound` of 0, a position in the path or a member, a wildcard in a member, an empty predicate list, an unknown `op`, a value no ordering takes, and a member whose full path the field does not order. The operator is listed only on a field that orders a path and whose type names the elements domain (`TargetDescriptor::with_elements_query_sql_domain`), so a host that does not name it, such as the Go guest today, sees no new operator. The heap hygiene suite watches the operator too. Its spy now searches for the data key where its lock holds it, and records the key in place: it used to copy the key onto the stack to compare, on every free during a watch, and a future built later at that depth carried the copy into its box. On this branch that made the `json_open (subtree)` watch fail on the spy's own copy, in `load_keysets`' future box (macOS arm64, debug); with the copy gone the watch passes and `load_keysets` is unchanged. Refs #1440 --- .../stack-encrypt/src/dynamic/json_query.rs | 291 +++++++- packages/stack-encrypt/src/dynamic/mod.rs | 12 + packages/stack-encrypt/src/dynamic/query.rs | 45 +- packages/stack-encrypt/src/dynamic/target.rs | 16 + packages/stack-encrypt/tests/heap_hygiene.rs | 51 +- .../stack-encrypt/tests/json_operators.rs | 686 +++++++++++++++++- 6 files changed, 1086 insertions(+), 15 deletions(-) diff --git a/packages/stack-encrypt/src/dynamic/json_query.rs b/packages/stack-encrypt/src/dynamic/json_query.rs index ea5e261f6..fcf4e9874 100644 --- a/packages/stack-encrypt/src/dynamic/json_query.rs +++ b/packages/stack-encrypt/src/dynamic/json_query.rs @@ -16,6 +16,7 @@ //! | `selector` | `{"path"}` | text: the selector `->` takes | //! | `subtree` | `{"path", "wildcard_bound"?}` | text array: the tokens `json_tree_subtree` takes | //! | `children`, `child_count` | `{"path"}`: a path with no wildcard | text: the node's token | +//! | `element_match` | `{"path", "wildcard_bound"?, "predicates"}`: an array's elements, by a path with a wildcard, and predicates on fields of an element at ordered paths | EQL: per predicate, the field's read tokens and its bounds (`eql_v4.query_json_elements`), compared as the orderings compare, so its `eq` is of folded values | //! //! Every JSON value in an input is UTF-8 JSON text, parsed by //! [`parse_text`](crate::json::parse_text)'s rules. A path is JSONPath text @@ -32,6 +33,52 @@ //! hold of the same array element; one containment query of a document //! holding both does. //! +//! # Element predicates +//! +//! `element_match` asks whether some one element of an array satisfies +//! every predicate, which two `eq_at` or ordering operands cannot: the +//! jsonpath `exists($.contacts[*] ? (@.relationship == "Sibling" && +//! @.calls > 5))`. Its input names the array's elements by a path with a +//! wildcard (`$.contacts[*]`), and each predicate a field of an element by +//! a `member` path relative to it (`relationship`, `address.postcode`, +//! `["first name"]`), an `op` and its value: +//! +//! ```json +//! {"path": "$.contacts[*]", "wildcard_bound": 16, "predicates": [ +//! {"member": "relationship", "op": "eq", "json": "\"Sibling\""}, +//! {"member": "calls", "op": "gt", "json": "5"}, +//! {"member": "age", "op": "between", "low": "18", "high": "40"} +//! ]} +//! ``` +//! +//! `op` is `eq`, `lt`, `lte`, `gt` or `gte`, with `json`, or `between`, +//! with `low` and `high`, inclusive. Each member's full path, the path and +//! the member joined, must be one the field orders. The operand holds, per +//! predicate, the tokens of a `subtree` read of the full path at the bound, +//! and the bounds as the orderings derive them: `gt` the leaf operand +//! `gt_at` derives, `between` the two `between_at` derives as `gte` and +//! `lte`, and `eq` one leaf operand as both `gte` and `lte`. So equality +//! here is the ordering term's: text compares case- and accent-folded, and +//! numbers as float64, so integers past 2^53 can tie. Exact equality at a +//! path is `eq_at`, of any element. +//! +//! Refused before anything is derived, each by its reason: a path with no +//! wildcard ([`Reason::JsonPathNoWildcard`]), a `wildcard_bound` of `0`, +//! whose operand could never be true ([`Reason::JsonWildcardBoundZero`]), +//! an empty list of predicates ([`Reason::JsonNoPredicates`]), an `op` +//! that is none of the six ([`Reason::JsonUnknownPredicateOp`]), a +//! predicate whose keys do not fit its `op` ([`Reason::JsonInput`]), a +//! wildcard in a member ([`Reason::JsonPathWildcard`]); and by the index's +//! own refusals, a position in the path or a member +//! ([`TermError::JsonOrderingAtPosition`]), a member whose full path the +//! field does not order ([`TermError::JsonPathNotOrdered`]), a value that +//! is not a number or a string ([`TermError::JsonLeafNotOrdered`]), and a +//! member that is not a path ([`TermError::JsonPath`], at an offset into +//! the member's own text). The +//! database refuses a row whose array is longer than the bound, as a +//! subtree read of it opens to a refusal: each member's tokens end with +//! the probe, and `json_tree_element_match` raises when it names a node. +//! //! # Reads //! //! [`json_open`] opens the rows a read returned, against the request that @@ -76,6 +123,8 @@ pub(super) enum Binds { Text, /// `text[]`. TextArray, + /// The target's element predicate domain (`eql_v4.query_json_elements`). + Elements, } /// What a JSON operator's operand binds as, or `None` for an operator @@ -90,6 +139,7 @@ pub(super) fn binds(op: OpSpec) -> Option { } OpSpec::Selector | OpSpec::Children | OpSpec::ChildCount => Some(Binds::Text), OpSpec::Subtree => Some(Binds::TextArray), + OpSpec::ElementMatch => Some(Binds::Elements), OpSpec::Eq | OpSpec::Matches | OpSpec::Lt @@ -108,6 +158,7 @@ pub(super) fn domain(target: &super::TargetDescriptor, binds: Binds) -> Option target.leaf_query_sql_domain.clone(), Binds::Text => Some("text".to_owned()), Binds::TextArray => Some("text[]".to_owned()), + Binds::Elements => target.elements_query_sql_domain.clone(), } } @@ -199,6 +250,53 @@ struct ReadInput { wildcard_bound: Option, } +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ElementsInput { + path: String, + #[serde(default)] + wildcard_bound: Option, + predicates: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct PredicateInput { + member: String, + op: String, + #[serde(default)] + json: Option, + #[serde(default)] + low: Option, + #[serde(default)] + high: Option, +} + +/// How a predicate of `element_match` compares its field with one value. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Compare { + Eq, + Lt, + Lte, + Gt, + Gte, +} + +/// A predicate's bounds, as its input gives them. +enum Bounds { + /// One value, compared by the predicate's `op`. + One(Compare, Parsed), + /// `between`: the low bound then the high, both inclusive. + Between(Parsed, Parsed), +} + +/// One predicate of `element_match`, checked: the full path of the +/// element's field, which the field orders, and its bounds. +struct Predicate { + path: JsonPath, + bounds: Bounds, +} + /// An operator's input, read from its frame and checked: everything /// [`derive`] would refuse before it derives anything. enum Input { @@ -207,6 +305,7 @@ enum Input { Between(JsonPath, Parsed, Parsed), Path(JsonPath), Read(JsonPath, u32), + Elements(u32, Vec), } fn read_frame Deserialize<'de>>(input: &ValueFrame) -> Result { @@ -280,10 +379,97 @@ fn read_input(op: OpSpec, options: &JsonOptions, input: &ValueFrame) -> Result read_elements(options, input), _ => Err(Error::Internal), } } +/// `element_match`'s input: an array's elements, by a path with a wildcard +/// and no position, and one or more predicates on their fields. +fn read_elements(options: &JsonOptions, input: &ValueFrame) -> Result { + let elements: ElementsInput = read_frame(input)?; + let base = path(&elements.path)?; + if base.is_positional() { + return Err(Error::bad_source(Reason::JsonPathNoWildcard)); + } + if base.names_a_position() { + return Err(term(TermError::JsonOrderingAtPosition)); + } + let bound = elements + .wildcard_bound + .unwrap_or(JsonSubtreeKey::DEFAULT_BOUND); + if bound == 0 { + return Err(Error::bad_source(Reason::JsonWildcardBoundZero)); + } + if elements.predicates.is_empty() { + return Err(Error::bad_source(Reason::JsonNoPredicates)); + } + let predicates = elements + .predicates + .iter() + .map(|predicate| read_predicate(options, &base, predicate)) + .collect::, _>>()?; + Ok(Input::Elements(bound, predicates)) +} + +/// One predicate of `element_match`: its member's full path below `base`, +/// which the field must order, and the bounds its `op` takes. +fn read_predicate( + options: &JsonOptions, + base: &JsonPath, + predicate: &PredicateInput, +) -> Result { + let compare = match predicate.op.as_str() { + "eq" => Some(Compare::Eq), + "lt" => Some(Compare::Lt), + "lte" => Some(Compare::Lte), + "gt" => Some(Compare::Gt), + "gte" => Some(Compare::Gte), + "between" => None, + _ => return Err(Error::bad_source(Reason::JsonUnknownPredicateOp)), + }; + let path = member_path(options, base, &predicate.member)?; + let value = |text: &JsonText| -> Result { + let value = text.parse()?; + leaf(&value.0)?; + Ok(value) + }; + let bounds = match (compare, &predicate.json, &predicate.low, &predicate.high) { + (Some(compare), Some(json), None, None) => Bounds::One(compare, value(json)?), + (None, None, Some(low), Some(high)) => Bounds::Between(value(low)?, value(high)?), + _ => return Err(Error::bad_source(Reason::JsonInput)), + }; + Ok(Predicate { path, bounds }) +} + +/// A member's full path: `base` and then the member's steps, read as a +/// path relative to an element (`name`, `address.city`, `["a b"]`). A +/// member names one field of each element, so it has no wildcard of its +/// own, and the full path is an ordering's: no position, and ordered. A +/// member that does not parse is refused at an offset into the member's +/// own text. +fn member_path(options: &JsonOptions, base: &JsonPath, member: &str) -> Result { + let prefix = if member.starts_with('[') { "$" } else { "$." }; + let relative = JsonPath::parse(&format!("{prefix}{member}")).map_err(|error| { + let error = crate::json::JsonPathError { + at: error.at.saturating_sub(prefix.len()), + ..error + }; + Error::Cipher(TermError::JsonPath(error).into()) + })?; + if !relative.is_positional() { + return Err(Error::bad_source(Reason::JsonPathWildcard)); + } + let full = relative + .segments() + .iter() + .fold(base.clone(), |at, segment| at.then(segment.clone())); + if full.segments().len() > crate::json::MAX_DEPTH { + return Err(term(TermError::JsonTooDeep)); + } + ordered_path_of(options, full) +} + /// `eq_at`'s input: a scalar at a path read through a selector. fn read_eq_at(input: &ValueFrame) -> Result { let at: AtInput = read_frame(input)?; @@ -324,7 +510,11 @@ fn read_between(options: &JsonOptions, input: &ValueFrame) -> Result Result { - let path = path(text)?; + ordered_path_of(options, path(text)?) +} + +/// [`ordered_path`] of a path already parsed. +fn ordered_path_of(options: &JsonOptions, path: JsonPath) -> Result { if path.names_a_position() { Err(term(TermError::JsonOrderingAtPosition)) } else if options.orders(&path) { @@ -421,9 +611,108 @@ pub(super) fn derive<'a, K: 'static>( let key = keyset.json_subtree_key_now(&path, options, bound, column())?; Pending::ready(keyset, Ok(Operand::TextList(key.tokens()))) } + Input::Elements(bound, predicates) => elements(keyset, label, options, bound, &predicates)?, }) } +/// One member of an `element_match` operand: the tokens of a read of the +/// member's full path, probes included, and its bounds, each a leaf +/// operand. `eql_v4.query_json_elements` holds a list of them as `{"m"}`. +#[derive(serde::Serialize)] +struct MemberOperand { + t: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + gt: Option, + #[serde(skip_serializing_if = "Option::is_none")] + gte: Option, + #[serde(skip_serializing_if = "Option::is_none")] + lt: Option, + #[serde(skip_serializing_if = "Option::is_none")] + lte: Option, +} + +/// `element_match`'s operand, `eql_v4.query_json_elements`. +#[derive(serde::Serialize)] +struct ElementsOperand { + m: Vec, +} + +impl MemberOperand { + /// The member reading `tokens` that compares as `compare`, with the + /// leaf operands of its values in order: one for a comparison, the + /// low's then the high's for `between` (`compare` `None`). Equality is + /// both inclusive bounds at its one value. + fn new(tokens: Vec, compare: Option, leaves: Vec) -> Self { + let mut member = Self { + t: tokens, + gt: None, + gte: None, + lt: None, + lte: None, + }; + let mut leaves = leaves.into_iter(); + match compare { + Some(Compare::Eq) => { + member.gte = leaves.next(); + member.lte = member.gte.clone(); + } + Some(Compare::Lt) => member.lt = leaves.next(), + Some(Compare::Lte) => member.lte = leaves.next(), + Some(Compare::Gt) => member.gt = leaves.next(), + Some(Compare::Gte) => member.gte = leaves.next(), + None => { + member.gte = leaves.next(); + member.lte = leaves.next(); + } + } + member + } +} + +/// `element_match`'s operand: for each predicate, the tokens of a subtree +/// read of its full path at `bound`, as `subtree` derives them, and the +/// leaf operand of each of its values, as the orderings derive them. The +/// reads share the path and the bound, so the `k`-th token of every member +/// names a field of one element, and each member's last names the element +/// at the bound: the probe. +fn elements<'a, K: 'static>( + keyset: &'a KeysetCipher<'_, K>, + label: &Label, + options: &JsonOptions, + bound: u32, + predicates: &[Predicate], +) -> Result, Error> { + let mut members = Vec::with_capacity(predicates.len()); + for predicate in predicates { + let tokens = keyset + .json_subtree_key_now( + &predicate.path, + options, + bound, + NonEmpty::from(label.clone()), + )? + .tokens(); + let leaf = |value: &Parsed| { + terms( + keyset, + options, + label, + predicate.path.holding(value.0.clone()), + ) + .map(|terms| JsonLeafQuery::of_leaf(&terms)) + }; + let (compare, leaves) = match &predicate.bounds { + Bounds::One(compare, value) => (Some(*compare), vec![leaf(value)]), + Bounds::Between(low, high) => (None, vec![leaf(low), leaf(high)]), + }; + members.push( + Pending::all(keyset, leaves) + .map(move |leaves| MemberOperand::new(tokens, compare, leaves)), + ); + } + Ok(Pending::all(keyset, members).try_map(|m| eql(&ElementsOperand { m }).map(Operand::One))) +} + /// A read to open, as the request that prepared it names it: the /// operator, the path and, for a subtree read, the wildcard bound. What a /// binding keeps from preparing the read and hands back with its rows; the diff --git a/packages/stack-encrypt/src/dynamic/mod.rs b/packages/stack-encrypt/src/dynamic/mod.rs index 7e57f7c16..e1f8b2f61 100644 --- a/packages/stack-encrypt/src/dynamic/mod.rs +++ b/packages/stack-encrypt/src/dynamic/mod.rs @@ -657,6 +657,18 @@ reasons! { /// finds a null by its term. A `"null": "sql"` field's nulls are SQL /// `NULL`, which `IS NULL` finds. NullQuery => "null_query", r#"a null is found by its term only with eq on a field with "null": "sealed"; a "null": "sql" field's nulls are SQL NULL"#; + /// An element predicate's path has no wildcard: the predicate is + /// over the elements of an array, which `[*]` names. + JsonPathNoWildcard => "json_path_no_wildcard", "an element predicate is over an array's elements, and the path names no wildcard"; + /// An element predicate has no predicates: it would be true of + /// every row with an element. + JsonNoPredicates => "json_no_predicates", "an element predicate needs at least one predicate"; + /// An element predicate's `wildcard_bound` is `0`: its reads would + /// send the probe alone, so the predicate could never be true. + JsonWildcardBoundZero => "json_wildcard_bound_zero", "an element predicate reads at least one element, and a wildcard_bound of 0 reads none"; + /// A predicate of an element predicate has an `op` that is not + /// `eq`, `lt`, `lte`, `gt`, `gte` or `between`. + JsonUnknownPredicateOp => "json_unknown_predicate_op", r#"a predicate's "op" is not eq, lt, lte, gt, gte or between"#; } } diff --git a/packages/stack-encrypt/src/dynamic/query.rs b/packages/stack-encrypt/src/dynamic/query.rs index c08895572..c6161a570 100644 --- a/packages/stack-encrypt/src/dynamic/query.rs +++ b/packages/stack-encrypt/src/dynamic/query.rs @@ -154,12 +154,17 @@ pub enum OpSpec { /// `"child_count"`: the same token, which `json_tree_child_count` /// takes. ChildCount, + /// `"element_match"`: a predicate on one element of an array, each + /// member a field of the element at an ordered path with its bounds, + /// which `json_tree_element_match` takes. True of a row where some one + /// element satisfies every member. + ElementMatch, } impl OpSpec { /// Every operator, in declaration order: the order a catalog lists them /// in. - pub const ALL: [OpSpec; 19] = [ + pub const ALL: [OpSpec; 20] = [ OpSpec::Eq, OpSpec::Lt, OpSpec::Lte, @@ -179,6 +184,7 @@ impl OpSpec { OpSpec::Subtree, OpSpec::Children, OpSpec::ChildCount, + OpSpec::ElementMatch, ]; /// The operator a name names, or `None` for a name that is not one. @@ -209,6 +215,7 @@ impl OpSpec { OpSpec::Subtree => "subtree", OpSpec::Children => "children", OpSpec::ChildCount => "child_count", + OpSpec::ElementMatch => "element_match", } } @@ -222,8 +229,9 @@ impl OpSpec { /// The shape of the operator's input, as a catalog names it: `value` /// (one value of the field's kind), `pair` (two, low then high), /// `json` (`{"json"}`), `path_json` (`{"path", "json"}`), `path_pair` - /// (`{"path", "low", "high"}`), `path` (`{"path"}`) or `path_bound` - /// (`{"path", "wildcard_bound"?}`). + /// (`{"path", "low", "high"}`), `path` (`{"path"}`), `path_bound` + /// (`{"path", "wildcard_bound"?}`) or `path_predicates` (`{"path", + /// "wildcard_bound"?, "predicates"}`). pub fn input(self) -> &'static str { match self { OpSpec::Between => "pair", @@ -234,6 +242,7 @@ impl OpSpec { OpSpec::BetweenAt => "path_pair", OpSpec::Selector | OpSpec::Children | OpSpec::ChildCount => "path", OpSpec::Subtree => "path_bound", + OpSpec::ElementMatch => "path_predicates", OpSpec::Eq | OpSpec::Lt | OpSpec::Lte | OpSpec::Gt | OpSpec::Gte | OpSpec::Matches => { "value" } @@ -248,6 +257,14 @@ impl OpSpec { /// opens what the rows hold; see [`json_open`]). Equality at a wildcard /// path matches when any element does, and an ordering compared through /// `col -> sel` at one sees an unspecified element of each row. + /// + /// The orderings at a path (`lt_at` … `between_at`) and `element_match` + /// compare ordering terms, and `exact` is of the terms: the server + /// answers the comparison of the terms without false positives, and the + /// terms order text case- and accent-folded and numbers as float64. So + /// two values the plaintext tells apart can compare equal, and + /// `element_match`'s `eq`, which is `gte` and `lte` at one value, is an + /// equality of folded values, where `eq_at` is exact on the value. pub fn fidelity(self) -> Fidelity { match self { OpSpec::Matches => Fidelity::Probabilistic, @@ -281,13 +298,19 @@ impl OpSpec { let declared = |wanted: fn(&IndexSpec) -> bool| indexes.iter().find(|index| wanted(index)); if self.is_json() { // A document field's JSON index answers the document operators; - // the orderings at a path only when it orders some path. + // the orderings at a path, and the element predicates built of + // them, only when it orders some path. if kind != ValueKind::Json { return None; } let ordering = matches!( self, - OpSpec::LtAt | OpSpec::LteAt | OpSpec::GtAt | OpSpec::GteAt | OpSpec::BetweenAt + OpSpec::LtAt + | OpSpec::LteAt + | OpSpec::GtAt + | OpSpec::GteAt + | OpSpec::BetweenAt + | OpSpec::ElementMatch ); return indexes.iter().find(|index| match index { IndexSpec::Json(options) => !ordering || !options.ordered.is_empty(), @@ -317,7 +340,8 @@ impl OpSpec { | OpSpec::Selector | OpSpec::Subtree | OpSpec::Children - | OpSpec::ChildCount => None, + | OpSpec::ChildCount + | OpSpec::ElementMatch => None, } .filter(|index| admits(kind, index)) } @@ -3174,7 +3198,7 @@ mod wire_bytes { assert_eq!(written, PINNED); } - const PINNED: [&str; 156] = [ + const PINNED: [&str; 163] = [ "0a020000006571", "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0500000076616c7565", "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0500000076616c7565", @@ -3195,6 +3219,7 @@ mod wire_bytes { "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0a000000706174685f626f756e64", "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a0d000000656c656d656e745f6d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0f000000706174685f70726564696361746573", "0a050000006d61746368", "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", @@ -3215,6 +3240,7 @@ mod wire_bytes { "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0a000000706174685f626f756e64", "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a0d000000656c656d656e745f6d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0f000000706174685f70726564696361746573", "f101000000050000006d61746368f10400000009000000746f6b656e697a65720a080000007374616e6461726408000000646f776e6361736502010000006b070400000000000000010000006d070002000000000000", "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", @@ -3235,6 +3261,7 @@ mod wire_bytes { "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0a000000706174685f626f756e64", "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a0d000000656c656d656e745f6d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0f000000706174685f70726564696361746573", "f101000000050000006d61746368f10400000009000000746f6b656e697a6572f101000000050000006e6772616d07040000000000000008000000646f776e6361736503010000006b070300000000000000010000006d070001000000000000", "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", @@ -3255,6 +3282,7 @@ mod wire_bytes { "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0a000000706174685f626f756e64", "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a0d000000656c656d656e745f6d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0f000000706174685f70726564696361746573", "0a030000006f7265", "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0500000076616c7565", "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0500000076616c7565", @@ -3275,6 +3303,7 @@ mod wire_bytes { "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0a000000706174685f626f756e64", "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a0d000000656c656d656e745f6d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0f000000706174685f70726564696361746573", "0a030000006f7065", "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0500000076616c7565", "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0500000076616c7565", @@ -3295,6 +3324,7 @@ mod wire_bytes { "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0a000000706174685f626f756e64", "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a0d000000656c656d656e745f6d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0f000000706174685f70726564696361746573", "0a040000006a736f6e", "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0500000076616c7565", "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0500000076616c7565", @@ -3315,6 +3345,7 @@ mod wire_bytes { "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0a000000706174685f626f756e64", "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a0d000000656c656d656e745f6d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0f000000706174685f70726564696361746573", "f106000000080000006f70657261746f720a02000000657108000000666964656c6974790a05000000657861637405000000696e6465780a02000000657106000000646f6d61696e0008000000656e636f64696e670a050000006279746573040000007465726d0b03000000010203", "f106000000080000006f70657261746f720a070000006265747765656e08000000666964656c6974790a05000000657861637405000000696e6465780a030000006f706506000000646f6d61696e0a1000000065716c5f76342e71756572795f696e7408000000656e636f64696e670a0300000065716c040000007465726df0020000000b01000000040b020000000506", "f106000000080000006f70657261746f720a070000006d61746368657308000000666964656c6974790a0d00000070726f626162696c697374696305000000696e6465780a050000006d6174636806000000646f6d61696e0008000000656e636f64696e670a050000006279746573040000007465726d0b00000000", diff --git a/packages/stack-encrypt/src/dynamic/target.rs b/packages/stack-encrypt/src/dynamic/target.rs index 197c51cc4..77555ec42 100644 --- a/packages/stack-encrypt/src/dynamic/target.rs +++ b/packages/stack-encrypt/src/dynamic/target.rs @@ -110,6 +110,13 @@ pub struct TargetDescriptor { /// [`query_sql_domain`](Self::query_sql_domain) names. Not on the /// wire; a generator reads it from the operators' terms. pub leaf_query_sql_domain: Option, + /// The PostgreSQL domain of the type's element-scoped predicate, for a + /// type that stores a JSON document + /// ([`with_elements_query_sql_domain`](Self::with_elements_query_sql_domain)): + /// what `element_match` binds as. Not on the wire, as + /// [`leaf_query_sql_domain`](Self::leaf_query_sql_domain) is not; a type + /// that names none does not answer `element_match`. + pub elements_query_sql_domain: Option, } /// The descriptor in its wire form: the object a guest's `se_targets` @@ -178,6 +185,7 @@ impl TargetDescriptor { reason, match_options: None, leaf_query_sql_domain: None, + elements_query_sql_domain: None, } } @@ -189,6 +197,14 @@ impl TargetDescriptor { self } + /// The descriptor of a type that stores a JSON document, whose + /// element-scoped predicate is the domain `domain`. + #[must_use] + pub fn with_elements_query_sql_domain(mut self, domain: impl Into) -> Self { + self.elements_query_sql_domain = Some(domain.into()); + self + } + /// The descriptor of a type whose match index is built under `options`: /// such a type answers `matches` through its query twin /// ([`operators`](Self::operators)). A type with a match index and no diff --git a/packages/stack-encrypt/tests/heap_hygiene.rs b/packages/stack-encrypt/tests/heap_hygiene.rs index 72a533386..91c226c93 100644 --- a/packages/stack-encrypt/tests/heap_hygiene.rs +++ b/packages/stack-encrypt/tests/heap_hygiene.rs @@ -29,6 +29,10 @@ //! first zeroes the stack below it ([`scrub_stack`]); otherwise residue from //! earlier work (a hash's block buffer after a PRF over a marked key name) //! would reach a heap block and be blamed on the operation that boxed it. +//! For the same reason the spy never copies what it looks for onto the +//! stack: it runs on every free, from inside the watched operation, and a +//! copy left in its dead frame would be picked up the same way by a future +//! built later at that depth (see [`holds_key`]). //! //! Unix only: the spy scans a freed block with libc's `memmem`, not as a //! Rust slice (see [`holds_mark`]). @@ -94,14 +98,23 @@ unsafe fn holds_mark(ptr: *const u8, len: usize) -> bool { /// Whether the `len` bytes at `ptr` hold the data key in [`KEY`], read as /// [`holds_mark`] reads. /// +/// The key is searched for where [`KEY`] holds it, under the lock, and never +/// copied out. This runs on every free during a watch, at whatever depth +/// the operation freed from; a copy in this frame would stay on the dead +/// stack, and a future the operation builds there next (a `JoinAll` whose +/// unused variant space the build leaves as it found it) would carry the +/// key into its box, which the spy would then report as the operation's. +/// /// # Safety /// /// `ptr` must point to `len` allocated bytes. unsafe fn holds_key(ptr: *const u8, len: usize) -> bool { BUSY.with(|busy| busy.set(true)); - let key = KEY.lock().map(|key| *key).unwrap_or(None); + let found = KEY + .lock() + .is_ok_and(|key| key.as_ref().is_some_and(|key| find(ptr, len, key))); BUSY.with(|busy| busy.set(false)); - key.is_some_and(|key| find(ptr, len, &key)) + found } /// [`holds_mark`] for any `needle`. @@ -412,7 +425,8 @@ impl stack_encrypt::dynamic::TargetResolver for Documents { true, None, ) - .with_leaf_query_sql_domain("eql_v4.query_json_leaf")] + .with_leaf_query_sql_domain("eql_v4.query_json_leaf") + .with_elements_query_sql_domain("eql_v4.query_json_elements")] } fn encrypt<'a, K: 'static>( &self, @@ -580,8 +594,13 @@ mod json_index { /// [`KEY`] so the spy can look for it. struct Recorded(DeterministicSource); + /// Written into [`KEY`] in place, not built on the stack and moved in, + /// for the reason [`holds_key`] reads it in place. fn record(key: &DataKey) { - *KEY.lock().unwrap() = Some(*key.key()); + KEY.lock() + .unwrap() + .get_or_insert([0; 32]) + .copy_from_slice(key.key()); } impl DataKeySource for Recorded { @@ -815,7 +834,7 @@ mod json_index { let keyset = cipher.default_keyset(); let wire = json!({ "profile": {"context": ["users", "profile"], "type": "json", "null": "sql", - "target": {"name": "JsonSearch", "options": {"ordered": ["$.age"]}}} + "target": {"name": "JsonSearch", "options": {"ordered": ["$.age", "$.visits[*].at"]}}} }); let plan = record::plan_with( &stack_encrypt::dynamic::frame::serde::to_vec(&wire).unwrap(), @@ -853,6 +872,28 @@ mod json_index { ("high", frame(quoted.as_str())), ]), ), + ( + OpSpec::ElementMatch, + object(vec![ + ("path", frame("$.visits[*]")), + ( + "predicates", + frame(vec![ + object(vec![ + ("member", frame("at")), + ("op", frame("eq")), + ("json", frame(quoted.as_str())), + ]), + object(vec![ + ("member", frame("at")), + ("op", frame("between")), + ("low", frame(quoted.as_str())), + ("high", frame(quoted.as_str())), + ]), + ]), + ), + ]), + ), ]; for (operator, input) in &inputs { let term = watched(&format!("query_term ({operator:?})"), || { diff --git a/packages/stack-encrypt/tests/json_operators.rs b/packages/stack-encrypt/tests/json_operators.rs index d9e1b7ac6..4bfc7e870 100644 --- a/packages/stack-encrypt/tests/json_operators.rs +++ b/packages/stack-encrypt/tests/json_operators.rs @@ -1,7 +1,7 @@ //! A document field's JSON operators and reads through the data plan, with //! the engine's own JSON index behind the target: what a binding names by //! operator (`contains`, `eq_at`, the orderings at a path, `selector`, -//! `subtree`, `children`, `child_count`), the operand each derives and the +//! `subtree`, `children`, `child_count`, `element_match`), the operand each derives and the //! type it binds as, the refusals made before anything is derived, and //! reads opened against the request that prepared them. //! @@ -51,7 +51,8 @@ impl TargetResolver for Documents { true, None, ) - .with_leaf_query_sql_domain("eql_v4.query_json_leaf")] + .with_leaf_query_sql_domain("eql_v4.query_json_leaf") + .with_elements_query_sql_domain("eql_v4.query_json_elements")] } fn encrypt<'a, K: 'static>( @@ -921,3 +922,684 @@ fn a_malformed_read_is_refused_before_any_key() { .unwrap_err(); assert_eq!(error.reason(), Some(Reason::OperatorNotAnswered)); } + +/// `element_match`: a predicate on one element of an array, its operand the +/// members' subtree reads and their bounds, as the `subtree` operator and +/// the orderings derive them. `json_tree_element_match` is modelled over +/// the rows `JsonDocument::subtree` returns, as the subtree tests model +/// `json_tree_subtree`; `packages/eql/tests/encryption` runs the SQL. +mod element_match { + use super::*; + + use proptest::prelude::*; + use proptest::test_runner::{Config, TestCaseError, TestRunner}; + + /// `profile`, ordering a contact's `relationship`, `calls`, + /// `address.postcode` and `["first name"]`, but not its `note`. + fn contacts() -> record::Plan { + plan_of(json!({ + "name": "JsonSearch", + "options": {"ordered": [ + "$.contacts[*].relationship", + "$.contacts[*].calls", + "$.contacts[*].address.postcode", + "$.contacts[*][\"first name\"]", + ]} + })) + } + + fn input(bound: u32, predicates: Value) -> Value { + json!({"path": "$.contacts[*]", "wildcard_bound": bound, "predicates": predicates}) + } + + /// The operand's JSON, `eql_v4.query_json_elements`. + async fn operand( + keyset: &KeysetCipher<'_, K>, + plan: &record::Plan, + input: Value, + ) -> Value { + let term = term(keyset, plan, "element_match", input).await; + assert_eq!(term.encoding(), "eql", "an EQL operand"); + assert_eq!( + term.domain(), + Some("eql_v4.query_json_elements"), + "bound as the target's element predicate domain" + ); + match term.operand() { + Operand::One(bytes) => serde_json::from_slice(bytes).unwrap(), + other => panic!("one operand, got {other:?}"), + } + } + + /// The one leaf operand an ordering at `path` derives for `json`. + async fn leaf( + keyset: &KeysetCipher<'_, K>, + plan: &record::Plan, + operator: &str, + path: &str, + json: &str, + ) -> Value { + let term = term(keyset, plan, operator, json!({"path": path, "json": json})).await; + match term.operand() { + Operand::One(bytes) => serde_json::from_slice(bytes).unwrap(), + other => panic!("{operator}: {other:?}"), + } + } + + /// The tokens a `subtree` read of `path` at `bound` sends. + async fn tokens( + keyset: &KeysetCipher<'_, K>, + plan: &record::Plan, + path: &str, + bound: u32, + ) -> Value { + let term = term( + keyset, + plan, + "subtree", + json!({"path": path, "wildcard_bound": bound}), + ) + .await; + match term.operand() { + Operand::TextList(tokens) => json!(tokens), + other => panic!("{other:?}"), + } + } + + /// What `json_tree_element_match(doc, operand)` answers: whether one + /// `k` has, for every member, the node its `k`-th token names at the + /// member's selector and inside its bounds, compared as the SQL + /// compares ordering terms (bytes); or `None` where a token names a + /// node at another selector, a probe's hit, which the SQL raises on. + fn element_match(doc: &JsonDocument, operand: &Value) -> Option { + let members = operand["m"].as_array().unwrap(); + let width = members[0]["t"].as_array().unwrap().len(); + let term = |leaf: &Value| Base64::decode_vec(leaf["op"].as_str().unwrap()).unwrap(); + let mut truncated = false; + let mut matched = false; + for k in 0..width { + let mut all = true; + for member in members { + let bounds: Vec<(&str, &Value)> = ["gt", "gte", "lt", "lte"] + .into_iter() + .filter_map(|key| member.get(key).map(|leaf| (key, leaf))) + .collect(); + let selector = tag(bounds[0].1["s"].as_str().unwrap()); + let token = tag(member["t"][k].as_str().unwrap()); + let Some((j, entry, _)) = doc.subtree(&[token]).into_iter().next() else { + all = false; + continue; + }; + if entry.selector() != &selector { + truncated = true; + all = false; + continue; + } + let node = serde_json::to_value(doc.node(j).unwrap()).unwrap(); + if node.get("op").is_none() { + all = false; + continue; + } + let stored = term(&node); + all &= bounds.iter().all(|(key, leaf)| { + let bound = term(leaf); + match *key { + "gt" => stored > bound, + "gte" => stored >= bound, + "lt" => stored < bound, + _ => stored <= bound, + } + }); + } + matched |= all; + } + (!truncated).then_some(matched) + } + + mod given_a_predicate_per_comparison { + use super::*; + + /// Each `op` derives the bound the ordering of the same name + /// derives, under its own key; `eq` is `gte` and `lte` at one + /// value, and `between` is `between_at`'s two. + #[tokio::test] + async fn each_bound_is_the_orderings_leaf_operand() { + let cipher = common::stack_cipher().await; + let keyset = cipher.default_keyset(); + let plan = contacts(); + let calls = "$.contacts[*].calls"; + let operand = operand( + &keyset, + &plan, + input( + 2, + json!([ + {"member": "calls", "op": "eq", "json": "5"}, + {"member": "calls", "op": "lt", "json": "5"}, + {"member": "calls", "op": "lte", "json": "5"}, + {"member": "calls", "op": "gt", "json": "5"}, + {"member": "calls", "op": "gte", "json": "5"}, + {"member": "calls", "op": "between", "low": "1", "high": "9"}, + ]), + ), + ) + .await; + let m = operand["m"].as_array().unwrap(); + assert_eq!(m.len(), 6, "one member per predicate: {operand}"); + let at_five = leaf(&keyset, &plan, "gte_at", calls, "5").await; + assert_eq!( + leaf(&keyset, &plan, "lt_at", calls, "5").await, + at_five, + "a leaf operand is one term, whichever ordering asks" + ); + let keys = |member: &Value| -> Vec { + member.as_object().unwrap().keys().cloned().collect() + }; + assert_eq!(keys(&m[0]), ["gte", "lte", "t"], "eq: {}", m[0]); + assert_eq!(m[0]["gte"], at_five, "eq's low bound is the value's"); + assert_eq!(m[0]["lte"], m[0]["gte"], "eq is lo = hi"); + for (member, key) in [(&m[1], "lt"), (&m[2], "lte"), (&m[3], "gt"), (&m[4], "gte")] { + assert_eq!(keys(member), [key, "t"], "{member}"); + assert_eq!(member[key], at_five, "{key}: the value's leaf operand"); + } + let between = term( + &keyset, + &plan, + "between_at", + json!({"path": calls, "low": "1", "high": "9"}), + ) + .await; + let Operand::Range { lo, hi } = between.operand() else { + panic!("between_at is a range") + }; + let json = |bytes: &[u8]| serde_json::from_slice::(bytes).unwrap(); + assert_eq!(keys(&m[5]), ["gte", "lte", "t"], "between: {}", m[5]); + assert_eq!(m[5]["gte"], json(lo), "between's low bound is between_at's"); + assert_eq!( + m[5]["lte"], + json(hi), + "between's high bound is between_at's" + ); + assert_ne!(m[5]["gte"], m[5]["lte"], "two values, two terms"); + } + + /// Every member's tokens are the `subtree` read of its full path at + /// the bound, the default bound when the input names none: one per + /// element below it, then the probe, the element at the bound + /// itself, which every member shares. + #[tokio::test] + async fn each_member_reads_its_full_path_and_ends_with_the_probe() { + let cipher = common::stack_cipher().await; + let keyset = cipher.default_keyset(); + let plan = contacts(); + let read = operand( + &keyset, + &plan, + input( + 3, + json!([ + {"member": "relationship", "op": "eq", "json": "\"Sibling\""}, + {"member": "address.postcode", "op": "gte", "json": "3000"}, + {"member": "[\"first name\"]", "op": "lt", "json": "\"M\""}, + ]), + ), + ) + .await; + let m = read["m"].as_array().unwrap(); + for (member, path) in [ + (&m[0], "$.contacts[*].relationship"), + (&m[1], "$.contacts[*].address.postcode"), + (&m[2], "$.contacts[*][\"first name\"]"), + ] { + assert_eq!( + member["t"], + tokens(&keyset, &plan, path, 3).await, + "{path}: the subtree read's tokens" + ); + assert_eq!(member["t"].as_array().unwrap().len(), 3 + 1, "{path}"); + } + assert_eq!(m[0]["t"][3], m[1]["t"][3], "the probe is the element's"); + assert_eq!(m[0]["t"][3], m[2]["t"][3], "the probe is the element's"); + assert_ne!(m[0]["t"][0], m[1]["t"][0], "a field's token is its own"); + + let defaulted = operand( + &keyset, + &plan, + json!({"path": "$.contacts[*]", "predicates": [ + {"member": "calls", "op": "gt", "json": "1"} + ]}), + ) + .await; + assert_eq!( + defaulted["m"][0]["t"], + tokens(&keyset, &plan, "$.contacts[*].calls", 64).await, + "no bound named is the subtree read's default" + ); + } + } + + mod given_documents { + use super::*; + + fn sibling_with_more_than_five_calls(bound: u32) -> Value { + input( + bound, + json!([ + {"member": "relationship", "op": "eq", "json": "\"Sibling\""}, + {"member": "calls", "op": "gt", "json": "5"}, + ]), + ) + } + + /// A Sibling with more than five calls. Two contacts each matching + /// half of it are not a match, one matching both is, and an array + /// longer than the bound is the probe's hit, not an answer. + #[tokio::test] + async fn one_element_must_satisfy_every_member() { + let cipher = common::stack_cipher().await; + let keyset = cipher.default_keyset(); + let plan = contacts(); + let operand = operand(&keyset, &plan, sibling_with_more_than_five_calls(2)).await; + for (document, want, why) in [ + ( + json!({"contacts": [ + {"relationship": "Sibling", "calls": 3}, + {"relationship": "Friend", "calls": 7} + ]}), + Some(false), + "each half in another element", + ), + ( + json!({"contacts": [ + {"relationship": "Friend", "calls": 9}, + {"relationship": "Sibling", "calls": 7} + ]}), + Some(true), + "both halves in the second element", + ), + ( + json!({"contacts": [{"relationship": "Sibling"}, {"calls": 8}]}), + Some(false), + "a field absent from an element satisfies none of its bounds", + ), + (json!({"contacts": []}), Some(false), "no element"), + (json!({"other": 1}), Some(false), "no array"), + ( + json!({"contacts": [ + {"relationship": "Friend", "calls": 1}, + {"relationship": "Friend", "calls": 1}, + {"relationship": "Sibling", "calls": 9} + ]}), + None, + "three elements past a bound of two: the probe names a node", + ), + ] { + let doc = seal(&keyset, &plan, &document).await; + assert_eq!(element_match(&doc, &operand), want, "{why}: {document}"); + } + } + + /// Against plaintext jsonpath over random arrays: within the bound + /// the operand answers `exists($.contacts[*] ? (@.relationship == + /// $r && @.calls $n))`, and past it the probe names a node. + #[test] + fn answers_as_the_jsonpath_over_the_plaintext() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let cipher = runtime.block_on(common::stack_cipher()); + let keyset = cipher.default_keyset(); + let plan = contacts(); + let bound = 3u32; + let names = ["Sibling", "Friend", "Parent"]; + let contact = (0..names.len(), proptest::option::of(0u64..8)); + let ops = prop::sample::select(vec!["eq", "lt", "lte", "gt", "gte", "between"]); + let strategy = ( + prop::collection::vec(contact, 0..=4), + 0..names.len(), + ops, + 0u64..8, + 0u64..8, + ); + let mut runner = TestRunner::new(Config { + cases: 48, + ..Config::default() + }); + runner + .run(&strategy, |(contacts, wanted, op, n, m)| { + let document = json!({"contacts": contacts + .iter() + .map(|(name, calls)| match calls { + Some(calls) => json!({"relationship": names[*name], "calls": calls}), + None => json!({"relationship": names[*name]}), + }) + .collect::>()}); + let (low, high) = (n.min(m), n.max(m)); + let calls = match op { + "between" => json!({ + "member": "calls", "op": op, + "low": low.to_string(), "high": high.to_string() + }), + _ => json!({"member": "calls", "op": op, "json": n.to_string()}), + }; + let predicates = json!([ + {"member": "relationship", "op": "eq", "json": json!(names[wanted]).to_string()}, + calls, + ]); + let (doc, operand) = runtime.block_on(async { + ( + seal(&keyset, &plan, &document).await, + operand(&keyset, &plan, input(bound, predicates)).await, + ) + }); + let holds = |calls: u64| match op { + "eq" => calls == n, + "lt" => calls < n, + "lte" => calls <= n, + "gt" => calls > n, + "gte" => calls >= n, + _ => (low..=high).contains(&calls), + }; + let within = contacts.len() <= usize::try_from(bound).unwrap(); + let want = within.then(|| { + contacts + .iter() + .any(|(name, calls)| *name == wanted && calls.is_some_and(holds)) + }); + let got = element_match(&doc, &operand); + if got != want { + return Err(TestCaseError::fail(format!( + "{document} {op} {n} {m}: got {got:?}, want {want:?}" + ))); + } + Ok(()) + }) + .unwrap(); + } + } + + mod given_an_input_it_cannot_answer { + use super::*; + + /// The refusals made before anything is derived, each by its + /// reason, or the index's term error by name. + #[test] + fn is_refused_with_its_reason() { + let plan = contacts(); + let check = |input: &Value| { + dynamic::check_query( + &plan, + "profile", + OpSpec::ElementMatch, + json_frame(input), + &Documents, + ) + }; + let reason = |input: &Value| -> String { + let error = check(input).unwrap_err(); + match (&error, error.reason()) { + (_, Some(reason)) => reason.as_str().to_owned(), + (dynamic::Error::Cipher(stack_encrypt::Error::Term(term)), None) => { + format!("{term:?}") + .split(['(', ' ', '{']) + .next() + .unwrap_or_default() + .to_owned() + } + (other, None) => panic!("neither a reason nor a term error: {other:?}"), + } + }; + let calls = |op: &str| json!([{"member": "calls", "op": op, "json": "1"}]); + let one = |predicate: Value| input(2, json!([predicate])); + let deep_member = |steps: usize| { + one(json!({"member": vec!["a"; steps].join("."), "op": "gt", "json": "1"})) + }; + for (input, want) in [ + (input(2, json!([])), "json_no_predicates"), + ( + json!({"path": "$.contacts", "predicates": calls("gt")}), + "json_path_no_wildcard", + ), + ( + json!({"path": "$.contacts[0]", "predicates": calls("gt")}), + "json_path_no_wildcard", + ), + ( + json!({"path": "$.contacts[1].tags[*]", "predicates": calls("gt")}), + "JsonOrderingAtPosition", + ), + ( + one(json!({"member": "note", "op": "eq", "json": "\"x\""})), + "JsonPathNotOrdered", + ), + ( + one(json!({"member": "address", "op": "gt", "json": "1"})), + "JsonPathNotOrdered", + ), + ( + one(json!({"member": "address[0].postcode", "op": "gt", "json": "1"})), + "JsonOrderingAtPosition", + ), + ( + one(json!({"member": "phones[*]", "op": "gt", "json": "1"})), + "json_path_wildcard", + ), + (input(2, calls("ne")), "json_unknown_predicate_op"), + (input(2, calls("between_at")), "json_unknown_predicate_op"), + (input(0, calls("gt")), "json_wildcard_bound_zero"), + (one(json!({"member": "calls", "op": "gt"})), "json_input"), + ( + one(json!({"member": "calls", "op": "eq", "low": "1", "high": "2"})), + "json_input", + ), + ( + one(json!({"member": "calls", "op": "gt", "json": "1", "low": "1"})), + "json_input", + ), + ( + one(json!({"member": "calls", "op": "gt", "json": "1", "high": "1"})), + "json_input", + ), + ( + one(json!({"member": "calls", "op": "between", "low": "1"})), + "json_input", + ), + ( + one(json!({"member": "calls", "op": "between", "high": "1"})), + "json_input", + ), + ( + one( + json!({"member": "calls", "op": "between", "json": "1", "low": "1", "high": "2"}), + ), + "json_input", + ), + ( + one(json!({"member": "calls", "op": "gt", "json": "1", "extra": 1})), + "json_input", + ), + (json!({"path": "$.contacts[*]"}), "json_input"), + ( + one(json!({"member": "calls", "op": "eq", "json": "true"})), + "JsonLeafNotOrdered", + ), + ( + one(json!({"member": "calls", "op": "between", "low": "1", "high": "null"})), + "JsonLeafNotOrdered", + ), + ( + one(json!({"member": "calls", "op": "between", "low": "[1]", "high": "2"})), + "JsonLeafNotOrdered", + ), + ( + one(json!({"member": "", "op": "gt", "json": "1"})), + "JsonPath", + ), + ( + json!({"path": "secret", "predicates": calls("gt")}), + "JsonPath", + ), + // The path has two steps, so a member of 125 is a full path + // of 127, as deep as a document goes, and one of 126 deeper. + (deep_member(125), "JsonPathNotOrdered"), + (deep_member(126), "JsonTooDeep"), + ] { + assert_eq!(reason(&input), want, "{input}"); + } + // A member that does not parse is refused at an offset into the + // member's own text, whichever form it starts with. + for (member, at) in [("a.b[", 4), ("[\"a\"].", 6), ("a..b", 2), ("[x]", 1)] { + let error = + check(&one(json!({"member": member, "op": "gt", "json": "1"}))).unwrap_err(); + let dynamic::Error::Cipher(stack_encrypt::Error::Term( + stack_encrypt::sem::TermError::JsonPath(error), + )) = error + else { + panic!("{member}: not a path error: {error:?}") + }; + assert_eq!(error.at, at, "{member}: {error}"); + } + for input in [ + input(1, calls("gte")), + input(2, calls("gte")), + input( + 2, + json!([ + {"member": "relationship", "op": "eq", "json": "\"a\""}, + {"member": "calls", "op": "between", "low": "1", "high": "2"}, + ]), + ), + ] { + assert!(check(&input).is_ok(), "{input}"); + } + } + } + + mod given_the_fields_type { + use super::*; + + /// A resolver whose `JsonSearch` names no element predicate domain: + /// what a host presents that has not been taught the operand. + struct Unnamed; + + impl TargetResolver for Unnamed { + fn targets(&self) -> Vec { + let mut targets = Documents.targets(); + targets[0].elements_query_sql_domain = None; + targets + } + fn encrypt<'a, K: 'static>( + &self, + name: &str, + options: Option<&TargetOptions>, + keyset: &'a KeysetCipher<'_, K>, + label: &Label, + plaintext: ValueFrame, + ) -> Result, K>, TargetError> { + Documents.encrypt(name, options, keyset, label, plaintext) + } + fn decrypt<'a, K: 'static>( + &self, + name: &str, + cipher: &'a StackCipher, + label: &Label, + bytes: &[u8], + ) -> Result, TargetError> { + Documents.decrypt(name, cipher, label, bytes) + } + fn query<'a, K: 'static>( + &self, + name: &str, + index: &IndexSpec, + keyset: &'a KeysetCipher<'_, K>, + label: &Label, + plaintext: ValueFrame, + ) -> Result, K>, TargetError> { + Documents.query(name, index, keyset, label, plaintext) + } + } + + fn names(plan: &record::Plan, resolver: &impl TargetResolver) -> Vec<&'static str> { + dynamic::field_operators(plan, "profile", resolver) + .unwrap() + .into_iter() + .map(|(op, _)| op.name()) + .collect() + } + + /// Answered, through the JSON index, on a field that orders a path + /// and whose type names the operand's domain; its input is + /// `path_predicates`. + #[test] + fn lists_it_when_a_path_is_ordered_and_the_domain_named() { + let plan = contacts(); + assert_eq!( + names(&plan, &Documents).last(), + Some(&"element_match"), + "the last operator, in OpSpec::ALL's order" + ); + let (_, index) = dynamic::field_operators(&plan, "profile", &Documents) + .unwrap() + .into_iter() + .find(|(op, _)| *op == OpSpec::ElementMatch) + .unwrap(); + assert!( + matches!(index, IndexSpec::Json(_)), + "the JSON index answers it: {index:?}" + ); + assert_eq!( + OpSpec::parse("element_match"), + Some(OpSpec::ElementMatch), + "the wire name parses" + ); + assert_eq!( + OpSpec::ElementMatch.input(), + "path_predicates", + "its input shape" + ); + assert_eq!( + OpSpec::ElementMatch.fidelity(), + OpSpec::BetweenAt.fidelity(), + "compared as the orderings compare" + ); + } + + #[test] + fn does_not_list_it_without_an_ordered_path() { + let names = names(&plan_of(json!("JsonSearch")), &Documents); + assert!( + !names.contains(&"element_match"), + "no ordering term to compare: {names:?}" + ); + } + + #[test] + fn does_not_answer_it_when_the_type_names_no_domain() { + let wire = json!({ + "profile": {"context": ["users", "profile"], "type": "json", "null": "sql", + "target": {"name": "JsonSearch", "options": {"ordered": ["$.a[*].b"]}}} + }); + let plan = record::plan_with(&dynamic::frame::serde::to_vec(&wire).unwrap(), &Unnamed) + .unwrap(); + let names = names(&plan, &Unnamed); + assert!(names.contains(&"lt_at"), "the orderings still: {names:?}"); + assert!(!names.contains(&"element_match"), "{names:?}"); + let error = dynamic::check_query( + &plan, + "profile", + OpSpec::ElementMatch, + json_frame(&json!({"path": "$.a[*]", "predicates": [ + {"member": "b", "op": "gt", "json": "1"} + ]})), + &Unnamed, + ) + .unwrap_err(); + assert_eq!( + error.reason(), + Some(Reason::OperatorNotAnswered), + "{error:?}" + ); + } + } +} From b7fa5a6006d3eee136e6e8e0aaab84a7d56c2d90 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 30 Oct 2026 00:18:02 +1100 Subject: [PATCH 3/3] test(eql): run the element predicate over Stack Encrypt documents in PostgreSQL Seals contact lists with the JSON index, derives `element_match` through a data plan over eql-bindings' target table, and queries the column: two contacts each holding half of the predicate are returned by the superset query (`@>` and an ordering over `json_tree_path_query`) and not by `json_tree_element_match`; one contact holding both is returned by both; a folded spelling ties under the element predicate and not under exact containment; a bound below the longest array raises; and `json_tree_subtree` equals the indexed read without `k` under every budget, the sealed-root fallback at `k = 0`. The rest of the SQL's contract is pinned over the same documents and operands: the operand domain's CHECK member by member, over a derived operand and its parts; one element must satisfy every member under the sealed root and the open one; each bound, `between` and `eq` compare as the entry operators do, and a field with no ordering term or no field satisfies none; a probe's hit raises, including a token whose node was moved to the element's selector; and `k` is the place of the token whose node a row sits in. EQL tests run over real encrypted data, and the SQLx suite has no way to produce a JSON tree document, so these live here. Refs #1440 --- .../eql/tests/encryption/tests/json_tree.rs | 4 + .../encryption/tests/json_tree/elements.rs | 1038 +++++++++++++++++ 2 files changed, 1042 insertions(+) create mode 100644 packages/eql/tests/encryption/tests/json_tree/elements.rs diff --git a/packages/eql/tests/encryption/tests/json_tree.rs b/packages/eql/tests/encryption/tests/json_tree.rs index 965613f98..a1d506cf5 100644 --- a/packages/eql/tests/encryption/tests/json_tree.rs +++ b/packages/eql/tests/encryption/tests/json_tree.rs @@ -516,6 +516,10 @@ mod generated; #[path = "json_tree/native_functions.rs"] mod native_functions; +/// The element-scoped predicate, through a data plan, against the column. +#[path = "json_tree/elements.rs"] +mod elements; + /// Needs an EQL v4 installation. Extraction by selector opens to the leaf; /// the subtree walk by a client-derived token returns exactly the nodes /// under it and reassembles; the array functions count and list elements; diff --git a/packages/eql/tests/encryption/tests/json_tree/elements.rs b/packages/eql/tests/encryption/tests/json_tree/elements.rs new file mode 100644 index 000000000..3b9c01abe --- /dev/null +++ b/packages/eql/tests/encryption/tests/json_tree/elements.rs @@ -0,0 +1,1038 @@ +//! The element-scoped predicate end to end: documents sealed by Stack +//! Encrypt's JSON index, the `element_match` operand derived through a data +//! plan over eql-bindings' target table (the path the Go guest takes), and +//! `eql_v4.json_tree_element_match` over the stored column. +//! +//! These tests pin: +//! +//! - the operand domain's CHECK, member by member, over a derived operand +//! and its parts; +//! - that one element must satisfy every member: two elements each matching +//! half of the predicate are not a match, where the component orderings +//! over `json_tree_path_query` are, under the sealed root and the open one; +//! - each bound's comparison, a field with no ordering term, and an absent +//! field; +//! - that a probe's hit raises: an array longer than the bound, and a token +//! that names a node at another selector than its member's; +//! - that `json_tree_subtree` is `json_tree_subtree_indexed` without `k`, +//! under every budget, and what `k` is, the sealed-root fallback's `0` +//! included. + +use super::*; + +use eql_bindings::encryption::targets::{self as eql_targets, Target}; +use stack_encrypt::dynamic::{ + self, record, OpSpec, Operand, TargetDescriptor, TargetError, TargetResolver, ValueFrame, +}; +use stack_encrypt::{KeysetCipher, Label, Pending, StackCipher}; + +/// eql-bindings' table as the engine's resolver, naming each type's leaf +/// and element predicate domains, as a host that answers `element_match` +/// describes it. +struct Eql; + +fn describe(target: &Target) -> TargetDescriptor { + let mut descriptor = TargetDescriptor::new( + target.name, + target.family, + target.suffix, + target.plaintext_kind(), + target.sql_domain, + target.indexes.iter().map(|key| (*key).to_owned()).collect(), + target.query.map(str::to_owned), + target.query_sql_domain.map(str::to_owned), + target.producible, + target.reason.map(str::to_owned), + ); + if let Some(domain) = target.leaf_query_sql_domain() { + descriptor = descriptor.with_leaf_query_sql_domain(domain); + } + if let Some(domain) = target.elements_query_sql_domain() { + descriptor = descriptor.with_elements_query_sql_domain(domain); + } + descriptor +} + +impl TargetResolver for Eql { + fn targets(&self) -> Vec { + eql_targets::targets().iter().map(describe).collect() + } + fn encrypt<'a, K: 'static>( + &self, + name: &str, + options: Option<&dynamic::TargetOptions>, + keyset: &'a KeysetCipher<'_, K>, + label: &Label, + plaintext: ValueFrame, + ) -> Result, K>, TargetError> { + eql_targets::encrypt_with(name, options, keyset, label, plaintext) + .map_err(|error| TargetError::Other(Box::new(error))) + } + fn decrypt<'a, K: 'static>( + &self, + name: &str, + cipher: &'a StackCipher, + label: &Label, + stored: &[u8], + ) -> Result, TargetError> { + eql_targets::decrypt(name, cipher, label, stored) + .map_err(|error| TargetError::Other(Box::new(error))) + } + fn query<'a, K: 'static>( + &self, + name: &str, + index: &stack_encrypt::target::IndexSpec, + keyset: &'a KeysetCipher<'_, K>, + label: &Label, + plaintext: ValueFrame, + ) -> Result, K>, TargetError> { + eql_targets::query(name, index, keyset, label, plaintext) + .map_err(|error| TargetError::Other(Box::new(error))) + } +} + +const RELATIONSHIP: &str = "$.extra.emergency_contacts[*].relationship"; +const CALLS: &str = "$.extra.emergency_contacts[*].previous_call_count"; + +/// The documents' index options: each contact's relationship and call +/// count carry ordering terms. +fn contact_options() -> JsonOptions { + JsonOptions::default() + .ordering([RELATIONSHIP, CALLS]) + .unwrap() +} + +/// `profile` under `users/profile`, the column the documents are sealed +/// under, naming `JsonSearch` with [`contact_options`]. +fn contact_plan() -> record::Plan { + let wire = json!({ + "profile": { + "context": ["users", "profile"], + "type": "json", + "null": "sql", + "target": {"name": "JsonSearch", "options": {"ordered": [RELATIONSHIP, CALLS]}} + } + }); + record::plan_with(&dynamic::frame::serde::to_vec(&wire).unwrap(), &Eql).unwrap() +} + +fn frame_of(value: &Value) -> ValueFrame { + ValueFrame::from_bytes(&dynamic::frame::serde::to_vec(value).unwrap()).unwrap() +} + +/// The operand of `operator` on `profile` for `input`, as the SQL binds +/// it: an EQL operand's JSON text, or a list of tokens. +fn derive( + runtime: &tokio::runtime::Runtime, + keyset: &KeysetCipher<'_, common::Observed>, + plan: &record::Plan, + operator: &str, + input: Value, +) -> Operand { + let term = wait( + runtime, + dynamic::query_term( + keyset, + plan, + "profile", + OpSpec::parse(operator).unwrap(), + frame_of(&input), + &Eql, + ) + .unwrap(), + ) + .unwrap(); + term.operand().clone() +} + +fn eql_text(operand: &Operand) -> String { + match operand { + Operand::One(bytes) => String::from_utf8(bytes.clone()).unwrap(), + other => panic!("one EQL operand, got {other:?}"), + } +} + +/// The contacts array's elements, which every `element_match` here reads. +const ELEMENTS: &str = "$.extra.emergency_contacts[*]"; + +/// SQLSTATE `check_violation`, a value its domain's CHECK refuses. +const CHECK_VIOLATION: &str = "23514"; +/// SQLSTATE `invalid_parameter_value`, which a probe's hit raises with. +const INVALID_PARAMETER_VALUE: &str = "22023"; + +/// [`contact_options`] with the root sealed or open. +fn layout_options(sealed: bool) -> JsonOptions { + let mut options = contact_options(); + options.sealed_root = sealed; + options +} + +/// [`contact_plan`] with the root sealed or open. +fn layout_plan(sealed: bool) -> record::Plan { + let wire = json!({ + "profile": { + "context": ["users", "profile"], + "type": "json", + "null": "sql", + "target": {"name": "JsonSearch", "options": { + "sealed_root": sealed, + "ordered": [RELATIONSHIP, CALLS] + }} + } + }); + record::plan_with(&dynamic::frame::serde::to_vec(&wire).unwrap(), &Eql).unwrap() +} + +/// A contact with each field present when given. +fn contact(relationship: Option<&str>, calls: Option) -> Value { + let mut fields = Map::new(); + if let Some(relationship) = relationship { + let _ = fields.insert("relationship".into(), json!(relationship)); + } + if let Some(calls) = calls { + let _ = fields.insert("previous_call_count".into(), json!(calls)); + } + Value::Object(fields) +} + +/// A document holding `list` as its contacts. +fn contacts_document(list: Vec) -> Value { + json!({"name": "x", "extra": {"emergency_contacts": list}}) +} + +/// The `element_match` operand over the contacts at `bound`, as JSON text. +fn element_match( + runtime: &tokio::runtime::Runtime, + keyset: &KeysetCipher<'_, common::Observed>, + plan: &record::Plan, + bound: u32, + predicates: Value, +) -> String { + eql_text(&derive( + runtime, + keyset, + plan, + "element_match", + json!({"path": ELEMENTS, "wildcard_bound": bound, "predicates": predicates}), + )) +} + +/// "relationship is Sibling and more than five calls", read to `bound`. +fn sibling_with_more_than_five_calls( + runtime: &tokio::runtime::Runtime, + keyset: &KeysetCipher<'_, common::Observed>, + plan: &record::Plan, + bound: u32, +) -> String { + element_match( + runtime, + keyset, + plan, + bound, + json!([ + {"member": "relationship", "op": "eq", "json": "\"Sibling\""}, + {"member": "previous_call_count", "op": "gt", "json": "5"} + ]), + ) +} + +/// The ids `json_tree_element_match` holds of in `table`, for `operand`. +fn matching(db: &mut postgres::Client, table: &str, operand: &str) -> Vec { + db.query( + &format!( + "SELECT id FROM {table} \ + WHERE eql_v4.json_tree_element_match(doc, $1::text::jsonb::eql_v4.query_json_elements) ORDER BY id" + ), + &[&operand], + ) + .unwrap_or_else(|error| panic!("{table}: {error}")) + .iter() + .map(|row| row.get(0)) + .collect() +} + +/// The SQLSTATE a statement fails with, or `None` if it succeeds. +fn sqlstate( + db: &mut postgres::Client, + sql: &str, + params: &[&(dyn postgres::types::ToSql + Sync)], +) -> Option { + match db.query(sql, params) { + Ok(_) => None, + Err(error) => match error.as_db_error() { + Some(e) => Some(e.code().code().to_owned()), + None => panic!("{sql}: not a database error: {error}"), + }, + } +} + +/// The one token of a read of the concrete `path` under `options`. +fn token( + runtime: &tokio::runtime::Runtime, + keyset: &KeysetCipher<'_, common::Observed>, + options: &JsonOptions, + path: &str, +) -> String { + let key = wait( + runtime, + keyset.json_subtree_key(&JsonPath::parse(path).unwrap(), options, column()), + ) + .unwrap(); + let tokens = key.tokens(); + assert_eq!(tokens.len(), 1, "{path} names one node"); + tokens[0].clone() +} + +/// Needs an EQL v4 installation. The customer's predicate, a Sibling with +/// more than five calls in one contact: the element predicate answers it, +/// where `@>` and an ordering over `json_tree_path_query` answer "some +/// contact is a Sibling and some has more than five calls". A row with a +/// contact past the bound fails the query. `json_tree_subtree` is the +/// indexed read without `k`, and `k` pairs a contact's two fields. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_an_element_predicate_holds_of_one_element() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = contact_plan(); + let mut db = connect(); + let contacts = |list: Value| json!({"name": "x", "extra": {"emergency_contacts": list}}); + let documents = [ + // Each contact holds one half of the predicate. + contacts(json!([ + {"relationship": "Sibling", "previous_call_count": 3}, + {"relationship": "Friend", "previous_call_count": 7} + ])), + // One contact holds both. + contacts(json!([ + {"relationship": "Friend", "previous_call_count": 9}, + {"relationship": "Sibling", "previous_call_count": 7} + ])), + // Neither. + contacts(json!([{"relationship": "Friend", "previous_call_count": 9}])), + // No contacts. + json!({"name": "y"}), + // Folded equality: "sibling" ties with "Sibling". + contacts(json!([{"relationship": "sibling", "previous_call_count": 6}])), + ]; + store( + &mut db, + &runtime, + &cipher, + "json_tree_elements", + &contact_options(), + &documents, + ); + let ids = |db: &mut postgres::Client, + sql: &str, + params: &[&(dyn postgres::types::ToSql + Sync)]| + -> Vec { + db.query(sql, params) + .unwrap_or_else(|error| panic!("{sql}: {error}")) + .iter() + .map(|row| row.get(0)) + .collect() + }; + + let predicate = |bound: u32| { + eql_text(&derive( + &runtime, + &keyset, + &plan, + "element_match", + json!({"path": "$.extra.emergency_contacts[*]", "wildcard_bound": bound, "predicates": [ + {"member": "relationship", "op": "eq", "json": "\"Sibling\""}, + {"member": "previous_call_count", "op": "gt", "json": "5"} + ]}), + )) + }; + let element_scoped = "SELECT id FROM json_tree_elements \ + WHERE eql_v4.json_tree_element_match(doc, $1::text::jsonb::eql_v4.query_json_elements) ORDER BY id"; + assert_eq!( + ids(&mut db, element_scoped, &[&predicate(4)]), + [2, 5], + "one contact a Sibling with more than five calls, the folded spelling included" + ); + + // The superset: containment on the relationship, an ordering over every + // call count. + let needle = eql_text(&derive( + &runtime, + &keyset, + &plan, + "eq_at", + json!({"path": RELATIONSHIP, "json": "\"Sibling\""}), + )); + let Operand::Text(calls_selector) = + derive(&runtime, &keyset, &plan, "selector", json!({"path": CALLS})) + else { + panic!("a selector is text") + }; + let five = eql_text(&derive( + &runtime, + &keyset, + &plan, + "gt_at", + json!({"path": CALLS, "json": "5"}), + )); + let superset = "SELECT id FROM json_tree_elements \ + WHERE doc @> $1::text::jsonb::eql_v4.query_json \ + AND EXISTS (SELECT 1 FROM eql_v4.json_tree_path_query(doc, $2::text) AS e \ + WHERE e > $3::text::jsonb::eql_v4.query_json_leaf) ORDER BY id"; + assert_eq!( + ids(&mut db, superset, &[&needle, &calls_selector, &five]), + [1, 2], + "some contact a Sibling and some over five: row 1's halves answer it, and \ + containment is exact, so the folded spelling does not" + ); + + // A bound below the longest array: the probe names the element at it. + let error = db.query(element_scoped, &[&predicate(1)]).unwrap_err(); + assert_eq!( + error.code(), + Some(&postgres::error::SqlState::INVALID_PARAMETER_VALUE), + "{error}" + ); + let message = error + .as_db_error() + .map(|db| db.message()) + .unwrap_or_default(); + assert!( + message.contains("longer than the bound"), + "the message says why: {message}" + ); + // At exactly the longest array's length, it answers. + assert_eq!(ids(&mut db, element_scoped, &[&predicate(2)]), [2, 5]); + + // `json_tree_subtree` is the indexed read without `k`, and the `k`-th + // token of each field's read names a field of one contact. + let tokens = |path: &str| match derive( + &runtime, + &keyset, + &plan, + "subtree", + json!({"path": path, "wildcard_bound": 2}), + ) { + Operand::TextList(tokens) => tokens, + other => panic!("{other:?}"), + }; + let (relationships, counts) = (tokens(RELATIONSHIP), tokens(CALLS)); + for budget in [0, 1, 64] { + for id in 1..=5 { + let plain: Vec = db + .query( + "SELECT r::jsonb::text FROM json_tree_elements, eql_v4.json_tree_subtree(doc, $1::text[], $2::int) AS r WHERE id = $3", + &[&relationships, &budget, &id], + ) + .unwrap() + .iter() + .map(|row| row.get(0)) + .collect(); + let indexed: Vec = db + .query( + "SELECT r.entry::jsonb::text FROM json_tree_elements, eql_v4.json_tree_subtree_indexed(doc, $1::text[], $2::int) AS r WHERE id = $3", + &[&relationships, &budget, &id], + ) + .unwrap() + .iter() + .map(|row| row.get(0)) + .collect(); + assert_eq!(plain, indexed, "row {id}, budget {budget}"); + } + } + let paired: Vec<(i32, i32)> = db + .query( + "SELECT r.k, c.k FROM json_tree_elements, + eql_v4.json_tree_subtree_indexed(doc, $1::text[], 0) AS r + JOIN eql_v4.json_tree_subtree_indexed(doc, $2::text[], 0) AS c ON c.k = r.k + WHERE id = 2 ORDER BY r.k", + &[&relationships, &counts], + ) + .unwrap() + .iter() + .map(|row| (row.get(0), row.get(1))) + .collect(); + assert_eq!(paired, [(1, 1), (2, 2)], "one pair per contact, by k"); + let fallback: Vec = db + .query( + "SELECT r.k FROM json_tree_elements, eql_v4.json_tree_subtree_indexed(doc, $1::text[], 1) AS r WHERE id = 2", + &[&relationships], + ) + .unwrap() + .iter() + .map(|row| row.get(0)) + .collect(); + assert_eq!( + fallback, + [0], + "past the budget, the sealed root alone, at k = 0" + ); + + // A member whose full path the field does not order is refused before + // anything is derived. + let error = dynamic::check_query( + &plan, + "profile", + OpSpec::ElementMatch, + frame_of( + &json!({"path": "$.extra.emergency_contacts[*]", "predicates": [ + {"member": "name", "op": "eq", "json": "\"x\""} + ]}), + ), + &Eql, + ) + .unwrap_err(); + assert!( + matches!( + error, + dynamic::Error::Cipher(stack_encrypt::Error::Term( + stack_encrypt::sem::TermError::JsonPathNotOrdered + )) + ), + "{error:?}" + ); +} + +/// Needs an EQL v4 installation. The operand domain's CHECK takes a list +/// of members, each tokens and one or two bounds at one selector, and every +/// member's tokens as many as every other's. The values are a derived +/// operand and its parts: two of its tokens, and the leaf operands it holds +/// at each field. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_the_elements_operand_domain_checks_each_member() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = contact_plan(); + let mut db = connect(); + let derived = sibling_with_more_than_five_calls(&runtime, &keyset, &plan, 1); + let operand: Value = serde_json::from_str(&derived).unwrap(); + let (relationship, calls) = (&operand["m"][0], &operand["m"][1]); + // The relationship's leaf, and the call count's: two selectors. + let (leaf, other) = (&relationship["gte"], &calls["gt"]); + // The read at bound 1 is one element and the probe: two tokens. + let (a, b) = (&relationship["t"][0], &relationship["t"][1]); + assert!(a.is_string() && b.is_string() && a != b); + let without = |key: &str| { + let mut bound = leaf.clone(); + let _ = bound.as_object_mut().unwrap().remove(key); + bound + }; + let mut leaf_and_more = leaf.clone(); + let _ = leaf_and_more + .as_object_mut() + .unwrap() + .insert("x".into(), json!(1)); + + let valid = [ + operand.clone(), + json!({"m": [{"t": [a, b], "gt": leaf}]}), + json!({"m": [{"t": [a], "gte": leaf, "lte": leaf}]}), + json!({"m": [{"t": [a], "gt": leaf, "lt": leaf}, {"t": [b], "lte": other}]}), + ]; + let invalid = [ + ("not an object", json!([])), + ("no members", json!({"m": []})), + ("members not a list", json!({"m": {}})), + ("no m", json!({})), + ( + "another key", + json!({"m": [{"t": [a], "gt": leaf}], "x": 1}), + ), + ("a member not an object", json!({"m": [a]})), + ("no tokens", json!({"m": [{"gt": leaf}]})), + ("empty tokens", json!({"m": [{"t": [], "gt": leaf}]})), + ("a token not text", json!({"m": [{"t": [1], "gt": leaf}]})), + ("tokens not a list", json!({"m": [{"t": a, "gt": leaf}]})), + ("no bound", json!({"m": [{"t": [a]}]})), + ( + "two lower bounds", + json!({"m": [{"t": [a], "gt": leaf, "gte": leaf}]}), + ), + ( + "two upper bounds", + json!({"m": [{"t": [a], "lt": leaf, "lte": leaf}]}), + ), + ( + "a bound by another name", + json!({"m": [{"t": [a], "eq": leaf}]}), + ), + ( + "a bound with no term", + json!({"m": [{"t": [a], "gt": without("op")}]}), + ), + ( + "a bound with no selector", + json!({"m": [{"t": [a], "gt": without("s")}]}), + ), + ( + "a bound with another key", + json!({"m": [{"t": [a], "gt": leaf_and_more}]}), + ), + ( + "a bound not an object", + json!({"m": [{"t": [a], "gt": leaf["s"]}]}), + ), + ( + "bounds at two selectors", + json!({"m": [{"t": [a], "gt": leaf, "lt": other}]}), + ), + ( + "members of two widths", + json!({"m": [{"t": [a], "gt": leaf}, {"t": [a, b], "gt": other}]}), + ), + ]; + let cast = "SELECT $1::text::jsonb::eql_v4.query_json_elements"; + for value in &valid { + assert_eq!( + sqlstate(&mut db, cast, &[&value.to_string()]), + None, + "valid: {value}" + ); + } + for (why, value) in &invalid { + assert_eq!( + sqlstate(&mut db, cast, &[&value.to_string()]).as_deref(), + Some(CHECK_VIOLATION), + "{why}: {value}" + ); + } +} + +/// Needs an EQL v4 installation. A Sibling with more than five calls, as +/// the element predicate and as the component orderings, under the sealed +/// root and the open one: only a row where one contact is both answers the +/// first, where the second also answers a row whose two contacts each hold +/// half, or whose fields sit in different contacts. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_one_element_must_satisfy_every_member() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let mut db = connect(); + let documents = [ + contacts_document(vec![ + contact(Some("Sibling"), Some(3)), + contact(Some("Friend"), Some(7)), + ]), + contacts_document(vec![ + contact(Some("Friend"), Some(9)), + contact(Some("Sibling"), Some(7)), + ]), + contacts_document(vec![contact(Some("Friend"), Some(9))]), + contacts_document(vec![contact(Some("Sibling"), None), contact(None, Some(8))]), + contacts_document(vec![]), + contacts_document(vec![contact(Some("Sibling"), Some(6))]), + ]; + for sealed in [true, false] { + let table = if sealed { + "sealed_contacts" + } else { + "open_contacts" + }; + let plan = layout_plan(sealed); + store( + &mut db, + &runtime, + &cipher, + table, + &layout_options(sealed), + &documents, + ); + let predicate = sibling_with_more_than_five_calls(&runtime, &keyset, &plan, 2); + assert_eq!( + matching(&mut db, table, &predicate), + [2, 6], + "{table}: one contact both a Sibling and over five calls" + ); + + let selector = |path: &str| match derive( + &runtime, + &keyset, + &plan, + "selector", + json!({"path": path}), + ) { + Operand::Text(selector) => selector, + other => panic!("a selector is text, got {other:?}"), + }; + let leaf = |operator: &str, path: &str, value: &str| { + eql_text(&derive( + &runtime, + &keyset, + &plan, + operator, + json!({"path": path, "json": value}), + )) + }; + let superset: Vec = db + .query( + &format!( + "SELECT id FROM {table} + WHERE EXISTS (SELECT 1 FROM eql_v4.json_tree_path_query(doc, $1::text) AS e + WHERE e >= $2::text::jsonb::eql_v4.query_json_leaf + AND e <= $3::text::jsonb::eql_v4.query_json_leaf) + AND EXISTS (SELECT 1 FROM eql_v4.json_tree_path_query(doc, $4::text) AS e + WHERE e > $5::text::jsonb::eql_v4.query_json_leaf) + ORDER BY id" + ), + &[ + &selector(RELATIONSHIP), + &leaf("gte_at", RELATIONSHIP, "\"Sibling\""), + &leaf("lte_at", RELATIONSHIP, "\"Sibling\""), + &selector(CALLS), + &leaf("gt_at", CALLS, "5"), + ], + ) + .unwrap() + .iter() + .map(|row| row.get(0)) + .collect(); + assert_eq!( + superset, + [1, 2, 4, 6], + "{table}: the component orderings, some contact a Sibling and some over five" + ); + + let nulls: Option = db + .query_one( + "SELECT eql_v4.json_tree_element_match(NULL, $1::text::jsonb::eql_v4.query_json_elements)", + &[&predicate], + ) + .unwrap() + .get(0); + assert_eq!(nulls, None, "NULL in, NULL out"); + } +} + +/// Needs an EQL v4 installation. Each bound compares as its operator does +/// on an entry, `between` and `eq` at one value are equality, and a field +/// with no ordering term, or no field at all, satisfies no bound. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_each_bound_compares_as_its_operator() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = contact_plan(); + let mut db = connect(); + store( + &mut db, + &runtime, + &cipher, + "five_calls", + &contact_options(), + &[contacts_document(vec![contact(Some("Sibling"), Some(5))])], + ); + let calls = |op: &str, value: i64| json!({"member": "previous_call_count", "op": op, "json": value.to_string()}); + let between = |low: i64, high: i64| { + json!({"member": "previous_call_count", "op": "between", + "low": low.to_string(), "high": high.to_string()}) + }; + for (predicates, want) in [ + (json!([calls("gt", 5)]), false), + (json!([calls("gt", 4)]), true), + (json!([calls("gte", 5)]), true), + (json!([calls("gte", 6)]), false), + (json!([calls("lt", 5)]), false), + (json!([calls("lt", 6)]), true), + (json!([calls("lte", 5)]), true), + (json!([calls("lte", 4)]), false), + (json!([between(5, 5)]), true), + (json!([between(4, 4)]), false), + (json!([calls("eq", 5)]), true), + (json!([calls("eq", 4)]), false), + (json!([calls("gt", 4), calls("lt", 6)]), true), + (json!([calls("gt", 5), calls("lt", 6)]), false), + (json!([calls("gt", 4), calls("lt", 5)]), false), + ] { + let operand = element_match(&runtime, &keyset, &plan, 1, predicates.clone()); + let got = !matching(&mut db, "five_calls", &operand).is_empty(); + assert_eq!(got, want, "{predicates}"); + } + + // A call count with no ordering term, in a column that orders only the + // relationship; and a contact with no call count. + let relationship_only = JsonOptions::default().ordering([RELATIONSHIP]).unwrap(); + store( + &mut db, + &runtime, + &cipher, + "unordered_calls", + &relationship_only, + &[contacts_document(vec![contact(Some("Sibling"), Some(5))])], + ); + store( + &mut db, + &runtime, + &cipher, + "no_calls", + &contact_options(), + &[contacts_document(vec![contact(Some("Sibling"), None)])], + ); + let any_calls = element_match(&runtime, &keyset, &plan, 1, json!([calls("gte", 0)])); + // The member's first token names the unordered call count, which holds + // no term. + let member: Value = serde_json::from_str(&any_calls).unwrap(); + let tokens: Vec = serde_json::from_value(member["m"][0]["t"].clone()).unwrap(); + let found: Vec = db + .query( + "SELECT r.entry::jsonb ? 'op' FROM unordered_calls, + eql_v4.json_tree_subtree_indexed(doc, $1::text[], 0) AS r WHERE r.k = 1", + &[&tokens], + ) + .unwrap() + .iter() + .map(|row| row.get(0)) + .collect(); + assert_eq!(found, [false], "the call count is found, with no term"); + for table in ["unordered_calls", "no_calls"] { + assert_eq!( + matching(&mut db, table, &any_calls), + Vec::::new(), + "{table}: no term, or no field: no bound holds" + ); + } +} + +/// Needs an EQL v4 installation. A member's token that names a node at +/// another selector than its bounds' is the probe's hit: the array is +/// longer than the bound, and the predicate raises rather than answer over +/// part of it. An array exactly as long as the bound does not. A node at +/// the element's selector raises wherever its token sits in the member's +/// list: here, a document whose call count's entry is moved to the +/// element's selector, which the member's first token then names. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_a_probe_hit_raises() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = contact_plan(); + let mut db = connect(); + let predicate = sibling_with_more_than_five_calls(&runtime, &keyset, &plan, 2); + let two_contacts = contacts_document(vec![ + contact(Some("Friend"), Some(1)), + contact(Some("Sibling"), Some(9)), + ]); + store( + &mut db, + &runtime, + &cipher, + "within_bound", + &contact_options(), + std::slice::from_ref(&two_contacts), + ); + assert_eq!( + matching(&mut db, "within_bound", &predicate), + [1], + "within the bound" + ); + + // A third contact: the element at the bound is a node, at the + // element's selector, which the probe names. + store( + &mut db, + &runtime, + &cipher, + "past_bound", + &contact_options(), + &[ + two_contacts, + contacts_document(vec![ + contact(Some("Sibling"), Some(9)), + contact(Some("Friend"), Some(1)), + contact(Some("Friend"), Some(1)), + ]), + ], + ); + let query = "SELECT id FROM past_bound \ + WHERE eql_v4.json_tree_element_match(doc, $1::text::jsonb::eql_v4.query_json_elements) ORDER BY id"; + let error = db.query(query, &[&predicate]).unwrap_err(); + assert_eq!( + error.code(), + Some(&postgres::error::SqlState::INVALID_PARAMETER_VALUE), + "a row past the bound fails the query, though an element within it matches: {error}" + ); + let message = error + .as_db_error() + .map(|db| db.message()) + .unwrap_or_default(); + assert!( + message.contains("longer than the bound"), + "the message says why: {message}" + ); + + // A node at the element's selector, named by a member's first token. + store( + &mut db, + &runtime, + &cipher, + "forged_selector", + &contact_options(), + &[contacts_document(vec![contact(Some("Sibling"), Some(9))])], + ); + let only_calls = element_match( + &runtime, + &keyset, + &plan, + 2, + json!([{"member": "previous_call_count", "op": "gt", "json": "5"}]), + ); + let member: Value = serde_json::from_str(&only_calls).unwrap(); + let first = member["m"][0]["t"][0].as_str().unwrap().to_owned(); + let Operand::Text(element_selector) = derive( + &runtime, + &keyset, + &plan, + "selector", + json!({"path": ELEMENTS}), + ) else { + panic!("a selector is text") + }; + let moved = db + .execute( + "UPDATE forged_selector SET doc = jsonb_set(doc::jsonb, ARRAY['sv', ( + SELECT (e.o - 1)::text + FROM jsonb_array_elements(doc::jsonb -> 'sv') WITH ORDINALITY AS e(v, o) + WHERE e.v ->> 'x' = eql_v4_internal.json_tree_self_tag( + eql_v4_internal.json_tree_iv(doc::jsonb ->> 'h'), decode($1::text, 'base64'))), + 's'], to_jsonb($2::text))::public.eql_v4_json_search", + &[&first, &element_selector], + ) + .unwrap(); + assert_eq!(moved, 1); + assert_eq!( + sqlstate( + &mut db, + "SELECT eql_v4.json_tree_element_match(doc, $1::text::jsonb::eql_v4.query_json_elements) FROM forged_selector", + &[&only_calls], + ) + .as_deref(), + Some(INVALID_PARAMETER_VALUE), + "a node at the element's selector is never a member's field" + ); +} + +/// Needs an EQL v4 installation. `json_tree_subtree` returns what +/// `json_tree_subtree_indexed` does, without `k`, under every budget; every +/// row carries the place of the token that found its node, the rows +/// beneath that node included; and the sealed root's fallback comes back +/// alone with `k = 0`. The tokens name nothing (`$.missing`), an object +/// with one member (`$.obj`), the two contacts' relationships, and the +/// element past the last, which the document does not hold. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_the_indexed_read_is_the_subtree_read_with_k() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let mut db = connect(); + let document = json!({ + "obj": {"child": "c"}, + "extra": {"emergency_contacts": [ + {"relationship": "Sibling", "previous_call_count": 1}, + {"relationship": "Friend", "previous_call_count": 2} + ]} + }); + for sealed in [false, true] { + let table = if sealed { "sealed_read" } else { "open_read" }; + let options = layout_options(sealed); + store( + &mut db, + &runtime, + &cipher, + table, + &options, + std::slice::from_ref(&document), + ); + let tokens: Vec = [ + "$.missing", + "$.obj", + "$.extra.emergency_contacts[1].relationship", + "$.extra.emergency_contacts[0].relationship", + "$.extra.emergency_contacts[2]", + ] + .iter() + .map(|path| token(&runtime, &keyset, &options, path)) + .collect(); + for budget in [0, 1, 2, 3, 64] { + let read: Vec = db + .query( + &format!( + "SELECT r::jsonb::text FROM {table}, eql_v4.json_tree_subtree(doc, $1::text[], $2::int) AS r" + ), + &[&tokens, &budget], + ) + .unwrap() + .iter() + .map(|row| serde_json::from_str(&row.get::<_, String>(0)).unwrap()) + .collect(); + let indexed: Vec<(i32, Value)> = db + .query( + &format!( + "SELECT r.k, r.entry::jsonb::text FROM {table}, eql_v4.json_tree_subtree_indexed(doc, $1::text[], $2::int) AS r" + ), + &[&tokens, &budget], + ) + .unwrap() + .iter() + .map(|row| { + ( + row.get(0), + serde_json::from_str(&row.get::<_, String>(1)).unwrap(), + ) + }) + .collect(); + let entries: Vec = indexed.iter().map(|(_, entry)| entry.clone()).collect(); + assert_eq!( + read, entries, + "{table}, budget {budget}: the same rows in order" + ); + let ks: Vec = indexed.iter().map(|(k, _)| *k).collect(); + let mut sorted = ks.clone(); + sorted.sort_unstable(); + assert_eq!(ks, sorted, "{table}, budget {budget}: ordered by k"); + // Four nodes, at d = 0 and 1, from four of the five tokens. + if sealed && budget > 0 && budget < 4 { + assert_eq!(indexed.len(), 1, "{table}, budget {budget}: the root alone"); + assert_eq!(ks, [0], "the root is found by no token"); + assert_eq!(indexed[0].1["j"], 0, "the root entry"); + } else { + // Within one `k` the rows come in the order the entries + // are stored, which sealing draws at random. + let mut found: Vec<(i32, i64)> = indexed + .iter() + .map(|(k, entry)| (*k, entry["d"].as_i64().unwrap())) + .collect(); + found.sort_unstable(); + assert_eq!( + found, + [(2, 0), (2, 1), (3, 0), (4, 0)], + "{table}, budget {budget}: k is the place of the token whose node a row sits in" + ); + } + } + } +}