diff --git a/.changeset/eql-json-tree-match.md b/.changeset/eql-json-tree-match.md new file mode 100644 index 000000000..5e9eb11d9 --- /dev/null +++ b/.changeset/eql-json-tree-match.md @@ -0,0 +1,9 @@ +--- +"@cipherstash/eql": minor +--- + +EQL v4 `json_tree`: a whole filter as one parameter, `eql_v4.json_tree_match(col, $1::eql_v4.query_json_predicate)` and `col @@ $1::eql_v4.query_json_predicate` (#1441). + +An application that compiles a filter language into SQL targets `jsonb_path_match(col, $1, $2)` on a plaintext column: one predicate, one set of values. On a `public.eql_v4_json_search` column each question is its own operator, so the same filter had to be assembled by hand, one operand per leaf. The new domain `eql_v4.query_json_predicate` is a boolean tree (`and`, `or`, `not`) whose leaves are the operands those operators already take (a `contains` needle, an `exists` selector, `cmp` ordering bounds and an `elements` predicate), each checked by its own domain's rule, and `eql_v4.json_tree_match` evaluates it with SQL's three-valued logic, the answer the hand-written WHERE gives. Stack Encrypt's `match` query operator compiles a jsonpath predicate in the subset the terms can answer into the tree. The tree is evaluated inside a function, so it filters a scan and engages no index; the operators written out per leaf are the fast path. The operand must carry its type: an untyped operand on the right of `@@` is refused as ambiguous, and `@@` with a `jsonpath` is still blocked. + +`eql_v4.json_tree_element_match` now reads an opened document through `eql_v4_internal.json_tree_element_match_opened`, which `json_tree_match` shares; its answers are unchanged. diff --git a/languages/golang/encrypt/eql/eql_gen.go b/languages/golang/encrypt/eql/eql_gen.go index 5094e8087..d29569c59 100644 --- a/languages/golang/encrypt/eql/eql_gen.go +++ b/languages/golang/encrypt/eql/eql_gen.go @@ -2252,6 +2252,29 @@ func (v *JSONSearchElementsQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } +// JSONSearchPredicateQuery is the json_tree family's compiled predicate, a boolean tree of the other operands, which eql_v4.json_tree_match and `@@` evaluate: the value eql_v4.query_json_predicate holds. +type JSONSearchPredicateQuery []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v JSONSearchPredicateQuery) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a eql_v4.query_json_predicate column. +func (v *JSONSearchPredicateQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "JSONSearchPredicateQuery") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v JSONSearchPredicateQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "JSONSearchPredicateQuery") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *JSONSearchPredicateQuery) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} + // JSONEntry is the json_tree family's entry: one node of a document, as `->` and the walks return it: the value public.eql_v4_json_entry holds. type JSONEntry []byte diff --git a/languages/golang/encrypt/guest/src/targets.rs b/languages/golang/encrypt/guest/src/targets.rs index 0754a9908..4dc74b805 100644 --- a/languages/golang/encrypt/guest/src/targets.rs +++ b/languages/golang/encrypt/guest/src/targets.rs @@ -268,6 +268,35 @@ mod tests { )); } + /// The JSON document's query domains this guest names, and so the JSON + /// operators the Go SDK is offered: the leaf operand, for the orderings + /// at a path, and not the element-scoped or compiled predicate, whose + /// operators (`element_match`, `match`) the Go side does not parse yet. + /// Naming one is a change to the Go SDK's operator list: update this + /// test with the Go side that takes the operator. + #[cfg(feature = "eql")] + #[test] + fn a_document_names_the_leaf_domain_and_not_the_predicate_domains() { + let targets = resolver().targets(); + let document = targets + .iter() + .find(|t| t.name == "JsonSearch") + .expect("JsonSearch"); + assert_eq!( + document.leaf_query_sql_domain.as_deref(), + Some("eql_v4.query_json_leaf") + ); + assert_eq!(document.elements_query_sql_domain, None); + assert_eq!(document.predicate_query_sql_domain, None); + let operators = document.operators(); + for absent in [ + stack_encrypt::dynamic::OpSpec::ElementMatch, + stack_encrypt::dynamic::OpSpec::Match, + ] { + assert!(!operators.contains(&absent), "{absent:?}: {operators:?}"); + } + } + #[cfg(feature = "eql")] #[test] fn eql_refusals_convert_to_the_engines_with_the_field_left_to_the_lowering() { diff --git a/packages/eql/AGENTS.md b/packages/eql/AGENTS.md index b742b9d9c..bed9617df 100644 --- a/packages/eql/AGENTS.md +++ b/packages/eql/AGENTS.md @@ -60,7 +60,7 @@ Where the two versions' SQL must differ, the source holds both forms in **versio #### Files confined to one EQL version -A source file belongs to every version unless it carries the line `-- ONLY EQL VERSION: N` (the payload version, not the schema name, which the source may not write). Three surfaces do today. The generated range surface: `src/v3/scalars//query__range_types.sql` and `query__range_functions.sql` hold the range query domains (`eql_v4.query__range_ore` / `_range_ope`), their `lo` / `hi` accessors and `eql_v4.between(column, range)` (ADR-0009 decision 7). And the hand-written JSON tree family under `src/v3/json_tree/` (`types.sql`, `functions.sql`, `operators.sql`, `elements.sql`, `blockers.sql`): `public.eql_v4_json_search`, `public.eql_v4_json_entry`, `eql_v4.query_json`, `eql_v4.query_json_leaf` and `eql_v4.query_json_elements`, the document Stack Encrypt's JSON index writes, under the names v3 gives its searchable JSON document but not a port of the SteVec surface (design: `docs/plans/2026-10-17-json-tree-index.md` at the repository root; bindings: `crates/eql-bindings/src/v4/json_tree.rs`, hand-written like the SteVec structs; catalog: `eql_domains::DOCUMENT_FAMILIES`, whose stored type is the target `JsonSearch`). Both are `-- ONLY EQL VERSION: 4`. The third surface goes the other way: the SteVec SQL under `src/v3/json/` and the generated `json_entry_` cross files under `src/v3/scalars//` are `-- ONLY EQL VERSION: 3`, because in v4 their domain names belong to the JSON tree family. EQL v3 is what cipherstash-client writes, nothing there produces a range or a JSON tree document, and its SQL must not change: the v3 bundle differs from an unconfined build only by those directive comment lines. `tasks/build.sh` runs the install order through `eql-codegen select <3|4>` before rendering each version, `eql-codegen render` refuses to render a confined file as the other version, `verify_installer_complete.sh` filters its disk side the same way with its own `grep`, the symbol-order checker runs on the v4 order too (on a copy mapped back to `eql_v3` names), the Doxygen filter reads a confined file from its own version's rendered tree, and `docs:validate:documented-sql` runs the whole v4 install order (`build/render/eql_v4/deps-ordered.txt`) in one rolled-back transaction, so every statement of the confined files runs after what it requires, besides running each file of the v3 render on its own. `eql-codegen order` still orders every file, so `src/deps-ordered-v3.txt` lists the files confined to either version; the per-version lists are `build/render/eql_v/deps-ordered.txt`. +A source file belongs to every version unless it carries the line `-- ONLY EQL VERSION: N` (the payload version, not the schema name, which the source may not write). Three surfaces do today. The generated range surface: `src/v3/scalars//query__range_types.sql` and `query__range_functions.sql` hold the range query domains (`eql_v4.query__range_ore` / `_range_ope`), their `lo` / `hi` accessors and `eql_v4.between(column, range)` (ADR-0009 decision 7). And the hand-written JSON tree family under `src/v3/json_tree/` (`types.sql`, `functions.sql`, `operators.sql`, `elements.sql`, `predicate.sql`, `blockers.sql`): `public.eql_v4_json_search`, `public.eql_v4_json_entry`, `eql_v4.query_json`, `eql_v4.query_json_leaf`, `eql_v4.query_json_elements` and `eql_v4.query_json_predicate`, the document Stack Encrypt's JSON index writes, under the names v3 gives its searchable JSON document but not a port of the SteVec surface (design: `docs/plans/2026-10-17-json-tree-index.md` at the repository root; bindings: `crates/eql-bindings/src/v4/json_tree.rs`, hand-written like the SteVec structs; catalog: `eql_domains::DOCUMENT_FAMILIES`, whose stored type is the target `JsonSearch`). Both are `-- ONLY EQL VERSION: 4`. The third surface goes the other way: the SteVec SQL under `src/v3/json/` and the generated `json_entry_` cross files under `src/v3/scalars//` are `-- ONLY EQL VERSION: 3`, because in v4 their domain names belong to the JSON tree family. EQL v3 is what cipherstash-client writes, nothing there produces a range or a JSON tree document, and its SQL must not change: the v3 bundle differs from an unconfined build only by those directive comment lines. `tasks/build.sh` runs the install order through `eql-codegen select <3|4>` before rendering each version, `eql-codegen render` refuses to render a confined file as the other version, `verify_installer_complete.sh` filters its disk side the same way with its own `grep`, the symbol-order checker runs on the v4 order too (on a copy mapped back to `eql_v3` names), the Doxygen filter reads a confined file from its own version's rendered tree, and `docs:validate:documented-sql` runs the whole v4 install order (`build/render/eql_v4/deps-ordered.txt`) in one rolled-back transaction, so every statement of the confined files runs after what it requires, besides running each file of the v3 render on its own. `eql-codegen order` still orders every file, so `src/deps-ordered-v3.txt` lists the files confined to either version; the per-version lists are `build/render/eql_v/deps-ordered.txt`. Anything that needs the source as real SQL reads the rendered tree, not `src/`: `docs:validate:documented-sql` runs psql over `build/render/eql_v3/src`, and the Doxygen input filter (`tasks/docs/doxygen-filter.sh`) reads each file's rendered copy from `build/render/eql_v3/`. A new tool that parses `src/v3` must do the same, and depend on `build`; do not substitute the placeholders a second way. diff --git a/packages/eql/crates/eql-bindings/src/encryption/targets.rs b/packages/eql/crates/eql-bindings/src/encryption/targets.rs index aed81c9d9..a1cb06544 100644 --- a/packages/eql/crates/eql-bindings/src/encryption/targets.rs +++ b/packages/eql/crates/eql-bindings/src/encryption/targets.rs @@ -164,6 +164,18 @@ impl Target { .map(|_| crate::v4::json_tree::JsonTreeElementsQuery::sql_domain_static()) } + /// The PostgreSQL domain of the type's compiled predicate, for a type + /// that stores a JSON document (`JsonSearch`: + /// `eql_v4.query_json_predicate`); `None` for every other type. Stack + /// Encrypt binds `match` as it + /// (`TargetDescriptor::with_predicate_query_sql_domain`), and answers + /// `match` only on a type that names it. + pub fn predicate_query_sql_domain(&self) -> Option<&'static str> { + use crate::v4::DomainType; + document(self.name) + .map(|_| crate::v4::json_tree::JsonTreePredicateQuery::sql_domain_static()) + } + /// The options this type's match index is built under, for a type that /// carries one (`TextMatch`): [`EqlMatch`](super::EqlMatch)'s, the one /// match configuration every EQL type's filter is built under. `None` diff --git a/packages/eql/crates/eql-bindings/src/v4/inventory.rs b/packages/eql/crates/eql-bindings/src/v4/inventory.rs index 613a60377..fc1ba1cfa 100644 --- a/packages/eql/crates/eql-bindings/src/v4/inventory.rs +++ b/packages/eql/crates/eql-bindings/src/v4/inventory.rs @@ -107,6 +107,7 @@ pub fn all_query() -> Vec> { Box::new(PhantomData::), Box::new(PhantomData::), Box::new(PhantomData::), + Box::new(PhantomData::), ] } /// Every v4 RANGE query operand (`eql_v4.query__range_`), diff --git a/packages/eql/crates/eql-bindings/src/v4/json_tree.rs b/packages/eql/crates/eql-bindings/src/v4/json_tree.rs index 1c2d56c2a..b2673d234 100644 --- a/packages/eql/crates/eql-bindings/src/v4/json_tree.rs +++ b/packages/eql/crates/eql-bindings/src/v4/json_tree.rs @@ -219,6 +219,57 @@ pub struct JsonTreeElementsQuery { pub m: Vec, } +/// The bounds of an ordering at one path: one or two ordering operands at +/// the path's selector, `gt` or `gte` the lower and `lt` or `lte` the upper. +/// The `cmp` leaf of a [`JsonTreePredicateQuery`]. The domain's CHECK also +/// holds what this type does not: at least one bound, at most one of each +/// side, and every bound at one selector. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct JsonTreeBounds { + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub gt: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub gte: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub lt: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub lte: Option, +} + +/// `eql_v4.query_json_predicate`: a compiled predicate, a boolean tree +/// whose leaves are the family's other query operands, which +/// `eql_v4.json_tree_match` and `col @@ $1` evaluate. What Stack Encrypt's +/// `match` operator compiles a jsonpath predicate into. Each node is an +/// object of one key naming its kind: `{"and": [node, …]}`, `{"or": […]}`, +/// `{"not": node}`, `{"contains": needle}`, `{"exists": selector}`, +/// `{"cmp": bounds}` and `{"elements": predicate}`. The domain's CHECK also +/// holds what this type does not: at least one child under `and` and `or`. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] +#[serde(rename_all = "snake_case", deny_unknown_fields)] +pub enum JsonTreePredicateQuery { + /// True when every child is; false when one is false; else NULL. + And(Vec), + /// True when one child is; false when every child is false; else NULL. + Or(Vec), + /// The child's negation, NULL of NULL. + Not(Box), + /// The document contains the needle, `col @> needle`. + Contains(JsonTreeQuery), + /// The document has a node at the path, `json_tree_path_exists`. + Exists(JsonTreeSelector), + /// The first entry at the bounds' selector against each bound, + /// `(col -> s) > gt AND (col -> s) < lt`. + Cmp(JsonTreeBounds), + /// One element of an array satisfies every member, + /// `json_tree_element_match`. + Elements(JsonTreeElementsQuery), +} + /// The entry `col -> selector` returned, with the selector that asked for /// it: opens to the scalar the entry holds, or to the whole document for a /// sealed root asked for at `$`. The only constructor, @@ -330,6 +381,7 @@ json_tree_domain_type!(JsonTreeEntry, "public.eql_v4_json_entry"); json_tree_domain_type!(JsonTreeQuery, "eql_v4.query_json"); json_tree_domain_type!(JsonTreeLeafQuery, "eql_v4.query_json_leaf"); json_tree_domain_type!(JsonTreeElementsQuery, "eql_v4.query_json_elements"); +json_tree_domain_type!(JsonTreePredicateQuery, "eql_v4.query_json_predicate"); // ============================================================================= // Conversions to and from the engine's types, through their one wire form @@ -594,6 +646,47 @@ mod tests { JsonTreeElementsQuery::sql_domain_static(), "eql_v4.query_json_elements" ); + assert_eq!( + JsonTreePredicateQuery::sql_domain_static(), + "eql_v4.query_json_predicate" + ); + } + + #[test] + fn a_predicate_query_is_a_tree_of_one_key_nodes() { + let leaf = serde_json::json!({"s": b64(0xaa), "op": "AQI="}); + let needle = serde_json::json!({"sv": [{"s": b64(1), "vs": b64(2)}], "el": []}); + let elements = serde_json::json!({"m": [{"t": [b64(1)], "gt": leaf}]}); + let query = serde_json::json!({"and": [ + {"exists": b64(0xaa)}, + {"not": {"cmp": {"gte": leaf, "lt": leaf}}}, + {"or": [{"contains": needle}, {"elements": elements}]}, + ]}); + let parsed: JsonTreePredicateQuery = serde_json::from_value(query.clone()).unwrap(); + let JsonTreePredicateQuery::And(children) = &parsed else { + panic!("the root is an and: {parsed:?}") + }; + assert!( + matches!(children[0], JsonTreePredicateQuery::Exists(_)), + "{children:?}" + ); + assert_eq!( + serde_json::to_value(&parsed).unwrap(), + query, + "a tree reads and writes as itself" + ); + for wrong in [ + serde_json::json!({"exists": b64(1), "not": {"exists": b64(1)}}), + serde_json::json!({"xor": []}), + serde_json::json!({"cmp": {"eq": leaf}}), + serde_json::json!({"exists": 1}), + serde_json::json!([]), + ] { + assert!( + serde_json::from_value::(wrong.clone()).is_err(), + "{wrong} should not deserialize" + ); + } } #[test] diff --git a/packages/eql/crates/eql-codegen/src/bindings.rs b/packages/eql/crates/eql-codegen/src/bindings.rs index 627be79f5..69caee2d2 100644 --- a/packages/eql/crates/eql-codegen/src/bindings.rs +++ b/packages/eql/crates/eql-codegen/src/bindings.rs @@ -1084,6 +1084,7 @@ pub fn render_v4_inventory_rs() -> String { eql_domains::DocumentRole::Query, eql_domains::DocumentRole::LeafQuery, eql_domains::DocumentRole::ElementsQuery, + eql_domains::DocumentRole::PredicateQuery, ]); quote! { #query_entries #documents } }; diff --git a/packages/eql/crates/eql-codegen/src/go_eql.rs b/packages/eql/crates/eql-codegen/src/go_eql.rs index 9dbc4636b..ef1771d76 100644 --- a/packages/eql/crates/eql-codegen/src/go_eql.rs +++ b/packages/eql/crates/eql-codegen/src/go_eql.rs @@ -189,6 +189,9 @@ fn document_types(out: &mut String) { DocumentRole::ElementsQuery => { "element-scoped predicate, which eql_v4.json_tree_element_match tests" } + DocumentRole::PredicateQuery => { + "compiled predicate, a boolean tree of the other operands, which eql_v4.json_tree_match and `@@` evaluate" + } DocumentRole::Entry => { "entry: one node of a document, as `->` and the walks return it" } @@ -275,6 +278,7 @@ mod tests { "JSONSearchQuery", "JSONSearchLeafQuery", "JSONSearchElementsQuery", + "JSONSearchPredicateQuery", "JSONEntry", ] { assert_eq!( diff --git a/packages/eql/crates/eql-domains/src/lib.rs b/packages/eql/crates/eql-domains/src/lib.rs index 1e7738eaa..7c431f0c6 100644 --- a/packages/eql/crates/eql-domains/src/lib.rs +++ b/packages/eql/crates/eql-domains/src/lib.rs @@ -567,6 +567,9 @@ pub enum DocumentRole { /// The element-scoped predicate: fields of one array element, each /// with its read tokens and its bounds. ElementsQuery, + /// The compiled predicate: a boolean tree whose leaves are the other + /// query operands. + PredicateQuery, /// One node, as `->` and the walks return it. Entry, } @@ -625,8 +628,10 @@ impl DocumentFamily { /// `JsonSearch` over `public.eql_v4_json_search`, its needle /// `JsonSearchQuery` (`eql_v4.query_json`), its ordering operand /// `JsonSearchLeafQuery` (`eql_v4.query_json_leaf`), its element-scoped -/// predicate `JsonSearchElementsQuery` (`eql_v4.query_json_elements`) and -/// its entry `JsonEntry` (`public.eql_v4_json_entry`). The storage-only `Json` +/// predicate `JsonSearchElementsQuery` (`eql_v4.query_json_elements`), its +/// compiled predicate `JsonSearchPredicateQuery` +/// (`eql_v4.query_json_predicate`) and its entry `JsonEntry` +/// (`public.eql_v4_json_entry`). The storage-only `Json` /// (`public.eql_v4_json`) is the `json` family's. The Rust structs keep the /// implementation's name, as v3's `SteVecDocument` does. pub const JSON_TREE: DocumentFamily = DocumentFamily { @@ -658,6 +663,12 @@ pub const JSON_TREE: DocumentFamily = DocumentFamily { rust: "JsonTreeElementsQuery", sql_domain: "eql_v4.query_json_elements", }, + DocumentDomain { + role: DocumentRole::PredicateQuery, + name: "JsonSearchPredicateQuery", + rust: "JsonTreePredicateQuery", + sql_domain: "eql_v4.query_json_predicate", + }, DocumentDomain { role: DocumentRole::Entry, name: "JsonEntry", diff --git a/packages/eql/crates/eql-domains/src/tests.rs b/packages/eql/crates/eql-domains/src/tests.rs index 103414893..9547707b8 100644 --- a/packages/eql/crates/eql-domains/src/tests.rs +++ b/packages/eql/crates/eql-domains/src/tests.rs @@ -1634,6 +1634,7 @@ mod document_families { DocumentRole::Query, DocumentRole::LeafQuery, DocumentRole::ElementsQuery, + DocumentRole::PredicateQuery, DocumentRole::Entry, ] { assert_eq!( @@ -1646,9 +1647,10 @@ mod document_families { for domain in family.domains { let schema = match domain.role { DocumentRole::Stored | DocumentRole::Entry => "public.eql_v4_", - DocumentRole::Query | DocumentRole::LeafQuery | DocumentRole::ElementsQuery => { - "eql_v4.query_" - } + DocumentRole::Query + | DocumentRole::LeafQuery + | DocumentRole::ElementsQuery + | DocumentRole::PredicateQuery => "eql_v4.query_", }; assert!( domain.sql_domain.starts_with(schema), @@ -1685,6 +1687,11 @@ mod document_families { "JsonSearchElementsQuery", "eql_v4.query_json_elements", ), + ( + DocumentRole::PredicateQuery, + "JsonSearchPredicateQuery", + "eql_v4.query_json_predicate", + ), (DocumentRole::Entry, "JsonEntry", "public.eql_v4_json_entry"), ] { let domain = JSON_TREE.domain(role).unwrap(); diff --git a/packages/eql/docs/reference/eql-functions.md b/packages/eql/docs/reference/eql-functions.md index 7eaf74266..3615dd2bb 100644 --- a/packages/eql/docs/reference/eql-functions.md +++ b/packages/eql/docs/reference/eql-functions.md @@ -12,6 +12,7 @@ A reference for the functions and operators EQL exposes for querying encrypted d - [Encrypted JSON (`public.eql_v3_json_search`)](#encrypted-json-publiceql_v3_json_search) - [Range queries in EQL v4: `eql_v4.between`](#range-queries-in-eql-v4-eql_v4between) - [One array element in EQL v4: `eql_v4.json_tree_element_match`](#one-array-element-in-eql-v4-eql_v4json_tree_element_match) +- [A compiled predicate in EQL v4: `eql_v4.json_tree_match`](#a-compiled-predicate-in-eql-v4-eql_v4json_tree_match) - [Aggregate Functions](#aggregate-functions) - [Diagnostics](#diagnostics) @@ -219,6 +220,40 @@ That form does not check the probe: on a row whose array is longer than the boun --- +## A compiled predicate in EQL v4: `eql_v4.json_tree_match` + +On plaintext `jsonb`, one function answers a whole filter: `jsonb_path_match(col, $1, $2)`, a jsonpath predicate and its variables. An application that compiles a filter language into SQL can target that shape with one template and two parameters. On an encrypted JSON document (`public.eql_v4_json_search`) the database cannot evaluate a path or a value, so `@@` with a `jsonpath` is blocked, `jsonb_path_match` matches nothing ([Native jsonb functions read the envelope](json-tree-leakage.md#native-jsonb-functions-read-the-envelope)), and each question is its own operator. `eql_v4.json_tree_match` takes the whole filter as one parameter again: Stack Encrypt's `match` query operator compiles the jsonpath predicate on the client into a tree of those operators' operands, and the function evaluates the tree. + +```sql +eql_v4.json_tree_match(e public.eql_v4_json_search, p eql_v4.query_json_predicate) RETURNS boolean + +SELECT id FROM people WHERE eql_v4.json_tree_match(profile, $1::eql_v4.query_json_predicate); +SELECT id FROM people WHERE profile @@ $1::eql_v4.query_json_predicate; +``` + +The operand `eql_v4.query_json_predicate` is a boolean tree, serialised as JSON. Each node is an object with one key, which names its kind: + +| node | true when | the same question, written out | +|---|---|---| +| `{"and": [node, …]}`, `{"or": [node, …]}` | every child is, or one child is | `… AND …`, `… OR …` | +| `{"not": node}` | the child is false | `NOT …` | +| `{"contains": needle}` | the document contains the needle, an `eql_v4.query_json` | `col @> needle` | +| `{"exists": selector}` | the document has a node at the path | `eql_v4.json_tree_path_exists(col, selector)` | +| `{"cmp": {"gt" \| "gte": leaf, "lt" \| "lte": leaf}}` | the first entry at the bounds' selector is inside every bound, each an `eql_v4.query_json_leaf` | `(col -> s) > gt AND (col -> s) < lt` | +| `{"elements": predicate}` | one element of an array satisfies every member, an `eql_v4.query_json_elements` | `eql_v4.json_tree_element_match(col, predicate)` | + +Each leaf is checked by the CHECK of its own domain, and `and` and `or` take at least one child. + +- **The answer is the hand-written WHERE's, NULL included.** `and`, `or` and `not` follow SQL's three-valued logic, and a `cmp` leaf is NULL where the path is absent or its node has no ordering term, as `(col -> s) < $1` is. Stack Encrypt compiles a jsonpath comparison so that the tree answers as `jsonb_path_match` does over the plaintext: `$.age < $n` is a node at `$.age`, not a JSON `null`, whose leaf compares; `$.age != $n` is a node there that is not the value. So an absent path is false under every comparison and `null` false under an ordering, as jsonpath has them, and an ordering of a boolean is unknown (NULL), as jsonpath's comparison of two kinds is. +- **Where the tree and jsonpath part.** In four places, each a property of the stored document or of the ordering terms. A path is read as written, so an array where the predicate names no `[*]` is not unwrapped as jsonpath's lax mode unwraps it (`$.tags == "a"` over `{"tags": ["a"]}` is true to jsonpath, false here). Nor is a value that is not an array wrapped where the predicate does name `[*]`: `$.tags[*] == "a"` over `{"tags": "a"}`, `exists($.tags[*])` over a scalar `tags` and `exists($.items[*] ? (@.sku == "x"))` over `{"items": {"sku": "x"}}` are true to jsonpath and false here, since `$.tags[*]` and `$.tags` have different selectors. A comparison of a string with a number, which jsonpath answers unknown, is false under `==` and true under `!=`, since the terms do not record a value's kind. An ordering compares the ordering terms, so text is case- and accent-folded, as every ordering at a path is; so does each condition of a filter with more than one, which compiles to an element leaf. +- **Cast the operand.** `col @@ $1::eql_v4.query_json_predicate`. An untyped operand on the right of `@@` matches both this operator and the blocked `jsonpath` one, and PostgreSQL refuses the statement with SQLSTATE 42725 (`operator is not unique`). A `jsonpath` operand still raises `operator @@ is not supported`. +- **Element leaves raise as their function does.** An array longer than the bound the predicate was compiled to raises SQLSTATE `22023`. Children are read in order and the read stops at the first child that decides the answer, so an element leaf the answer does not depend on is not read and does not raise. +- **The document is opened once.** It is decompressed once per row, and indexed once when the tree has an element leaf, however many leaves read it. +- **No index serves it.** The tree is evaluated inside a PL/pgSQL function, which the planner cannot see into: `json_tree_match` and `@@` filter the rows they are given, a sequential scan unless something else narrows them. The same predicate written out per operator (the right-hand column above) engages the GIN index on `eql_v4.json_tree_query(col)` and the functional btrees on `eql_v4.ord_term(col -> s)`, and is the fast path. Use `match` where one template matters more than the plan, and put an indexed predicate beside it where you can. +- **What it reveals** is what its leaves reveal sent one by one, and the boolean shape of the filter. See [What a query shows](./json-tree-leakage.md#what-a-query-shows). + +--- + ## Aggregate Functions ### `eql_v3.min()` / `eql_v3.max()` (per-domain) diff --git a/packages/eql/docs/reference/json-tree-leakage.md b/packages/eql/docs/reference/json-tree-leakage.md index 9bf763e10..694245163 100644 --- a/packages/eql/docs/reference/json-tree-leakage.md +++ b/packages/eql/docs/reference/json-tree-leakage.md @@ -52,6 +52,7 @@ A query hands the database a term derived from the plaintext it asks about, and - **The root.** A read of `$` sends the root's token, and a read of anything below `$` does not. With the sealed root on (the default) that token covers no document's structure: it finds the root entry, whose ciphertext is the document, and nothing beneath it, since the top-level fields chain from nothing. With it off, the root's token covers every row's whole document. - **`json_tree_children` and `json_tree_child_count`** take one token, of any node a path names, and send nothing else. Under the sealed root the root's token enumerates nothing (its count is NULL), for the same reason. - **An element-scoped predicate (`json_tree_element_match`).** For each field it tests, the tokens of a wildcard read of that field's path, probes included, and the bounds' ordering terms, as a subtree read and an ordering send them. So the database learns what those reads and orderings learn, and one thing more, which the answer itself is: in which rows one element holds every field inside its bounds. It does not open a value. Equality there is the ordering term's, of folded values (see `op` above). +- **A compiled predicate (`json_tree_match`, `@@` with an `eql_v4.query_json_predicate`).** What its leaves reveal sent separately (the selectors of the paths it tests, the (path, value) selectors of its equalities, the bounds' ordering terms, and the element leaves' tokens), plus the boolean shape of the filter: which leaves are joined by `and`, `or` and `not`. An ordering compiled from jsonpath carries a `null` needle at its path beside its bound, so the database also learns which rows hold `null` there, as `== null` sent alone shows it. The database holds every leaf and can evaluate each on every row, so it can learn each leaf's answer per row, as it can of the leaves sent separately. ## Native jsonb functions read the envelope @@ -60,7 +61,7 @@ A query hands the database a term derived from the plaintext it asks about, and - **Operators are blocked.** `?`, `?|`, `?&`, `@?`, `@@`, `#>`, `#>>`, `-`, `#-`, `||`, `->` and `->>` by position, root-document comparisons, and containment against a plain `jsonb` operand raise `operator … is not supported for public.eql_v4_json_search`. `col @@ '…'` with an untyped right operand is refused as ambiguous (`operator is not unique`). - **Functions are not blocked, and fail silently.** `jsonb_path_match(col, '$.score == 3')` returns false for every row, with no error, and so does `jsonb_path_exists(col, '$.score')`, whether the path is a literal, cast, or bound as a parameter. `jsonb_extract_path(col, 'score')` returns NULL. A key of the envelope returns the envelope's own field instead: `jsonb_extract_path(col, 'v')` is `4`, and `jsonb_object_keys(col)` lists `h`, `i`, `q`, `sv` and `v`. The jsonpath functions find nothing on an entry extracted with `->` either. Only `jsonb_array_elements`, `jsonb_array_elements_text` and `jsonb_array_length` raise, because the envelope and an entry are objects. -Query the document with EQL instead: `@>` with an `eql_v4.query_json` needle for equality, `col -> selector` compared with an `eql_v4.query_json_leaf` for ranges, and `col -> selector`, `eql_v4.json_tree_path_query` or `eql_v4.json_tree_subtree` to extract, then decrypt in the client. +Query the document with EQL instead: `@>` with an `eql_v4.query_json` needle for equality, `col -> selector` compared with an `eql_v4.query_json_leaf` for ranges, `col @@ $1::eql_v4.query_json_predicate` (`eql_v4.json_tree_match`) for a whole jsonpath filter compiled by Stack Encrypt, and `col -> selector`, `eql_v4.json_tree_path_query` or `eql_v4.json_tree_subtree` to extract, then decrypt in the client. EQL cannot make these functions raise. An overload taking the domain as its first argument would be a second candidate for every call whose first argument has no type, such as `jsonb_typeof($1)` from a driver that sends untyped parameters, so installing EQL would make those calls ambiguous on plain `jsonb` too, in the whole database. And it would still lose to the native function whenever any other argument is untyped, as the path is in `jsonb_path_match(col, '$.a == 1')`. A type of its own that `jsonb` functions do not accept would close this, but PostgreSQL lets only a superuser create one, and EQL installs without superuser. diff --git a/packages/eql/src/v3/json_tree/elements.sql b/packages/eql/src/v3/json_tree/elements.sql index 0b9856222..a9bd6ed1f 100644 --- a/packages/eql/src/v3/json_tree/elements.sql +++ b/packages/eql/src/v3/json_tree/elements.sql @@ -8,6 +8,94 @@ --! @brief Element-scoped predicates on public.{{prefix}}_json_search: tests --! that one element of an array satisfies them all. +--! @brief Whether an entry is inside every bound of an ordering at one +--! path: what an element predicate's member and a compiled +--! predicate's `cmp` leaf each test of the entry they find. +--! @internal +--! +--! `gt` by `>`, `gte` by `>=`, `lt` by `<` and `lte` by `<=`, the ordering +--! operators an entry has against a {{prefix}}.query_json_leaf, joined by +--! SQL `AND`; a bound `bounds` does not carry is no test. So it is NULL +--! where a bound's comparison is: an entry that is NULL (no node at the +--! path) or has no ordering term. +--! +--! @param entry public.{{prefix}}_json_entry The entry. +--! @param bounds jsonb Bounds, as {{prefix}}_internal.json_tree_is_valid_bounds +--! validates them; other keys (a member's `t`) are not read. +--! @return boolean Whether the entry satisfies every bound. +--! @note LANGUAGE sql and not STRICT, so it inlines into its callers. +CREATE FUNCTION {{prefix}}_internal.json_tree_within_bounds(entry public.{{prefix}}_json_entry, bounds jsonb) + RETURNS boolean + LANGUAGE sql IMMUTABLE PARALLEL SAFE +AS $$ + SELECT (NOT bounds ? 'gt' OR {{prefix}}.gt(entry, (bounds -> 'gt')::{{prefix}}.query_json_leaf)) + AND (NOT bounds ? 'gte' OR {{prefix}}.gte(entry, (bounds -> 'gte')::{{prefix}}.query_json_leaf)) + AND (NOT bounds ? 'lt' OR {{prefix}}.lt(entry, (bounds -> 'lt')::{{prefix}}.query_json_leaf)) + AND (NOT bounds ? 'lte' OR {{prefix}}.lte(entry, (bounds -> 'lte')::{{prefix}}.query_json_leaf)) +$$; + +--! @brief {{prefix}}.json_tree_element_match over a document already opened +--! and indexed: what that function and {{prefix}}.json_tree_match +--! each call, so a predicate with several element-scoped leaves +--! opens and indexes its row once. +--! @internal +--! @param iv bytea The document's `iv`, which salts its self tags. +--! @param sv jsonb The document's entries. +--! @param tags jsonb The document's self tags, each mapped to the entry's +--! ordinal in `sv`: the `nodes` {{prefix}}_internal.json_tree_index +--! builds. +--! @param p jsonb The predicate, a valid {{prefix}}.query_json_elements. +--! @return boolean As {{prefix}}.json_tree_element_match. +--! @throws invalid_parameter_value As {{prefix}}.json_tree_element_match. +CREATE FUNCTION {{prefix}}_internal.json_tree_element_match_opened(iv bytea, sv jsonb, tags jsonb, p jsonb) + RETURNS boolean + LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE +AS $$ +DECLARE + truncated boolean; + matched boolean; +BEGIN + WITH + member AS MATERIALIZED ( + SELECT m.i, + m.v, + -- The selector of the member's field: every bound carries it. + COALESCE(m.v -> 'gt', m.v -> 'gte', m.v -> 'lt', m.v -> 'lte') ->> 's' AS s + FROM jsonb_array_elements(p -> 'm') WITH ORDINALITY AS m(v, i) + ), + -- The node each member's `k`-th token names, by its self tag; a token + -- that names none yields no row. + node AS MATERIALIZED ( + SELECT member.i, member.v, member.s, t.k, sv -> found.j AS entry + FROM member + CROSS JOIN LATERAL jsonb_array_elements_text(member.v -> 't') WITH ORDINALITY AS t(token, k) + CROSS JOIN LATERAL ( + SELECT (tags ->> {{prefix}}_internal.json_tree_self_tag(iv, decode(t.token, 'base64')))::integer AS j + ) AS found + WHERE found.j IS NOT NULL + ), + satisfied AS ( + SELECT node.k + FROM node + CROSS JOIN LATERAL (SELECT node.entry::public.{{prefix}}_json_entry AS entry) AS typed + WHERE node.entry ->> 's' = node.s + AND {{prefix}}_internal.json_tree_within_bounds(typed.entry, node.v) + GROUP BY node.k + HAVING count(DISTINCT node.i) = (SELECT count(*) FROM member) + ) + SELECT EXISTS (SELECT 1 FROM node WHERE node.entry ->> 's' IS DISTINCT FROM node.s), + EXISTS (SELECT 1 FROM satisfied) + INTO truncated, matched; + + IF truncated THEN + RAISE EXCEPTION 'an array the element predicate reads is longer than the bound it was derived to, so the predicate would answer over part of it' + USING ERRCODE = 'invalid_parameter_value', + HINT = 'Derive the predicate (element_match) with a wildcard_bound past the longest array.'; + END IF; + RETURN matched; +END; +$$; + --! @brief Whether some one element of an array in the document satisfies --! every member of an element-scoped predicate. --! @@ -53,68 +141,18 @@ --! @note plpgsql, to raise, and so opaque to the planner: the call filters --! a scan, as a subtree read in a predicate does. Narrow the rows with --! an indexed predicate first, such as `@>` on the same values. +--! @see {{prefix}}_internal.json_tree_element_match_opened CREATE FUNCTION {{prefix}}.json_tree_element_match(e public.{{prefix}}_json_search, p {{prefix}}.query_json_elements) RETURNS boolean LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE AS $$ DECLARE - truncated boolean; - matched boolean; + -- The document, decompressed once. + d jsonb := e::jsonb #> '{}'::text[]; + tags jsonb; BEGIN - WITH - -- The document, decompressed and indexed once for every member. - raw AS MATERIALIZED (SELECT e::jsonb #> '{}'::text[] AS d), - doc AS MATERIALIZED ( - SELECT {{prefix}}_internal.json_tree_iv(raw.d ->> 'h') AS iv, - raw.d -> 'sv' AS sv, - idx.nodes - FROM raw - CROSS JOIN LATERAL {{prefix}}_internal.json_tree_index(raw.d) AS idx - ), - member AS MATERIALIZED ( - SELECT m.i, - m.v, - -- The selector of the member's field: every bound carries it. - COALESCE(m.v -> 'gt', m.v -> 'gte', m.v -> 'lt', m.v -> 'lte') ->> 's' AS s - FROM jsonb_array_elements(p::jsonb -> 'm') WITH ORDINALITY AS m(v, i) - ), - -- The node each member's `k`-th token names, by its self tag; a token - -- that names none yields no row. - node AS MATERIALIZED ( - SELECT member.i, member.v, member.s, t.k, doc.sv -> found.j AS entry - FROM member - CROSS JOIN doc - CROSS JOIN LATERAL jsonb_array_elements_text(member.v -> 't') WITH ORDINALITY AS t(token, k) - CROSS JOIN LATERAL ( - SELECT (doc.nodes ->> {{prefix}}_internal.json_tree_self_tag(doc.iv, decode(t.token, 'base64')))::integer AS j - ) AS found - WHERE found.j IS NOT NULL - ), - satisfied AS ( - SELECT node.k - FROM node - CROSS JOIN LATERAL (SELECT node.entry::public.{{prefix}}_json_entry AS entry) AS typed - WHERE node.entry ->> 's' = node.s - AND (NOT node.v ? 'gt' - OR {{prefix}}.gt(typed.entry, (node.v -> 'gt')::{{prefix}}.query_json_leaf)) - AND (NOT node.v ? 'gte' - OR {{prefix}}.gte(typed.entry, (node.v -> 'gte')::{{prefix}}.query_json_leaf)) - AND (NOT node.v ? 'lt' - OR {{prefix}}.lt(typed.entry, (node.v -> 'lt')::{{prefix}}.query_json_leaf)) - AND (NOT node.v ? 'lte' - OR {{prefix}}.lte(typed.entry, (node.v -> 'lte')::{{prefix}}.query_json_leaf)) - GROUP BY node.k - HAVING count(DISTINCT node.i) = (SELECT count(*) FROM member) - ) - SELECT EXISTS (SELECT 1 FROM node WHERE node.entry ->> 's' IS DISTINCT FROM node.s), - EXISTS (SELECT 1 FROM satisfied) - INTO truncated, matched; - - IF truncated THEN - RAISE EXCEPTION 'an array the element predicate reads is longer than the bound it was derived to, so the predicate would answer over part of it' - USING ERRCODE = 'invalid_parameter_value', - HINT = 'Derive the predicate (element_match) with a wildcard_bound past the longest array.'; - END IF; - RETURN matched; + SELECT idx.nodes INTO tags FROM {{prefix}}_internal.json_tree_index(d) AS idx; + RETURN {{prefix}}_internal.json_tree_element_match_opened( + {{prefix}}_internal.json_tree_iv(d ->> 'h'), d -> 'sv', tags, p::jsonb); END; $$; diff --git a/packages/eql/src/v3/json_tree/predicate.sql b/packages/eql/src/v3/json_tree/predicate.sql new file mode 100644 index 000000000..d28caed8f --- /dev/null +++ b/packages/eql/src/v3/json_tree/predicate.sql @@ -0,0 +1,294 @@ +-- ONLY EQL VERSION: 4 +-- REQUIRE: src/v3/schema.sql +-- REQUIRE: src/v3/json_tree/types.sql +-- REQUIRE: src/v3/json_tree/functions.sql +-- REQUIRE: src/v3/json_tree/operators.sql +-- REQUIRE: src/v3/json_tree/elements.sql + +--! @file v3/json_tree/predicate.sql +--! @brief A compiled predicate on public.{{prefix}}_json_search: a boolean +--! tree of the family's query operands, bound as one parameter and +--! evaluated by {{prefix}}.json_tree_match or `@@`. +--! +--! A filter compiler that targets plaintext `jsonb_path_match(col, $1, $2)` +--! binds one predicate and its values. The database cannot evaluate a +--! jsonpath over ciphertext, so Stack Encrypt's `match` operator compiles +--! the jsonpath on the client into this tree, whose leaves are the operands +--! the family's operators already take, and the server evaluates the tree. +--! +--! @note The tree is evaluated inside a function, which is opaque to the +--! planner: `col @@ $1` filters a scan, and engages no index. The +--! operators the leaves stand for, written out in the WHERE clause +--! (`col @> $1`, `(col -> $2) < $3`), engage the GIN and btree indexes +--! and are the fast path. + +--! @brief Validate a compiled predicate, recursively. +--! @internal +--! +--! A node is an object of exactly one key, which names its kind: +--! - `{"and": [node, …]}`, `{"or": [node, …]}`: at least one node; +--! - `{"not": node}`; +--! - `{"contains": needle}`: a containment needle, as +--! public.{{prefix}}_is_valid_json_tree_query_payload validates the +--! {{prefix}}.query_json CHECK; +--! - `{"exists": "…"}`: a path selector, base64; +--! - `{"cmp": bounds}`: one or two ordering operands at one selector, as +--! {{prefix}}_internal.json_tree_is_valid_bounds validates them; +--! - `{"elements": predicate}`: an element-scoped predicate, as +--! {{prefix}}_internal.json_tree_is_valid_elements_payload validates the +--! {{prefix}}.query_json_elements CHECK. +--! +--! @param val jsonb Candidate node. +--! @return boolean True when `val` is a node. +--! @note plpgsql: the recursion and the subqueries over the children cannot +--! be written as a CHECK expression. +CREATE OR REPLACE FUNCTION {{prefix}}_internal.json_tree_is_valid_predicate_payload(val jsonb) + RETURNS boolean + LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE +AS $$ +DECLARE + kind text; +BEGIN + IF jsonb_typeof(val) IS DISTINCT FROM 'object' + OR (SELECT count(*) FROM jsonb_object_keys(val)) <> 1 THEN + RETURN false; + END IF; + SELECT key INTO kind FROM jsonb_object_keys(val) AS key; + RETURN COALESCE( + CASE kind + WHEN 'and' THEN + jsonb_typeof(val -> 'and') = 'array' + AND jsonb_array_length(val -> 'and') >= 1 + AND NOT EXISTS ( + SELECT 1 FROM jsonb_array_elements(val -> 'and') AS child + WHERE NOT {{prefix}}_internal.json_tree_is_valid_predicate_payload(child) + ) + WHEN 'or' THEN + jsonb_typeof(val -> 'or') = 'array' + AND jsonb_array_length(val -> 'or') >= 1 + AND NOT EXISTS ( + SELECT 1 FROM jsonb_array_elements(val -> 'or') AS child + WHERE NOT {{prefix}}_internal.json_tree_is_valid_predicate_payload(child) + ) + WHEN 'not' THEN + {{prefix}}_internal.json_tree_is_valid_predicate_payload(val -> 'not') + WHEN 'contains' THEN + public.{{prefix}}_is_valid_json_tree_query_payload(val -> 'contains') + WHEN 'exists' THEN + jsonb_typeof(val -> 'exists') = 'string' + WHEN 'cmp' THEN + {{prefix}}_internal.json_tree_is_valid_bounds(val -> 'cmp') + WHEN 'elements' THEN + {{prefix}}_internal.json_tree_is_valid_elements_payload(val -> 'elements') + ELSE false + END, + false + ); +END; +$$; + +--! @brief Domain type for a compiled predicate on a JSON tree document. +--! +--! A boolean tree, serialised as JSON, whose leaves are the family's query +--! operands, as Stack Encrypt's `match` operator compiles a jsonpath +--! predicate into it: +--! - `{"and": [node, …]}`, `{"or": [node, …]}`, `{"not": node}`; +--! - `{"contains": {{prefix}}.query_json}`: the document contains the +--! needle, `col @> needle`; +--! - `{"exists": selector}`: the document has a node at the path, +--! `{{prefix}}.json_tree_path_exists(col, selector)`; +--! - `{"cmp": {"gt" | "gte": leaf, "lt" | "lte": leaf}}`: the first entry +--! at the bounds' selector against each bound, each leaf a +--! {{prefix}}.query_json_leaf, `(col -> s) > gt AND (col -> s) < lt`; +--! - `{"elements": {{prefix}}.query_json_elements}`: one element of an +--! array satisfies every member, +--! `{{prefix}}.json_tree_element_match(col, elements)`. +--! {{prefix}}.json_tree_match reads it. +--! +--! @note Construct from inline JSON via the DOMAIN cast: +--! `'{"and":[{"exists":"…"},{"contains":{"sv":[…],"el":[]}}]}'::{{prefix}}.query_json_predicate`. +--! An untyped literal on the right of `@@` does not resolve to the +--! domain: write the cast, `col @@ $1::{{prefix}}.query_json_predicate`. +--! @see {{prefix}}.json_tree_match +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_type + WHERE typname = 'query_json_predicate' AND typnamespace = '{{prefix}}'::regnamespace + ) THEN + CREATE DOMAIN {{prefix}}.query_json_predicate AS jsonb + CHECK ( + {{prefix}}_internal.json_tree_is_valid_predicate_payload(VALUE) + ); + END IF; + + COMMENT ON DOMAIN {{prefix}}.query_json_predicate IS 'EQL JSON tree query operand (compiled predicate)'; +END +$$; + +--! @brief One node of a compiled predicate, over a document already opened. +--! @internal +--! +--! `and`, `or` and `not` follow SQL's three-valued logic, as the operators +--! the leaves stand for would combine in a hand-written WHERE: `and` is +--! false when a child is false, else NULL when a child is NULL, else true; +--! `or` the dual; `not` NULL of NULL. Children are read in order and the +--! read stops at the first child that decides the answer, so an element +--! leaf the answer does not depend on is not read and cannot raise. +--! +--! Each leaf answers as its operator does on the column: +--! - `contains`: native `jsonb @>` of the document's projection over the +--! needle, and over its `alt` when it carries one, the rule +--! {{prefix}}.json_tree_contains applies (`col @> needle`), here over +--! the projection the caller read once; +--! - `exists`: {{prefix}}.json_tree_path_exists; +--! - `cmp`: the first entry at the selector, +--! {{prefix}}.json_tree_path_query_first (what `col -> s` returns), +--! against its bounds by {{prefix}}_internal.json_tree_within_bounds, +--! as an element member's entry is: NULL when the document has no node +--! at the path or the node has no ordering term; +--! - `elements`: {{prefix}}_internal.json_tree_element_match_opened over +--! the index the caller built once. +--! +--! @param d jsonb The document, decompressed. +--! @param sv jsonb Its entries, or NULL when the predicate has no element +--! leaf. +--! @param q jsonb Its containment projection. +--! @param iv bytea Its `iv`, or NULL when the predicate has no element +--! leaf. +--! @param tags jsonb Its self tags, as {{prefix}}_internal.json_tree_index +--! builds them, or NULL when the predicate has no element leaf. +--! @param node jsonb The node, valid. +--! @return boolean The node's answer: true, false or NULL. +--! @throws invalid_parameter_value As {{prefix}}.json_tree_element_match, +--! from an element leaf that is read. +--! @note Not STRICT: the parts a predicate does not need are NULL. +CREATE FUNCTION {{prefix}}_internal.json_tree_match_node(d jsonb, sv jsonb, q jsonb, iv bytea, tags jsonb, node jsonb) + RETURNS boolean + LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE +AS $$ +DECLARE + child jsonb; + answer boolean; + result boolean; + needle jsonb; + bounds jsonb; + entry public.{{prefix}}_json_entry; +BEGIN + IF node ? 'and' THEN + result := true; + FOR child IN SELECT jsonb_array_elements(node -> 'and') LOOP + answer := {{prefix}}_internal.json_tree_match_node(d, sv, q, iv, tags, child); + IF answer IS FALSE THEN + RETURN false; + ELSIF answer IS NULL THEN + result := NULL; + END IF; + END LOOP; + RETURN result; + ELSIF node ? 'or' THEN + result := false; + FOR child IN SELECT jsonb_array_elements(node -> 'or') LOOP + answer := {{prefix}}_internal.json_tree_match_node(d, sv, q, iv, tags, child); + IF answer IS TRUE THEN + RETURN true; + ELSIF answer IS NULL THEN + result := NULL; + END IF; + END LOOP; + RETURN result; + ELSIF node ? 'not' THEN + RETURN NOT {{prefix}}_internal.json_tree_match_node(d, sv, q, iv, tags, node -> 'not'); + ELSIF node ? 'contains' THEN + needle := node -> 'contains'; + RETURN q @> (needle - 'alt') + OR ((needle ? 'alt') AND q @> (needle -> 'alt')); + ELSIF node ? 'exists' THEN + RETURN {{prefix}}.json_tree_path_exists(d, node ->> 'exists'); + ELSIF node ? 'cmp' THEN + bounds := node -> 'cmp'; + entry := {{prefix}}.json_tree_path_query_first( + d, COALESCE(bounds -> 'gt', bounds -> 'gte', bounds -> 'lt', bounds -> 'lte') ->> 's'); + RETURN {{prefix}}_internal.json_tree_within_bounds(entry, bounds); + ELSE + RETURN {{prefix}}_internal.json_tree_element_match_opened(iv, sv, tags, node -> 'elements'); + END IF; +END; +$$; + +--! @brief Whether a document satisfies a compiled predicate. +--! +--! The one-parameter form of a filter over a JSON tree column, for a +--! compiler that targets plaintext `jsonb_path_match(col, $1, $2)`: Stack +--! Encrypt's `match` operator compiles the jsonpath predicate and its +--! values into `p`, and the tree is evaluated here, each leaf as its +--! operator answers on the column (see +--! {{prefix}}_internal.json_tree_match_node). The answer is the +--! hand-written WHERE's: `{"and": [{"exists": s}, {"cmp": {"lt": l}}]}` is +--! `{{prefix}}.json_tree_path_exists(col, s) AND (col -> s) < l`, NULL +--! where that is NULL. +--! +--! The document is decompressed once, and indexed once when the predicate +--! has an element leaf, however many leaves read it. The `exists` and +--! `cmp` leaves each scan the decompressed entries, as their functions do. +--! Whether the predicate has an element leaf is one walk of the predicate, +--! which is small, per row: the projection `q` is read without asking, as +--! one key of the decompressed document costs less than the walk. +--! +--! @param e public.{{prefix}}_json_search The stored document. +--! @param p {{prefix}}.query_json_predicate The predicate. +--! @return boolean The predicate's answer: true, false, or NULL as SQL's +--! three-valued logic gives it; NULL when either argument is NULL. +--! @throws invalid_parameter_value As {{prefix}}.json_tree_element_match, +--! from an element leaf that is read. +--! @note plpgsql, and so opaque to the planner: the call filters a scan, +--! and engages no index. Written out per operator, the same predicate +--! engages the GIN index on {{prefix}}.json_tree_query(col) and the +--! functional btrees on {{prefix}}.ord_term(col -> s), so narrow the +--! rows with an indexed predicate beside it, or write the operators. +--! @see {{prefix}}.query_json_predicate +CREATE FUNCTION {{prefix}}.json_tree_match(e public.{{prefix}}_json_search, p {{prefix}}.query_json_predicate) + RETURNS boolean + LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE +AS $$ +DECLARE + -- The document, decompressed once. + d jsonb := e::jsonb #> '{}'::text[]; + node jsonb := p::jsonb; + sv jsonb; + q jsonb; + iv bytea; + tags jsonb; +BEGIN + q := d -> 'q'; + IF node @? '$.**.elements' THEN + sv := d -> 'sv'; + iv := {{prefix}}_internal.json_tree_iv(d ->> 'h'); + SELECT idx.nodes INTO tags FROM {{prefix}}_internal.json_tree_index(d) AS idx; + END IF; + RETURN {{prefix}}_internal.json_tree_match_node(d, sv, q, iv, tags, node); +END; +$$; + +--! @brief @@ operator with a compiled predicate: {{prefix}}.json_tree_match. +--! +--! `col @@ $1::{{prefix}}.query_json_predicate`. The operand MUST carry the +--! domain's type, as a typed parameter or an explicit cast; `@@` with a +--! `jsonpath` operand is blocked (blockers.sql), since the database cannot +--! evaluate a path over ciphertext. +--! +--! @param a public.{{prefix}}_json_search The stored document. +--! @param b {{prefix}}.query_json_predicate The predicate. +--! @return boolean As {{prefix}}.json_tree_match. +CREATE FUNCTION {{prefix}}."@@"(a public.{{prefix}}_json_search, b {{prefix}}.query_json_predicate) + RETURNS boolean + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT {{prefix}}.json_tree_match(a, b) +$$; + +CREATE OPERATOR @@( + FUNCTION={{prefix}}."@@", + LEFTARG=public.{{prefix}}_json_search, + RIGHTARG={{prefix}}.query_json_predicate +); diff --git a/packages/eql/src/v3/json_tree/types.sql b/packages/eql/src/v3/json_tree/types.sql index e1cb324c0..7ba6d6d1a 100644 --- a/packages/eql/src/v3/json_tree/types.sql +++ b/packages/eql/src/v3/json_tree/types.sql @@ -20,6 +20,9 @@ --! - {{prefix}}.query_json_elements — an element-scoped predicate: per --! member of an array's elements, the --! member's read tokens and its bounds. +--! A sixth, {{prefix}}.query_json_predicate, a boolean tree whose leaves +--! are these operands, is in predicate.sql beside the function that reads +--! it. --! --! The design is `docs/plans/2026-10-17-json-tree-index.md` --! at the repository root. The document's per-entry fields: @@ -301,21 +304,56 @@ BEGIN END $$; +--! @brief Validate the bounds of an ordering at one path. +--! @internal +--! +--! Bounds are `{"gt" | "gte": leaf, "lt" | "lte": leaf}`: one or two +--! ordering operands of the shape `{{prefix}}.query_json_leaf` holds +--! (`{"s", "op"}`). At most one lower bound (`gt` or `gte`) and at most one +--! upper (`lt` or `lte`), at least one in all, and every bound at the one +--! selector: the path's. A member of an element-scoped predicate is bounds +--! with its tokens beside them, and the `cmp` leaf of a +--! `{{prefix}}.query_json_predicate` is bounds alone. +--! +--! @param val jsonb Candidate bounds. +--! @return boolean True when `val` is bounds. +--! @note plpgsql, for the subqueries over the bounds, which a CHECK +--! expression cannot hold. +CREATE OR REPLACE FUNCTION {{prefix}}_internal.json_tree_is_valid_bounds(val jsonb) + RETURNS boolean + LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE +AS $$ +BEGIN + RETURN COALESCE( + jsonb_typeof(val) = 'object' + AND val - ARRAY['gt', 'gte', 'lt', 'lte']::text[] = '{}'::jsonb + AND val ?| ARRAY['gt', 'gte', 'lt', 'lte']::text[] + AND NOT (val ? 'gt' AND val ? 'gte') + AND NOT (val ? 'lt' AND val ? 'lte') + -- Each bound by the `{{prefix}}.query_json_leaf` CHECK's own rule. + AND NOT EXISTS ( + SELECT 1 FROM jsonb_each(val) AS bound + WHERE NOT {{prefix}}_internal.json_tree_is_valid_leaf_query_payload(bound.value) + ) + AND (SELECT count(DISTINCT bound.value ->> 's') FROM jsonb_each(val) AS bound) = 1, + false + ); +END; +$$; + --! @brief Validate one member of an element-scoped predicate. --! @internal --! --! A member is `{"t": ["…", …], "gt" | "gte": leaf, "lt" | "lte": leaf}`: --! the tokens of a read of the member's path, one per element below the ---! wildcard bound in element order and then the probe, and one or two ---! bounds, each an ordering operand of the shape `{{prefix}}.query_json_leaf` ---! holds (`{"s", "op"}`). At most one lower bound (`gt` or `gte`) and at most ---! one upper (`lt` or `lte`), at least one in all, and every bound of the ---! member at the one selector: the member's path's. +--! wildcard bound in element order and then the probe, and the bounds of +--! the member's path beside them, as +--! {{prefix}}_internal.json_tree_is_valid_bounds validates them. --! --! @param val jsonb Candidate member. --! @return boolean True when `val` is a member. ---! @note plpgsql, for the subqueries over the tokens and bounds, which a ---! CHECK expression cannot hold. +--! @note plpgsql, for the subquery over the tokens, which a CHECK +--! expression cannot hold. CREATE OR REPLACE FUNCTION {{prefix}}_internal.json_tree_is_valid_elements_member(val jsonb) RETURNS boolean LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE @@ -323,22 +361,13 @@ AS $$ BEGIN RETURN COALESCE( jsonb_typeof(val) = 'object' - AND val - ARRAY['t', 'gt', 'gte', 'lt', 'lte']::text[] = '{}'::jsonb AND jsonb_typeof(val -> 't') = 'array' AND jsonb_array_length(val -> 't') >= 1 AND NOT EXISTS ( SELECT 1 FROM jsonb_array_elements(val -> 't') AS token WHERE jsonb_typeof(token) <> 'string' ) - AND val ?| ARRAY['gt', 'gte', 'lt', 'lte']::text[] - AND NOT (val ? 'gt' AND val ? 'gte') - AND NOT (val ? 'lt' AND val ? 'lte') - -- Each bound by the `{{prefix}}.query_json_leaf` CHECK's own rule. - AND NOT EXISTS ( - SELECT 1 FROM jsonb_each(val - 't') AS bound - WHERE NOT {{prefix}}_internal.json_tree_is_valid_leaf_query_payload(bound.value) - ) - AND (SELECT count(DISTINCT bound.value ->> 's') FROM jsonb_each(val - 't') AS bound) = 1, + AND {{prefix}}_internal.json_tree_is_valid_bounds(val - 't'), false ); END; diff --git a/packages/eql/tests/encryption/tests/json_tree.rs b/packages/eql/tests/encryption/tests/json_tree.rs index 965613f98..e3c69c9a8 100644 --- a/packages/eql/tests/encryption/tests/json_tree.rs +++ b/packages/eql/tests/encryption/tests/json_tree.rs @@ -516,6 +516,15 @@ mod generated; #[path = "json_tree/native_functions.rs"] mod native_functions; +/// The element-scoped predicate, through a data plan, against the column. +#[path = "json_tree/elements.rs"] +mod elements; + +/// The compiled predicate, through a data plan, against the column and +/// against `jsonb_path_match` over the plaintexts. +#[path = "json_tree/predicate.rs"] +mod predicate; + /// Needs an EQL v4 installation. Extraction by selector opens to the leaf; /// the subtree walk by a client-derived token returns exactly the nodes /// under it and reassembles; the array functions count and list elements; diff --git a/packages/eql/tests/encryption/tests/json_tree/elements.rs b/packages/eql/tests/encryption/tests/json_tree/elements.rs new file mode 100644 index 000000000..5e4ea4359 --- /dev/null +++ b/packages/eql/tests/encryption/tests/json_tree/elements.rs @@ -0,0 +1,1047 @@ +//! The element-scoped predicate end to end: documents sealed by Stack +//! Encrypt's JSON index, the `element_match` operand derived through a data +//! plan over eql-bindings' target table, and +//! `eql_v4.json_tree_element_match` over the stored column. The resolver +//! here ([`Eql`]) names the document's leaf, element-scoped and compiled +//! predicate domains, so the engine answers `element_match` and `match` +//! through it. The Go guest's resolver names the leaf domain alone (its +//! `targets` tests pin that), so the Go SDK is not offered either operator +//! until the Go side takes them. +//! +//! These tests pin: +//! +//! - the operand domain's CHECK, member by member, over a derived operand +//! and its parts; +//! - that one element must satisfy every member: two elements each matching +//! half of the predicate are not a match, where the component orderings +//! over `json_tree_path_query` are, under the sealed root and the open one; +//! - each bound's comparison, a field with no ordering term, and an absent +//! field; +//! - that a probe's hit raises: an array longer than the bound, and a token +//! that names a node at another selector than its member's; +//! - that `json_tree_subtree` is `json_tree_subtree_indexed` without `k`, +//! under every budget, and what `k` is, the sealed-root fallback's `0` +//! included. + +use super::*; + +use eql_bindings::encryption::targets::{self as eql_targets, Target}; +use stack_encrypt::dynamic::{ + self, record, OpSpec, Operand, TargetDescriptor, TargetError, TargetResolver, ValueFrame, +}; +use stack_encrypt::{KeysetCipher, Label, Pending, StackCipher}; + +/// eql-bindings' table as the engine's resolver, naming each type's leaf, +/// element predicate and compiled predicate domains: a host that answers +/// `element_match` and `match`. Not the Go guest's resolver, which names +/// the leaf domain alone. +pub(super) struct Eql; + +fn describe(target: &Target) -> TargetDescriptor { + let mut descriptor = TargetDescriptor::new( + target.name, + target.family, + target.suffix, + target.plaintext_kind(), + target.sql_domain, + target.indexes.iter().map(|key| (*key).to_owned()).collect(), + target.query.map(str::to_owned), + target.query_sql_domain.map(str::to_owned), + target.producible, + target.reason.map(str::to_owned), + ); + if let Some(domain) = target.leaf_query_sql_domain() { + descriptor = descriptor.with_leaf_query_sql_domain(domain); + } + if let Some(domain) = target.elements_query_sql_domain() { + descriptor = descriptor.with_elements_query_sql_domain(domain); + } + if let Some(domain) = target.predicate_query_sql_domain() { + descriptor = descriptor.with_predicate_query_sql_domain(domain); + } + descriptor +} + +impl TargetResolver for Eql { + fn targets(&self) -> Vec { + eql_targets::targets().iter().map(describe).collect() + } + fn encrypt<'a, K: 'static>( + &self, + name: &str, + options: Option<&dynamic::TargetOptions>, + keyset: &'a KeysetCipher<'_, K>, + label: &Label, + plaintext: ValueFrame, + ) -> Result, K>, TargetError> { + eql_targets::encrypt_with(name, options, keyset, label, plaintext) + .map_err(|error| TargetError::Other(Box::new(error))) + } + fn decrypt<'a, K: 'static>( + &self, + name: &str, + cipher: &'a StackCipher, + label: &Label, + stored: &[u8], + ) -> Result, TargetError> { + eql_targets::decrypt(name, cipher, label, stored) + .map_err(|error| TargetError::Other(Box::new(error))) + } + fn query<'a, K: 'static>( + &self, + name: &str, + index: &stack_encrypt::target::IndexSpec, + keyset: &'a KeysetCipher<'_, K>, + label: &Label, + plaintext: ValueFrame, + ) -> Result, K>, TargetError> { + eql_targets::query(name, index, keyset, label, plaintext) + .map_err(|error| TargetError::Other(Box::new(error))) + } +} + +const RELATIONSHIP: &str = "$.extra.emergency_contacts[*].relationship"; +const CALLS: &str = "$.extra.emergency_contacts[*].previous_call_count"; + +/// The documents' index options: each contact's relationship and call +/// count carry ordering terms. +fn contact_options() -> JsonOptions { + JsonOptions::default() + .ordering([RELATIONSHIP, CALLS]) + .unwrap() +} + +/// `profile` under `users/profile`, the column the documents are sealed +/// under, naming `JsonSearch` with [`contact_options`]. +fn contact_plan() -> record::Plan { + let wire = json!({ + "profile": { + "context": ["users", "profile"], + "type": "json", + "null": "sql", + "target": {"name": "JsonSearch", "options": {"ordered": [RELATIONSHIP, CALLS]}} + } + }); + record::plan_with(&dynamic::frame::serde::to_vec(&wire).unwrap(), &Eql).unwrap() +} + +pub(super) fn frame_of(value: &Value) -> ValueFrame { + ValueFrame::from_bytes(&dynamic::frame::serde::to_vec(value).unwrap()).unwrap() +} + +/// The operand of `operator` on `profile` for `input`, as the SQL binds +/// it: an EQL operand's JSON text, or a list of tokens. +pub(super) fn derive( + runtime: &tokio::runtime::Runtime, + keyset: &KeysetCipher<'_, common::Observed>, + plan: &record::Plan, + operator: &str, + input: Value, +) -> Operand { + let term = wait( + runtime, + dynamic::query_term( + keyset, + plan, + "profile", + OpSpec::parse(operator).unwrap(), + frame_of(&input), + &Eql, + ) + .unwrap(), + ) + .unwrap(); + term.operand().clone() +} + +pub(super) fn eql_text(operand: &Operand) -> String { + match operand { + Operand::One(bytes) => String::from_utf8(bytes.clone()).unwrap(), + other => panic!("one EQL operand, got {other:?}"), + } +} + +/// The contacts array's elements, which every `element_match` here reads. +const ELEMENTS: &str = "$.extra.emergency_contacts[*]"; + +/// SQLSTATE `check_violation`, a value its domain's CHECK refuses. +const CHECK_VIOLATION: &str = "23514"; +/// SQLSTATE `invalid_parameter_value`, which a probe's hit raises with. +const INVALID_PARAMETER_VALUE: &str = "22023"; + +/// [`contact_options`] with the root sealed or open. +fn layout_options(sealed: bool) -> JsonOptions { + let mut options = contact_options(); + options.sealed_root = sealed; + options +} + +/// [`contact_plan`] with the root sealed or open. +fn layout_plan(sealed: bool) -> record::Plan { + let wire = json!({ + "profile": { + "context": ["users", "profile"], + "type": "json", + "null": "sql", + "target": {"name": "JsonSearch", "options": { + "sealed_root": sealed, + "ordered": [RELATIONSHIP, CALLS] + }} + } + }); + record::plan_with(&dynamic::frame::serde::to_vec(&wire).unwrap(), &Eql).unwrap() +} + +/// A contact with each field present when given. +fn contact(relationship: Option<&str>, calls: Option) -> Value { + let mut fields = Map::new(); + if let Some(relationship) = relationship { + let _ = fields.insert("relationship".into(), json!(relationship)); + } + if let Some(calls) = calls { + let _ = fields.insert("previous_call_count".into(), json!(calls)); + } + Value::Object(fields) +} + +/// A document holding `list` as its contacts. +fn contacts_document(list: Vec) -> Value { + json!({"name": "x", "extra": {"emergency_contacts": list}}) +} + +/// The `element_match` operand over the contacts at `bound`, as JSON text. +fn element_match( + runtime: &tokio::runtime::Runtime, + keyset: &KeysetCipher<'_, common::Observed>, + plan: &record::Plan, + bound: u32, + predicates: Value, +) -> String { + eql_text(&derive( + runtime, + keyset, + plan, + "element_match", + json!({"path": ELEMENTS, "wildcard_bound": bound, "predicates": predicates}), + )) +} + +/// "relationship is Sibling and more than five calls", read to `bound`. +fn sibling_with_more_than_five_calls( + runtime: &tokio::runtime::Runtime, + keyset: &KeysetCipher<'_, common::Observed>, + plan: &record::Plan, + bound: u32, +) -> String { + element_match( + runtime, + keyset, + plan, + bound, + json!([ + {"member": "relationship", "op": "eq", "json": "\"Sibling\""}, + {"member": "previous_call_count", "op": "gt", "json": "5"} + ]), + ) +} + +/// The ids `json_tree_element_match` holds of in `table`, for `operand`. +fn matching(db: &mut postgres::Client, table: &str, operand: &str) -> Vec { + db.query( + &format!( + "SELECT id FROM {table} \ + WHERE eql_v4.json_tree_element_match(doc, $1::text::jsonb::eql_v4.query_json_elements) ORDER BY id" + ), + &[&operand], + ) + .unwrap_or_else(|error| panic!("{table}: {error}")) + .iter() + .map(|row| row.get(0)) + .collect() +} + +/// The SQLSTATE a statement fails with, or `None` if it succeeds. +fn sqlstate( + db: &mut postgres::Client, + sql: &str, + params: &[&(dyn postgres::types::ToSql + Sync)], +) -> Option { + match db.query(sql, params) { + Ok(_) => None, + Err(error) => match error.as_db_error() { + Some(e) => Some(e.code().code().to_owned()), + None => panic!("{sql}: not a database error: {error}"), + }, + } +} + +/// The one token of a read of the concrete `path` under `options`. +fn token( + runtime: &tokio::runtime::Runtime, + keyset: &KeysetCipher<'_, common::Observed>, + options: &JsonOptions, + path: &str, +) -> String { + let key = wait( + runtime, + keyset.json_subtree_key(&JsonPath::parse(path).unwrap(), options, column()), + ) + .unwrap(); + let tokens = key.tokens(); + assert_eq!(tokens.len(), 1, "{path} names one node"); + tokens[0].clone() +} + +/// Needs an EQL v4 installation. The customer's predicate, a Sibling with +/// more than five calls in one contact: the element predicate answers it, +/// where `@>` and an ordering over `json_tree_path_query` answer "some +/// contact is a Sibling and some has more than five calls". A row with a +/// contact past the bound fails the query. `json_tree_subtree` is the +/// indexed read without `k`, and `k` pairs a contact's two fields. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_an_element_predicate_holds_of_one_element() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = contact_plan(); + let mut db = connect(); + let contacts = |list: Value| json!({"name": "x", "extra": {"emergency_contacts": list}}); + let documents = [ + // Each contact holds one half of the predicate. + contacts(json!([ + {"relationship": "Sibling", "previous_call_count": 3}, + {"relationship": "Friend", "previous_call_count": 7} + ])), + // One contact holds both. + contacts(json!([ + {"relationship": "Friend", "previous_call_count": 9}, + {"relationship": "Sibling", "previous_call_count": 7} + ])), + // Neither. + contacts(json!([{"relationship": "Friend", "previous_call_count": 9}])), + // No contacts. + json!({"name": "y"}), + // Folded equality: "sibling" ties with "Sibling". + contacts(json!([{"relationship": "sibling", "previous_call_count": 6}])), + ]; + store( + &mut db, + &runtime, + &cipher, + "json_tree_elements", + &contact_options(), + &documents, + ); + let ids = |db: &mut postgres::Client, + sql: &str, + params: &[&(dyn postgres::types::ToSql + Sync)]| + -> Vec { + db.query(sql, params) + .unwrap_or_else(|error| panic!("{sql}: {error}")) + .iter() + .map(|row| row.get(0)) + .collect() + }; + + let predicate = |bound: u32| { + eql_text(&derive( + &runtime, + &keyset, + &plan, + "element_match", + json!({"path": "$.extra.emergency_contacts[*]", "wildcard_bound": bound, "predicates": [ + {"member": "relationship", "op": "eq", "json": "\"Sibling\""}, + {"member": "previous_call_count", "op": "gt", "json": "5"} + ]}), + )) + }; + let element_scoped = "SELECT id FROM json_tree_elements \ + WHERE eql_v4.json_tree_element_match(doc, $1::text::jsonb::eql_v4.query_json_elements) ORDER BY id"; + assert_eq!( + ids(&mut db, element_scoped, &[&predicate(4)]), + [2, 5], + "one contact a Sibling with more than five calls, the folded spelling included" + ); + + // The superset: containment on the relationship, an ordering over every + // call count. + let needle = eql_text(&derive( + &runtime, + &keyset, + &plan, + "eq_at", + json!({"path": RELATIONSHIP, "json": "\"Sibling\""}), + )); + let Operand::Text(calls_selector) = + derive(&runtime, &keyset, &plan, "selector", json!({"path": CALLS})) + else { + panic!("a selector is text") + }; + let five = eql_text(&derive( + &runtime, + &keyset, + &plan, + "gt_at", + json!({"path": CALLS, "json": "5"}), + )); + let superset = "SELECT id FROM json_tree_elements \ + WHERE doc @> $1::text::jsonb::eql_v4.query_json \ + AND EXISTS (SELECT 1 FROM eql_v4.json_tree_path_query(doc, $2::text) AS e \ + WHERE e > $3::text::jsonb::eql_v4.query_json_leaf) ORDER BY id"; + assert_eq!( + ids(&mut db, superset, &[&needle, &calls_selector, &five]), + [1, 2], + "some contact a Sibling and some over five: row 1's halves answer it, and \ + containment is exact, so the folded spelling does not" + ); + + // A bound below the longest array: the probe names the element at it. + let error = db.query(element_scoped, &[&predicate(1)]).unwrap_err(); + assert_eq!( + error.code(), + Some(&postgres::error::SqlState::INVALID_PARAMETER_VALUE), + "{error}" + ); + let message = error + .as_db_error() + .map(|db| db.message()) + .unwrap_or_default(); + assert!( + message.contains("longer than the bound"), + "the message says why: {message}" + ); + // At exactly the longest array's length, it answers. + assert_eq!(ids(&mut db, element_scoped, &[&predicate(2)]), [2, 5]); + + // `json_tree_subtree` is the indexed read without `k`, and the `k`-th + // token of each field's read names a field of one contact. + let tokens = |path: &str| match derive( + &runtime, + &keyset, + &plan, + "subtree", + json!({"path": path, "wildcard_bound": 2}), + ) { + Operand::TextList(tokens) => tokens, + other => panic!("{other:?}"), + }; + let (relationships, counts) = (tokens(RELATIONSHIP), tokens(CALLS)); + for budget in [0, 1, 64] { + for id in 1..=5 { + let plain: Vec = db + .query( + "SELECT r::jsonb::text FROM json_tree_elements, eql_v4.json_tree_subtree(doc, $1::text[], $2::int) AS r WHERE id = $3", + &[&relationships, &budget, &id], + ) + .unwrap() + .iter() + .map(|row| row.get(0)) + .collect(); + let indexed: Vec = db + .query( + "SELECT r.entry::jsonb::text FROM json_tree_elements, eql_v4.json_tree_subtree_indexed(doc, $1::text[], $2::int) AS r WHERE id = $3", + &[&relationships, &budget, &id], + ) + .unwrap() + .iter() + .map(|row| row.get(0)) + .collect(); + assert_eq!(plain, indexed, "row {id}, budget {budget}"); + } + } + let paired: Vec<(i32, i32)> = db + .query( + "SELECT r.k, c.k FROM json_tree_elements, + eql_v4.json_tree_subtree_indexed(doc, $1::text[], 0) AS r + JOIN eql_v4.json_tree_subtree_indexed(doc, $2::text[], 0) AS c ON c.k = r.k + WHERE id = 2 ORDER BY r.k", + &[&relationships, &counts], + ) + .unwrap() + .iter() + .map(|row| (row.get(0), row.get(1))) + .collect(); + assert_eq!(paired, [(1, 1), (2, 2)], "one pair per contact, by k"); + let fallback: Vec = db + .query( + "SELECT r.k FROM json_tree_elements, eql_v4.json_tree_subtree_indexed(doc, $1::text[], 1) AS r WHERE id = 2", + &[&relationships], + ) + .unwrap() + .iter() + .map(|row| row.get(0)) + .collect(); + assert_eq!( + fallback, + [0], + "past the budget, the sealed root alone, at k = 0" + ); + + // A member whose full path the field does not order is refused before + // anything is derived. + let error = dynamic::check_query( + &plan, + "profile", + OpSpec::ElementMatch, + frame_of( + &json!({"path": "$.extra.emergency_contacts[*]", "predicates": [ + {"member": "name", "op": "eq", "json": "\"x\""} + ]}), + ), + &Eql, + ) + .unwrap_err(); + assert!( + matches!( + error, + dynamic::Error::Cipher(stack_encrypt::Error::Term( + stack_encrypt::sem::TermError::JsonPathNotOrdered + )) + ), + "{error:?}" + ); +} + +/// Needs an EQL v4 installation. The operand domain's CHECK takes a list +/// of members, each tokens and one or two bounds at one selector, and every +/// member's tokens as many as every other's. The values are a derived +/// operand and its parts: two of its tokens, and the leaf operands it holds +/// at each field. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_the_elements_operand_domain_checks_each_member() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = contact_plan(); + let mut db = connect(); + let derived = sibling_with_more_than_five_calls(&runtime, &keyset, &plan, 1); + let operand: Value = serde_json::from_str(&derived).unwrap(); + let (relationship, calls) = (&operand["m"][0], &operand["m"][1]); + // The relationship's leaf, and the call count's: two selectors. + let (leaf, other) = (&relationship["gte"], &calls["gt"]); + // The read at bound 1 is one element and the probe: two tokens. + let (a, b) = (&relationship["t"][0], &relationship["t"][1]); + assert!(a.is_string() && b.is_string() && a != b); + let without = |key: &str| { + let mut bound = leaf.clone(); + let _ = bound.as_object_mut().unwrap().remove(key); + bound + }; + let mut leaf_and_more = leaf.clone(); + let _ = leaf_and_more + .as_object_mut() + .unwrap() + .insert("x".into(), json!(1)); + + let valid = [ + operand.clone(), + json!({"m": [{"t": [a, b], "gt": leaf}]}), + json!({"m": [{"t": [a], "gte": leaf, "lte": leaf}]}), + json!({"m": [{"t": [a], "gt": leaf, "lt": leaf}, {"t": [b], "lte": other}]}), + ]; + let invalid = [ + ("not an object", json!([])), + ("no members", json!({"m": []})), + ("members not a list", json!({"m": {}})), + ("no m", json!({})), + ( + "another key", + json!({"m": [{"t": [a], "gt": leaf}], "x": 1}), + ), + ("a member not an object", json!({"m": [a]})), + ("no tokens", json!({"m": [{"gt": leaf}]})), + ("empty tokens", json!({"m": [{"t": [], "gt": leaf}]})), + ("a token not text", json!({"m": [{"t": [1], "gt": leaf}]})), + ("tokens not a list", json!({"m": [{"t": a, "gt": leaf}]})), + ("no bound", json!({"m": [{"t": [a]}]})), + ( + "two lower bounds", + json!({"m": [{"t": [a], "gt": leaf, "gte": leaf}]}), + ), + ( + "two upper bounds", + json!({"m": [{"t": [a], "lt": leaf, "lte": leaf}]}), + ), + ( + "a bound by another name", + json!({"m": [{"t": [a], "eq": leaf}]}), + ), + ( + "a bound with no term", + json!({"m": [{"t": [a], "gt": without("op")}]}), + ), + ( + "a bound with no selector", + json!({"m": [{"t": [a], "gt": without("s")}]}), + ), + ( + "a bound with another key", + json!({"m": [{"t": [a], "gt": leaf_and_more}]}), + ), + ( + "a bound not an object", + json!({"m": [{"t": [a], "gt": leaf["s"]}]}), + ), + ( + "bounds at two selectors", + json!({"m": [{"t": [a], "gt": leaf, "lt": other}]}), + ), + ( + "members of two widths", + json!({"m": [{"t": [a], "gt": leaf}, {"t": [a, b], "gt": other}]}), + ), + ]; + let cast = "SELECT $1::text::jsonb::eql_v4.query_json_elements"; + for value in &valid { + assert_eq!( + sqlstate(&mut db, cast, &[&value.to_string()]), + None, + "valid: {value}" + ); + } + for (why, value) in &invalid { + assert_eq!( + sqlstate(&mut db, cast, &[&value.to_string()]).as_deref(), + Some(CHECK_VIOLATION), + "{why}: {value}" + ); + } +} + +/// Needs an EQL v4 installation. A Sibling with more than five calls, as +/// the element predicate and as the component orderings, under the sealed +/// root and the open one: only a row where one contact is both answers the +/// first, where the second also answers a row whose two contacts each hold +/// half, or whose fields sit in different contacts. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_one_element_must_satisfy_every_member() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let mut db = connect(); + let documents = [ + contacts_document(vec![ + contact(Some("Sibling"), Some(3)), + contact(Some("Friend"), Some(7)), + ]), + contacts_document(vec![ + contact(Some("Friend"), Some(9)), + contact(Some("Sibling"), Some(7)), + ]), + contacts_document(vec![contact(Some("Friend"), Some(9))]), + contacts_document(vec![contact(Some("Sibling"), None), contact(None, Some(8))]), + contacts_document(vec![]), + contacts_document(vec![contact(Some("Sibling"), Some(6))]), + ]; + for sealed in [true, false] { + let table = if sealed { + "sealed_contacts" + } else { + "open_contacts" + }; + let plan = layout_plan(sealed); + store( + &mut db, + &runtime, + &cipher, + table, + &layout_options(sealed), + &documents, + ); + let predicate = sibling_with_more_than_five_calls(&runtime, &keyset, &plan, 2); + assert_eq!( + matching(&mut db, table, &predicate), + [2, 6], + "{table}: one contact both a Sibling and over five calls" + ); + + let selector = |path: &str| match derive( + &runtime, + &keyset, + &plan, + "selector", + json!({"path": path}), + ) { + Operand::Text(selector) => selector, + other => panic!("a selector is text, got {other:?}"), + }; + let leaf = |operator: &str, path: &str, value: &str| { + eql_text(&derive( + &runtime, + &keyset, + &plan, + operator, + json!({"path": path, "json": value}), + )) + }; + let superset: Vec = db + .query( + &format!( + "SELECT id FROM {table} + WHERE EXISTS (SELECT 1 FROM eql_v4.json_tree_path_query(doc, $1::text) AS e + WHERE e >= $2::text::jsonb::eql_v4.query_json_leaf + AND e <= $3::text::jsonb::eql_v4.query_json_leaf) + AND EXISTS (SELECT 1 FROM eql_v4.json_tree_path_query(doc, $4::text) AS e + WHERE e > $5::text::jsonb::eql_v4.query_json_leaf) + ORDER BY id" + ), + &[ + &selector(RELATIONSHIP), + &leaf("gte_at", RELATIONSHIP, "\"Sibling\""), + &leaf("lte_at", RELATIONSHIP, "\"Sibling\""), + &selector(CALLS), + &leaf("gt_at", CALLS, "5"), + ], + ) + .unwrap() + .iter() + .map(|row| row.get(0)) + .collect(); + assert_eq!( + superset, + [1, 2, 4, 6], + "{table}: the component orderings, some contact a Sibling and some over five" + ); + + let nulls: Option = db + .query_one( + "SELECT eql_v4.json_tree_element_match(NULL, $1::text::jsonb::eql_v4.query_json_elements)", + &[&predicate], + ) + .unwrap() + .get(0); + assert_eq!(nulls, None, "NULL in, NULL out"); + } +} + +/// Needs an EQL v4 installation. Each bound compares as its operator does +/// on an entry, `between` and `eq` at one value are equality, and a field +/// with no ordering term, or no field at all, satisfies no bound. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_each_bound_compares_as_its_operator() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = contact_plan(); + let mut db = connect(); + store( + &mut db, + &runtime, + &cipher, + "five_calls", + &contact_options(), + &[contacts_document(vec![contact(Some("Sibling"), Some(5))])], + ); + let calls = |op: &str, value: i64| json!({"member": "previous_call_count", "op": op, "json": value.to_string()}); + let between = |low: i64, high: i64| { + json!({"member": "previous_call_count", "op": "between", + "low": low.to_string(), "high": high.to_string()}) + }; + for (predicates, want) in [ + (json!([calls("gt", 5)]), false), + (json!([calls("gt", 4)]), true), + (json!([calls("gte", 5)]), true), + (json!([calls("gte", 6)]), false), + (json!([calls("lt", 5)]), false), + (json!([calls("lt", 6)]), true), + (json!([calls("lte", 5)]), true), + (json!([calls("lte", 4)]), false), + (json!([between(5, 5)]), true), + (json!([between(4, 4)]), false), + (json!([calls("eq", 5)]), true), + (json!([calls("eq", 4)]), false), + (json!([calls("gt", 4), calls("lt", 6)]), true), + (json!([calls("gt", 5), calls("lt", 6)]), false), + (json!([calls("gt", 4), calls("lt", 5)]), false), + ] { + let operand = element_match(&runtime, &keyset, &plan, 1, predicates.clone()); + let got = !matching(&mut db, "five_calls", &operand).is_empty(); + assert_eq!(got, want, "{predicates}"); + } + + // A call count with no ordering term, in a column that orders only the + // relationship; and a contact with no call count. + let relationship_only = JsonOptions::default().ordering([RELATIONSHIP]).unwrap(); + store( + &mut db, + &runtime, + &cipher, + "unordered_calls", + &relationship_only, + &[contacts_document(vec![contact(Some("Sibling"), Some(5))])], + ); + store( + &mut db, + &runtime, + &cipher, + "no_calls", + &contact_options(), + &[contacts_document(vec![contact(Some("Sibling"), None)])], + ); + let any_calls = element_match(&runtime, &keyset, &plan, 1, json!([calls("gte", 0)])); + // The member's first token names the unordered call count, which holds + // no term. + let member: Value = serde_json::from_str(&any_calls).unwrap(); + let tokens: Vec = serde_json::from_value(member["m"][0]["t"].clone()).unwrap(); + let found: Vec = db + .query( + "SELECT r.entry::jsonb ? 'op' FROM unordered_calls, + eql_v4.json_tree_subtree_indexed(doc, $1::text[], 0) AS r WHERE r.k = 1", + &[&tokens], + ) + .unwrap() + .iter() + .map(|row| row.get(0)) + .collect(); + assert_eq!(found, [false], "the call count is found, with no term"); + for table in ["unordered_calls", "no_calls"] { + assert_eq!( + matching(&mut db, table, &any_calls), + Vec::::new(), + "{table}: no term, or no field: no bound holds" + ); + } +} + +/// Needs an EQL v4 installation. A member's token that names a node at +/// another selector than its bounds' is the probe's hit: the array is +/// longer than the bound, and the predicate raises rather than answer over +/// part of it. An array exactly as long as the bound does not. A node at +/// the element's selector raises wherever its token sits in the member's +/// list: here, a document whose call count's entry is moved to the +/// element's selector, which the member's first token then names. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_a_probe_hit_raises() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = contact_plan(); + let mut db = connect(); + let predicate = sibling_with_more_than_five_calls(&runtime, &keyset, &plan, 2); + let two_contacts = contacts_document(vec![ + contact(Some("Friend"), Some(1)), + contact(Some("Sibling"), Some(9)), + ]); + store( + &mut db, + &runtime, + &cipher, + "within_bound", + &contact_options(), + std::slice::from_ref(&two_contacts), + ); + assert_eq!( + matching(&mut db, "within_bound", &predicate), + [1], + "within the bound" + ); + + // A third contact: the element at the bound is a node, at the + // element's selector, which the probe names. + store( + &mut db, + &runtime, + &cipher, + "past_bound", + &contact_options(), + &[ + two_contacts, + contacts_document(vec![ + contact(Some("Sibling"), Some(9)), + contact(Some("Friend"), Some(1)), + contact(Some("Friend"), Some(1)), + ]), + ], + ); + let query = "SELECT id FROM past_bound \ + WHERE eql_v4.json_tree_element_match(doc, $1::text::jsonb::eql_v4.query_json_elements) ORDER BY id"; + let error = db.query(query, &[&predicate]).unwrap_err(); + assert_eq!( + error.code(), + Some(&postgres::error::SqlState::INVALID_PARAMETER_VALUE), + "a row past the bound fails the query, though an element within it matches: {error}" + ); + let message = error + .as_db_error() + .map(|db| db.message()) + .unwrap_or_default(); + assert!( + message.contains("longer than the bound"), + "the message says why: {message}" + ); + + // A node at the element's selector, named by a member's first token. + store( + &mut db, + &runtime, + &cipher, + "forged_selector", + &contact_options(), + &[contacts_document(vec![contact(Some("Sibling"), Some(9))])], + ); + let only_calls = element_match( + &runtime, + &keyset, + &plan, + 2, + json!([{"member": "previous_call_count", "op": "gt", "json": "5"}]), + ); + let member: Value = serde_json::from_str(&only_calls).unwrap(); + let first = member["m"][0]["t"][0].as_str().unwrap().to_owned(); + let Operand::Text(element_selector) = derive( + &runtime, + &keyset, + &plan, + "selector", + json!({"path": ELEMENTS}), + ) else { + panic!("a selector is text") + }; + let moved = db + .execute( + "UPDATE forged_selector SET doc = jsonb_set(doc::jsonb, ARRAY['sv', ( + SELECT (e.o - 1)::text + FROM jsonb_array_elements(doc::jsonb -> 'sv') WITH ORDINALITY AS e(v, o) + WHERE e.v ->> 'x' = eql_v4_internal.json_tree_self_tag( + eql_v4_internal.json_tree_iv(doc::jsonb ->> 'h'), decode($1::text, 'base64'))), + 's'], to_jsonb($2::text))::public.eql_v4_json_search", + &[&first, &element_selector], + ) + .unwrap(); + assert_eq!(moved, 1); + assert_eq!( + sqlstate( + &mut db, + "SELECT eql_v4.json_tree_element_match(doc, $1::text::jsonb::eql_v4.query_json_elements) FROM forged_selector", + &[&only_calls], + ) + .as_deref(), + Some(INVALID_PARAMETER_VALUE), + "a node at the element's selector is never a member's field" + ); +} + +/// Needs an EQL v4 installation. `json_tree_subtree` returns what +/// `json_tree_subtree_indexed` does, without `k`, under every budget; every +/// row carries the place of the token that found its node, the rows +/// beneath that node included; and the sealed root's fallback comes back +/// alone with `k = 0`. The tokens name nothing (`$.missing`), an object +/// with one member (`$.obj`), the two contacts' relationships, and the +/// element past the last, which the document does not hold. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_the_indexed_read_is_the_subtree_read_with_k() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let mut db = connect(); + let document = json!({ + "obj": {"child": "c"}, + "extra": {"emergency_contacts": [ + {"relationship": "Sibling", "previous_call_count": 1}, + {"relationship": "Friend", "previous_call_count": 2} + ]} + }); + for sealed in [false, true] { + let table = if sealed { "sealed_read" } else { "open_read" }; + let options = layout_options(sealed); + store( + &mut db, + &runtime, + &cipher, + table, + &options, + std::slice::from_ref(&document), + ); + let tokens: Vec = [ + "$.missing", + "$.obj", + "$.extra.emergency_contacts[1].relationship", + "$.extra.emergency_contacts[0].relationship", + "$.extra.emergency_contacts[2]", + ] + .iter() + .map(|path| token(&runtime, &keyset, &options, path)) + .collect(); + for budget in [0, 1, 2, 3, 64] { + let read: Vec = db + .query( + &format!( + "SELECT r::jsonb::text FROM {table}, eql_v4.json_tree_subtree(doc, $1::text[], $2::int) AS r" + ), + &[&tokens, &budget], + ) + .unwrap() + .iter() + .map(|row| serde_json::from_str(&row.get::<_, String>(0)).unwrap()) + .collect(); + let indexed: Vec<(i32, Value)> = db + .query( + &format!( + "SELECT r.k, r.entry::jsonb::text FROM {table}, eql_v4.json_tree_subtree_indexed(doc, $1::text[], $2::int) AS r" + ), + &[&tokens, &budget], + ) + .unwrap() + .iter() + .map(|row| { + ( + row.get(0), + serde_json::from_str(&row.get::<_, String>(1)).unwrap(), + ) + }) + .collect(); + let entries: Vec = indexed.iter().map(|(_, entry)| entry.clone()).collect(); + assert_eq!( + read, entries, + "{table}, budget {budget}: the same rows in order" + ); + let ks: Vec = indexed.iter().map(|(k, _)| *k).collect(); + let mut sorted = ks.clone(); + sorted.sort_unstable(); + assert_eq!(ks, sorted, "{table}, budget {budget}: ordered by k"); + // Four nodes, at d = 0 and 1, from four of the five tokens. + if sealed && budget > 0 && budget < 4 { + assert_eq!(indexed.len(), 1, "{table}, budget {budget}: the root alone"); + assert_eq!(ks, [0], "the root is found by no token"); + assert_eq!(indexed[0].1["j"], 0, "the root entry"); + } else { + // Within one `k` the rows come in the order the entries + // are stored, which sealing draws at random. + let mut found: Vec<(i32, i64)> = indexed + .iter() + .map(|(k, entry)| (*k, entry["d"].as_i64().unwrap())) + .collect(); + found.sort_unstable(); + assert_eq!( + found, + [(2, 0), (2, 1), (3, 0), (4, 0)], + "{table}, budget {budget}: k is the place of the token whose node a row sits in" + ); + } + } + } +} diff --git a/packages/eql/tests/encryption/tests/json_tree/predicate.rs b/packages/eql/tests/encryption/tests/json_tree/predicate.rs new file mode 100644 index 000000000..5d9c75626 --- /dev/null +++ b/packages/eql/tests/encryption/tests/json_tree/predicate.rs @@ -0,0 +1,967 @@ +//! The compiled predicate end to end: documents sealed by Stack Encrypt's +//! JSON index, jsonpath predicates compiled by the `match` operator through +//! a data plan over eql-bindings' target table, with a resolver that names +//! the compiled predicate domain ([`Eql`]; the Go guest's does not yet), +//! and `eql_v4.json_tree_match` and `@@` over the stored column, +//! against PostgreSQL's own `jsonb_path_match` over the plaintexts and +//! against the WHERE a hand writes from the operators the leaves stand for. + +use super::elements::{derive, eql_text, frame_of, Eql}; +use super::*; + +use stack_encrypt::dynamic::{self, record, OpSpec, Operand}; +use stack_encrypt::KeysetCipher; + +/// The paths the documents order: what the predicates' ranges ask at. +const ORDERED: [&str; 4] = [ + "$.age", + "$.name", + "$.contacts[*].calls", + "$.contacts[*].relationship", +]; + +fn people_options() -> JsonOptions { + JsonOptions::default().ordering(ORDERED).unwrap() +} + +/// `profile` under `users/profile`, the column the documents are sealed +/// under, naming `JsonSearch` with [`people_options`]. +fn people_plan() -> record::Plan { + let wire = json!({ + "profile": { + "context": ["users", "profile"], + "type": "json", + "null": "sql", + "target": {"name": "JsonSearch", "options": {"ordered": ORDERED}} + } + }); + record::plan_with(&dynamic::frame::serde::to_vec(&wire).unwrap(), &Eql).unwrap() +} + +/// Documents with every field absent, `null` or a value of the one type its +/// path holds, and arrays only where the predicates name `[*]`. +fn people() -> Vec { + vec![ + json!({"age": 34, "name": "ann", "tags": ["admin", "ops"], + "contacts": [{"relationship": "sibling", "calls": 3}, + {"relationship": "friend", "calls": 7}]}), + json!({"age": 17, "name": "bob", + "contacts": [{"relationship": "friend", "calls": 9}, + {"relationship": "sibling", "calls": 7}]}), + json!({"age": null, "name": "cat", "tags": []}), + json!({"name": null, "contacts": []}), + json!({}), + json!({"age": 60, "tags": ["ops"], "contacts": [{"relationship": "sibling"}]}), + json!({"age": 18, "name": "dan", "ok": true}), + ] +} + +/// Each row's answer under `sql`, which takes `params`, NULL as `None`. +fn answers( + db: &mut postgres::Client, + sql: &str, + params: &[&(dyn postgres::types::ToSql + Sync)], +) -> Vec<(i32, Option)> { + db.query(sql, params) + .unwrap_or_else(|error| panic!("{sql}: {error}")) + .iter() + .map(|row| (row.get(0), row.get(1))) + .collect() +} + +/// `match`'s operand for `predicate` under `vars`. +fn compiled( + runtime: &tokio::runtime::Runtime, + keyset: &KeysetCipher<'_, common::Observed>, + plan: &record::Plan, + predicate: &str, + vars: &Value, +) -> String { + eql_text(&derive( + runtime, + keyset, + plan, + "match", + json!({"predicate": predicate, "vars": vars.to_string(), "wildcard_bound": 4}), + )) +} + +/// Needs an EQL v4 installation. For each predicate, `json_tree_match` and +/// `@@` over the encrypted column answer what `jsonb_path_match` answers of +/// the same predicate and variables over the plaintexts, row by row, NULL +/// included: equality, `!=` and the orderings with absent paths and JSON +/// `null`s, wildcards, filters, and `&&`, `||` and `!` over them. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_a_compiled_predicate_answers_as_jsonb_path_match() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = people_plan(); + let mut db = connect(); + store( + &mut db, + &runtime, + &cipher, + "json_tree_people", + &people_options(), + &people(), + ); + let vars = json!({"min": 18, "who": "ann", "r": "sibling", "n": 5, "tag": "ops"}); + for predicate in [ + "$.age >= $min", + "!($.age >= $min)", + "$.age < 18", + "$.age == null", + "$.age != null", + "$.age != 34", + "$.name == $who", + "$.name != $who", + "$.name > \"bob\"", + "!($.name <= \"bob\")", + "$.ok == true", + "exists($.tags)", + "exists($.tags[*])", + "$.tags[*] == $tag", + "exists($.tags[*] ? (@ == $tag))", + "$.contacts[*].calls > $n", + "$.contacts[*].relationship == $r", + "exists($.contacts[*] ? (@.relationship == $r))", + "exists($.contacts[*] ? (@.relationship == $r && @.calls > $n))", + "!exists($.contacts[*] ? (@.relationship == $r && @.calls > $n))", + "$.age >= $min && exists($.contacts[*] ? (@.relationship == $r && @.calls > $n))", + "$.name == $who || $.tags[*] == $tag", + "!($.age < $min || $.name == null) && exists($.name)", + "$.age > 20 && $.age < 50", + ] { + let operand = compiled(&runtime, &keyset, &plan, predicate, &vars); + let matched = answers( + &mut db, + "SELECT id, eql_v4.json_tree_match(doc, $1::text::jsonb::eql_v4.query_json_predicate) \ + FROM json_tree_people ORDER BY id", + &[&operand], + ); + let by_operator = answers( + &mut db, + "SELECT id, doc @@ $1::text::jsonb::eql_v4.query_json_predicate \ + FROM json_tree_people ORDER BY id", + &[&operand], + ); + let plaintext = answers( + &mut db, + "SELECT id, jsonb_path_match(plain, $1::text::jsonpath, $2::text::jsonb) \ + FROM json_tree_people ORDER BY id", + &[&predicate, &vars.to_string()], + ); + assert_eq!(matched, plaintext, "{predicate}: against jsonb_path_match"); + assert_eq!(by_operator, matched, "{predicate}: @@ is json_tree_match"); + } +} + +/// Needs an EQL v4 installation. A compiled predicate answers what the +/// hand-written WHERE answers, NULL included, each leaf by the operand its +/// own operator derives: `json_tree_path_exists` for the path, `@>` for +/// the `null` needle and the equalities, `->` and `>=` for the ordering, +/// and `json_tree_element_match` for the filter. And what `match` refuses +/// it refuses before anything is derived, at the offset of the construct. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_a_compiled_predicate_answers_as_its_operators() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = people_plan(); + let mut db = connect(); + store( + &mut db, + &runtime, + &cipher, + "json_tree_operators", + &people_options(), + &people(), + ); + let one = + |operator: &str, input: Value| eql_text(&derive(&runtime, &keyset, &plan, operator, input)); + let Operand::Text(age) = derive( + &runtime, + &keyset, + &plan, + "selector", + json!({"path": "$.age"}), + ) else { + panic!("a selector is text") + }; + let null_age = one("eq_at", json!({"path": "$.age", "json": "null"})); + let at_least = one("gte_at", json!({"path": "$.age", "json": "18"})); + let sibling_over_five = one( + "element_match", + json!({"path": "$.contacts[*]", "wildcard_bound": 4, "predicates": [ + {"member": "relationship", "op": "eq", "json": "\"sibling\""}, + {"member": "calls", "op": "gt", "json": "5"}, + ]}), + ); + let ann = one("eq_at", json!({"path": "$.name", "json": "\"ann\""})); + + let predicate = + "!($.age >= 18 && exists($.contacts[*] ? (@.relationship == \"sibling\" && @.calls > 5))) \ + || $.name == \"ann\""; + let operand = compiled(&runtime, &keyset, &plan, predicate, &json!({})); + let matched = answers( + &mut db, + "SELECT id, eql_v4.json_tree_match(doc, $1::text::jsonb::eql_v4.query_json_predicate) \ + FROM json_tree_operators ORDER BY id", + &[&operand], + ); + let written = answers( + &mut db, + "SELECT id, (NOT (eql_v4.json_tree_path_exists(doc, $1::text) \ + AND NOT (doc @> $2::text::jsonb::eql_v4.query_json) \ + AND (doc -> $1::text) >= $3::text::jsonb::eql_v4.query_json_leaf \ + AND eql_v4.json_tree_element_match(doc, $4::text::jsonb::eql_v4.query_json_elements))) \ + OR doc @> $5::text::jsonb::eql_v4.query_json \ + FROM json_tree_operators ORDER BY id", + &[&age, &null_age, &at_least, &sibling_over_five, &ann], + ); + assert_eq!( + matched, written, + "the tree against the operators written out" + ); + let plaintext = answers( + &mut db, + "SELECT id, jsonb_path_match(plain, $1::text::jsonpath) FROM json_tree_operators ORDER BY id", + &[&predicate], + ); + assert_eq!(matched, plaintext, "and against jsonb_path_match"); + + for (text, reason, at) in [ + ("$.age + 1 > 18", "json_predicate_unsupported", 6), + ("$.name like_regex \"^a\"", "json_predicate_unsupported", 7), + ("$.age.size() > 1", "json_predicate_unsupported", 5), + ( + "$.tags[*] != \"ops\"", + "json_predicate_not_equal_at_wildcard", + 10, + ), + ("$.age > $missing", "json_predicate_unknown_variable", 8), + ] { + let error = dynamic::check_query( + &plan, + "profile", + OpSpec::Match, + frame_of(&json!({"predicate": text})), + &Eql, + ) + .unwrap_err(); + let payload = stack_encrypt::ErrorPayload::payload(&error); + assert_eq!(payload["reason"], json!(reason), "{text}: {payload:?}"); + assert_eq!(payload["at"], json!(at), "{text}: {payload:?}"); + } + let error = dynamic::check_query( + &plan, + "profile", + OpSpec::Match, + frame_of(&json!({"predicate": "$.height > 1"})), + &Eql, + ) + .unwrap_err(); + assert!( + matches!( + error, + dynamic::Error::Cipher(stack_encrypt::Error::Term( + stack_encrypt::sem::TermError::JsonPathNotOrdered + )) + ), + "a range at a path the column does not order: {error:?}" + ); +} + +/// SQLSTATE `check_violation`, a value its domain's CHECK refuses. +const CHECK_VIOLATION: &str = "23514"; +/// SQLSTATE `raise_exception`, which an operator blocker raises. +const RAISE_EXCEPTION: &str = "P0001"; +/// SQLSTATE `ambiguous_function`, an operator PostgreSQL cannot choose. +const AMBIGUOUS_FUNCTION: &str = "42725"; +/// SQLSTATE `invalid_parameter_value`, which a probe's hit raises with. +const INVALID_PARAMETER_VALUE: &str = "22023"; + +/// The SQLSTATE a statement fails with, or `None` if it succeeds. +fn sqlstate( + db: &mut postgres::Client, + sql: &str, + params: &[&(dyn postgres::types::ToSql + Sync)], +) -> Option { + match db.query(sql, params) { + Ok(_) => None, + Err(error) => match error.as_db_error() { + Some(e) => Some(e.code().code().to_owned()), + None => panic!("{sql}: not a database error: {error}"), + }, + } +} + +/// A JSON value as an SQL string literal. +fn literal(value: &Value) -> String { + let text = match value { + Value::String(text) => text.clone(), + other => other.to_string(), + }; + format!("'{}'", text.replace('\'', "''")) +} + +/// The WHERE a hand writes from the operators a compiled tree's leaves +/// stand for, over the column `doc`, each leaf's operand inlined: `@>` +/// for `contains`, `json_tree_path_exists` for `exists`, `->` and the +/// orderings for `cmp`, and `json_tree_element_match` for `elements`. +fn written(tree: &Value) -> String { + let node = tree.as_object().expect("a node is an object"); + assert_eq!(node.len(), 1, "a node has one key: {tree}"); + let (kind, operand) = node.iter().next().unwrap(); + let list = |join: &str| { + let parts: Vec = operand.as_array().unwrap().iter().map(written).collect(); + format!("({})", parts.join(join)) + }; + match kind.as_str() { + "and" => list(" AND "), + "or" => list(" OR "), + "not" => format!("(NOT {})", written(operand)), + "contains" => format!("(doc @> {}::jsonb::eql_v4.query_json)", literal(operand)), + "exists" => format!( + "eql_v4.json_tree_path_exists(doc, {}::text)", + literal(operand) + ), + "cmp" => { + let parts: Vec = operand + .as_object() + .unwrap() + .iter() + .map(|(bound, leaf)| { + let op = match bound.as_str() { + "gt" => ">", + "gte" => ">=", + "lt" => "<", + "lte" => "<=", + other => panic!("a bound named {other}"), + }; + format!( + "((doc -> {}::text) {op} {}::jsonb::eql_v4.query_json_leaf)", + literal(&leaf["s"]), + literal(leaf) + ) + }) + .collect(); + format!("({})", parts.join(" AND ")) + } + "elements" => format!( + "eql_v4.json_tree_element_match(doc, {}::jsonb::eql_v4.query_json_elements)", + literal(operand) + ), + other => panic!("a node of kind {other}"), + } +} + +/// The first node of `kind` in `tree`, depth first. +fn find<'a>(tree: &'a Value, kind: &str) -> Option<&'a Value> { + let (key, operand) = tree.as_object()?.iter().next()?; + if key == kind { + return Some(tree); + } + match operand { + Value::Array(children) => children.iter().find_map(|child| find(child, kind)), + Value::Object(_) if key == "not" => find(operand, kind), + _ => None, + } +} + +/// Each row's answer under the compiled `operand` by `json_tree_match`, +/// by `@@`, and under the WHERE written from it, in `table`. +fn three_ways( + db: &mut postgres::Client, + table: &str, + operand: &str, +) -> [Vec<(i32, Option)>; 3] { + let tree: Value = serde_json::from_str(operand).unwrap(); + [ + answers( + db, + &format!( + "SELECT id, eql_v4.json_tree_match(doc, $1::text::jsonb::eql_v4.query_json_predicate) \ + FROM {table} ORDER BY id" + ), + &[&operand], + ), + answers( + db, + &format!( + "SELECT id, doc @@ $1::text::jsonb::eql_v4.query_json_predicate FROM {table} ORDER BY id" + ), + &[&operand], + ), + answers( + db, + &format!("SELECT id, {} FROM {table} ORDER BY id", written(&tree)), + &[], + ), + ] +} + +/// Each row's answer under `jsonb_path_match` of `predicate` over the +/// plaintexts in `table`. +fn plaintext(db: &mut postgres::Client, table: &str, predicate: &str) -> Vec<(i32, Option)> { + answers( + db, + &format!("SELECT id, jsonb_path_match(plain, $1::text::jsonpath) FROM {table} ORDER BY id"), + &[&predicate], + ) +} + +/// Needs an EQL v4 installation. The predicate domain's CHECK takes +/// one-key nodes, each kind's operand by its own domain's rule, and refuses +/// everything else. The values are operands the engine derived and their +/// parts: selectors, a needle, a root scalar's needle with its `alt`, leaf +/// operands at two paths, an element predicate and one of its tokens. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_the_predicate_domain_checks_each_node() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = people_plan(); + let mut db = connect(); + let one = |operator: &str, input: Value| -> Value { + serde_json::from_str(&eql_text(&derive( + &runtime, &keyset, &plan, operator, input, + ))) + .unwrap() + }; + let Operand::Text(selector) = derive( + &runtime, + &keyset, + &plan, + "selector", + json!({"path": "$.age"}), + ) else { + panic!("a selector is text") + }; + let needle = one("eq_at", json!({"path": "$.name", "json": "\"ann\""})); + let root_one = one("contains", json!({"json": "1"})); + assert!( + root_one.get("alt").is_some(), + "a root scalar's needle has an alt" + ); + let leaf = one("gt_at", json!({"path": "$.age", "json": "18"})); + let other = one("gt_at", json!({"path": "$.name", "json": "\"a\""})); + let elements = one( + "element_match", + json!({"path": "$.contacts[*]", "wildcard_bound": 1, "predicates": [ + {"member": "calls", "op": "gt", "json": "5"} + ]}), + ); + let token = &elements["m"][0]["t"][0]; + let compiled_tree: Value = serde_json::from_str(&compiled( + &runtime, + &keyset, + &plan, + "$.age >= 18 && (exists($.tags) || !($.name == \"ann\")) \ + && exists($.contacts[*] ? (@.calls > 5))", + &json!({}), + )) + .unwrap(); + let without = |value: &Value, key: &str| { + let mut value = value.clone(); + let _ = value.as_object_mut().unwrap().remove(key); + value + }; + + let valid = [ + compiled_tree, + json!({"exists": selector}), + json!({"contains": needle}), + json!({"contains": root_one}), + json!({"cmp": {"gt": leaf}}), + json!({"cmp": {"gte": leaf, "lt": leaf}}), + json!({"elements": elements}), + json!({"not": {"exists": selector}}), + json!({"and": [{"exists": selector}, {"or": [{"cmp": {"lte": leaf}}]}]}), + ]; + let invalid = [ + ("not an object", json!([])), + ("no key", json!({})), + ( + "two keys", + json!({"exists": selector, "not": {"exists": selector}}), + ), + ("an unknown kind", json!({"xor": []})), + ("an empty and", json!({"and": []})), + ("an empty or", json!({"or": []})), + ("an and not a list", json!({"and": {"exists": selector}})), + ("an invalid child", json!({"or": [{"exists": 1}]})), + ( + "a deep invalid child", + json!({"not": {"not": {"and": [{}]}}}), + ), + ("not of a list", json!({"not": [{"exists": selector}]})), + ("exists not text", json!({"exists": [selector]})), + ( + "a needle missing el", + json!({"contains": without(&needle, "el")}), + ), + ( + "a needle with a bad pair", + json!({"contains": {"sv": [{"s": 1}], "el": []}}), + ), + ("no bound", json!({"cmp": {}})), + ("a bound by another name", json!({"cmp": {"eq": leaf}})), + ( + "two lower bounds", + json!({"cmp": {"gt": leaf, "gte": leaf}}), + ), + ( + "bounds at two selectors", + json!({"cmp": {"gt": leaf, "lt": other}}), + ), + ( + "a bound with no term", + json!({"cmp": {"gt": without(&leaf, "op")}}), + ), + ( + "cmp with tokens", + json!({"cmp": {"t": [token], "gt": leaf}}), + ), + ("elements with no members", json!({"elements": {"m": []}})), + ( + "elements of bounds alone", + json!({"elements": {"m": [{"gt": leaf}]}}), + ), + ]; + let cast = "SELECT $1::text::jsonb::eql_v4.query_json_predicate"; + for value in &valid { + assert_eq!( + sqlstate(&mut db, cast, &[&value.to_string()]), + None, + "valid: {value}" + ); + } + for (why, value) in &invalid { + assert_eq!( + sqlstate(&mut db, cast, &[&value.to_string()]).as_deref(), + Some(CHECK_VIOLATION), + "{why}: {value}" + ); + } + // The element member's CHECK, through the bounds validator the two + // domains share, refuses what it refused. + let mut member_and_more = elements["m"][0].clone(); + let _ = member_and_more + .as_object_mut() + .unwrap() + .insert("x".into(), json!(1)); + for (why, value) in [ + ("no tokens", json!({"m": [{"gt": leaf}]})), + ("no bound", json!({"m": [{"t": [token]}]})), + ("another key", json!({"m": [member_and_more]})), + ] { + assert_eq!( + sqlstate( + &mut db, + "SELECT $1::text::jsonb::eql_v4.query_json_elements", + &[&value.to_string()] + ) + .as_deref(), + Some(CHECK_VIOLATION), + "{why}: {value}" + ); + } +} + +/// Needs an EQL v4 installation. For every predicate over every document: +/// `json_tree_match` and `@@` answer what the WHERE written from the +/// compiled tree's leaves answers, NULL included, and `jsonb_path_match` +/// what all three answer, over the plaintext the document was sealed +/// from. Every field is absent, `null` or a value of the one type its path +/// holds. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_a_compiled_predicate_answers_as_the_operators_written_out() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = people_plan(); + let mut db = connect(); + store( + &mut db, + &runtime, + &cipher, + "json_tree_written", + &people_options(), + &[ + json!({"age": 1, "name": "ann", "tags": ["a", "b"]}), + json!({"age": 3, "name": "bob"}), + json!({"age": 5, "tags": []}), + json!({"age": null, "name": null}), + json!({}), + json!({"name": "dan", "tags": ["c"]}), + json!({"age": 3, "ok": true, "tags": ["a"]}), + ], + ); + for predicate in [ + "$.age == 3", + "$.age != 3", + "$.age == null", + "$.age != null", + "$.age < 3", + "!($.age < 3)", + "$.age >= 3", + "$.name > \"ann\"", + "$.name != \"ann\"", + "$.ok == true", + "$.ok != true", + "exists($.tags)", + "exists($.tags[*])", + "$.tags[*] == \"a\"", + "exists($.tags[*] ? (@ == \"c\"))", + "$.age > 1 && $.name == \"bob\" || !exists($.tags)", + "$.age >= 1 && $.age <= 3", + "!($.age >= 2 || $.name == \"dan\")", + ] { + let operand = compiled(&runtime, &keyset, &plan, predicate, &json!({})); + let [matched, by_operator, by_hand] = three_ways(&mut db, "json_tree_written", &operand); + assert_eq!(matched, by_hand, "{predicate}: against the WHERE"); + assert_eq!(by_operator, matched, "{predicate}: @@ is json_tree_match"); + assert_eq!( + matched, + plaintext(&mut db, "json_tree_written", predicate), + "{predicate}: against jsonb_path_match" + ); + } +} + +/// Needs an EQL v4 installation. A NULL leaf follows SQL's three-valued +/// logic: an ordering of a node with no ordering term (a boolean at an +/// ordered path) is NULL, and so is `not` of it; `and` with a false child +/// is false and `or` with a true one true. jsonpath answers each the same: +/// a boolean against a number is unknown. The ordering leaf alone, without +/// the guards `match` puts beside it, is NULL also where the path is +/// absent, as `(col -> s) < $1` is. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_a_null_leaf_is_unknown_under_and_or_and_not() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = people_plan(); + let mut db = connect(); + store( + &mut db, + &runtime, + &cipher, + "json_tree_unknown", + &people_options(), + &[json!({"age": true, "name": "eve"})], + ); + for (predicate, want) in [ + ("$.age < 3", None), + ("!($.age < 3)", None), + ("$.age < 3 && exists($.name)", None), + ("$.age < 3 && $.name == \"x\"", Some(false)), + ("$.age < 3 || exists($.name)", Some(true)), + ("$.age < 3 || $.name == \"x\"", None), + ] { + let operand = compiled(&runtime, &keyset, &plan, predicate, &json!({})); + let [matched, by_operator, by_hand] = three_ways(&mut db, "json_tree_unknown", &operand); + assert_eq!(matched, [(1, want)], "{predicate}"); + assert_eq!(by_operator, matched, "{predicate}: @@ is json_tree_match"); + assert_eq!(by_hand, matched, "{predicate}: against the WHERE"); + assert_eq!( + plaintext(&mut db, "json_tree_unknown", predicate), + matched, + "{predicate}: against jsonb_path_match" + ); + } + + let compiled_lt: Value = + serde_json::from_str(&compiled(&runtime, &keyset, &plan, "$.age < 3", &json!({}))).unwrap(); + let cmp = find(&compiled_lt, "cmp") + .expect("an ordering compiles to a cmp leaf") + .to_string(); + store( + &mut db, + &runtime, + &cipher, + "json_tree_cmp_alone", + &people_options(), + &[ + json!({}), + json!({"age": null}), + json!({"age": 1}), + json!({"age": 4}), + ], + ); + assert_eq!( + answers( + &mut db, + "SELECT id, eql_v4.json_tree_match(doc, $1::text::jsonb::eql_v4.query_json_predicate) \ + FROM json_tree_cmp_alone ORDER BY id", + &[&cmp], + ), + [(1, None), (2, None), (3, Some(true)), (4, Some(false))], + "the cmp leaf alone" + ); +} + +/// Needs an EQL v4 installation. A `contains` leaf is `@>`, the needle's +/// `alt` included: a root scalar matches a root array that holds it. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_a_contains_leaf_is_the_containment_operator() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = people_plan(); + let mut db = connect(); + store( + &mut db, + &runtime, + &cipher, + "json_tree_roots", + &people_options(), + &[json!([1, 2]), json!(1), json!({"a": [1]}), json!([[1]])], + ); + let needle = |json: &str| { + eql_text(&derive( + &runtime, + &keyset, + &plan, + "contains", + json!({"json": json}), + )) + }; + let root_one = needle("1"); + for json in ["1", "[1]", "{\"a\": [1]}", "[[1]]", "[]"] { + let held = needle(json); + let tree = format!(r#"{{"contains": {held}}}"#); + let matched = answers( + &mut db, + "SELECT id, eql_v4.json_tree_match(doc, $1::text::jsonb::eql_v4.query_json_predicate) \ + FROM json_tree_roots ORDER BY id", + &[&tree], + ); + let contained = answers( + &mut db, + "SELECT id, doc @> $1::text::jsonb::eql_v4.query_json FROM json_tree_roots ORDER BY id", + &[&held], + ); + assert_eq!(matched, contained, "{json}"); + } + assert_eq!( + answers( + &mut db, + "SELECT id, eql_v4.json_tree_match(doc, $1::text::jsonb::eql_v4.query_json_predicate) \ + FROM json_tree_roots ORDER BY id", + &[&format!(r#"{{"contains": {root_one}}}"#)], + ), + [ + (1, Some(true)), + (2, Some(true)), + (3, Some(false)), + (4, Some(false)) + ], + "the root scalar, and a root array holding it through `alt`" + ); +} + +/// Needs an EQL v4 installation. An `elements` leaf is +/// `json_tree_element_match` over the document the tree opened once; a leaf +/// the answer does not depend on is not read, so it cannot raise, and one +/// the answer depends on raises as the function does. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_an_elements_leaf_is_the_element_match() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = people_plan(); + let mut db = connect(); + store( + &mut db, + &runtime, + &cipher, + "json_tree_calls", + &people_options(), + &[ + json!({"contacts": [{"calls": 3}, {"calls": 7}]}), + json!({"contacts": [{"calls": 1}]}), + json!({"contacts": []}), + ], + ); + let calls_over = |n: i64, bound: u32| { + eql_text(&derive( + &runtime, + &keyset, + &plan, + "element_match", + json!({"path": "$.contacts[*]", "wildcard_bound": bound, "predicates": [ + {"member": "calls", "op": "gt", "json": n.to_string()} + ]}), + )) + }; + let selector = + |path: &str| match derive(&runtime, &keyset, &plan, "selector", json!({"path": path})) { + Operand::Text(selector) => selector, + other => panic!("a selector is text, got {other:?}"), + }; + let over_five = calls_over(5, 2); + let tree_match = |db: &mut postgres::Client, tree: &str| { + answers( + db, + "SELECT id, eql_v4.json_tree_match(doc, $1::text::jsonb::eql_v4.query_json_predicate) \ + FROM json_tree_calls ORDER BY id", + &[&tree], + ) + }; + let matched = tree_match(&mut db, &format!(r#"{{"elements": {over_five}}}"#)); + let element = answers( + &mut db, + "SELECT id, eql_v4.json_tree_element_match(doc, $1::text::jsonb::eql_v4.query_json_elements) \ + FROM json_tree_calls ORDER BY id", + &[&over_five], + ); + assert_eq!(matched, element, "the element match itself"); + assert_eq!( + matched, + [(1, Some(true)), (2, Some(false)), (3, Some(false))] + ); + let two_leaves = format!( + r#"{{"and": [{{"elements": {over_five}}}, {{"not": {{"elements": {}}}}}]}}"#, + calls_over(7, 2) + ); + assert_eq!( + tree_match(&mut db, &two_leaves), + [(1, Some(true)), (2, Some(false)), (3, Some(false))], + "two element leaves over one opened document" + ); + + // A bound of one: the two-contact row is longer than the bound. + let raising = format!(r#"{{"elements": {}}}"#, calls_over(5, 1)); + let present = json!({"exists": selector("$.contacts")}).to_string(); + let absent = json!({"exists": selector("$.none")}).to_string(); + let on_row_one = + "SELECT eql_v4.json_tree_match(doc, $1::text::jsonb::eql_v4.query_json_predicate) \ + FROM json_tree_calls WHERE id = 1"; + for (tree, want) in [ + (format!(r#"{{"and": [{absent}, {raising}]}}"#), None), + (format!(r#"{{"or": [{present}, {raising}]}}"#), None), + ( + format!(r#"{{"and": [{present}, {raising}]}}"#), + Some(INVALID_PARAMETER_VALUE), + ), + (raising.clone(), Some(INVALID_PARAMETER_VALUE)), + ] { + assert_eq!( + sqlstate(&mut db, on_row_one, &[&tree]).as_deref(), + want, + "{tree}" + ); + } +} + +/// Needs an EQL v4 installation. `@@` takes a typed predicate; with a +/// `jsonpath` it is still blocked; an untyped operand resolves to neither +/// silently; and a NULL argument is NULL. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_the_operator_takes_a_typed_predicate_and_jsonpath_stays_blocked() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = people_plan(); + let mut db = connect(); + store( + &mut db, + &runtime, + &cipher, + "json_tree_typed", + &people_options(), + &[json!({"age": 3})], + ); + let exists = + |path: &str| match derive(&runtime, &keyset, &plan, "selector", json!({"path": path})) { + Operand::Text(selector) => json!({"exists": selector}).to_string(), + other => panic!("a selector is text, got {other:?}"), + }; + let (age, none) = (exists("$.age"), exists("$.none")); + let answer: Option = db + .query_one( + "SELECT doc @@ $1::text::jsonb::eql_v4.query_json_predicate FROM json_tree_typed", + &[&age], + ) + .unwrap() + .get(0); + assert_eq!(answer, Some(true)); + let bound: Option = db + .query_typed( + "SELECT doc @@ $1::eql_v4.query_json_predicate FROM json_tree_typed", + &[(&none, postgres::types::Type::TEXT)], + ) + .unwrap()[0] + .get(0); + assert_eq!(bound, Some(false), "a bound parameter cast to the domain"); + assert_eq!( + sqlstate( + &mut db, + "SELECT doc @@ '$.age == 3'::jsonpath FROM json_tree_typed", + &[] + ) + .as_deref(), + Some(RAISE_EXCEPTION), + "a jsonpath is blocked" + ); + for untyped in [ + format!( + "SELECT doc @@ {} FROM json_tree_typed", + literal(&json!(none)) + ), + "SELECT doc @@ '$.age == 3' FROM json_tree_typed".to_owned(), + ] { + assert_eq!( + sqlstate(&mut db, &untyped, &[]).as_deref(), + Some(AMBIGUOUS_FUNCTION), + "an untyped operand is refused, not read as native jsonb: {untyped}" + ); + } + let predicate = format!( + "{}::jsonb::eql_v4.query_json_predicate", + literal(&json!(none)) + ); + for sql in [ + format!("SELECT eql_v4.json_tree_match(NULL, {predicate})"), + "SELECT eql_v4.json_tree_match(doc, NULL) FROM json_tree_typed".to_owned(), + format!("SELECT NULL::public.eql_v4_json_search @@ {predicate}"), + ] { + let answer: Option = db.query_one(&sql, &[]).unwrap().get(0); + assert_eq!(answer, None, "{sql}"); + } +} diff --git a/packages/stack-encrypt/src/codes.rs b/packages/stack-encrypt/src/codes.rs index c1cb4a3de..77c45aa03 100644 --- a/packages/stack-encrypt/src/codes.rs +++ b/packages/stack-encrypt/src/codes.rs @@ -212,6 +212,12 @@ mod tests { Error::UntypedIndex { .. } => Error::UntypedIndex { field: name() }, Error::Source { .. } => Error::bad_source(Reason::FieldMissing), Error::Record { .. } => Error::bad_record(Reason::NoCiphertextNode), + Error::Predicate { .. } => Error::Predicate { + field: Some(name()), + reason: Reason::JsonPredicateUnsupported, + construct: "`like_regex`", + at: 4, + }, Error::Internal => Error::Internal, Error::Target(_) => Error::Target(TargetError::NoTargets { name: target() }), Error::Cipher(_) => Error::Cipher(crate::Error::Aead), @@ -596,6 +602,23 @@ mod tests { Box::new(Error::UntypedIndex { field: name() }), json!({ "field": "email" }), ), + ( + Box::new( + Error::Predicate { + field: None, + reason: Reason::JsonPredicateUnsupported, + construct: "`like_regex`", + at: 4, + } + .in_field("email"), + ), + json!({ + "field": "email", + "reason": "json_predicate_unsupported", + "construct": "`like_regex`", + "at": 4 + }), + ), (Box::new(Error::Internal), json!({})), ( Box::new(TargetError::NoTargets { name: target() }), diff --git a/packages/stack-encrypt/src/dynamic/json_predicate.rs b/packages/stack-encrypt/src/dynamic/json_predicate.rs new file mode 100644 index 000000000..368c465ca --- /dev/null +++ b/packages/stack-encrypt/src/dynamic/json_predicate.rs @@ -0,0 +1,1819 @@ +//! `match`'s predicate: the SQL/JSON path predicates the JSON index can +//! answer, parsed from the caller's text and compiled onto the inputs of +//! the other JSON operators. +//! +//! The text is PostgreSQL's SQL/JSON path language, as `jsonb_path_match` +//! takes it ([SQL/JSON path language](https://www.postgresql.org/docs/current/functions-json.html#FUNCTIONS-SQLJSON-PATH)), +//! restricted to predicates the terms can answer: +//! +//! ```text +//! $.age >= $min a comparison with a variable +//! $.name == "ann" || $."first name" != null literals and a quoted key +//! !($.age < 18) && exists($.email) negation and presence +//! $.tags[*] == $tag any element equals +//! $.orders[*].total > 100 any element's field orders +//! exists($.contacts[*] ? (@.relationship == $r && @.calls > 5)) +//! one element satisfies both +//! ``` +//! +//! `OpSpec::Match`'s documentation runs one through the data plan. The +//! grammar and the compilation are documented with the operator, in +//! the [`json_query`](super::json_query) module docs. This module holds the +//! parser ([`parse`]), which reads the text into an [`Ast`] and refuses +//! what the grammar does not read at the byte where it stops, and the +//! compiler ([`compile`]), which binds the values and maps each comparison +//! onto the inputs `eq_at`, `selector`, the orderings and `element_match` +//! take, so that every leaf is derived by that operator's own code +//! ([`derive`]). +//! +//! A refusal names a construct in the grammar's words and an offset, never +//! a slice of the text: a key name or a value in it can be customer data. +//! The text itself is the caller's buffer, wiped when it drops; the tree +//! holds offsets into it, the key names in [`JsonPath`]s, which wipe them, +//! and the values in [`Parsed`], which wipes them. + +use serde_json::Value; +use zeroize::Zeroizing; + +use super::json_query::{ + derive_input, eql, json_reason, leaf, member_of, ordered_path_of, Bounds, Compare, Input, + Parsed, Predicate, +}; +use super::query::{OpSpec, Operand}; +use super::{Error, Reason}; +use crate::json::{parse_text, JsonOptions, JsonPath, Segment, MAX_DEPTH}; +use crate::{KeysetCipher, Label, Pending}; + +/// How deeply a predicate may nest parentheses, `!` and `exists`. The +/// parser descends once per level, so the bound keeps a hostile predicate +/// from overflowing the stack; the SQL evaluates the compiled tree by +/// recursion too. +pub(super) const MAX_NESTING: usize = 32; + +// The constructs a refusal names, in the grammar's own words. +const C_TEXT: &str = "text the grammar does not read"; +const C_TRAILING: &str = "text after the predicate"; +const C_STRING: &str = "a string literal"; +const C_NUMBER: &str = "a numeric literal"; +const C_MODE: &str = "a mode (`strict`, `lax`)"; +const C_ARITHMETIC: &str = "arithmetic (`+`, `-`, `*`, `/`, `%`)"; +const C_LIKE_REGEX: &str = "`like_regex`"; +const C_STARTS_WITH: &str = "`starts with`"; +const C_IS_UNKNOWN: &str = "`is unknown`"; +const C_METHOD: &str = "a method (`.name()`)"; +const C_ANY_MEMBER: &str = "`.*`"; +const C_ANY_DEPTH: &str = "`.**`"; +const C_SLICE: &str = "an array slice or list (`[a to b]`, `[a, b]`)"; +const C_LAST: &str = "`last`"; +const C_SUBSCRIPT: &str = "a subscript other than `[*]`"; +const C_POSITION: &str = "a position (`[2]`)"; +const C_AT_OUTSIDE: &str = "`@` outside a filter"; +const C_DOLLAR_IN_FILTER: &str = "a `$` path in a filter"; +const C_VARIABLE_PATH: &str = "an accessor on a variable"; +const C_TWO_PATHS: &str = "a comparison of two paths"; +const C_TWO_VALUES: &str = "a comparison of two values"; +const C_ALONE: &str = "an operand with no comparison"; +const C_FILTER_OUTSIDE: &str = "a filter outside `exists`"; +const C_AFTER_FILTER: &str = "an accessor after a filter"; +const C_IN_EXISTS: &str = "a comparison inside `exists`"; +const C_OR_IN_FILTER: &str = "`||` in a filter"; +const C_NOT_IN_FILTER: &str = "`!` in a filter"; +const C_NE_IN_FILTER: &str = "`!=` in a filter"; +const C_EXISTS_IN_FILTER: &str = "`exists` in a filter"; +const C_ELEMENTS_THEMSELVES: &str = "an ordering of an array's elements themselves"; +const C_WHOLE_ELEMENT: &str = "`@` beside another condition"; +const C_FILTER_WILDCARD: &str = "a filter on a path with no wildcard"; +const C_MEMBER_WILDCARD: &str = "a wildcard in a filter's member"; +const C_NOT_EQUAL: &str = "`!=` at a path with a wildcard"; +const C_VALUE: &str = "a value"; +const C_VARIABLE: &str = "a variable"; +const C_NESTING: &str = "nesting"; + +/// A refusal at a byte of the predicate's text. +#[derive(Debug, PartialEq, Eq)] +struct Refusal { + reason: Reason, + construct: &'static str, + at: usize, +} + +impl Refusal { + fn syntax(construct: &'static str, at: usize) -> Self { + Self { + reason: Reason::JsonPredicateSyntax, + construct, + at, + } + } + + fn unsupported(construct: &'static str, at: usize) -> Self { + Self { + reason: Reason::JsonPredicateUnsupported, + construct, + at, + } + } +} + +impl From for Error { + fn from(refusal: Refusal) -> Self { + Error::Predicate { + field: None, + reason: refusal.reason, + construct: refusal.construct, + at: refusal.at, + } + } +} + +/// A comparison operator, as the text spells it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Op { + Eq, + Ne, + Lt, + Lte, + Gt, + Gte, +} + +impl Op { + /// The operator with its operands swapped: `1 < $.a` is `$.a > 1`. + fn flipped(self) -> Self { + match self { + Op::Lt => Op::Gt, + Op::Lte => Op::Gte, + Op::Gt => Op::Lt, + Op::Gte => Op::Lte, + same => same, + } + } + + /// How an element-scoped filter's condition compares; `None` for + /// `!=`, which a filter does not take. + fn compare(self) -> Option { + match self { + Op::Eq => Some(Compare::Eq), + Op::Lt => Some(Compare::Lt), + Op::Lte => Some(Compare::Lte), + Op::Gt => Some(Compare::Gt), + Op::Gte => Some(Compare::Gte), + Op::Ne => None, + } + } +} + +/// A path in the text: `$` or `@` and the steps after it, and where it +/// starts. +#[derive(Debug)] +struct PathAt { + path: JsonPath, + at: usize, +} + +/// A value in the text: a literal, by its bytes, or a variable, by name. +#[derive(Debug, PartialEq, Eq)] +enum ValueAt { + /// `"…"`, a number, `true`, `false` or `null`: the literal's bytes, + /// `start..end`, which are JSON text. + Literal { start: usize, end: usize }, + /// `$name` or `$"name"`. + Variable { name: String, at: usize }, +} + +impl ValueAt { + fn at(&self) -> usize { + match self { + ValueAt::Literal { start, .. } => *start, + ValueAt::Variable { at, .. } => *at, + } + } +} + +/// One side of a comparison. +#[derive(Debug)] +enum Side { + /// A path from `$`. + Root(PathAt), + /// A path from `@`, inside a filter. + Current(PathAt), + Value(ValueAt), +} + +/// A comparison of a path with a value, the path on the left. +#[derive(Debug)] +struct Comparison { + path: PathAt, + op: Op, + value: ValueAt, + /// Where the operator is. + at: usize, +} + +/// A predicate as the text writes it. +#[derive(Debug)] +enum Ast { + And(Vec), + Or(Vec), + Not(Box), + Compare(Comparison), + /// `exists(path)`, or `exists(path ? (conditions))`, each condition a + /// comparison of a path from `@`. + Exists { + path: PathAt, + filter: Option>, + }, +} + +/// The characters that end a name and are not a name's: what +/// PostgreSQL's jsonpath scanner reads as punctuation. +fn special(c: char) -> bool { + c.is_whitespace() || "?%$.[]{}()|&!=<>@#,*:-+/\\\"'".contains(c) +} + +/// A recursive descent over the text, holding the offset it has read to. +struct Parser<'t> { + text: &'t str, + pos: usize, + depth: usize, +} + +impl<'t> Parser<'t> { + fn rest(&self) -> &'t str { + &self.text[self.pos..] + } + + /// Skip whitespace and `/* … */` comments. + fn space(&mut self) -> Result<(), Refusal> { + loop { + let rest = self.rest(); + let trimmed = rest.trim_start(); + self.pos += rest.len() - trimmed.len(); + if let Some(comment) = trimmed.strip_prefix("/*") { + let Some(end) = comment.find("*/") else { + return Err(Refusal::syntax(C_TEXT, self.pos)); + }; + self.pos += 2 + end + 2; + } else { + return Ok(()); + } + } + } + + /// The next character after any space, unread. + fn peek(&mut self) -> Result, Refusal> { + self.space()?; + Ok(self.rest().chars().next()) + } + + /// Read `token` if it is next. + fn eat(&mut self, token: &str) -> Result { + self.space()?; + if self.rest().starts_with(token) { + self.pos += token.len(); + Ok(true) + } else { + Ok(false) + } + } + + fn expect(&mut self, token: &str) -> Result<(), Refusal> { + if self.eat(token)? { + Ok(()) + } else { + Err(Refusal::syntax(C_TEXT, self.pos)) + } + } + + /// The name at the cursor, unread: a run of characters that are not + /// punctuation, not starting with a digit. + fn word(&mut self) -> Result, Refusal> { + self.space()?; + let rest = self.rest(); + if rest.starts_with(|c: char| c.is_ascii_digit()) { + return Ok(None); + } + let end = rest.find(special).unwrap_or(rest.len()); + Ok((end > 0).then(|| &rest[..end])) + } + + /// One more level of nesting, refused past [`MAX_NESTING`]. + fn descend(&mut self) -> Result<(), Refusal> { + self.depth += 1; + if self.depth > MAX_NESTING { + return Err(Refusal { + reason: Reason::JsonPredicateTooDeep, + construct: C_NESTING, + at: self.pos, + }); + } + Ok(()) + } + + fn ascend(&mut self) { + self.depth -= 1; + } + + /// `or := and ('||' and)*`. + fn or(&mut self) -> Result { + let mut terms = vec![self.and()?]; + while self.eat("||")? { + terms.push(self.and()?); + } + Ok(if terms.len() == 1 { + terms.remove(0) + } else { + Ast::Or(terms) + }) + } + + /// `and := unary ('&&' unary)*`. + fn and(&mut self) -> Result { + let mut terms = vec![self.unary()?]; + while self.eat("&&")? { + terms.push(self.unary()?); + } + Ok(if terms.len() == 1 { + terms.remove(0) + } else { + Ast::And(terms) + }) + } + + /// `unary := '!' primary | primary`. As in PostgreSQL, `!` takes a + /// parenthesised predicate or `exists`, not a bare comparison. + fn unary(&mut self) -> Result { + self.space()?; + if self.rest().starts_with('!') && !self.rest().starts_with("!=") { + self.pos += 1; + self.descend()?; + let at = self.pos; + let negated = match self.peek()? { + Some('(') => self.primary()?, + _ if self.keyword("exists")? => self.primary()?, + _ => return Err(Refusal::syntax(C_TEXT, at)), + }; + self.ascend(); + return Ok(Ast::Not(Box::new(negated))); + } + self.primary() + } + + /// Whether the word at the cursor is `keyword`, in any case, as + /// PostgreSQL reads `exists`. + fn keyword(&mut self, keyword: &str) -> Result { + Ok(self + .word()? + .is_some_and(|word| word.eq_ignore_ascii_case(keyword))) + } + + /// `primary := '(' or ')' | 'exists' '(' path filter? ')' | comparison`. + fn primary(&mut self) -> Result { + if self.eat("(")? { + self.descend()?; + let inner = self.or()?; + self.expect(")")?; + self.ascend(); + return Ok(inner); + } + if self.keyword("exists")? { + self.pos += "exists".len(); + if self.eat("(")? { + return self.exists(); + } + return Err(Refusal::syntax(C_TEXT, self.pos)); + } + let comparison = self.comparison(false)?; + Ok(Ast::Compare(comparison)) + } + + /// The inside of `exists(…)`, the `(` read. + fn exists(&mut self) -> Result { + self.descend()?; + self.space()?; + let at = self.pos; + let path = match self.operand(false)? { + Side::Root(path) => path, + Side::Current(_) => return Err(Refusal::unsupported(C_AT_OUTSIDE, at)), + Side::Value(_) => return Err(Refusal::syntax(C_TEXT, at)), + }; + let filter = if self.eat("?")? { + self.expect("(")?; + self.descend()?; + let conditions = self.conditions()?; + self.expect(")")?; + self.ascend(); + let at = self.pos; + match self.peek()? { + Some('.' | '[') => return Err(Refusal::unsupported(C_AFTER_FILTER, at)), + Some('?') => return Err(Refusal::unsupported(C_FILTER_OUTSIDE, at)), + _ => {} + } + Some(conditions) + } else { + None + }; + if let Some(refusal) = self.after_operand()? { + return Err(refusal); + } + self.space()?; + if self.op()?.is_some() { + return Err(Refusal::unsupported(C_IN_EXISTS, at)); + } + self.expect(")")?; + self.ascend(); + Ok(Ast::Exists { path, filter }) + } + + /// A filter's conditions: comparisons of a path from `@`, joined by + /// `&&`, with parentheses. + fn conditions(&mut self) -> Result, Refusal> { + let mut conditions = Vec::new(); + loop { + self.space()?; + let at = self.pos; + if self.rest().starts_with('!') && !self.rest().starts_with("!=") { + return Err(Refusal::unsupported(C_NOT_IN_FILTER, at)); + } + if self.keyword("exists")? { + return Err(Refusal::unsupported(C_EXISTS_IN_FILTER, at)); + } + if self.eat("(")? { + self.descend()?; + conditions.extend(self.conditions()?); + self.expect(")")?; + self.ascend(); + } else { + conditions.push(self.comparison(true)?); + } + self.space()?; + if self.rest().starts_with("||") { + return Err(Refusal::unsupported(C_OR_IN_FILTER, self.pos)); + } + if !self.eat("&&")? { + return Ok(conditions); + } + } + } + + /// A comparison operator at the cursor, read, and where it was. + fn op(&mut self) -> Result, Refusal> { + self.space()?; + let at = self.pos; + let rest = self.rest(); + let (op, len) = if rest.starts_with("==") { + (Op::Eq, 2) + } else if rest.starts_with("!=") || rest.starts_with("<>") { + (Op::Ne, 2) + } else if rest.starts_with("<=") { + (Op::Lte, 2) + } else if rest.starts_with(">=") { + (Op::Gte, 2) + } else if rest.starts_with('<') { + (Op::Lt, 1) + } else if rest.starts_with('>') { + (Op::Gt, 1) + } else { + return Ok(None); + }; + self.pos += len; + Ok(Some((op, at))) + } + + /// The construct at the cursor that follows an operand and that the + /// index cannot answer, if one is there. + fn after_operand(&mut self) -> Result, Refusal> { + self.space()?; + let at = self.pos; + let rest = self.rest(); + if rest.starts_with(['+', '-', '*', '/', '%']) { + return Ok(Some(Refusal::unsupported(C_ARITHMETIC, at))); + } + if rest.starts_with('?') { + return Ok(Some(Refusal::unsupported(C_FILTER_OUTSIDE, at))); + } + let word = self.word()?.unwrap_or_default(); + Ok(if word.eq_ignore_ascii_case("like_regex") { + Some(Refusal::unsupported(C_LIKE_REGEX, at)) + } else if word.eq_ignore_ascii_case("starts") { + Some(Refusal::unsupported(C_STARTS_WITH, at)) + } else if word.eq_ignore_ascii_case("is") { + Some(Refusal::unsupported(C_IS_UNKNOWN, at)) + } else { + None + }) + } + + /// `comparison := operand op operand`, one side a path and the other a + /// value; in a filter the path is from `@`. + fn comparison(&mut self, in_filter: bool) -> Result { + self.space()?; + let left = self.operand(in_filter)?; + if let Some(refusal) = self.after_operand()? { + return Err(refusal); + } + let Some((op, at)) = self.op()? else { + self.space()?; + let at = self.pos; + let ends = self.rest().is_empty() || self.rest().starts_with([')', '&', '|']); + return Err(if ends { + Refusal::unsupported(C_ALONE, at) + } else { + Refusal::syntax(C_TEXT, at) + }); + }; + if in_filter && op == Op::Ne { + return Err(Refusal::unsupported(C_NE_IN_FILTER, at)); + } + self.space()?; + let right = self.operand(in_filter)?; + if let Some(refusal) = self.after_operand()? { + return Err(refusal); + } + let (path, op, value) = match (left, right) { + (Side::Root(path) | Side::Current(path), Side::Value(value)) => (path, op, value), + (Side::Value(value), Side::Root(path) | Side::Current(path)) => { + (path, op.flipped(), value) + } + (Side::Value(_), Side::Value(_)) => return Err(Refusal::unsupported(C_TWO_VALUES, at)), + _ => return Err(Refusal::unsupported(C_TWO_PATHS, at)), + }; + Ok(Comparison { + path, + op, + value, + at, + }) + } + + /// A path, a variable or a literal. In a filter a path is from `@`; + /// outside one, from `$`. + fn operand(&mut self, in_filter: bool) -> Result { + self.space()?; + let at = self.pos; + let rest = self.rest(); + let Some(first) = rest.chars().next() else { + return Err(Refusal::syntax(C_TEXT, at)); + }; + match first { + '$' => { + let after = &rest[1..]; + if after.starts_with('"') { + let (mut name, len) = self.string_at(at + 1)?; + self.pos = at + 1 + len; + return self.variable(std::mem::take(&mut *name), at); + } + if after.starts_with(|c: char| !special(c)) { + self.pos += 1; + let Some(name) = self.word()? else { + return Err(Refusal::syntax(C_TEXT, self.pos)); + }; + self.pos += name.len(); + return self.variable(name.to_owned(), at); + } + if in_filter { + return Err(Refusal::unsupported(C_DOLLAR_IN_FILTER, at)); + } + self.pos += 1; + Ok(Side::Root(self.steps(at)?)) + } + '@' => { + if !in_filter { + return Err(Refusal::unsupported(C_AT_OUTSIDE, at)); + } + self.pos += 1; + Ok(Side::Current(self.steps(at)?)) + } + '"' => { + let (_, len) = self.string_at(at)?; + self.pos += len; + Ok(Side::Value(ValueAt::Literal { + start: at, + end: at + len, + })) + } + '-' | '0'..='9' => { + let len = number_len(rest).ok_or(Refusal::syntax(C_NUMBER, at))?; + self.pos += len; + Ok(Side::Value(ValueAt::Literal { + start: at, + end: at + len, + })) + } + '+' => Err(Refusal::unsupported(C_ARITHMETIC, at)), + _ => match self.word()? { + Some(word @ ("true" | "false" | "null")) => { + self.pos += word.len(); + Ok(Side::Value(ValueAt::Literal { + start: at, + end: at + word.len(), + })) + } + Some(word) if word.eq_ignore_ascii_case("last") => { + Err(Refusal::unsupported(C_LAST, at)) + } + Some(word) if is_mode(word) => Err(Refusal::unsupported(C_MODE, at)), + _ => Err(Refusal::syntax(C_TEXT, at)), + }, + } + } + + /// A variable named `name`, which takes no accessor. + fn variable(&mut self, name: String, at: usize) -> Result { + self.space()?; + if self.rest().starts_with(['.', '[']) { + return Err(Refusal::unsupported(C_VARIABLE_PATH, self.pos)); + } + Ok(Side::Value(ValueAt::Variable { name, at })) + } + + /// The JSON string at `at`, decoded, and its length in the text. Its + /// escapes are JSON's. The decoded text is wiped when it drops: a + /// literal's is a value, read only to check it, and a key's is moved + /// into its path, which wipes it. + fn string_at(&self, at: usize) -> Result<(Zeroizing, usize), Refusal> { + let text = &self.text[at..]; + let mut escaped = false; + for (end, byte) in text.bytes().enumerate().skip(1) { + match (escaped, byte) { + (true, _) => escaped = false, + (false, b'\\') => escaped = true, + (false, b'"') => { + let decoded = serde_json::from_str(&text[..=end]) + .map(Zeroizing::new) + .map_err(|_| Refusal::syntax(C_STRING, at))?; + return Ok((decoded, end + 1)); + } + (false, _) => {} + } + } + Err(Refusal::syntax(C_STRING, at)) + } + + /// The steps after `$` or `@`: `.name`, `."name"`, `[*]`, and the + /// accessors refused by name. A step past a document's depth is + /// refused before it is read, so the path held is never deeper than + /// [`MAX_DEPTH`]. + fn steps(&mut self, start: usize) -> Result { + let mut segments = Vec::new(); + loop { + self.space()?; + let at = self.pos; + let rest = self.rest(); + if !rest.starts_with(['.', '[']) { + break; + } + if segments.len() >= MAX_DEPTH { + return Err(Refusal { + reason: Reason::JsonPredicateTooDeep, + construct: C_NESTING, + at: start, + }); + } + if let Some(after) = rest.strip_prefix('.') { + let after_space = after.trim_start(); + let gap = after.len() - after_space.len(); + if after_space.starts_with("**") { + return Err(Refusal::unsupported(C_ANY_DEPTH, at)); + } + if after_space.starts_with('*') { + return Err(Refusal::unsupported(C_ANY_MEMBER, at)); + } + if after_space.starts_with('"') { + let (mut key, len) = self.string_at(at + 1 + gap)?; + self.pos = at + 1 + gap + len; + segments.push(Segment::Key(std::mem::take(&mut *key))); + continue; + } + self.pos = at + 1; + let Some(name) = self.word()? else { + return Err(Refusal::syntax(C_TEXT, self.pos)); + }; + self.pos += name.len(); + if self.peek()? == Some('(') { + return Err(Refusal::unsupported(C_METHOD, at)); + } + segments.push(Segment::Key(name.to_owned())); + } else { + self.pos += 1; + self.space()?; + if self.eat("*")? { + self.expect("]")?; + segments.push(Segment::Wildcard); + } else if self.rest().starts_with(|c: char| c.is_ascii_digit()) { + let digits = self + .rest() + .find(|c: char| !c.is_ascii_digit()) + .unwrap_or(self.rest().len()); + self.pos += digits; + self.space()?; + if self.rest().starts_with(']') { + return Err(Refusal { + reason: Reason::JsonPathPosition, + construct: C_POSITION, + at, + }); + } + return Err(Refusal::unsupported(C_SLICE, at)); + } else if self.keyword("last")? { + return Err(Refusal::unsupported(C_LAST, at)); + } else { + return Err(Refusal::unsupported(C_SUBSCRIPT, at)); + } + } + } + Ok(PathAt { + path: JsonPath::of(segments), + at: start, + }) + } +} + +/// The length of the JSON number at the front of `text`: +/// `-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)?`, followed by +/// something that does not continue it. `None` for text that is not one. +fn number_len(text: &str) -> Option { + let bytes = text.as_bytes(); + let mut i = usize::from(bytes.first() == Some(&b'-')); + let digits = |from: usize| { + bytes[from..] + .iter() + .take_while(|b| b.is_ascii_digit()) + .count() + }; + match bytes.get(i) { + Some(b'0') => i += 1, + Some(b'1'..=b'9') => i += digits(i), + _ => return None, + } + if bytes.get(i) == Some(&b'.') { + let fraction = digits(i + 1); + if fraction == 0 { + return None; + } + i += 1 + fraction; + } + if matches!(bytes.get(i), Some(b'e' | b'E')) { + let mut j = i + 1; + if matches!(bytes.get(j), Some(b'+' | b'-')) { + j += 1; + } + let exponent = digits(j); + if exponent == 0 { + return None; + } + i = j + exponent; + } + match text[i..].chars().next() { + Some(c) if !special(c) || c == '.' => None, + _ => Some(i), + } +} + +/// Whether `word` is a mode word, `strict` or `lax`, in any case. +fn is_mode(word: &str) -> bool { + word.eq_ignore_ascii_case("strict") || word.eq_ignore_ascii_case("lax") +} + +/// Parse a predicate's text. +fn parse(text: &str) -> Result { + let mut parser = Parser { + text, + pos: 0, + depth: 0, + }; + parser.space()?; + let at = parser.pos; + if parser.word()?.is_some_and(is_mode) { + return Err(Refusal::unsupported(C_MODE, at)); + } + let ast = parser.or()?; + parser.space()?; + if !parser.rest().is_empty() { + return Err(Refusal::syntax(C_TRAILING, parser.pos)); + } + Ok(ast) +} + +/// A predicate compiled: a tree of the inputs of the operators whose +/// operands are its leaves. +#[derive(PartialEq)] +pub(super) enum Node { + And(Vec), + Or(Vec), + Not(Box), + /// One leaf: an operator and the input it derives from, checked. + Leaf(OpSpec, Input), +} + +impl std::fmt::Debug for Node { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Node::And(children) => f.debug_tuple("And").field(children).finish(), + Node::Or(children) => f.debug_tuple("Or").field(children).finish(), + Node::Not(child) => f.debug_tuple("Not").field(child).finish(), + // The input holds the predicate's values: the operator alone. + Node::Leaf(op, _) => f.debug_tuple("Leaf").field(op).finish(), + } + } +} + +/// What a compilation reads besides the tree: the text the tree's +/// offsets are into, the variables, the field's options and the bound an +/// element leaf is read to. +struct Compiler<'c> { + text: &'c str, + vars: &'c serde_json::Map, + options: &'c JsonOptions, + bound: u32, +} + +/// Compile a predicate's text under `vars`, a JSON object, for a field +/// under `options`, its element leaves read to `bound`. +/// +/// # Errors +/// +/// [`Error::Predicate`] for text the grammar does not read or a construct +/// it reads and the index cannot answer, at its offset; and the refusals +/// the leaves' own operators make of their inputs, such as +/// [`TermError::JsonPathNotOrdered`](crate::sem::TermError::JsonPathNotOrdered) for an ordering at a path the field +/// does not order. +pub(super) fn compile( + text: &str, + vars: &Value, + options: &JsonOptions, + bound: u32, +) -> Result { + let empty = serde_json::Map::new(); + let compiler = Compiler { + text, + vars: vars.as_object().unwrap_or(&empty), + options, + bound, + }; + let ast = parse(text)?; + Ok(simplify(compiler.node(ast)?)) +} + +impl Compiler<'_> { + fn node(&self, ast: Ast) -> Result { + Ok(match ast { + Ast::And(terms) => Node::And( + terms + .into_iter() + .map(|t| self.node(t)) + .collect::>()?, + ), + Ast::Or(terms) => Node::Or( + terms + .into_iter() + .map(|t| self.node(t)) + .collect::>()?, + ), + Ast::Not(inner) => Node::Not(Box::new(self.node(*inner)?)), + Ast::Compare(comparison) => self.comparison(comparison)?, + Ast::Exists { path, filter: None } => { + Node::Leaf(OpSpec::Selector, Input::Path(path.path)) + } + Ast::Exists { + path, + filter: Some(conditions), + } => self.filter(path, conditions)?, + }) + } + + /// The value a literal or a variable holds. + fn value(&self, value: &ValueAt) -> Result { + match value { + ValueAt::Literal { start, end } => parse_text(&self.text.as_bytes()[*start..*end]) + .map(Parsed) + .map_err(|error| { + Refusal { + reason: json_reason(error), + construct: C_VALUE, + at: *start, + } + .into() + }), + ValueAt::Variable { name, at } => self + .vars + .get(name) + .map(|value| Parsed(value.clone())) + .ok_or_else(|| { + Refusal { + reason: Reason::JsonPredicateUnknownVariable, + construct: C_VARIABLE, + at: *at, + } + .into() + }), + } + } + + /// The value of `at`, a scalar: what equality takes. + fn scalar(&self, at: &ValueAt) -> Result { + let value = self.value(at)?; + if matches!(value.0, Value::Object(_) | Value::Array(_)) { + return Err(value_type(at)); + } + Ok(value) + } + + /// The value of `at`, a number or a string: what an ordering takes. + fn ordered(&self, at: &ValueAt) -> Result { + let value = self.value(at)?; + leaf(&value.0).map_err(|_| value_type(at))?; + Ok(value) + } + + /// `path op value`. + fn comparison(&self, comparison: Comparison) -> Result { + let Comparison { + path, + op, + value, + at, + } = comparison; + let wild = !path.path.is_positional(); + let (compare, operator) = match op { + // `==`: containment of the value at the path, exact; at a + // wildcard, of any element. + Op::Eq => { + return Ok(Node::Leaf( + OpSpec::EqAt, + Input::At(path.path, self.scalar(&value)?), + )) + } + // `!=` over several elements is true when one differs, which + // containment cannot say. + Op::Ne if wild => { + return Err(Refusal { + reason: Reason::JsonPredicateNotEqualAtWildcard, + construct: C_NOT_EQUAL, + at, + } + .into()) + } + // `!=` at one node: a node there that is not the value. + Op::Ne => { + let value = self.scalar(&value)?; + return Ok(Node::And(vec![ + Node::Leaf(OpSpec::Selector, Input::Path(path.path.clone())), + Node::Not(Box::new(Node::Leaf( + OpSpec::EqAt, + Input::At(path.path, value), + ))), + ])); + } + Op::Lt => (Compare::Lt, OpSpec::LtAt), + Op::Lte => (Compare::Lte, OpSpec::LteAt), + Op::Gt => (Compare::Gt, OpSpec::GtAt), + Op::Gte => (Compare::Gte, OpSpec::GteAt), + }; + Ok(if wild { + // An ordering at a wildcard: some element's field, which is + // `element_match` on the array through the last wildcard. + let value = self.ordered(&value)?; + let mut segments = path.path.segments().to_vec(); + let last = segments + .iter() + .rposition(|segment| *segment == Segment::Wildcard) + .unwrap_or_default(); + let member = segments.split_off(last + 1); + if member.is_empty() { + return Err(Refusal::unsupported(C_ELEMENTS_THEMSELVES, at).into()); + } + let base = JsonPath::of(segments); + let predicate = self.member(&base, JsonPath::of(member), compare, value)?; + Node::Leaf( + OpSpec::ElementMatch, + Input::Elements(base, self.bound, vec![predicate]), + ) + } else { + // An ordering at one node: a node there, not a JSON null, whose + // leaf compares. jsonpath answers false of an absent path and of + // a null, where the ordering alone is NULL. + let value = self.ordered(&value)?; + let path = ordered_path_of(self.options, path.path)?; + Node::And(vec![ + Node::Leaf(OpSpec::Selector, Input::Path(path.clone())), + Node::Not(Box::new(Node::Leaf( + OpSpec::EqAt, + Input::At(path.clone(), Parsed(Value::Null)), + ))), + Node::Leaf(operator, Input::At(path, value)), + ]) + }) + } + + /// One field of an element-scoped predicate: its full path, which the + /// field orders, and its bound. + fn member( + &self, + base: &JsonPath, + member: JsonPath, + compare: Compare, + value: Parsed, + ) -> Result { + let (path, member) = member_of(self.options, base, member)?; + Ok(Predicate { + path, + member, + bounds: Bounds::One(compare, value), + }) + } + + /// `exists(path ? (conditions))`: some element at the path satisfies + /// every condition. One equality is containment at the condition's + /// path, exact; anything else is `element_match`, every condition on a + /// field of one element. + fn filter(&self, path: PathAt, conditions: Vec) -> Result { + if path.path.is_positional() { + return Err(Refusal { + reason: Reason::JsonPathNoWildcard, + construct: C_FILTER_WILDCARD, + at: path.at, + } + .into()); + } + let base = path.path; + for condition in &conditions { + if base.segments().len() + condition.path.path.segments().len() > MAX_DEPTH { + return Err(Refusal { + reason: Reason::JsonPredicateTooDeep, + construct: C_NESTING, + at: path.at, + } + .into()); + } + if !condition.path.path.is_positional() { + return Err(Refusal { + reason: Reason::JsonPathWildcard, + construct: C_MEMBER_WILDCARD, + at: condition.path.at, + } + .into()); + } + } + if let [only] = conditions.as_slice() { + if only.op == Op::Eq { + let full = only.path.path.segments().iter().fold( + base.segments().to_vec(), + |mut at, segment| { + at.push(segment.clone()); + at + }, + ); + return Ok(Node::Leaf( + OpSpec::EqAt, + Input::At(JsonPath::of(full), self.scalar(&only.value)?), + )); + } + } + let mut predicates = Vec::with_capacity(conditions.len()); + for condition in conditions { + let Some(compare) = condition.op.compare() else { + return Err(Refusal::unsupported(C_NE_IN_FILTER, condition.at).into()); + }; + if condition.path.path.segments().is_empty() { + let construct = if compare == Compare::Eq { + C_WHOLE_ELEMENT + } else { + C_ELEMENTS_THEMSELVES + }; + return Err(Refusal::unsupported(construct, condition.path.at).into()); + } + let value = self.ordered(&condition.value)?; + predicates.push(self.member(&base, condition.path.path, compare, value)?); + } + Ok(Node::Leaf( + OpSpec::ElementMatch, + Input::Elements(base, self.bound, predicates), + )) + } +} + +fn value_type(at: &ValueAt) -> Error { + Refusal { + reason: Reason::JsonPredicateValueType, + construct: C_VALUE, + at: at.at(), + } + .into() +} + +/// The tree with every `and` in an `and` and every `or` in an `or` lifted +/// into its parent, a child equal to an earlier sibling dropped, a list of +/// one replaced by its child, and `not` of `not` by its child: each +/// preserves SQL's three-valued answer. +fn simplify(node: Node) -> Node { + fn gather(children: Vec, lift: fn(Node) -> Result, Node>) -> Vec { + let mut out: Vec = Vec::with_capacity(children.len()); + for child in children { + let lifted = match lift(simplify(child)) { + Ok(grandchildren) => grandchildren, + Err(child) => vec![child], + }; + for child in lifted { + if !out.contains(&child) { + out.push(child); + } + } + } + out + } + let one_or = |mut children: Vec, wrap: fn(Vec) -> Node| { + if children.len() == 1 { + children.remove(0) + } else { + wrap(children) + } + }; + match node { + Node::And(children) => one_or( + gather(children, |node| match node { + Node::And(children) => Ok(children), + other => Err(other), + }), + Node::And, + ), + Node::Or(children) => one_or( + gather(children, |node| match node { + Node::Or(children) => Ok(children), + other => Err(other), + }), + Node::Or, + ), + Node::Not(child) => match simplify(*child) { + Node::Not(inner) => *inner, + child => Node::Not(Box::new(child)), + }, + leaf => leaf, + } +} + +/// The shape of a compiled tree with its leaves taken out, in the order +/// they were: what the leaves' operands are put back into. +enum Shape { + And(Vec), + Or(Vec), + Not(Box), + /// A leaf's key in the tree (`contains`, `exists`, `cmp`, `elements`) + /// and, for `cmp`, its bound's. + Leaf(&'static str, Option<&'static str>), +} + +impl Shape { + /// The shape of `node`, its leaves pushed to `leaves` in order. + fn of(node: Node, leaves: &mut Vec<(OpSpec, Input)>) -> Result { + Ok(match node { + Node::And(children) => Shape::And( + children + .into_iter() + .map(|c| Self::of(c, leaves)) + .collect::>()?, + ), + Node::Or(children) => Shape::Or( + children + .into_iter() + .map(|c| Self::of(c, leaves)) + .collect::>()?, + ), + Node::Not(child) => Shape::Not(Box::new(Self::of(*child, leaves)?)), + Node::Leaf(op, input) => { + let shape = match op { + OpSpec::EqAt => Shape::Leaf("contains", None), + OpSpec::Selector => Shape::Leaf("exists", None), + OpSpec::LtAt => Shape::Leaf("cmp", Some("lt")), + OpSpec::LteAt => Shape::Leaf("cmp", Some("lte")), + OpSpec::GtAt => Shape::Leaf("cmp", Some("gt")), + OpSpec::GteAt => Shape::Leaf("cmp", Some("gte")), + OpSpec::ElementMatch => Shape::Leaf("elements", None), + _ => return Err(Error::Internal), + }; + leaves.push((op, input)); + shape + } + }) + } + + /// The tree, each leaf's operand taken from `operands` in order: the + /// `eql_v4.query_json_predicate` operand. + fn fill(self, operands: &mut std::vec::IntoIter) -> Result { + let missing = || crate::Error::Other("a compiled predicate lost a leaf".into()); + let list = |shapes: Vec, operands: &mut std::vec::IntoIter| { + shapes + .into_iter() + .map(|shape| shape.fill(operands)) + .collect::, _>>() + }; + Ok(match self { + Shape::And(children) => serde_json::json!({ "and": list(children, operands)? }), + Shape::Or(children) => serde_json::json!({ "or": list(children, operands)? }), + Shape::Not(child) => serde_json::json!({ "not": child.fill(operands)? }), + Shape::Leaf(key, None) => { + serde_json::json!({ key: operands.next().ok_or_else(missing)? }) + } + Shape::Leaf(key, Some(bound)) => { + serde_json::json!({ key: { bound: operands.next().ok_or_else(missing)? } }) + } + }) + } +} + +/// A leaf's operand as JSON: the EQL value's, or a selector's text. +fn operand_value(operand: Operand) -> Result { + match operand { + Operand::One(bytes) => { + serde_json::from_slice(&bytes).map_err(|error| crate::Error::Other(Box::new(error))) + } + Operand::Text(text) => Ok(Value::String(text)), + _ => Err(crate::Error::Other( + "a predicate leaf derives one operand".into(), + )), + } +} + +/// Derive a compiled predicate's operand: each leaf by its own operator's +/// derivation ([`derive_input`]), all in one pending, put back into the +/// tree. +pub(super) fn derive<'a, K: 'static>( + keyset: &'a KeysetCipher<'_, K>, + label: &Label, + options: &JsonOptions, + node: Node, +) -> Result, Error> { + let mut leaves = Vec::new(); + let shape = Shape::of(node, &mut leaves)?; + let operands = leaves + .into_iter() + .map(|(op, input)| { + derive_input(keyset, label, options, op, input).map(|p| p.try_map(operand_value)) + }) + .collect::, _>>()?; + Ok(Pending::all(keyset, operands) + .try_map(move |operands| eql(&shape.fill(&mut operands.into_iter())?).map(Operand::One))) +} + +#[cfg(test)] +mod tests { + use super::*; + + use crate::sem::TermError; + + use serde_json::json; + + /// A field ordering `$.age`, `$.name` and every contact's `calls` and + /// `relationship`. + fn options() -> JsonOptions { + JsonOptions::new() + .ordering([ + "$.age", + "$.name", + "$.contacts[*].calls", + "$.contacts[*].relationship", + ]) + .unwrap() + } + + fn compiled(text: &str, vars: Value) -> Result { + compile(text, &vars, &options(), 4) + } + + /// A compiled tree written out: each leaf its operator and its input. + fn written(node: &Node) -> String { + let list = |children: &[Node]| children.iter().map(written).collect::>().join(", "); + match node { + Node::And(children) => format!("and({})", list(children)), + Node::Or(children) => format!("or({})", list(children)), + Node::Not(child) => format!("not({})", written(child)), + Node::Leaf(op, Input::At(path, value)) => format!("{}({path}, {})", op.name(), value.0), + Node::Leaf(op, Input::Path(path)) => format!("{}({path})", op.name()), + Node::Leaf(op, Input::Elements(base, bound, predicates)) => { + let members: Vec = predicates + .iter() + .map(|p| match &p.bounds { + Bounds::One(compare, value) => { + format!("{} {compare:?} {}", p.member, value.0) + } + Bounds::Between(low, high) => { + format!("{} in {}..{}", p.member, low.0, high.0) + } + }) + .collect(); + format!("{}({base}, {bound}, [{}])", op.name(), members.join("; ")) + } + Node::Leaf(op, _) => op.name().to_owned(), + } + } + + fn compiles_to(text: &str, vars: Value, want: &str) { + let node = compiled(text, vars).unwrap_or_else(|error| panic!("{text}: {error}")); + assert_eq!(written(&node), want, "{text}"); + } + + /// The refusal `text` meets: its reason, construct and offset. + fn refusal(text: &str, vars: Value) -> (Reason, &'static str, usize) { + match compiled(text, vars) { + Err(Error::Predicate { + reason, + construct, + at, + .. + }) => (reason, construct, at), + Err(other) => panic!("{text}: not a predicate refusal: {other:?}"), + Ok(node) => panic!("{text}: compiled to {}", written(&node)), + } + } + + mod given_the_grammar_it_reads { + use super::*; + + /// Each comparison compiles onto the operators whose operands + /// answer it as jsonpath does: `==` containment; `!=` a node that is + /// not the value; an ordering a node, not null, whose leaf compares; + /// and at a wildcard, an element's field. + #[test] + fn each_comparison_compiles_onto_its_operators() { + for (text, want) in [ + (r#"$.name == "Ann""#, r#"eq_at($.name, "Ann")"#), + ("$.age == 3", "eq_at($.age, 3)"), + ("$.ok == true", "eq_at($.ok, true)"), + ("$.gone == null", "eq_at($.gone, null)"), + ("$.age != 3", "and(selector($.age), not(eq_at($.age, 3)))"), + ( + "$.age != null", + "and(selector($.age), not(eq_at($.age, null)))", + ), + ( + "$.age < 3", + "and(selector($.age), not(eq_at($.age, null)), lt_at($.age, 3))", + ), + ( + "$.age <= 3", + "and(selector($.age), not(eq_at($.age, null)), lte_at($.age, 3))", + ), + ( + "$.age > 3", + "and(selector($.age), not(eq_at($.age, null)), gt_at($.age, 3))", + ), + ( + "$.age >= 3", + "and(selector($.age), not(eq_at($.age, null)), gte_at($.age, 3))", + ), + ("$.tags[*] == \"x\"", r#"eq_at($.tags[*], "x")"#), + ( + "$.contacts[*].calls > 5", + "element_match($.contacts[*], 4, [$.calls Gt 5])", + ), + ("exists($.a.b)", "selector($.a.b)"), + ("exists($.tags[*])", "selector($.tags[*])"), + ] { + compiles_to(text, json!({}), want); + } + } + + /// A value on the left is the comparison turned round, `<>` is + /// `!=`, and a variable is its value in `vars`, of any JSON type a + /// comparison takes. + #[test] + fn operands_are_read_in_either_order_and_by_variable() { + for (text, want) in [ + ( + "3 < $.age", + "and(selector($.age), not(eq_at($.age, null)), gt_at($.age, 3))", + ), + ( + "3 >= $.age", + "and(selector($.age), not(eq_at($.age, null)), lte_at($.age, 3))", + ), + ("3 == $.age", "eq_at($.age, 3)"), + ("$.age <> 3", "and(selector($.age), not(eq_at($.age, 3)))"), + ("$.name == $who", r#"eq_at($.name, "Ann")"#), + (r#"$.name == $"who""#, r#"eq_at($.name, "Ann")"#), + ( + "$.age < $n", + "and(selector($.age), not(eq_at($.age, null)), lt_at($.age, 7))", + ), + ("$.ok == $yes", "eq_at($.ok, true)"), + ("$.gone == $nothing", "eq_at($.gone, null)"), + ] { + compiles_to( + text, + json!({"who": "Ann", "n": 7, "yes": true, "nothing": null}), + want, + ); + } + } + + /// Paths read as PostgreSQL reads them: quoted keys with JSON's + /// escapes, keys that are words of the grammar, and space between + /// steps; and literals as JSON reads them. + #[test] + fn paths_and_literals_read_as_postgres_reads_them() { + for (text, want) in [ + (r#"$."first name" == 1"#, "eq_at($.first name, 1)"), + (r#"$. "a b".c == 1"#, "eq_at($.a b.c, 1)"), + (r#"$."a\"b" == 1"#, r#"eq_at($["a\"b"], 1)"#), + (r#"$."a" == 1"#, "eq_at($.a, 1)"), + ("$.exists == 1", "eq_at($.exists, 1)"), + ("$.size == 1", "eq_at($.size, 1)"), + ("$.true == 1", "eq_at($.true, 1)"), + ("$.é == 1", "eq_at($.é, 1)"), + ("$ . a [ * ] . b == 1", "eq_at($.a[*].b, 1)"), + (" $.a==1 ", "eq_at($.a, 1)"), + ("$.a == 1 /* why */", "eq_at($.a, 1)"), + ("$.a == -1.5e3", "eq_at($.a, -1500.0)"), + ("$.a == 0", "eq_at($.a, 0)"), + ("$.a == 1.25", "eq_at($.a, 1.25)"), + ("$.a == 10.125E+2", "eq_at($.a, 1012.5)"), + ("$.a == 2e-1", "eq_at($.a, 0.2)"), + (r#"$.a == "x\ty""#, r#"eq_at($.a, "x\ty")"#), + ("$ == 1", "eq_at($, 1)"), + ] { + compiles_to(text, json!({}), want); + } + } + + /// `&&` binds tighter than `||`, parentheses group, `!` takes a + /// group or `exists`, and the tree is flattened: an `and` in an + /// `and` lifted, a repeated child dropped, `not` of `not` removed. + #[test] + fn the_boolean_structure_is_kept_and_flattened() { + for (text, want) in [ + ( + "$.a == 1 && $.b == 2 || $.c == 3", + "or(and(eq_at($.a, 1), eq_at($.b, 2)), eq_at($.c, 3))", + ), + ( + "$.a == 1 && ($.b == 2 || $.c == 3)", + "and(eq_at($.a, 1), or(eq_at($.b, 2), eq_at($.c, 3)))", + ), + ("!($.a == 1)", "not(eq_at($.a, 1))"), + ("! exists($.a)", "not(selector($.a))"), + ("!(!($.a == 1))", "eq_at($.a, 1)"), + ("((($.a == 1)))", "eq_at($.a, 1)"), + ( + "$.a == 1 && ($.b == 2 && $.c == 3)", + "and(eq_at($.a, 1), eq_at($.b, 2), eq_at($.c, 3))", + ), + ( + "$.a == 1 || ($.b == 2 || $.c == 3)", + "or(eq_at($.a, 1), eq_at($.b, 2), eq_at($.c, 3))", + ), + ("$.a == 1 || $.a == 1", "eq_at($.a, 1)"), + ("$.a == 1 && $.a == 1", "eq_at($.a, 1)"), + ( + "$.age > 1 && $.age < 9", + "and(selector($.age), not(eq_at($.age, null)), gt_at($.age, 1), lt_at($.age, 9))", + ), + ( + "EXISTS($.a) && Exists($.b)", + "and(selector($.a), selector($.b))", + ), + ] { + compiles_to(text, json!({}), want); + } + } + + /// A compiled tree prints its structure and operators, never the + /// values or paths it holds. + #[test] + fn a_compiled_tree_prints_no_value() { + let node = compiled( + r#"!($."zq-secret" == "zq-secret") || exists($.age)"#, + json!({}), + ) + .unwrap(); + let shown = format!("{node:?}"); + assert_eq!(shown, "Or([Not(Leaf(EqAt)), Leaf(Selector)])"); + } + + /// `exists(path ? (…))`: one equality is containment at its field, + /// exact; anything else is `element_match`, every condition on one + /// element, read to the input's bound. + #[test] + fn a_filter_compiles_to_containment_or_an_element_match() { + for (text, want) in [ + ( + "exists($.contacts[*] ? (@.relationship == $r))", + r#"eq_at($.contacts[*].relationship, "Sibling")"#, + ), + ( + "exists($.tags[*] ? (@ == $r))", + r#"eq_at($.tags[*], "Sibling")"#, + ), + ("exists($.tags[*] ? (1 == @))", "eq_at($.tags[*], 1)"), + ( + "exists($.contacts[*] ? (@.relationship == $r && @.calls > 5))", + r#"element_match($.contacts[*], 4, [$.relationship Eq "Sibling"; $.calls Gt 5])"#, + ), + ( + "exists($.contacts[*] ? ((@.calls >= 1) && (@.calls < 9)))", + "element_match($.contacts[*], 4, [$.calls Gte 1; $.calls Lt 9])", + ), + ( + "exists($.contacts[*] ? (@.calls < 2))", + "element_match($.contacts[*], 4, [$.calls Lt 2])", + ), + ] { + compiles_to(text, json!({"r": "Sibling"}), want); + } + } + } + + mod given_text_it_refuses { + use super::*; + + use Reason::{ + JsonPathNoWildcard, JsonPathPosition, JsonPathWildcard, + JsonPredicateNotEqualAtWildcard, JsonPredicateSyntax, JsonPredicateTooDeep, + JsonPredicateUnknownVariable, JsonPredicateUnsupported, JsonPredicateValueType, + JsonSyntax, + }; + + /// Each construct the index cannot answer is refused by name, at + /// the byte in the caller's text where it starts. + #[test] + fn names_each_unsupported_construct_at_its_offset() { + for (text, construct, at) in [ + ("strict $.a == 1", C_MODE, 0), + ("STRICT $.a == 1", C_MODE, 0), + ("$.a == Lax", C_MODE, 7), + (r#"$.a LIKE_REGEX "x""#, C_LIKE_REGEX, 4), + (r#"$.a Starts with "x""#, C_STARTS_WITH, 4), + ("$.a IS unknown", C_IS_UNKNOWN, 4), + ("$.a[LAST] == 1", C_LAST, 3), + ("$.a == LAST", C_LAST, 7), + (r#"exists($.a like_regex "x")"#, C_LIKE_REGEX, 11), + ("exists($.a + 1)", C_ARITHMETIC, 11), + ( + "exists($.a[*] ? (@ == 1) starts with \"x\")", + C_STARTS_WITH, + 25, + ), + (" lax $.a == 1", C_MODE, 2), + ("$.a + 1 == 2", C_ARITHMETIC, 4), + ("$.a == 1 - 2", C_ARITHMETIC, 9), + ("$.a == +1", C_ARITHMETIC, 7), + (r#"$.a like_regex "x""#, C_LIKE_REGEX, 4), + (r#"$.a starts with "x""#, C_STARTS_WITH, 4), + ("$.a is unknown", C_IS_UNKNOWN, 4), + ("$.a.size() == 1", C_METHOD, 3), + ("$.a.type () == 1", C_METHOD, 3), + ("$.* == 1", C_ANY_MEMBER, 1), + ("$.** == 1", C_ANY_DEPTH, 1), + ("$.a[1 to 2] == 1", C_SLICE, 3), + ("$.a[1, 2] == 1", C_SLICE, 3), + ("$.a[last] == 1", C_LAST, 3), + ("$.a == last", C_LAST, 7), + ("$.a == lax", C_MODE, 7), + ("$.a[$i] == 1", C_SUBSCRIPT, 3), + ("@ == 1", C_AT_OUTSIDE, 0), + ("$.a == $.b", C_TWO_PATHS, 4), + ("1 == 1", C_TWO_VALUES, 2), + ("$x == $y", C_TWO_VALUES, 3), + ("$.a", C_ALONE, 3), + ("$.a && $.b == 1", C_ALONE, 4), + ("true", C_ALONE, 4), + ("$.a[*] ? (@ == 1) == 1", C_FILTER_OUTSIDE, 7), + ("exists($.a[*] ? (@ == 1).b)", C_AFTER_FILTER, 24), + ("exists($.a[*] ? (@ == 1) ? (@ == 2))", C_FILTER_OUTSIDE, 24), + ("exists($.a == 1)", C_IN_EXISTS, 7), + ("exists(@)", C_AT_OUTSIDE, 7), + ( + "exists($.a[*] ? (@.b == 1 || @.c == 2))", + C_OR_IN_FILTER, + 26, + ), + ("exists($.a[*] ? (!(@.b == 1)))", C_NOT_IN_FILTER, 17), + ("exists($.a[*] ? (@.b != 1))", C_NE_IN_FILTER, 21), + ("exists($.a[*] ? (exists(@.b)))", C_EXISTS_IN_FILTER, 17), + ("exists($.a[*] ? ($.b == 1))", C_DOLLAR_IN_FILTER, 17), + ("$.v == $x.y", C_VARIABLE_PATH, 9), + ("$.tags[*] < 1", C_ELEMENTS_THEMSELVES, 10), + ( + "exists($.contacts[*] ? (@ > 1 && @.calls < 2))", + C_ELEMENTS_THEMSELVES, + 24, + ), + ( + "exists($.contacts[*] ? (@ == 1 && @.calls < 2))", + C_WHOLE_ELEMENT, + 24, + ), + ] { + assert_eq!( + refusal(text, json!({"x": 1, "y": 2})), + (JsonPredicateUnsupported, construct, at), + "{text}" + ); + } + } + + /// Text that is not a predicate is refused where reading stops. + #[test] + fn refuses_text_that_is_not_a_predicate_where_reading_stops() { + for (text, construct, at) in [ + ("", C_TEXT, 0), + ("$.a == 1 &&", C_TEXT, 11), + ("$.a = 1", C_TEXT, 4), + ("$.a === 1", C_TEXT, 6), + ("$.a > 1 > 2", C_TRAILING, 8), + ("$.a == 1 2", C_TRAILING, 9), + ("($.a == 1) is unknown", C_TRAILING, 11), + ("!$.a == 1", C_TEXT, 1), + ("!!($.a == 1)", C_TEXT, 1), + ("($.a == 1", C_TEXT, 9), + ("$.a == 01", C_NUMBER, 7), + ("$.a == 1.", C_NUMBER, 7), + ("$.a == 1e", C_NUMBER, 7), + ("$.a == .5", C_TEXT, 7), + ("$.a == 'x'", C_TEXT, 7), + (r#"$.a == "x"#, C_STRING, 7), + (r#"$."a == 1"#, C_STRING, 2), + ("$.a == TRUE", C_TEXT, 7), + ("$.1a == 1", C_TEXT, 2), + ("$. == 1", C_TEXT, 3), + ("$.a == 1 /* open", C_TEXT, 9), + ("exists $.a", C_TEXT, 7), + ("$.a == $1", C_TEXT, 8), + ] { + assert_eq!( + refusal(text, json!({})), + (JsonPredicateSyntax, construct, at), + "{text}" + ); + } + } + + /// A value is refused at its offset when no variable holds it, its + /// type is not one the comparison takes, or its text is not JSON. + #[test] + fn refuses_a_value_the_comparison_cannot_take_at_its_offset() { + let vars = json!({"doc": {"a": 1}, "list": [1], "n": 1, "flag": true, "none": null}); + for (text, reason, at) in [ + ("$.a == $missing", JsonPredicateUnknownVariable, 7), + ("$missing < $.age", JsonPredicateUnknownVariable, 0), + ("$.a == $doc", JsonPredicateValueType, 7), + ("$.a != $list", JsonPredicateValueType, 7), + ("$.age < true", JsonPredicateValueType, 8), + ("$.age < $flag", JsonPredicateValueType, 8), + ("$.age >= null", JsonPredicateValueType, 9), + ("$.age >= $none", JsonPredicateValueType, 9), + ( + "exists($.contacts[*] ? (@.calls == $n && @.relationship == $flag))", + JsonPredicateValueType, + 59, + ), + (r#"$.a == "\x41""#, JsonPredicateSyntax, 7), + (r#"$.a == 1e999"#, JsonSyntax, 7), + ] { + let (got, _, offset) = refusal(text, vars.clone()); + assert_eq!((got, offset), (reason, at), "{text}"); + } + } + + /// What the index's paths cannot name: a position, `!=` at a + /// wildcard, a filter on a path with no wildcard, a wildcard in a + /// filter's member, and nesting or paths past the limits. + #[test] + fn refuses_paths_the_index_cannot_name() { + let deep = format!( + "{} $.a == 1{}", + "(".repeat(MAX_NESTING + 1), + ")".repeat(MAX_NESTING + 1) + ); + let long = format!("$.{} == 1", vec!["a"; MAX_DEPTH + 1].join(".")); + for (text, want) in [ + ("$.a[2] == 1", (JsonPathPosition, C_POSITION, 3)), + ("exists($.a[ 0 ])", (JsonPathPosition, C_POSITION, 10)), + ( + "$.a[*] != 1", + (JsonPredicateNotEqualAtWildcard, C_NOT_EQUAL, 7), + ), + ( + "$.a[*].b != null", + (JsonPredicateNotEqualAtWildcard, C_NOT_EQUAL, 9), + ), + ( + "exists($.a ? (@ == 1))", + (JsonPathNoWildcard, C_FILTER_WILDCARD, 7), + ), + ( + "exists($.a[*] ? (@.b[*] == 1))", + (JsonPathWildcard, C_MEMBER_WILDCARD, 17), + ), + ( + deep.as_str(), + (JsonPredicateTooDeep, C_NESTING, MAX_NESTING + 1), + ), + (long.as_str(), (JsonPredicateTooDeep, C_NESTING, 0)), + ] { + assert_eq!(refusal(text, json!({})), want, "{text}"); + } + let at_limit = format!( + "{}$.a == 1{}", + "(".repeat(MAX_NESTING), + ")".repeat(MAX_NESTING) + ); + assert!(compiled(&at_limit, json!({})).is_ok(), "the limit is read"); + // A filter's equality is at the base and the member joined, as + // deep as a document at most. + let member = + |steps: usize| format!("exists($.a[*] ? (@.{} == 1))", vec!["b"; steps].join(".")); + assert!( + compiled(&member(MAX_DEPTH - 2), json!({})).is_ok(), + "a filter's full path as deep as a document" + ); + assert_eq!( + refusal(&member(MAX_DEPTH - 1), json!({})), + (JsonPredicateTooDeep, C_NESTING, 7), + "one equality: a filter's full path deeper than a document" + ); + let members = |steps: usize| { + format!( + "exists($.contacts[*] ? (@.calls > 1 && @.{} < 2))", + vec!["b"; steps].join(".") + ) + }; + assert_eq!( + refusal(&members(MAX_DEPTH - 1), json!({})), + (JsonPredicateTooDeep, C_NESTING, 7), + "two conditions: a member's full path deeper than a document" + ); + assert!( + matches!( + compiled(&members(MAX_DEPTH - 2), json!({})), + Err(Error::Cipher(crate::Error::Term( + TermError::JsonPathNotOrdered + ))) + ), + "as deep as a document is read, and refused only as unordered" + ); + let deepest = format!("$.{} == 1", vec!["a"; MAX_DEPTH].join(".")); + assert!( + compiled(&deepest, json!({})).is_ok(), + "a path as deep as a document" + ); + // A step past a document's depth is refused before it is read. + let past = format!("$.{}.** == 1", vec!["a"; MAX_DEPTH].join(".")); + assert_eq!( + refusal(&past, json!({})), + (JsonPredicateTooDeep, C_NESTING, 0), + "the step past the depth is not read" + ); + // Nesting is depth, not count: siblings past the limit, each + // in its own group or under its own `!` or `exists`, are read. + for sibling in ["($.a == 1)", "!($.a == 1)", "exists($.a)"] { + let wide = vec![sibling; MAX_NESTING + 1].join(" && "); + assert!(compiled(&wide, json!({})).is_ok(), "{sibling} side by side"); + } + let filters = + vec!["exists($.contacts[*] ? ((@.calls > 1)))"; MAX_NESTING + 1].join(" || "); + assert!( + compiled(&filters, json!({})).is_ok(), + "filters side by side" + ); + } + + /// The index's own refusals of the leaves' inputs come through as + /// the operators give them: an ordering at a path the field does not + /// order, here or at an element's field. + #[test] + fn refuses_an_ordering_at_a_path_the_field_does_not_order() { + for text in [ + "$.height > 1", + "$.contacts[*].note > 1", + "exists($.contacts[*] ? (@.note > 1 && @.calls < 2))", + ] { + assert!( + matches!( + compiled(text, json!({})), + Err(Error::Cipher(crate::Error::Term( + TermError::JsonPathNotOrdered + ))) + ), + "{text}" + ); + } + } + + /// A refusal carries the construct and the offset, never text from + /// the predicate. + #[test] + fn a_refusal_quotes_no_text_of_the_predicate() { + let secret = "zq-secret-key-name"; + for text in [ + format!(r#"$."{secret}" like_regex "x""#), + format!(r#"$.{secret} == "{secret}" + 1"#), + format!(r#"$.{secret} == ${secret}"#), + format!(r#"$.{secret} == "{secret}" {secret}"#), + ] { + let error = compiled(&text, json!({})).unwrap_err(); + let shown = format!("{error} {:?}", crate::ErrorPayload::payload(&error)); + assert!(!shown.contains(secret), "{shown}"); + } + } + } +} diff --git a/packages/stack-encrypt/src/dynamic/json_query.rs b/packages/stack-encrypt/src/dynamic/json_query.rs index ea5e261f6..4693270f0 100644 --- a/packages/stack-encrypt/src/dynamic/json_query.rs +++ b/packages/stack-encrypt/src/dynamic/json_query.rs @@ -16,6 +16,8 @@ //! | `selector` | `{"path"}` | text: the selector `->` takes | //! | `subtree` | `{"path", "wildcard_bound"?}` | text array: the tokens `json_tree_subtree` takes | //! | `children`, `child_count` | `{"path"}`: a path with no wildcard | text: the node's token | +//! | `element_match` | `{"path", "wildcard_bound"?, "predicates"}`: an array's elements, by a path with a wildcard, and predicates on fields of an element at ordered paths | EQL: per predicate, the field's read tokens and its bounds (`eql_v4.query_json_elements`), compared as the orderings compare, so its `eq` is of folded values | +//! | `match` | `{"predicate", "vars"?, "wildcard_bound"?}`: a jsonpath predicate in the subset below, and its variables as one JSON object | EQL: the predicate as a boolean tree of the operands above (`eql_v4.query_json_predicate`), which `json_tree_match` and `@@` evaluate | //! //! Every JSON value in an input is UTF-8 JSON text, parsed by //! [`parse_text`](crate::json::parse_text)'s rules. A path is JSONPath text @@ -32,6 +34,145 @@ //! hold of the same array element; one containment query of a document //! holding both does. //! +//! # Element predicates +//! +//! `element_match` asks whether some one element of an array satisfies +//! every predicate, which two `eq_at` or ordering operands cannot: the +//! jsonpath `exists($.contacts[*] ? (@.relationship == "Sibling" && +//! @.calls > 5))`. Its input names the array's elements by a path with a +//! wildcard (`$.contacts[*]`), and each predicate a field of an element by +//! a `member` path relative to it (`relationship`, `address.postcode`, +//! `["first name"]`), an `op` and its value: +//! +//! ```json +//! {"path": "$.contacts[*]", "wildcard_bound": 16, "predicates": [ +//! {"member": "relationship", "op": "eq", "json": "\"Sibling\""}, +//! {"member": "calls", "op": "gt", "json": "5"}, +//! {"member": "age", "op": "between", "low": "18", "high": "40"} +//! ]} +//! ``` +//! +//! `op` is `eq`, `lt`, `lte`, `gt` or `gte`, with `json`, or `between`, +//! with `low` and `high`, inclusive. Each member's full path, the path and +//! the member joined, must be one the field orders. The operand holds, per +//! predicate, the tokens of a `subtree` read of the full path at the bound, +//! and the bounds as the orderings derive them: `gt` the leaf operand +//! `gt_at` derives, `between` the two `between_at` derives as `gte` and +//! `lte`, and `eq` one leaf operand as both `gte` and `lte`. So equality +//! here is the ordering term's: text compares case- and accent-folded, and +//! numbers as float64, so integers past 2^53 can tie. Exact equality at a +//! path is `eq_at`, of any element. +//! +//! Refused before anything is derived, each by its reason: a path with no +//! wildcard ([`Reason::JsonPathNoWildcard`]), a `wildcard_bound` of `0`, +//! whose operand could never be true ([`Reason::JsonWildcardBoundZero`]), +//! an empty list of predicates ([`Reason::JsonNoPredicates`]), an `op` +//! that is none of the six ([`Reason::JsonUnknownPredicateOp`]), a +//! predicate whose keys do not fit its `op` ([`Reason::JsonInput`]), a +//! wildcard in a member ([`Reason::JsonPathWildcard`]); and by the index's +//! own refusals, a position in the path or a member +//! ([`TermError::JsonOrderingAtPosition`]), a member whose full path the +//! field does not order ([`TermError::JsonPathNotOrdered`]), a value that +//! is not a number or a string ([`TermError::JsonLeafNotOrdered`]), and a +//! member that is not a path ([`TermError::JsonPath`], at an offset into +//! the member's own text). The +//! database refuses a row whose array is longer than the bound, as a +//! subtree read of it opens to a refusal: each member's tokens end with +//! the probe, and `json_tree_element_match` raises when it names a node. +//! +//! # Compiled predicates +//! +//! `match` takes what plaintext `jsonb_path_match(col, $1, $2)` takes: a +//! jsonpath predicate and its variables, `vars` the JSON text of one object +//! (`{}` when absent). It compiles the predicate into one operand, a tree +//! of `and`, `or` and `not` over the operands the operators above derive, +//! each leaf derived by its operator's own code: +//! +//! ```json +//! {"predicate": "$.age >= $min && exists($.contacts[*] ? (@.relationship == $r && @.calls > 5))", +//! "vars": "{\"min\": 18, \"r\": \"Sibling\"}", "wildcard_bound": 16} +//! ``` +//! +//! The grammar is the subset of SQL/JSON path predicates the terms answer, +//! as PostgreSQL reads it: a path is `$` and steps `.key`, `."quoted key"` +//! (JSON's escapes) and `[*]`; a value is a variable (`$v`, `$"v"`) or a +//! JSON literal (a string, a number, `true`, `false`, `null`); predicates +//! are comparisons of a path with a value (`==`, `!=` or `<>`, `<`, `<=`, +//! `>`, `>=`, either side first), `exists(path)` and `exists(path ? (…))` +//! with a filter of comparisons on `@` and `@.member` joined by `&&`, and +//! `&&`, `||`, `!(…)` and parentheses over them. Each maps onto operands +//! this way: +//! +//! | jsonpath | operand | +//! |---|---| +//! | `P == v`, `P == null` | `eq_at`'s containment needle | +//! | `P != v`, `P != null` | `selector` and not that needle: a node there that is not the value | +//! | `P < v` … `P >= v` | `selector`, not the `null` needle, and the ordering's leaf: a node there, not null, whose leaf compares | +//! | `P < v` … `P >= v`, `P` with a wildcard | `element_match` on the array through `P`'s last wildcard, the rest of `P` its member | +//! | `exists(P)` | `selector` | +//! | `exists(P ? (@.m == v))`, `exists(P ? (@ == v))` | `eq_at` at `P.m` or `P` | +//! | `exists(P ? (@.m op v && …))` | `element_match` on `P`'s elements, one predicate per condition | +//! +//! So the tree answers as `jsonb_path_match` would over the plaintext: true, +//! false or NULL (unknown) as SQL's three-valued logic combines them. An +//! absent path is false under every comparison and a JSON `null` false +//! under an ordering, as jsonpath has them, `!=` is true of a `null`, and +//! an ordering is unknown of a node with no ordering term (a boolean, a +//! container), as jsonpath's comparison of two kinds is. The tree's answer +//! and jsonpath's part in four places, each a property of the stored +//! document or of the ordering terms rather than of the predicate: +//! +//! - A path is read as written: lax mode unwraps an array the path names +//! no `[*]` for (`$.tags == "a"` over `{"tags": ["a"]}` is true), and +//! the tree does not. Write `[*]`. +//! - Lax mode also wraps a value that is not an array where the path names +//! `[*]`: `$.tags[*] == "a"` over `{"tags": "a"}`, `exists($.tags[*])` +//! over a scalar `tags` and `exists($.items[*] ? (@.sku == "x"))` over +//! `{"items": {"sku": "x"}}` are all true to jsonpath. The tree finds +//! only array elements at `[*]`, since `$.tags[*]` and `$.tags` have +//! different selectors, so each is false here. Keep an array an array. +//! - A comparison of a string with a number, or of a boolean with a +//! string or a number, is unknown to jsonpath. The terms do not record +//! a value's kind, so the tree answers it false under `==` (and in a +//! filter, as jsonpath's filter does), true under `!=`, and by the +//! ordering terms under an ordering. Keep one kind of value at a path. +//! - An ordering compares the ordering terms, as the orderings do: text +//! case- and accent-folded, numbers as float64. So does every condition +//! of a filter of more than one, `element_match`'s, whose equality is +//! therefore of folded values; `==` elsewhere is exact. +//! +//! An ordering at a path the field does not order is refused, as the +//! orderings refuse it. +//! +//! Refused, each at the byte of the caller's text where the construct +//! starts and naming the construct by the grammar's words and never the +//! text ([`Error::Predicate`]): text the grammar does not read +//! ([`Reason::JsonPredicateSyntax`]); arithmetic, `like_regex`, `starts +//! with`, `is unknown`, a method (`.size()`), a mode word (`strict`, +//! `lax`), `.*`, `.**`, a slice, `last`, a comparison of two paths or two +//! values, a filter outside `exists`, and `||`, `!`, `!=` or `exists` +//! inside a filter ([`Reason::JsonPredicateUnsupported`]); a variable +//! `vars` does not hold ([`Reason::JsonPredicateUnknownVariable`]); a +//! value of a type its comparison cannot take, an object or array +//! anywhere, and under an ordering or in a filter of more than one +//! condition anything but a number or a string +//! ([`Reason::JsonPredicateValueType`]); `!=` at a path with a wildcard +//! ([`Reason::JsonPredicateNotEqualAtWildcard`]); nesting past 32 levels or +//! a path deeper than a document ([`Reason::JsonPredicateTooDeep`]); a +//! position (`[2]`, [`Reason::JsonPathPosition`]); a filter on a path with +//! no wildcard ([`Reason::JsonPathNoWildcard`]); and a wildcard in a +//! filter's member ([`Reason::JsonPathWildcard`]). The leaves' operators +//! refuse their own inputs as they would alone: an ordering at a path the +//! field does not order ([`TermError::JsonPathNotOrdered`]). A +//! `wildcard_bound` of `0` is [`Reason::JsonWildcardBoundZero`], and `vars` +//! that are not an object [`Reason::JsonInput`]. +//! +//! The operand reveals what its leaves reveal when sent one by one, and +//! the boolean shape of the filter; an ordering's `null` needle is one +//! leaf more than the ordering alone sends. `json_tree_match` evaluates the +//! tree inside a function, which engages no index, so a predicate written +//! out per operator is the fast path. +//! //! # Reads //! //! [`json_open`] opens the rows a read returned, against the request that @@ -76,6 +217,10 @@ pub(super) enum Binds { Text, /// `text[]`. TextArray, + /// The target's element predicate domain (`eql_v4.query_json_elements`). + Elements, + /// The target's compiled predicate domain (`eql_v4.query_json_predicate`). + Predicate, } /// What a JSON operator's operand binds as, or `None` for an operator @@ -90,6 +235,8 @@ pub(super) fn binds(op: OpSpec) -> Option { } OpSpec::Selector | OpSpec::Children | OpSpec::ChildCount => Some(Binds::Text), OpSpec::Subtree => Some(Binds::TextArray), + OpSpec::ElementMatch => Some(Binds::Elements), + OpSpec::Match => Some(Binds::Predicate), OpSpec::Eq | OpSpec::Matches | OpSpec::Lt @@ -108,12 +255,15 @@ pub(super) fn domain(target: &super::TargetDescriptor, binds: Binds) -> Option target.leaf_query_sql_domain.clone(), Binds::Text => Some("text".to_owned()), Binds::TextArray => Some("text[]".to_owned()), + Binds::Elements => target.elements_query_sql_domain.clone(), + Binds::Predicate => target.predicate_query_sql_domain.clone(), } } /// UTF-8 JSON text in an input: a text or bytes leaf, held in a buffer -/// wiped on drop. -struct JsonText(Zeroizing>); +/// wiped on drop. Also how `match` holds its predicate's text, whose +/// literals are values. +pub(super) struct JsonText(pub(super) Zeroizing>); impl<'de> Deserialize<'de> for JsonText { fn deserialize>(d: D) -> Result { @@ -135,7 +285,8 @@ impl<'de> Deserialize<'de> for JsonText { } /// A document parsed from an input, wiped when it drops. -struct Parsed(Value); +#[derive(PartialEq)] +pub(super) struct Parsed(pub(super) Value); impl Drop for Parsed { fn drop(&mut self) { @@ -144,7 +295,7 @@ impl Drop for Parsed { } impl JsonText { - fn parse(&self) -> Result { + pub(super) fn parse(&self) -> Result { parse_text(&self.0) .map(Parsed) .map_err(|error| Error::Source { @@ -155,7 +306,7 @@ impl JsonText { } /// The reason a JSON text refusal is reported with. -fn json_reason(error: crate::json::JsonTextError) -> Reason { +pub(super) fn json_reason(error: crate::json::JsonTextError) -> Reason { use crate::json::JsonTextError as E; match error { E::Syntax { .. } => Reason::JsonSyntax, @@ -199,14 +350,79 @@ struct ReadInput { wildcard_bound: Option, } +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ElementsInput { + path: String, + #[serde(default)] + wildcard_bound: Option, + predicates: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct PredicateInput { + member: String, + op: String, + #[serde(default)] + json: Option, + #[serde(default)] + low: Option, + #[serde(default)] + high: Option, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct MatchInput { + predicate: JsonText, + #[serde(default)] + vars: Option, + #[serde(default)] + wildcard_bound: Option, +} + +/// How a predicate of `element_match` compares its field with one value. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Compare { + Eq, + Lt, + Lte, + Gt, + Gte, +} + +/// A predicate's bounds, as its input gives them. +#[derive(PartialEq)] +pub(super) enum Bounds { + /// One value, compared by the predicate's `op`. + One(Compare, Parsed), + /// `between`: the low bound then the high, both inclusive. + Between(Parsed, Parsed), +} + +/// One predicate of `element_match`, checked: the full path of the +/// element's field, which the field orders, the member's steps below the +/// element, and its bounds. +#[derive(PartialEq)] +pub(super) struct Predicate { + pub(super) path: JsonPath, + pub(super) member: JsonPath, + pub(super) bounds: Bounds, +} + /// An operator's input, read from its frame and checked: everything -/// [`derive`] would refuse before it derives anything. -enum Input { +/// [`derive`] would refuse before it derives anything. `match`'s is its +/// predicate compiled to a tree of the other operators' inputs. +#[derive(PartialEq)] +pub(super) enum Input { Document(Parsed), At(JsonPath, Parsed), Between(JsonPath, Parsed, Parsed), Path(JsonPath), Read(JsonPath, u32), + Elements(JsonPath, u32, Vec), + Predicate(Box), } fn read_frame Deserialize<'de>>(input: &ValueFrame) -> Result { @@ -280,10 +496,142 @@ fn read_input(op: OpSpec, options: &JsonOptions, input: &ValueFrame) -> Result read_elements(options, input), + OpSpec::Match => read_match(options, input), _ => Err(Error::Internal), } } +/// The wildcard bound an input names, or the default: never `0`, which +/// would send the probe alone, so an element predicate could never be true. +fn wildcard_bound(bound: Option) -> Result { + match bound.unwrap_or(JsonSubtreeKey::DEFAULT_BOUND) { + 0 => Err(Error::bad_source(Reason::JsonWildcardBoundZero)), + bound => Ok(bound), + } +} + +/// `match`'s input: a jsonpath predicate, its variables as one JSON object +/// (the `vars` jsonb `jsonb_path_match` takes, `{}` when absent) and the +/// bound its element-scoped leaves are read to. The predicate is compiled +/// here, so every refusal it can meet is met before anything is derived. +fn read_match(options: &JsonOptions, input: &ValueFrame) -> Result { + let input: MatchInput = read_frame(input)?; + let bound = wildcard_bound(input.wildcard_bound)?; + let vars = match &input.vars { + Some(vars) => vars.parse()?, + None => Parsed(Value::Object(serde_json::Map::new())), + }; + if !vars.0.is_object() { + return Err(Error::bad_source(Reason::JsonInput)); + } + let text = std::str::from_utf8(&input.predicate.0) + .map_err(|_| Error::bad_source(Reason::JsonInput))?; + Ok(Input::Predicate(Box::new(super::json_predicate::compile( + text, &vars.0, options, bound, + )?))) +} + +/// `element_match`'s input: an array's elements, by a path with a wildcard +/// and no position, and one or more predicates on their fields. +fn read_elements(options: &JsonOptions, input: &ValueFrame) -> Result { + let elements: ElementsInput = read_frame(input)?; + let base = path(&elements.path)?; + if base.is_positional() { + return Err(Error::bad_source(Reason::JsonPathNoWildcard)); + } + if base.names_a_position() { + return Err(term(TermError::JsonOrderingAtPosition)); + } + let bound = wildcard_bound(elements.wildcard_bound)?; + if elements.predicates.is_empty() { + return Err(Error::bad_source(Reason::JsonNoPredicates)); + } + let predicates = elements + .predicates + .iter() + .map(|predicate| read_predicate(options, &base, predicate)) + .collect::, _>>()?; + Ok(Input::Elements(base, bound, predicates)) +} + +/// One predicate of `element_match`: its member's full path below `base`, +/// which the field must order, and the bounds its `op` takes. +fn read_predicate( + options: &JsonOptions, + base: &JsonPath, + predicate: &PredicateInput, +) -> Result { + let compare = match predicate.op.as_str() { + "eq" => Some(Compare::Eq), + "lt" => Some(Compare::Lt), + "lte" => Some(Compare::Lte), + "gt" => Some(Compare::Gt), + "gte" => Some(Compare::Gte), + "between" => None, + _ => return Err(Error::bad_source(Reason::JsonUnknownPredicateOp)), + }; + let (path, member) = member_path(options, base, &predicate.member)?; + let value = |text: &JsonText| -> Result { + let value = text.parse()?; + leaf(&value.0)?; + Ok(value) + }; + let bounds = match (compare, &predicate.json, &predicate.low, &predicate.high) { + (Some(compare), Some(json), None, None) => Bounds::One(compare, value(json)?), + (None, None, Some(low), Some(high)) => Bounds::Between(value(low)?, value(high)?), + _ => return Err(Error::bad_source(Reason::JsonInput)), + }; + Ok(Predicate { + path, + member, + bounds, + }) +} + +/// A member's full path, `base` and then the member's steps, and the +/// steps themselves, read as a path relative to an element (`name`, +/// `address.city`, `["a b"]`). A member names one field of each element, +/// so it has no wildcard of its own, and the full path is an ordering's: +/// no position, and ordered. A member that does not parse is refused at +/// an offset into the member's own text. +fn member_path( + options: &JsonOptions, + base: &JsonPath, + member: &str, +) -> Result<(JsonPath, JsonPath), Error> { + let prefix = if member.starts_with('[') { "$" } else { "$." }; + let relative = JsonPath::parse(&format!("{prefix}{member}")).map_err(|error| { + let error = crate::json::JsonPathError { + at: error.at.saturating_sub(prefix.len()), + ..error + }; + Error::Cipher(TermError::JsonPath(error).into()) + })?; + member_of(options, base, relative) +} + +/// A member's full path and its steps, from the steps already parsed: the +/// checks [`member_path`] makes past parsing, which `match` makes of the +/// members of an element-scoped filter. +pub(super) fn member_of( + options: &JsonOptions, + base: &JsonPath, + relative: JsonPath, +) -> Result<(JsonPath, JsonPath), Error> { + if !relative.is_positional() { + return Err(Error::bad_source(Reason::JsonPathWildcard)); + } + let full = relative + .segments() + .iter() + .fold(base.clone(), |at, segment| at.then(segment.clone())); + if full.segments().len() > crate::json::MAX_DEPTH { + return Err(term(TermError::JsonTooDeep)); + } + Ok((ordered_path_of(options, full)?, relative)) +} + /// `eq_at`'s input: a scalar at a path read through a selector. fn read_eq_at(input: &ValueFrame) -> Result { let at: AtInput = read_frame(input)?; @@ -324,7 +672,11 @@ fn read_between(options: &JsonOptions, input: &ValueFrame) -> Result Result { - let path = path(text)?; + ordered_path_of(options, path(text)?) +} + +/// [`ordered_path`] of a path already parsed. +pub(super) fn ordered_path_of(options: &JsonOptions, path: JsonPath) -> Result { if path.names_a_position() { Err(term(TermError::JsonOrderingAtPosition)) } else if options.orders(&path) { @@ -335,7 +687,7 @@ fn ordered_path(options: &JsonOptions, text: &str) -> Result { } /// A leaf the index orders: a number or a string. -fn leaf(value: &Value) -> Result<(), Error> { +pub(super) fn leaf(value: &Value) -> Result<(), Error> { match value { Value::Number(_) | Value::String(_) => Ok(()), _ => Err(term(TermError::JsonLeafNotOrdered)), @@ -368,7 +720,7 @@ fn terms<'a, K: 'static>( ) } -fn eql(value: &T) -> Result, crate::Error> { +pub(super) fn eql(value: &T) -> Result, crate::Error> { serde_json::to_vec(value).map_err(|error| crate::Error::Other(Box::new(error))) } @@ -380,9 +732,22 @@ pub(super) fn derive<'a, K: 'static>( options: &JsonOptions, op: OpSpec, input: &ValueFrame, +) -> Result, Error> { + derive_input(keyset, label, options, op, read_input(op, options, input)?) +} + +/// Derive an operand for an input already read and checked: what +/// [`derive`] does past reading, and what `match` does for each of its +/// leaves. +pub(super) fn derive_input<'a, K: 'static>( + keyset: &'a KeysetCipher<'_, K>, + label: &Label, + options: &JsonOptions, + op: OpSpec, + input: Input, ) -> Result, Error> { let column = || NonEmpty::from(label.clone()); - Ok(match read_input(op, options, input)? { + Ok(match input { Input::Document(document) => terms(keyset, options, label, document.0.clone()) .try_map(|terms| eql(&JsonQuery::of_terms(terms)).map(Operand::One)), Input::At(path, value) if op == OpSpec::EqAt => { @@ -421,9 +786,110 @@ pub(super) fn derive<'a, K: 'static>( let key = keyset.json_subtree_key_now(&path, options, bound, column())?; Pending::ready(keyset, Ok(Operand::TextList(key.tokens()))) } + Input::Elements(base, bound, predicates) => { + elements(keyset, label, options, &base, bound, &predicates)? + } + Input::Predicate(node) => super::json_predicate::derive(keyset, label, options, *node)?, }) } +/// One member of an `element_match` operand: the tokens of a read of the +/// member's full path, probes included, and its bounds, each a leaf +/// operand. `eql_v4.query_json_elements` holds a list of them as `{"m"}`. +#[derive(serde::Serialize)] +struct MemberOperand { + t: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + gt: Option, + #[serde(skip_serializing_if = "Option::is_none")] + gte: Option, + #[serde(skip_serializing_if = "Option::is_none")] + lt: Option, + #[serde(skip_serializing_if = "Option::is_none")] + lte: Option, +} + +/// `element_match`'s operand, `eql_v4.query_json_elements`. +#[derive(serde::Serialize)] +struct ElementsOperand { + m: Vec, +} + +impl MemberOperand { + /// The member reading `tokens` that compares as `compare`, with the + /// leaf operands of its values in order: one for a comparison, the + /// low's then the high's for `between` (`compare` `None`). Equality is + /// both inclusive bounds at its one value. + fn new(tokens: Vec, compare: Option, leaves: Vec) -> Self { + let mut member = Self { + t: tokens, + gt: None, + gte: None, + lt: None, + lte: None, + }; + let mut leaves = leaves.into_iter(); + match compare { + Some(Compare::Eq) => { + member.gte = leaves.next(); + member.lte = member.gte.clone(); + } + Some(Compare::Lt) => member.lt = leaves.next(), + Some(Compare::Lte) => member.lte = leaves.next(), + Some(Compare::Gt) => member.gt = leaves.next(), + Some(Compare::Gte) => member.gte = leaves.next(), + None => { + member.gte = leaves.next(); + member.lte = leaves.next(); + } + } + member + } +} + +/// `element_match`'s operand: for each predicate, the tokens of a subtree +/// read of its full path at `bound`, as `subtree` derives them, and the +/// leaf operand of each of its values, as the orderings derive them. The +/// reads share the path and the bound, so the `k`-th token of every member +/// names a field of one element, and each member's last names the element +/// at the bound: the probe. The read of the elements themselves is derived +/// once, and each member's read extends its tokens by the member's steps +/// ([`JsonSubtreeKey::below`]), which is the key `subtree` derives for the +/// full path. +fn elements<'a, K: 'static>( + keyset: &'a KeysetCipher<'_, K>, + label: &Label, + options: &JsonOptions, + base: &JsonPath, + bound: u32, + predicates: &[Predicate], +) -> Result, Error> { + let elements = + keyset.json_subtree_key_now(base, options, bound, NonEmpty::from(label.clone()))?; + let mut members = Vec::with_capacity(predicates.len()); + for predicate in predicates { + let tokens = elements.below(&predicate.member).map_err(term)?.tokens(); + let leaf = |value: &Parsed| { + terms( + keyset, + options, + label, + predicate.path.holding(value.0.clone()), + ) + .map(|terms| JsonLeafQuery::of_leaf(&terms)) + }; + let (compare, leaves) = match &predicate.bounds { + Bounds::One(compare, value) => (Some(*compare), vec![leaf(value)]), + Bounds::Between(low, high) => (None, vec![leaf(low), leaf(high)]), + }; + members.push( + Pending::all(keyset, leaves) + .map(move |leaves| MemberOperand::new(tokens, compare, leaves)), + ); + } + Ok(Pending::all(keyset, members).try_map(|m| eql(&ElementsOperand { m }).map(Operand::One))) +} + /// A read to open, as the request that prepared it names it: the /// operator, the path and, for a subtree read, the wildcard bound. What a /// binding keeps from preparing the read and hands back with its rows; the diff --git a/packages/stack-encrypt/src/dynamic/mod.rs b/packages/stack-encrypt/src/dynamic/mod.rs index 7e57f7c16..7a9a7b633 100644 --- a/packages/stack-encrypt/src/dynamic/mod.rs +++ b/packages/stack-encrypt/src/dynamic/mod.rs @@ -89,6 +89,7 @@ //! reason, given on the type. mod context; mod json; +mod json_predicate; mod json_query; mod kind; mod query; @@ -314,6 +315,33 @@ pub enum Error { reason: Reason, }, + /// A `match` predicate is refused at a place in its text: text the + /// grammar does not read, a construct of SQL/JSON path the JSON index + /// cannot answer, a variable its `vars` do not hold, or a value of a + /// type its comparison cannot take. The error names the construct in + /// the grammar's own words and the byte offset in the caller's + /// predicate text, and nothing of the text itself: a key name or a + /// value in it can be customer data. + #[error( + "the predicate is refused at byte {at}: {reason} ({construct}){}", + in_field_text(field) + )] + #[diagnostic( + code(stack_encrypt::dynamic_predicate), + help("A predicate compares a path (`$.a.b`, `$.a[*].b`, `$.\"a b\"`) with a variable (`$v`) or a literal by `==`, `!=`, `<`, `<=`, `>` or `>=`, tests `exists($.path)` or `exists($.array[*] ? (@.x == $v && @.y > $w))`, and combines them with `&&`, `||`, `!` and parentheses.") + )] + Predicate { + /// The plan field the predicate is for, if one is named yet. + field: Option, + /// What was wrong. + reason: Reason, + /// The construct refused, in the grammar's words: never text from + /// the predicate. + construct: &'static str, + /// The byte offset in the predicate text the construct starts at. + at: usize, + }, + /// An invariant this module maintains did not hold — a slot count that /// did not line up, a re-proof that should not have been able to fail. /// Always a bug here, never a statement about the caller's data. @@ -391,6 +419,7 @@ impl Error { | Self::Plan { field, .. } | Self::Source { field, .. } | Self::Record { field, .. } + | Self::Predicate { field, .. } if field.is_none() => { *field = Some(name.to_owned()); @@ -407,19 +436,22 @@ impl Error { | Self::Term { field, .. } | Self::Plan { field, .. } | Self::Source { field, .. } - | Self::Record { field, .. } => field.as_deref(), + | Self::Record { field, .. } + | Self::Predicate { field, .. } => field.as_deref(), Self::UntypedIndex { field } => Some(field), Self::Internal | Self::Target(_) | Self::Cipher(_) => None, } } - /// What was wrong, for a context, plan, source or record error. + /// What was wrong, for a context, plan, source, record or predicate + /// error. pub fn reason(&self) -> Option { match self { Self::Context { reason, .. } | Self::Plan { reason, .. } | Self::Source { reason, .. } - | Self::Record { reason, .. } => Some(*reason), + | Self::Record { reason, .. } + | Self::Predicate { reason, .. } => Some(*reason), Self::Term { .. } | Self::UntypedIndex { .. } | Self::Internal @@ -435,6 +467,10 @@ impl crate::ErrorPayload for Error { Self::Target(error) => return error.payload(), Self::Cipher(error) => return error.payload(), Self::Term { kind, .. } => crate::diagnostic::payload([("index", kind.key().into())]), + Self::Predicate { construct, at, .. } => crate::diagnostic::payload([ + ("construct", (*construct).into()), + ("at", (*at as u64).into()), + ]), _ => serde_json::Map::new(), }; if let Some(field) = self.field() { @@ -657,6 +693,44 @@ reasons! { /// finds a null by its term. A `"null": "sql"` field's nulls are SQL /// `NULL`, which `IS NULL` finds. NullQuery => "null_query", r#"a null is found by its term only with eq on a field with "null": "sealed"; a "null": "sql" field's nulls are SQL NULL"#; + /// An element predicate's path has no wildcard: the predicate is + /// over the elements of an array, which `[*]` names. + JsonPathNoWildcard => "json_path_no_wildcard", "an element predicate is over an array's elements, and the path names no wildcard"; + /// An element predicate has no predicates: it would be true of + /// every row with an element. + JsonNoPredicates => "json_no_predicates", "an element predicate needs at least one predicate"; + /// An element predicate's `wildcard_bound` is `0`: its reads would + /// send the probe alone, so the predicate could never be true. + JsonWildcardBoundZero => "json_wildcard_bound_zero", "an element predicate reads at least one element, and a wildcard_bound of 0 reads none"; + /// A predicate of an element predicate has an `op` that is not + /// `eq`, `lt`, `lte`, `gt`, `gte` or `between`. + JsonUnknownPredicateOp => "json_unknown_predicate_op", r#"a predicate's "op" is not eq, lt, lte, gt, gte or between"#; + /// A `match` predicate is not text the SQL/JSON path predicate + /// grammar reads: an unexpected character, an unterminated string, + /// a literal that is not JSON, or text after the predicate. + JsonPredicateSyntax => "json_predicate_syntax", "the predicate is not SQL/JSON path predicate text"; + /// A `match` predicate uses a construct of SQL/JSON path the JSON + /// index cannot answer: arithmetic, `like_regex`, `starts with`, + /// `is unknown`, a method, a `strict` or `lax` mode, `.*`, `.**`, + /// a slice, `last`, a comparison of two paths or of two values, a + /// filter outside `exists`, or `||`, `!` or `!=` inside a filter. + JsonPredicateUnsupported => "json_predicate_unsupported", "the predicate uses a construct the JSON index cannot answer"; + /// A `match` predicate names a variable (`$v`) its `vars` do not + /// hold. + JsonPredicateUnknownVariable => "json_predicate_unknown_variable", "the predicate names a variable its vars do not hold"; + /// A value in a `match` predicate, a literal or a variable, is of a + /// JSON type its comparison cannot take: an object or array + /// anywhere, and anything but a number or a string beside `<`, + /// `<=`, `>`, `>=` or in an element-scoped filter of more than one + /// equality. + JsonPredicateValueType => "json_predicate_value_type", "a value in the predicate is of a type its comparison cannot take"; + /// `!=` at a path with a wildcard: jsonpath answers it true when + /// some element differs, which no term of the index can say. + JsonPredicateNotEqualAtWildcard => "json_predicate_not_equal_at_wildcard", "!= at a path with a wildcard asks whether some element differs, which the index cannot answer"; + /// A `match` predicate nests parentheses, `!` and `exists` deeper + /// than the engine reads, or names a path with more steps than a + /// document has levels. + JsonPredicateTooDeep => "json_predicate_too_deep", "the predicate nests too deep"; } } diff --git a/packages/stack-encrypt/src/dynamic/query.rs b/packages/stack-encrypt/src/dynamic/query.rs index c08895572..2dda9fed9 100644 --- a/packages/stack-encrypt/src/dynamic/query.rs +++ b/packages/stack-encrypt/src/dynamic/query.rs @@ -154,12 +154,93 @@ pub enum OpSpec { /// `"child_count"`: the same token, which `json_tree_child_count` /// takes. ChildCount, + /// `"element_match"`: a predicate on one element of an array, each + /// member a field of the element at an ordered path with its bounds, + /// which `json_tree_element_match` takes. True of a row where some one + /// element satisfies every member. + ElementMatch, + /// `"match"`: a jsonpath predicate and its variables, compiled to a + /// tree of the other operators' operands, which `json_tree_match` and + /// `@@` take: the one-parameter form of `jsonb_path_match`. Its input + /// is `{"predicate", "vars"?, "wildcard_bound"?}`, `vars` the JSON text + /// of the object `jsonb_path_match` takes. It reads the subset of + /// SQL/JSON path predicates the index's terms answer: comparisons of a + /// path (`$.a.b`, `$.a[*].b`, `$."a b"`) with a variable or a literal, + /// `exists(path)`, `exists(path[*] ? (@.x == $v && @.y > $w))`, `&&`, + /// `||`, `!` and parentheses; and refuses anything else at the byte + /// it starts ([`Error::Predicate`](super::Error::Predicate)). The text + /// is PostgreSQL's [SQL/JSON path language](https://www.postgresql.org/docs/current/functions-json.html#FUNCTIONS-SQLJSON-PATH). + /// The answer is jsonpath's, three-valued, save that a path is read as + /// written (lax mode's unwrapping of an array the path names no `[*]` + /// for, and its wrapping of a value that is not an array where it + /// does, are not applied), a comparison of two kinds of value is + /// decided by the terms where jsonpath answers unknown, and an ordering + /// or a filter of more than one condition compares by the ordering + /// terms, as `element_match` does. + /// + /// ``` + /// use serde_json::json; + /// use stack_encrypt::dynamic::{check_query, frame, record, OpSpec, Reason, ValueFrame}; + /// # use stack_encrypt::dynamic::{TargetDescriptor, TargetError, TargetOptions, TargetResolver, ValueKind}; + /// # use stack_encrypt::target::IndexSpec; + /// # use stack_encrypt::{KeysetCipher, Label, Pending, StackCipher}; + /// # /// A host whose `JsonSearch` names the compiled predicate's domain. + /// # struct Documents; + /// # impl TargetResolver for Documents { + /// # fn targets(&self) -> Vec { + /// # vec![TargetDescriptor::new( + /// # "JsonSearch", "json", "", Some(ValueKind::Json), "public.eql_v4_json_search", + /// # vec!["json".to_owned()], Some("JsonSearchQuery".to_owned()), + /// # Some("eql_v4.query_json".to_owned()), true, None, + /// # ) + /// # .with_predicate_query_sql_domain("eql_v4.query_json_predicate")] + /// # } + /// # fn encrypt<'a, K: 'static>(&self, _: &str, _: Option<&TargetOptions>, + /// # _: &'a KeysetCipher<'_, K>, _: &Label, _: ValueFrame, + /// # ) -> Result, K>, TargetError> { unreachable!() } + /// # fn decrypt<'a, K: 'static>(&self, _: &str, _: &'a StackCipher, _: &Label, _: &[u8], + /// # ) -> Result, TargetError> { unreachable!() } + /// # fn query<'a, K: 'static>(&self, _: &str, _: &IndexSpec, _: &'a KeysetCipher<'_, K>, + /// # _: &Label, _: ValueFrame, + /// # ) -> Result, K>, TargetError> { unreachable!() } + /// # } + /// // A document field that orders `$.age`. + /// let plan = record::plan_with( + /// &frame::serde::to_vec(&json!({"profile": { + /// "context": ["users", "profile"], "type": "json", "null": "sql", + /// "target": {"name": "JsonSearch", "options": {"ordered": ["$.age"]}} + /// }}))?, + /// &Documents, + /// )?; + /// // What `jsonb_path_match(col, $1, $2)` takes: the predicate, and its + /// // variables as the JSON text of one object. + /// let input = |predicate: &str| -> ValueFrame { + /// let input = json!({"predicate": predicate, "vars": r#"{"min": 18, "r": "sibling"}"#}); + /// ValueFrame::from_bytes(&frame::serde::to_vec(&input).unwrap()).unwrap() + /// }; + /// let check = |predicate: &str| { + /// check_query(&plan, "profile", OpSpec::Match, input(predicate), &Documents) + /// }; + /// + /// check("$.age >= $min && !(exists($.retired))")?; + /// check(r#"$.name == "ann" || $."first name" != null"#)?; + /// check("exists($.contacts[*] ? (@.relationship == $r))")?; + /// + /// // Refused at the byte the construct starts, by name, never quoting + /// // the text. + /// let error = check(r#"$.email like_regex "@""#).unwrap_err(); + /// assert_eq!(error.reason(), Some(Reason::JsonPredicateUnsupported)); + /// let error = check("$.age + 1 > $min").unwrap_err(); + /// assert!(error.to_string().contains("at byte 6"), "{error}"); + /// # Ok::<(), Box>(()) + /// ``` + Match, } impl OpSpec { /// Every operator, in declaration order: the order a catalog lists them /// in. - pub const ALL: [OpSpec; 19] = [ + pub const ALL: [OpSpec; 21] = [ OpSpec::Eq, OpSpec::Lt, OpSpec::Lte, @@ -179,6 +260,8 @@ impl OpSpec { OpSpec::Subtree, OpSpec::Children, OpSpec::ChildCount, + OpSpec::ElementMatch, + OpSpec::Match, ]; /// The operator a name names, or `None` for a name that is not one. @@ -209,6 +292,8 @@ impl OpSpec { OpSpec::Subtree => "subtree", OpSpec::Children => "children", OpSpec::ChildCount => "child_count", + OpSpec::ElementMatch => "element_match", + OpSpec::Match => "match", } } @@ -222,8 +307,10 @@ impl OpSpec { /// The shape of the operator's input, as a catalog names it: `value` /// (one value of the field's kind), `pair` (two, low then high), /// `json` (`{"json"}`), `path_json` (`{"path", "json"}`), `path_pair` - /// (`{"path", "low", "high"}`), `path` (`{"path"}`) or `path_bound` - /// (`{"path", "wildcard_bound"?}`). + /// (`{"path", "low", "high"}`), `path` (`{"path"}`), `path_bound` + /// (`{"path", "wildcard_bound"?}`), `path_predicates` (`{"path", + /// "wildcard_bound"?, "predicates"}`) or `predicate` (`{"predicate", + /// "vars"?, "wildcard_bound"?}`). pub fn input(self) -> &'static str { match self { OpSpec::Between => "pair", @@ -234,6 +321,8 @@ impl OpSpec { OpSpec::BetweenAt => "path_pair", OpSpec::Selector | OpSpec::Children | OpSpec::ChildCount => "path", OpSpec::Subtree => "path_bound", + OpSpec::ElementMatch => "path_predicates", + OpSpec::Match => "predicate", OpSpec::Eq | OpSpec::Lt | OpSpec::Lte | OpSpec::Gt | OpSpec::Gte | OpSpec::Matches => { "value" } @@ -248,6 +337,19 @@ impl OpSpec { /// opens what the rows hold; see [`json_open`]). Equality at a wildcard /// path matches when any element does, and an ordering compared through /// `col -> sel` at one sees an unspecified element of each row. + /// + /// The orderings at a path (`lt_at` … `between_at`) and `element_match` + /// compare ordering terms, and `exact` is of the terms: the server + /// answers the comparison of the terms without false positives, and the + /// terms order text case- and accent-folded and numbers as float64. So + /// two values the plaintext tells apart can compare equal, and + /// `element_match`'s `eq`, which is `gte` and `lte` at one value, is an + /// equality of folded values, where `eq_at` is exact on the value. + /// `match` is exact on the terms its leaves are: its equalities are + /// `eq_at`'s save inside an element-scoped filter of more than one + /// condition, which is `element_match`'s, and its orderings are the + /// orderings' ([`OpSpec::Match`] says where its answer and jsonpath's + /// part). pub fn fidelity(self) -> Fidelity { match self { OpSpec::Matches => Fidelity::Probabilistic, @@ -281,13 +383,19 @@ impl OpSpec { let declared = |wanted: fn(&IndexSpec) -> bool| indexes.iter().find(|index| wanted(index)); if self.is_json() { // A document field's JSON index answers the document operators; - // the orderings at a path only when it orders some path. + // the orderings at a path, and the element predicates built of + // them, only when it orders some path. if kind != ValueKind::Json { return None; } let ordering = matches!( self, - OpSpec::LtAt | OpSpec::LteAt | OpSpec::GtAt | OpSpec::GteAt | OpSpec::BetweenAt + OpSpec::LtAt + | OpSpec::LteAt + | OpSpec::GtAt + | OpSpec::GteAt + | OpSpec::BetweenAt + | OpSpec::ElementMatch ); return indexes.iter().find(|index| match index { IndexSpec::Json(options) => !ordering || !options.ordered.is_empty(), @@ -317,7 +425,9 @@ impl OpSpec { | OpSpec::Selector | OpSpec::Subtree | OpSpec::Children - | OpSpec::ChildCount => None, + | OpSpec::ChildCount + | OpSpec::ElementMatch + | OpSpec::Match => None, } .filter(|index| admits(kind, index)) } @@ -387,7 +497,8 @@ impl Fidelity { /// lists them: none for a type the engine does not produce or that has no /// query twin, and otherwise every operator of one value that one of the /// type's indexes answers for its plaintext kind, `matches` only when the -/// type's match options are known. The match index answers under those +/// type's match options are known, and a JSON operator only when the type +/// names the domain its operand binds as. The match index answers under those /// options. See the [module docs](self#which-operators-a-field-answers). pub(super) fn target_operators(target: &TargetDescriptor) -> Vec<(OpSpec, IndexSpec)> { let Some(kind) = target @@ -408,10 +519,16 @@ pub(super) fn target_operators(target: &TargetDescriptor) -> Vec<(OpSpec, IndexS ) .collect(); // `between` through an EQL type is not answered yet; `between_at` on a - // document is (its bounds are two leaf operands). + // document is (its bounds are two leaf operands). A JSON operator is + // listed only when the type names the SQL type its operand binds as, + // as a field's list has it. OpSpec::ALL .into_iter() .filter(|op| !op.takes_pair()) + .filter(|op| { + super::json_query::binds(*op) + .is_none_or(|binds| super::json_query::domain(target, binds).is_some()) + }) .filter_map(|op| { op.answered_by(kind, &indexes) .map(|index| (op, index.clone())) @@ -3174,7 +3291,7 @@ mod wire_bytes { assert_eq!(written, PINNED); } - const PINNED: [&str; 156] = [ + const PINNED: [&str; 170] = [ "0a020000006571", "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0500000076616c7565", "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0500000076616c7565", @@ -3195,6 +3312,8 @@ mod wire_bytes { "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0a000000706174685f626f756e64", "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a0d000000656c656d656e745f6d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0f000000706174685f70726564696361746573", + "f105000000080000006f70657261746f720a050000006d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a09000000707265646963617465", "0a050000006d61746368", "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", @@ -3215,6 +3334,8 @@ mod wire_bytes { "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0a000000706174685f626f756e64", "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a0d000000656c656d656e745f6d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0f000000706174685f70726564696361746573", + "f105000000080000006f70657261746f720a050000006d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000707265646963617465", "f101000000050000006d61746368f10400000009000000746f6b656e697a65720a080000007374616e6461726408000000646f776e6361736502010000006b070400000000000000010000006d070002000000000000", "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", @@ -3235,6 +3356,8 @@ mod wire_bytes { "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0a000000706174685f626f756e64", "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a0d000000656c656d656e745f6d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0f000000706174685f70726564696361746573", + "f105000000080000006f70657261746f720a050000006d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000707265646963617465", "f101000000050000006d61746368f10400000009000000746f6b656e697a6572f101000000050000006e6772616d07040000000000000008000000646f776e6361736503010000006b070300000000000000010000006d070001000000000000", "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", @@ -3255,6 +3378,8 @@ mod wire_bytes { "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0a000000706174685f626f756e64", "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a0d000000656c656d656e745f6d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0f000000706174685f70726564696361746573", + "f105000000080000006f70657261746f720a050000006d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000707265646963617465", "0a030000006f7265", "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0500000076616c7565", "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0500000076616c7565", @@ -3275,6 +3400,8 @@ mod wire_bytes { "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0a000000706174685f626f756e64", "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a0d000000656c656d656e745f6d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0f000000706174685f70726564696361746573", + "f105000000080000006f70657261746f720a050000006d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a09000000707265646963617465", "0a030000006f7065", "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0500000076616c7565", "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0500000076616c7565", @@ -3295,6 +3422,8 @@ mod wire_bytes { "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0a000000706174685f626f756e64", "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a0d000000656c656d656e745f6d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0f000000706174685f70726564696361746573", + "f105000000080000006f70657261746f720a050000006d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a09000000707265646963617465", "0a040000006a736f6e", "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0500000076616c7565", "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0500000076616c7565", @@ -3315,6 +3444,8 @@ mod wire_bytes { "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0a000000706174685f626f756e64", "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a0d000000656c656d656e745f6d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0f000000706174685f70726564696361746573", + "f105000000080000006f70657261746f720a050000006d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a09000000707265646963617465", "f106000000080000006f70657261746f720a02000000657108000000666964656c6974790a05000000657861637405000000696e6465780a02000000657106000000646f6d61696e0008000000656e636f64696e670a050000006279746573040000007465726d0b03000000010203", "f106000000080000006f70657261746f720a070000006265747765656e08000000666964656c6974790a05000000657861637405000000696e6465780a030000006f706506000000646f6d61696e0a1000000065716c5f76342e71756572795f696e7408000000656e636f64696e670a0300000065716c040000007465726df0020000000b01000000040b020000000506", "f106000000080000006f70657261746f720a070000006d61746368657308000000666964656c6974790a0d00000070726f626162696c697374696305000000696e6465780a050000006d6174636806000000646f6d61696e0008000000656e636f64696e670a050000006279746573040000007465726d0b00000000", diff --git a/packages/stack-encrypt/src/dynamic/target.rs b/packages/stack-encrypt/src/dynamic/target.rs index 197c51cc4..85db274bc 100644 --- a/packages/stack-encrypt/src/dynamic/target.rs +++ b/packages/stack-encrypt/src/dynamic/target.rs @@ -110,6 +110,19 @@ pub struct TargetDescriptor { /// [`query_sql_domain`](Self::query_sql_domain) names. Not on the /// wire; a generator reads it from the operators' terms. pub leaf_query_sql_domain: Option, + /// The PostgreSQL domain of the type's element-scoped predicate, for a + /// type that stores a JSON document + /// ([`with_elements_query_sql_domain`](Self::with_elements_query_sql_domain)): + /// what `element_match` binds as. Not on the wire, as + /// [`leaf_query_sql_domain`](Self::leaf_query_sql_domain) is not; a type + /// that names none does not answer `element_match`. + pub elements_query_sql_domain: Option, + /// The PostgreSQL domain of the type's compiled predicate, for a type + /// that stores a JSON document + /// ([`with_predicate_query_sql_domain`](Self::with_predicate_query_sql_domain)): + /// what `match` binds as. Not on the wire; a type that names none does + /// not answer `match`. + pub predicate_query_sql_domain: Option, } /// The descriptor in its wire form: the object a guest's `se_targets` @@ -178,6 +191,8 @@ impl TargetDescriptor { reason, match_options: None, leaf_query_sql_domain: None, + elements_query_sql_domain: None, + predicate_query_sql_domain: None, } } @@ -189,6 +204,22 @@ impl TargetDescriptor { self } + /// The descriptor of a type that stores a JSON document, whose + /// element-scoped predicate is the domain `domain`. + #[must_use] + pub fn with_elements_query_sql_domain(mut self, domain: impl Into) -> Self { + self.elements_query_sql_domain = Some(domain.into()); + self + } + + /// The descriptor of a type that stores a JSON document, whose compiled + /// predicate is the domain `domain`. + #[must_use] + pub fn with_predicate_query_sql_domain(mut self, domain: impl Into) -> Self { + self.predicate_query_sql_domain = Some(domain.into()); + self + } + /// The descriptor of a type whose match index is built under `options`: /// such a type answers `matches` through its query twin /// ([`operators`](Self::operators)). A type with a match index and no @@ -213,8 +244,11 @@ impl TargetDescriptor { /// in [`OpSpec::ALL`]'s order: none for a type it does not produce or /// that has no query twin, and otherwise each operator of one value /// that one of the type's indexes answers for its plaintext kind - /// ([`OpSpec::answered_by`]): the exact ones, and `matches` when the - /// type's match options are known ([`with_match_options`](Self::with_match_options)). + /// ([`OpSpec::answered_by`]): the exact ones, `matches` when the + /// type's match options are known ([`with_match_options`](Self::with_match_options)), + /// and a JSON operator when the type names the SQL type its operand + /// binds as (`match` with + /// [`with_predicate_query_sql_domain`](Self::with_predicate_query_sql_domain)). /// The query for each is the type's query twin. `between` is not /// answered through a type yet: see [`query_term`](super::query_term)'s /// module. diff --git a/packages/stack-encrypt/src/json/derive.rs b/packages/stack-encrypt/src/json/derive.rs index dd145f7f3..95379b245 100644 --- a/packages/stack-encrypt/src/json/derive.rs +++ b/packages/stack-encrypt/src/json/derive.rs @@ -2102,6 +2102,71 @@ fn subtree_key_for( }) } +impl JsonSubtreeKey { + /// The key that reads `steps` below every node this key reads, each + /// probe kept as it is: the key of this key's path followed by + /// `steps`, at the same bound, derived from this key's tokens instead + /// of from the root. Every token below a node derives from the node's + /// token by the member or element rule and the step's selector, which + /// reads a position as the wildcard, so each step's selector is derived + /// once for every node, and each node's token extended by one hash per + /// step. A probe names an element past the bound, which the reads below + /// it do not reach, so it stays the element's own token. + /// + /// # Errors + /// + /// [`TermError::JsonWildcardTooWide`] for a step that is a wildcard, + /// which names many nodes below each one and so is a read of its own; + /// [`TermError::JsonTooWide`] for a position past the last one a + /// document numbers. + #[cfg(feature = "dynamic")] + pub(crate) fn below(&self, steps: &JsonPath) -> Result { + let prf = &self.selectors.keyset.prf; + let context = &self.selectors.context; + let mut pattern = self.path.clone(); + let mut rules = Vec::with_capacity(steps.segments().len()); + for step in steps.segments() { + let index = match step { + Segment::Key(_) => None, + Segment::Index(index) => Some(fit_ordinal(*index)?), + Segment::Wildcard => return Err(TermError::JsonWildcardTooWide), + }; + pattern = pattern.then(step.clone()); + let selector = prf16(prf, SELECTOR_DOMAIN, context, &pattern.selector_bytes())?; + rules.push((step, selector, index)); + } + let reads = self + .reads + .iter() + .map(|read| { + if read.probe { + return read.clone(); + } + let mut token = read.token; + let mut path = read.path.clone(); + for (step, selector, index) in &rules { + token = match index { + None => member_token(&token, selector), + Some(index) => element_token(&token, selector, *index), + }; + path = path.then((*step).clone()); + } + Read { + token, + path, + probe: false, + } + }) + .collect(); + Ok(Self { + reads, + bound: self.bound, + path: pattern, + selectors: self.selectors.clone(), + }) + } +} + impl KeysetCipher<'_, K> { /// The selector of `path`, under the field `descriptor`: what `->` and /// the path functions take. Every element of an array shares its @@ -2509,6 +2574,47 @@ mod tests { } } + /// A key extended below its nodes is the key of the longer path, + /// tokens, probes, paths and order, under either root and through + /// nested wildcards; a wildcard below is refused. + #[cfg(feature = "dynamic")] + #[test] + fn a_key_extended_below_its_nodes_is_the_longer_paths_key() { + for options in [JsonOptions::default(), open_root()] { + for (base, steps, bound) in [ + ("$.a[*]", "$.b", 2), + ("$.a[*]", "$.b.c", 3), + ("$.a[*]", "$[\"x y\"]", 1), + ("$.a[*]", "$.b[2].c", 2), + ("$[*]", "$.x", 2), + ("$.a[*].b[*]", "$.c", 2), + ("$.a[*]", "$", 2), + ("$.a[*]", "$.b", 0), + ] { + let full = format!("{base}{}", steps.trim_start_matches('$')); + let extended = key_for(&options, base, bound) + .unwrap() + .below(&JsonPath::parse(steps).unwrap()) + .unwrap(); + assert_eq!( + extended, + key_for(&options, &full, bound).unwrap(), + "{base} then {steps} at {bound}, sealed root {}", + options.sealed_root + ); + } + } + let key = key_for(&JsonOptions::default(), "$.a[*]", 2).unwrap(); + assert!(matches!( + key.below(&JsonPath::parse("$.b[*]").unwrap()), + Err(TermError::JsonWildcardTooWide) + )); + assert!(matches!( + key.below(&JsonPath::parse("$[2147483648]").unwrap()), + Err(TermError::JsonTooWide) + )); + } + #[test] fn a_wildcard_head_expands_under_the_sealed_root() { let t = terms(&json!([{"x": 1}, {"x": 2}]), &JsonOptions::default()); diff --git a/packages/stack-encrypt/src/json/path.rs b/packages/stack-encrypt/src/json/path.rs index 95b327e89..0e0d06108 100644 --- a/packages/stack-encrypt/src/json/path.rs +++ b/packages/stack-encrypt/src/json/path.rs @@ -264,6 +264,12 @@ impl JsonPath { current } + /// The path of these steps, from the root down. + #[cfg(feature = "dynamic")] + pub(crate) fn of(segments: Vec) -> Self { + Self { segments } + } + /// This path with one more step. pub(crate) fn then(&self, segment: Segment) -> Self { let mut segments = Vec::with_capacity(self.segments.len() + 1); diff --git a/packages/stack-encrypt/src/target/pending.rs b/packages/stack-encrypt/src/target/pending.rs index b62935126..08b48ebb6 100644 --- a/packages/stack-encrypt/src/target/pending.rs +++ b/packages/stack-encrypt/src/target/pending.rs @@ -630,11 +630,17 @@ fn load_keysets<'c, K: DataKeySource + IndexKeySource>( distinct.push(*keyset_id); } } - let loading = futures::future::join_all( - distinct - .iter() - .map(|&id| Box::pin(async move { cipher.keyset_state(id).await.map_err(Arc::new) })), - ); + // Boxed too, so the returned future holds the joined loads as a pointer + // rather than inline. `dispatch` polls this future first after the + // retrieves have put the batch's data keys through the stack. Held + // inline, `tests/heap_hygiene.rs` observes this future's box, 256 bytes + // in a macOS arm64 debug build, freed holding a retrieved data key and + // stack addresses (the `json_open (subtree)` watch). Which write copies + // them in is not identified; with the loads boxed the residue is not + // observed, and the box costs one allocation per dispatch. + let loading = Box::pin(futures::future::join_all(distinct.iter().map(|&id| { + Box::pin(async move { cipher.keyset_state(id).await.map_err(Arc::new) }) + }))); async move { let by_keyset: HashMap, Arc>> = distinct.into_iter().zip(loading.await).collect(); diff --git a/packages/stack-encrypt/tests/heap_hygiene.rs b/packages/stack-encrypt/tests/heap_hygiene.rs index 72a533386..5cf4781c6 100644 --- a/packages/stack-encrypt/tests/heap_hygiene.rs +++ b/packages/stack-encrypt/tests/heap_hygiene.rs @@ -412,7 +412,9 @@ impl stack_encrypt::dynamic::TargetResolver for Documents { true, None, ) - .with_leaf_query_sql_domain("eql_v4.query_json_leaf")] + .with_leaf_query_sql_domain("eql_v4.query_json_leaf") + .with_elements_query_sql_domain("eql_v4.query_json_elements") + .with_predicate_query_sql_domain("eql_v4.query_json_predicate")] } fn encrypt<'a, K: 'static>( &self, @@ -815,7 +817,7 @@ mod json_index { let keyset = cipher.default_keyset(); let wire = json!({ "profile": {"context": ["users", "profile"], "type": "json", "null": "sql", - "target": {"name": "JsonSearch", "options": {"ordered": ["$.age"]}}} + "target": {"name": "JsonSearch", "options": {"ordered": ["$.age", "$.visits[*].at"]}}} }); let plan = record::plan_with( &stack_encrypt::dynamic::frame::serde::to_vec(&wire).unwrap(), @@ -826,6 +828,12 @@ mod json_index { let mark = marked("operand"); let quoted = format!("{mark:?}"); let document = format!(r#"{{{quoted}: [{quoted}], "name": {quoted}}}"#); + // Every leaf kind, with a literal and a variable carrying the mark. + let predicate = format!( + "$.name == {quoted} && $.name != $v && $.age > {quoted} && \ + exists($.visits[*] ? (@.at == $v && @.at < {quoted})) && $.visits[*].at >= $v" + ); + let vars = format!(r#"{{"v": {quoted}}}"#); let inputs = [ ( OpSpec::Contains, @@ -853,6 +861,35 @@ mod json_index { ("high", frame(quoted.as_str())), ]), ), + ( + OpSpec::ElementMatch, + object(vec![ + ("path", frame("$.visits[*]")), + ( + "predicates", + frame(vec![ + object(vec![ + ("member", frame("at")), + ("op", frame("eq")), + ("json", frame(quoted.as_str())), + ]), + object(vec![ + ("member", frame("at")), + ("op", frame("between")), + ("low", frame(quoted.as_str())), + ("high", frame(quoted.as_str())), + ]), + ]), + ), + ]), + ), + ( + OpSpec::Match, + object(vec![ + ("predicate", frame(predicate.as_str())), + ("vars", frame(vars.as_str())), + ]), + ), ]; for (operator, input) in &inputs { let term = watched(&format!("query_term ({operator:?})"), || { diff --git a/packages/stack-encrypt/tests/json_operators.rs b/packages/stack-encrypt/tests/json_operators.rs index d9e1b7ac6..39d144a41 100644 --- a/packages/stack-encrypt/tests/json_operators.rs +++ b/packages/stack-encrypt/tests/json_operators.rs @@ -1,7 +1,7 @@ //! A document field's JSON operators and reads through the data plan, with //! the engine's own JSON index behind the target: what a binding names by //! operator (`contains`, `eq_at`, the orderings at a path, `selector`, -//! `subtree`, `children`, `child_count`), the operand each derives and the +//! `subtree`, `children`, `child_count`, `element_match`, `match`), the operand each derives and the //! type it binds as, the refusals made before anything is derived, and //! reads opened against the request that prepared them. //! @@ -51,7 +51,9 @@ impl TargetResolver for Documents { true, None, ) - .with_leaf_query_sql_domain("eql_v4.query_json_leaf")] + .with_leaf_query_sql_domain("eql_v4.query_json_leaf") + .with_elements_query_sql_domain("eql_v4.query_json_elements") + .with_predicate_query_sql_domain("eql_v4.query_json_predicate")] } fn encrypt<'a, K: 'static>( @@ -399,7 +401,8 @@ fn the_orderings_follow_the_resolved_options() { "selector", "subtree", "children", - "child_count" + "child_count", + "match" ] ); let error = dynamic::check_query( @@ -921,3 +924,1417 @@ fn a_malformed_read_is_refused_before_any_key() { .unwrap_err(); assert_eq!(error.reason(), Some(Reason::OperatorNotAnswered)); } + +/// `element_match`: a predicate on one element of an array, its operand the +/// members' subtree reads and their bounds, as the `subtree` operator and +/// the orderings derive them. `json_tree_element_match` is modelled over +/// the rows `JsonDocument::subtree` returns, as the subtree tests model +/// `json_tree_subtree`; `packages/eql/tests/encryption` runs the SQL. +mod element_match { + use super::*; + + use proptest::prelude::*; + use proptest::test_runner::{Config, TestCaseError, TestRunner}; + + /// `profile`, ordering a contact's `relationship`, `calls`, + /// `address.postcode` and `["first name"]`, but not its `note`. + fn contacts() -> record::Plan { + plan_of(json!({ + "name": "JsonSearch", + "options": {"ordered": [ + "$.contacts[*].relationship", + "$.contacts[*].calls", + "$.contacts[*].address.postcode", + "$.contacts[*][\"first name\"]", + ]} + })) + } + + fn input(bound: u32, predicates: Value) -> Value { + json!({"path": "$.contacts[*]", "wildcard_bound": bound, "predicates": predicates}) + } + + /// The operand's JSON, `eql_v4.query_json_elements`. + async fn operand( + keyset: &KeysetCipher<'_, K>, + plan: &record::Plan, + input: Value, + ) -> Value { + let term = term(keyset, plan, "element_match", input).await; + assert_eq!(term.encoding(), "eql", "an EQL operand"); + assert_eq!( + term.domain(), + Some("eql_v4.query_json_elements"), + "bound as the target's element predicate domain" + ); + match term.operand() { + Operand::One(bytes) => serde_json::from_slice(bytes).unwrap(), + other => panic!("one operand, got {other:?}"), + } + } + + /// The one leaf operand an ordering at `path` derives for `json`. + async fn leaf( + keyset: &KeysetCipher<'_, K>, + plan: &record::Plan, + operator: &str, + path: &str, + json: &str, + ) -> Value { + let term = term(keyset, plan, operator, json!({"path": path, "json": json})).await; + match term.operand() { + Operand::One(bytes) => serde_json::from_slice(bytes).unwrap(), + other => panic!("{operator}: {other:?}"), + } + } + + /// The tokens a `subtree` read of `path` at `bound` sends. + async fn tokens( + keyset: &KeysetCipher<'_, K>, + plan: &record::Plan, + path: &str, + bound: u32, + ) -> Value { + let term = term( + keyset, + plan, + "subtree", + json!({"path": path, "wildcard_bound": bound}), + ) + .await; + match term.operand() { + Operand::TextList(tokens) => json!(tokens), + other => panic!("{other:?}"), + } + } + + /// What `json_tree_element_match(doc, operand)` answers: whether one + /// `k` has, for every member, the node its `k`-th token names at the + /// member's selector and inside its bounds, compared as the SQL + /// compares ordering terms (bytes); or `None` where a token names a + /// node at another selector, a probe's hit, which the SQL raises on. + pub(super) fn element_match(doc: &JsonDocument, operand: &Value) -> Option { + let members = operand["m"].as_array().unwrap(); + let width = members[0]["t"].as_array().unwrap().len(); + let term = |leaf: &Value| Base64::decode_vec(leaf["op"].as_str().unwrap()).unwrap(); + let mut truncated = false; + let mut matched = false; + for k in 0..width { + let mut all = true; + for member in members { + let bounds: Vec<(&str, &Value)> = ["gt", "gte", "lt", "lte"] + .into_iter() + .filter_map(|key| member.get(key).map(|leaf| (key, leaf))) + .collect(); + let selector = tag(bounds[0].1["s"].as_str().unwrap()); + let token = tag(member["t"][k].as_str().unwrap()); + let Some((j, entry, _)) = doc.subtree(&[token]).into_iter().next() else { + all = false; + continue; + }; + if entry.selector() != &selector { + truncated = true; + all = false; + continue; + } + let node = serde_json::to_value(doc.node(j).unwrap()).unwrap(); + if node.get("op").is_none() { + all = false; + continue; + } + let stored = term(&node); + all &= bounds.iter().all(|(key, leaf)| { + let bound = term(leaf); + match *key { + "gt" => stored > bound, + "gte" => stored >= bound, + "lt" => stored < bound, + _ => stored <= bound, + } + }); + } + matched |= all; + } + (!truncated).then_some(matched) + } + + mod given_a_predicate_per_comparison { + use super::*; + + /// Each `op` derives the bound the ordering of the same name + /// derives, under its own key; `eq` is `gte` and `lte` at one + /// value, and `between` is `between_at`'s two. + #[tokio::test] + async fn each_bound_is_the_orderings_leaf_operand() { + let cipher = common::stack_cipher().await; + let keyset = cipher.default_keyset(); + let plan = contacts(); + let calls = "$.contacts[*].calls"; + let operand = operand( + &keyset, + &plan, + input( + 2, + json!([ + {"member": "calls", "op": "eq", "json": "5"}, + {"member": "calls", "op": "lt", "json": "5"}, + {"member": "calls", "op": "lte", "json": "5"}, + {"member": "calls", "op": "gt", "json": "5"}, + {"member": "calls", "op": "gte", "json": "5"}, + {"member": "calls", "op": "between", "low": "1", "high": "9"}, + ]), + ), + ) + .await; + let m = operand["m"].as_array().unwrap(); + assert_eq!(m.len(), 6, "one member per predicate: {operand}"); + let at_five = leaf(&keyset, &plan, "gte_at", calls, "5").await; + assert_eq!( + leaf(&keyset, &plan, "lt_at", calls, "5").await, + at_five, + "a leaf operand is one term, whichever ordering asks" + ); + let keys = |member: &Value| -> Vec { + member.as_object().unwrap().keys().cloned().collect() + }; + assert_eq!(keys(&m[0]), ["gte", "lte", "t"], "eq: {}", m[0]); + assert_eq!(m[0]["gte"], at_five, "eq's low bound is the value's"); + assert_eq!(m[0]["lte"], m[0]["gte"], "eq is lo = hi"); + for (member, key) in [(&m[1], "lt"), (&m[2], "lte"), (&m[3], "gt"), (&m[4], "gte")] { + assert_eq!(keys(member), [key, "t"], "{member}"); + assert_eq!(member[key], at_five, "{key}: the value's leaf operand"); + } + let between = term( + &keyset, + &plan, + "between_at", + json!({"path": calls, "low": "1", "high": "9"}), + ) + .await; + let Operand::Range { lo, hi } = between.operand() else { + panic!("between_at is a range") + }; + let json = |bytes: &[u8]| serde_json::from_slice::(bytes).unwrap(); + assert_eq!(keys(&m[5]), ["gte", "lte", "t"], "between: {}", m[5]); + assert_eq!(m[5]["gte"], json(lo), "between's low bound is between_at's"); + assert_eq!( + m[5]["lte"], + json(hi), + "between's high bound is between_at's" + ); + assert_ne!(m[5]["gte"], m[5]["lte"], "two values, two terms"); + } + + /// Every member's tokens are the `subtree` read of its full path at + /// the bound, the default bound when the input names none: one per + /// element below it, then the probe, the element at the bound + /// itself, which every member shares. + #[tokio::test] + async fn each_member_reads_its_full_path_and_ends_with_the_probe() { + let cipher = common::stack_cipher().await; + let keyset = cipher.default_keyset(); + let plan = contacts(); + let read = operand( + &keyset, + &plan, + input( + 3, + json!([ + {"member": "relationship", "op": "eq", "json": "\"Sibling\""}, + {"member": "address.postcode", "op": "gte", "json": "3000"}, + {"member": "[\"first name\"]", "op": "lt", "json": "\"M\""}, + ]), + ), + ) + .await; + let m = read["m"].as_array().unwrap(); + for (member, path) in [ + (&m[0], "$.contacts[*].relationship"), + (&m[1], "$.contacts[*].address.postcode"), + (&m[2], "$.contacts[*][\"first name\"]"), + ] { + assert_eq!( + member["t"], + tokens(&keyset, &plan, path, 3).await, + "{path}: the subtree read's tokens" + ); + assert_eq!(member["t"].as_array().unwrap().len(), 3 + 1, "{path}"); + } + assert_eq!(m[0]["t"][3], m[1]["t"][3], "the probe is the element's"); + assert_eq!(m[0]["t"][3], m[2]["t"][3], "the probe is the element's"); + assert_ne!(m[0]["t"][0], m[1]["t"][0], "a field's token is its own"); + + let defaulted = operand( + &keyset, + &plan, + json!({"path": "$.contacts[*]", "predicates": [ + {"member": "calls", "op": "gt", "json": "1"} + ]}), + ) + .await; + assert_eq!( + defaulted["m"][0]["t"], + tokens(&keyset, &plan, "$.contacts[*].calls", 64).await, + "no bound named is the subtree read's default" + ); + } + } + + mod given_documents { + use super::*; + + fn sibling_with_more_than_five_calls(bound: u32) -> Value { + input( + bound, + json!([ + {"member": "relationship", "op": "eq", "json": "\"Sibling\""}, + {"member": "calls", "op": "gt", "json": "5"}, + ]), + ) + } + + /// A Sibling with more than five calls. Two contacts each matching + /// half of it are not a match, one matching both is, and an array + /// longer than the bound is the probe's hit, not an answer. + #[tokio::test] + async fn one_element_must_satisfy_every_member() { + let cipher = common::stack_cipher().await; + let keyset = cipher.default_keyset(); + let plan = contacts(); + let operand = operand(&keyset, &plan, sibling_with_more_than_five_calls(2)).await; + for (document, want, why) in [ + ( + json!({"contacts": [ + {"relationship": "Sibling", "calls": 3}, + {"relationship": "Friend", "calls": 7} + ]}), + Some(false), + "each half in another element", + ), + ( + json!({"contacts": [ + {"relationship": "Friend", "calls": 9}, + {"relationship": "Sibling", "calls": 7} + ]}), + Some(true), + "both halves in the second element", + ), + ( + json!({"contacts": [{"relationship": "Sibling"}, {"calls": 8}]}), + Some(false), + "a field absent from an element satisfies none of its bounds", + ), + (json!({"contacts": []}), Some(false), "no element"), + (json!({"other": 1}), Some(false), "no array"), + ( + json!({"contacts": [ + {"relationship": "Friend", "calls": 1}, + {"relationship": "Friend", "calls": 1}, + {"relationship": "Sibling", "calls": 9} + ]}), + None, + "three elements past a bound of two: the probe names a node", + ), + ] { + let doc = seal(&keyset, &plan, &document).await; + assert_eq!(element_match(&doc, &operand), want, "{why}: {document}"); + } + } + + /// Against plaintext jsonpath over random arrays: within the bound + /// the operand answers `exists($.contacts[*] ? (@.relationship == + /// $r && @.calls $n))`, and past it the probe names a node. + #[test] + fn answers_as_the_jsonpath_over_the_plaintext() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let cipher = runtime.block_on(common::stack_cipher()); + let keyset = cipher.default_keyset(); + let plan = contacts(); + let bound = 3u32; + let names = ["Sibling", "Friend", "Parent"]; + let contact = (0..names.len(), proptest::option::of(0u64..8)); + let ops = prop::sample::select(vec!["eq", "lt", "lte", "gt", "gte", "between"]); + let strategy = ( + prop::collection::vec(contact, 0..=4), + 0..names.len(), + ops, + 0u64..8, + 0u64..8, + ); + let mut runner = TestRunner::new(Config { + cases: 48, + ..Config::default() + }); + runner + .run(&strategy, |(contacts, wanted, op, n, m)| { + let document = json!({"contacts": contacts + .iter() + .map(|(name, calls)| match calls { + Some(calls) => json!({"relationship": names[*name], "calls": calls}), + None => json!({"relationship": names[*name]}), + }) + .collect::>()}); + let (low, high) = (n.min(m), n.max(m)); + let calls = match op { + "between" => json!({ + "member": "calls", "op": op, + "low": low.to_string(), "high": high.to_string() + }), + _ => json!({"member": "calls", "op": op, "json": n.to_string()}), + }; + let predicates = json!([ + {"member": "relationship", "op": "eq", "json": json!(names[wanted]).to_string()}, + calls, + ]); + let (doc, operand) = runtime.block_on(async { + ( + seal(&keyset, &plan, &document).await, + operand(&keyset, &plan, input(bound, predicates)).await, + ) + }); + let holds = |calls: u64| match op { + "eq" => calls == n, + "lt" => calls < n, + "lte" => calls <= n, + "gt" => calls > n, + "gte" => calls >= n, + _ => (low..=high).contains(&calls), + }; + let within = contacts.len() <= usize::try_from(bound).unwrap(); + let want = within.then(|| { + contacts + .iter() + .any(|(name, calls)| *name == wanted && calls.is_some_and(holds)) + }); + let got = element_match(&doc, &operand); + if got != want { + return Err(TestCaseError::fail(format!( + "{document} {op} {n} {m}: got {got:?}, want {want:?}" + ))); + } + Ok(()) + }) + .unwrap(); + } + } + + mod given_an_input_it_cannot_answer { + use super::*; + + /// The refusals made before anything is derived, each by its + /// reason, or the index's term error by name. + #[test] + fn is_refused_with_its_reason() { + let plan = contacts(); + let check = |input: &Value| { + dynamic::check_query( + &plan, + "profile", + OpSpec::ElementMatch, + json_frame(input), + &Documents, + ) + }; + let reason = |input: &Value| -> String { + let error = check(input).unwrap_err(); + match (&error, error.reason()) { + (_, Some(reason)) => reason.as_str().to_owned(), + (dynamic::Error::Cipher(stack_encrypt::Error::Term(term)), None) => { + format!("{term:?}") + .split(['(', ' ', '{']) + .next() + .unwrap_or_default() + .to_owned() + } + (other, None) => panic!("neither a reason nor a term error: {other:?}"), + } + }; + let calls = |op: &str| json!([{"member": "calls", "op": op, "json": "1"}]); + let one = |predicate: Value| input(2, json!([predicate])); + let deep_member = |steps: usize| { + one(json!({"member": vec!["a"; steps].join("."), "op": "gt", "json": "1"})) + }; + for (input, want) in [ + (input(2, json!([])), "json_no_predicates"), + ( + json!({"path": "$.contacts", "predicates": calls("gt")}), + "json_path_no_wildcard", + ), + ( + json!({"path": "$.contacts[0]", "predicates": calls("gt")}), + "json_path_no_wildcard", + ), + ( + json!({"path": "$.contacts[1].tags[*]", "predicates": calls("gt")}), + "JsonOrderingAtPosition", + ), + ( + one(json!({"member": "note", "op": "eq", "json": "\"x\""})), + "JsonPathNotOrdered", + ), + ( + one(json!({"member": "address", "op": "gt", "json": "1"})), + "JsonPathNotOrdered", + ), + ( + one(json!({"member": "address[0].postcode", "op": "gt", "json": "1"})), + "JsonOrderingAtPosition", + ), + ( + one(json!({"member": "phones[*]", "op": "gt", "json": "1"})), + "json_path_wildcard", + ), + (input(2, calls("ne")), "json_unknown_predicate_op"), + (input(2, calls("between_at")), "json_unknown_predicate_op"), + (input(0, calls("gt")), "json_wildcard_bound_zero"), + (one(json!({"member": "calls", "op": "gt"})), "json_input"), + ( + one(json!({"member": "calls", "op": "eq", "low": "1", "high": "2"})), + "json_input", + ), + ( + one(json!({"member": "calls", "op": "gt", "json": "1", "low": "1"})), + "json_input", + ), + ( + one(json!({"member": "calls", "op": "gt", "json": "1", "high": "1"})), + "json_input", + ), + ( + one(json!({"member": "calls", "op": "between", "low": "1"})), + "json_input", + ), + ( + one(json!({"member": "calls", "op": "between", "high": "1"})), + "json_input", + ), + ( + one( + json!({"member": "calls", "op": "between", "json": "1", "low": "1", "high": "2"}), + ), + "json_input", + ), + ( + one(json!({"member": "calls", "op": "gt", "json": "1", "extra": 1})), + "json_input", + ), + (json!({"path": "$.contacts[*]"}), "json_input"), + ( + one(json!({"member": "calls", "op": "eq", "json": "true"})), + "JsonLeafNotOrdered", + ), + ( + one(json!({"member": "calls", "op": "between", "low": "1", "high": "null"})), + "JsonLeafNotOrdered", + ), + ( + one(json!({"member": "calls", "op": "between", "low": "[1]", "high": "2"})), + "JsonLeafNotOrdered", + ), + ( + one(json!({"member": "", "op": "gt", "json": "1"})), + "JsonPath", + ), + ( + json!({"path": "secret", "predicates": calls("gt")}), + "JsonPath", + ), + // The path has two steps, so a member of 125 is a full path + // of 127, as deep as a document goes, and one of 126 deeper. + (deep_member(125), "JsonPathNotOrdered"), + (deep_member(126), "JsonTooDeep"), + ] { + assert_eq!(reason(&input), want, "{input}"); + } + // A member that does not parse is refused at an offset into the + // member's own text, whichever form it starts with. + for (member, at) in [("a.b[", 4), ("[\"a\"].", 6), ("a..b", 2), ("[x]", 1)] { + let error = + check(&one(json!({"member": member, "op": "gt", "json": "1"}))).unwrap_err(); + let dynamic::Error::Cipher(stack_encrypt::Error::Term( + stack_encrypt::sem::TermError::JsonPath(error), + )) = error + else { + panic!("{member}: not a path error: {error:?}") + }; + assert_eq!(error.at, at, "{member}: {error}"); + } + for input in [ + input(1, calls("gte")), + input(2, calls("gte")), + input( + 2, + json!([ + {"member": "relationship", "op": "eq", "json": "\"a\""}, + {"member": "calls", "op": "between", "low": "1", "high": "2"}, + ]), + ), + ] { + assert!(check(&input).is_ok(), "{input}"); + } + } + } + + mod given_the_fields_type { + use super::*; + + /// A resolver whose `JsonSearch` names no element predicate domain: + /// what a host presents that has not been taught the operand. + struct Unnamed; + + impl TargetResolver for Unnamed { + fn targets(&self) -> Vec { + let mut targets = Documents.targets(); + targets[0].elements_query_sql_domain = None; + targets + } + fn encrypt<'a, K: 'static>( + &self, + name: &str, + options: Option<&TargetOptions>, + keyset: &'a KeysetCipher<'_, K>, + label: &Label, + plaintext: ValueFrame, + ) -> Result, K>, TargetError> { + Documents.encrypt(name, options, keyset, label, plaintext) + } + fn decrypt<'a, K: 'static>( + &self, + name: &str, + cipher: &'a StackCipher, + label: &Label, + bytes: &[u8], + ) -> Result, TargetError> { + Documents.decrypt(name, cipher, label, bytes) + } + fn query<'a, K: 'static>( + &self, + name: &str, + index: &IndexSpec, + keyset: &'a KeysetCipher<'_, K>, + label: &Label, + plaintext: ValueFrame, + ) -> Result, K>, TargetError> { + Documents.query(name, index, keyset, label, plaintext) + } + } + + fn names(plan: &record::Plan, resolver: &impl TargetResolver) -> Vec<&'static str> { + dynamic::field_operators(plan, "profile", resolver) + .unwrap() + .into_iter() + .map(|(op, _)| op.name()) + .collect() + } + + /// Answered, through the JSON index, on a field that orders a path + /// and whose type names the operand's domain; its input is + /// `path_predicates`. + #[test] + fn lists_it_when_a_path_is_ordered_and_the_domain_named() { + let plan = contacts(); + assert!( + names(&plan, &Documents).contains(&"element_match"), + "listed" + ); + let (_, index) = dynamic::field_operators(&plan, "profile", &Documents) + .unwrap() + .into_iter() + .find(|(op, _)| *op == OpSpec::ElementMatch) + .unwrap(); + assert!( + matches!(index, IndexSpec::Json(_)), + "the JSON index answers it: {index:?}" + ); + assert_eq!( + OpSpec::parse("element_match"), + Some(OpSpec::ElementMatch), + "the wire name parses" + ); + assert_eq!( + OpSpec::ElementMatch.input(), + "path_predicates", + "its input shape" + ); + assert_eq!( + OpSpec::ElementMatch.fidelity(), + OpSpec::BetweenAt.fidelity(), + "compared as the orderings compare" + ); + } + + #[test] + fn does_not_list_it_without_an_ordered_path() { + let names = names(&plan_of(json!("JsonSearch")), &Documents); + assert!( + !names.contains(&"element_match"), + "no ordering term to compare: {names:?}" + ); + } + + #[test] + fn does_not_answer_it_when_the_type_names_no_domain() { + let wire = json!({ + "profile": {"context": ["users", "profile"], "type": "json", "null": "sql", + "target": {"name": "JsonSearch", "options": {"ordered": ["$.a[*].b"]}}} + }); + let plan = record::plan_with(&dynamic::frame::serde::to_vec(&wire).unwrap(), &Unnamed) + .unwrap(); + let names = names(&plan, &Unnamed); + assert!(names.contains(&"lt_at"), "the orderings still: {names:?}"); + assert!(!names.contains(&"element_match"), "{names:?}"); + let error = dynamic::check_query( + &plan, + "profile", + OpSpec::ElementMatch, + json_frame(&json!({"path": "$.a[*]", "predicates": [ + {"member": "b", "op": "gt", "json": "1"} + ]})), + &Unnamed, + ) + .unwrap_err(); + assert_eq!( + error.reason(), + Some(Reason::OperatorNotAnswered), + "{error:?}" + ); + } + } +} + +/// `match`: a jsonpath predicate compiled to one operand, a tree of the +/// other operators' operands. `json_tree_match` is modelled over sealed +/// documents (`evaluate`), as the element predicate's tests model +/// `json_tree_element_match`, and held to an oracle of jsonpath's own +/// semantics over the plaintext; `packages/eql/tests/encryption` runs the +/// SQL against PostgreSQL's `jsonb_path_match`. +mod compiled_match { + use super::*; + + use proptest::prelude::*; + use proptest::test_runner::{Config, TestCaseError, TestRunner}; + use stack_encrypt::json::{JsonPath, JsonQuery}; + + /// `profile`, ordering `$.age`, `$.name` and every contact's `calls` + /// and `relationship`. + fn people() -> record::Plan { + plan_of(json!({ + "name": "JsonSearch", + "options": {"ordered": [ + "$.age", + "$.name", + "$.contacts[*].calls", + "$.contacts[*].relationship", + ]} + })) + } + + /// The bound every test reads its element leaves to. + const BOUND: u32 = 4; + + fn input(predicate: &str, vars: Value) -> Value { + json!({"predicate": predicate, "vars": vars.to_string(), "wildcard_bound": BOUND}) + } + + /// The operand's JSON, `eql_v4.query_json_predicate`. + async fn operand( + keyset: &KeysetCipher<'_, K>, + plan: &record::Plan, + predicate: &str, + vars: Value, + ) -> Value { + let term = term(keyset, plan, "match", input(predicate, vars)).await; + assert_eq!(term.encoding(), "eql", "an EQL operand"); + assert_eq!( + term.domain(), + Some("eql_v4.query_json_predicate"), + "bound as the target's compiled predicate domain" + ); + match term.operand() { + Operand::One(bytes) => serde_json::from_slice(bytes).unwrap(), + other => panic!("one operand, got {other:?}"), + } + } + + /// The one operand `operator` derives for `input`, as JSON. + async fn one( + keyset: &KeysetCipher<'_, K>, + plan: &record::Plan, + operator: &str, + input: Value, + ) -> Value { + match term(keyset, plan, operator, input).await.operand() { + Operand::One(bytes) => serde_json::from_slice(bytes).unwrap(), + Operand::Text(text) => json!(text), + other => panic!("{operator}: {other:?}"), + } + } + + fn needle(value: &Value) -> JsonQuery { + serde_json::from_value(value.clone()).unwrap() + } + + /// What `json_tree_match(doc, operand)` answers, as the SQL evaluates + /// each node: `and`, `or` and `not` in three-valued logic; `contains` + /// as `@>` over the stored projection; `exists` as some entry at the + /// selector; `cmp` as the first entry at the selector against each + /// bound, compared as the SQL compares ordering terms (bytes), NULL with + /// no entry or no term; `elements` as `json_tree_element_match`. + fn evaluate(doc: &JsonDocument, node: &Value) -> Option { + let (kind, argument) = node.as_object().unwrap().iter().next().unwrap(); + let children = || argument.as_array().unwrap().iter(); + match kind.as_str() { + "and" => children() + .try_fold(Some(true), |answer, child| { + match (answer, evaluate(doc, child)) { + (_, Some(false)) => Err(()), + (Some(true), Some(true)) => Ok(Some(true)), + _ => Ok(None), + } + }) + .unwrap_or(Some(false)), + "or" => children() + .try_fold(Some(false), |answer, child| { + match (answer, evaluate(doc, child)) { + (_, Some(true)) => Err(()), + (Some(false), Some(false)) => Ok(Some(false)), + _ => Ok(None), + } + }) + .unwrap_or(Some(true)), + "not" => evaluate(doc, argument).map(|answer| !answer), + "contains" => Some(needle(argument).matches(doc.selector_set())), + "exists" => { + let selector = tag(argument.as_str().unwrap()); + Some(doc.entries().iter().any(|e| e.selector() == &selector)) + } + "cmp" => { + let bounds = argument.as_object().unwrap(); + let selector = tag(bounds.values().next().unwrap()["s"].as_str().unwrap()); + let entry = doc.entries().iter().find(|e| e.selector() == &selector)?; + let stored = entry.ordering_term()?; + Some(bounds.iter().all(|(key, leaf)| { + let bound = Base64::decode_vec(leaf["op"].as_str().unwrap()).unwrap(); + match key.as_str() { + "gt" => stored > bound.as_slice(), + "gte" => stored >= bound.as_slice(), + "lt" => stored < bound.as_slice(), + _ => stored <= bound.as_slice(), + } + })) + } + "elements" => Some( + super::element_match::element_match(doc, argument) + .expect("every array within the bound"), + ), + other => panic!("not a node: {other}"), + } + } + + mod given_a_predicate { + use super::*; + + /// Each leaf is the operand its operator derives for the same path + /// and value: `exists` the selector, `contains` `eq_at`'s needle, + /// `cmp` the ordering's leaf operand, `elements` `element_match`'s + /// operand; and an ordering carries its guards, a node there and + /// not a `null`. + #[tokio::test] + async fn each_leaf_is_its_operators_operand() { + let cipher = common::stack_cipher().await; + let keyset = cipher.default_keyset(); + let plan = people(); + let tree = operand( + &keyset, + &plan, + "$.age >= $min && $.name == \"ann\" \ + && exists($.contacts[*] ? (@.relationship == $r && @.calls > 5))", + json!({"min": 18, "r": "sibling"}), + ) + .await; + let leaves = tree["and"].as_array().unwrap(); + assert_eq!(leaves.len(), 5, "{tree}"); + let selector = one(&keyset, &plan, "selector", json!({"path": "$.age"})).await; + assert_eq!(leaves[0], json!({"exists": selector}), "the node is there"); + let null = one( + &keyset, + &plan, + "eq_at", + json!({"path": "$.age", "json": "null"}), + ) + .await; + assert!( + needle(&leaves[1]["not"]["contains"]) == needle(&null), + "and is not a null: {}", + leaves[1] + ); + assert_eq!( + leaves[2], + json!({"cmp": {"gte": one(&keyset, &plan, "gte_at", json!({"path": "$.age", "json": "18"})).await}}), + "the ordering's own leaf operand" + ); + let ann = one( + &keyset, + &plan, + "eq_at", + json!({"path": "$.name", "json": "\"ann\""}), + ) + .await; + assert!( + needle(&leaves[3]["contains"]) == needle(&ann), + "{}", + leaves[3] + ); + let elements = one( + &keyset, + &plan, + "element_match", + json!({"path": "$.contacts[*]", "wildcard_bound": BOUND, "predicates": [ + {"member": "relationship", "op": "eq", "json": "\"sibling\""}, + {"member": "calls", "op": "gt", "json": "5"}, + ]}), + ) + .await; + assert_eq!( + leaves[4], + json!({"elements": elements}), + "element_match's operand" + ); + } + + /// `||`, `!` and the operators' guards keep their structure, and an + /// ordering at a wildcard is an element predicate of one member. + #[tokio::test] + async fn the_boolean_structure_is_the_predicates() { + let cipher = common::stack_cipher().await; + let keyset = cipher.default_keyset(); + let plan = people(); + let tree = operand( + &keyset, + &plan, + "!(exists($.name)) || $.contacts[*].calls < 2", + json!({}), + ) + .await; + let either = tree["or"].as_array().unwrap(); + assert!(either[0]["not"]["exists"].is_string(), "{tree}"); + let m = either[1]["elements"]["m"].as_array().unwrap(); + assert_eq!(m.len(), 1, "one member: {tree}"); + assert!(m[0]["lt"].is_object(), "its bound: {tree}"); + assert_eq!( + m[0]["t"].as_array().unwrap().len(), + usize::try_from(BOUND).unwrap() + 1, + "read to the input's bound, then the probe" + ); + } + } + + mod given_documents { + use super::*; + + /// A document generated to the shape the predicates read: each + /// field absent, `null` or a value of the one type its path holds. + fn document() -> impl Strategy { + let names = prop::sample::select(vec!["ann", "bob", "cat"]); + let field = |values: BoxedStrategy| { + prop_oneof![ + 1 => Just(None), + 1 => Just(Some(Value::Null)), + 3 => values.prop_map(Some), + ] + }; + let contact = ( + proptest::option::of(names.clone()), + proptest::option::of(0u64..6), + ) + .prop_map(|(relationship, calls)| { + let mut contact = serde_json::Map::new(); + if let Some(relationship) = relationship { + contact.insert("relationship".into(), json!(relationship)); + } + if let Some(calls) = calls { + contact.insert("calls".into(), json!(calls)); + } + Value::Object(contact) + }); + ( + field((0u64..6).prop_map(|n| json!(n)).boxed()), + field(names.clone().prop_map(|n| json!(n)).boxed()), + field(any::().prop_map(|b| json!(b)).boxed()), + proptest::option::of(prop::collection::vec(names, 0..3)), + proptest::option::of(prop::collection::vec(contact, 0..=3)), + ) + .prop_map(|(age, name, ok, tags, contacts)| { + let mut document = serde_json::Map::new(); + for (key, value) in [("age", age), ("name", name), ("ok", ok)] { + if let Some(value) = value { + document.insert(key.into(), value); + } + } + if let Some(tags) = tags { + document.insert("tags".into(), json!(tags)); + } + if let Some(contacts) = contacts { + document.insert("contacts".into(), json!(contacts)); + } + Value::Object(document) + }) + } + + /// A jsonpath predicate in the subset `match` reads, as text and as + /// what jsonpath answers of it over a plaintext document. + #[derive(Clone, Debug)] + enum Plain { + /// `path op value`. + Compare(&'static str, &'static str, Value), + /// `exists(path)`. + Exists(&'static str), + /// `exists(base ? (@member op value && …))`. + Filter(&'static str, Vec<(&'static str, &'static str, Value)>), + And(Box, Box), + Or(Box, Box), + Not(Box), + } + + impl Plain { + fn text(&self) -> String { + match self { + Plain::Compare(path, op, value) => format!("{path} {op} {value}"), + Plain::Exists(path) => format!("exists({path})"), + Plain::Filter(base, conditions) => { + let conditions: Vec = conditions + .iter() + .map(|(member, op, value)| format!("@{member} {op} {value}")) + .collect(); + format!("exists({base} ? ({}))", conditions.join(" && ")) + } + Plain::And(a, b) => format!("({} && {})", a.text(), b.text()), + Plain::Or(a, b) => format!("({} || {})", a.text(), b.text()), + Plain::Not(a) => format!("!({})", a.text()), + } + } + + /// What jsonpath answers, in lax mode over a document with no + /// array where a path names none: `None` for unknown. + fn answer(&self, document: &Value) -> Option { + match self { + Plain::Compare(path, op, value) => { + let items = JsonPath::parse(path).unwrap(); + some_item( + items + .select(document) + .into_iter() + .map(|item| compare(item, op, value)), + ) + } + Plain::Exists(path) => { + Some(!JsonPath::parse(path).unwrap().select(document).is_empty()) + } + Plain::Filter(base, conditions) => Some( + JsonPath::parse(base) + .unwrap() + .select(document) + .into_iter() + .any(|element| { + conditions.iter().all(|(member, op, value)| { + let item = match member.strip_prefix('.') { + Some(key) => element.get(key), + None => Some(element), + }; + item.and_then(|item| compare(item, op, value)) == Some(true) + }) + }), + ), + Plain::And(a, b) => match (a.answer(document), b.answer(document)) { + (Some(false), _) | (_, Some(false)) => Some(false), + (Some(true), Some(true)) => Some(true), + _ => None, + }, + Plain::Or(a, b) => match (a.answer(document), b.answer(document)) { + (Some(true), _) | (_, Some(true)) => Some(true), + (Some(false), Some(false)) => Some(false), + _ => None, + }, + Plain::Not(a) => a.answer(document).map(|answer| !answer), + } + } + } + + /// jsonpath's `any`: true when one comparison is, else unknown + /// when one is, else false (and false over no items). + fn some_item(answers: impl Iterator>) -> Option { + let mut answer = Some(false); + for one in answers { + match one { + Some(true) => return Some(true), + None => answer = None, + Some(false) => {} + } + } + answer + } + + /// One jsonpath comparison: `null` equals only `null` and orders + /// below nothing; two numbers, two strings or two booleans compare; + /// anything else is unknown. + fn compare(item: &Value, op: &str, value: &Value) -> Option { + use std::cmp::Ordering; + let order = match (item, value) { + (Value::Null, Value::Null) => Some(Ordering::Equal), + (Value::Null, _) | (_, Value::Null) => { + return Some(match op { + "==" => false, + "!=" => true, + _ => false, + }) + } + (Value::Number(a), Value::Number(b)) => { + a.as_f64().unwrap().partial_cmp(&b.as_f64().unwrap()) + } + (Value::String(a), Value::String(b)) => Some(a.cmp(b)), + (Value::Bool(a), Value::Bool(b)) if matches!(op, "==" | "!=") => Some(a.cmp(b)), + _ => return None, + }?; + Some(match op { + "==" => order.is_eq(), + "!=" => order.is_ne(), + "<" => order.is_lt(), + "<=" => order.is_le(), + ">" => order.is_gt(), + _ => order.is_ge(), + }) + } + + /// The predicates: each comparison `match` compiles, on paths of + /// every shape the documents hold, and `&&`, `||` and `!` over them. + fn predicate() -> impl Strategy { + let ordering = prop::sample::select(vec!["<", "<=", ">", ">="]); + let any_op = prop::sample::select(vec!["==", "!=", "<", "<=", ">", ">="]); + let equality = prop::sample::select(vec!["==", "!="]); + let name = prop::sample::select(vec!["ann", "bob", "cat"]).prop_map(|n| json!(n)); + let number = (0u64..6).prop_map(|n| json!(n)); + let leaf = prop_oneof![ + (any_op.clone(), number.clone()).prop_map(|(op, n)| Plain::Compare("$.age", op, n)), + (any_op, name.clone()).prop_map(|(op, n)| Plain::Compare("$.name", op, n)), + (equality.clone(), any::()).prop_map(|(op, b)| Plain::Compare( + "$.ok", + op, + json!(b) + )), + ( + equality, + prop::sample::select(vec!["$.age", "$.name", "$.ok"]) + ) + .prop_map(|(op, path)| Plain::Compare(path, op, Value::Null)), + prop::sample::select(vec!["$.age", "$.name", "$.tags", "$.tags[*]", "$.contacts"]) + .prop_map(Plain::Exists), + name.clone() + .prop_map(|n| Plain::Compare("$.tags[*]", "==", n)), + ( + prop::sample::select(vec!["==", "<", "<=", ">", ">="]), + number.clone() + ) + .prop_map(|(op, n)| Plain::Compare( + "$.contacts[*].calls", + op, + n + )), + name.clone() + .prop_map(|n| Plain::Filter("$.tags[*]", vec![("", "==", n)])), + name.clone() + .prop_map(|n| Plain::Filter("$.contacts[*]", vec![(".relationship", "==", n)])), + (name, ordering, number).prop_map(|(n, op, m)| Plain::Filter( + "$.contacts[*]", + vec![(".relationship", "==", n), (".calls", op, m)] + )), + ]; + leaf.prop_recursive(3, 12, 2, |inner| { + prop_oneof![ + (inner.clone(), inner.clone()) + .prop_map(|(a, b)| Plain::And(Box::new(a), Box::new(b))), + (inner.clone(), inner.clone()) + .prop_map(|(a, b)| Plain::Or(Box::new(a), Box::new(b))), + inner.prop_map(|a| Plain::Not(Box::new(a))), + ] + }) + } + + /// Over random documents and predicates, the compiled operand + /// answers as jsonpath answers over the plaintext: true, false or + /// unknown (`None`, SQL NULL). + #[test] + fn answers_as_jsonpath_over_the_plaintext() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let cipher = runtime.block_on(common::stack_cipher()); + let keyset = cipher.default_keyset(); + let plan = people(); + let mut runner = TestRunner::new(Config { + cases: 64, + ..Config::default() + }); + runner + .run(&(document(), predicate()), |(document, predicate)| { + let text = predicate.text(); + let (doc, tree) = runtime.block_on(async { + ( + seal(&keyset, &plan, &document).await, + operand(&keyset, &plan, &text, json!({})).await, + ) + }); + let (got, want) = (evaluate(&doc, &tree), predicate.answer(&document)); + if got != want { + return Err(TestCaseError::fail(format!( + "{text} over {document}: got {got:?}, want {want:?}" + ))); + } + Ok(()) + }) + .unwrap(); + } + + /// The three-valued cases a hand-written WHERE has: an ordering is + /// unknown of a node with no ordering term (a boolean), and so is + /// `!` of it, where an absent path and a `null` are false; `!=` + /// is true of a `null`; and `== null` finds only a `null`. + #[tokio::test] + async fn unknown_null_and_absent_answer_as_jsonpath_does() { + let cipher = common::stack_cipher().await; + let keyset = cipher.default_keyset(); + let plan = people(); + for (document, predicate, want) in [ + (json!({"age": true}), "$.age < 3", None), + (json!({"age": true}), "!($.age < 3)", None), + ( + json!({"age": true}), + "!($.age < 3) || exists($.age)", + Some(true), + ), + ( + json!({"age": true}), + "!($.age < 3) && $.age == 1", + Some(false), + ), + (json!({"age": null}), "$.age < 3", Some(false)), + (json!({"age": null}), "!($.age < 3)", Some(true)), + (json!({}), "$.age < 3", Some(false)), + (json!({}), "!($.age < 3)", Some(true)), + (json!({"age": null}), "$.age != 3", Some(true)), + (json!({}), "$.age != 3", Some(false)), + (json!({"age": null}), "$.age != null", Some(false)), + (json!({"age": 1}), "$.age != null", Some(true)), + (json!({"age": null}), "$.age == null", Some(true)), + (json!({}), "$.age == null", Some(false)), + (json!({"age": 1}), "$.age == null", Some(false)), + ] { + let doc = seal(&keyset, &plan, &document).await; + let tree = operand(&keyset, &plan, predicate, json!({})).await; + assert_eq!(evaluate(&doc, &tree), want, "{predicate} over {document}"); + } + } + } + + mod given_an_input_it_cannot_answer { + use super::*; + + fn check(input: &Value) -> Result<(), dynamic::Error> { + dynamic::check_query( + &people(), + "profile", + OpSpec::Match, + json_frame(input), + &Documents, + ) + } + + /// A refusal names its reason, the construct and its offset in the + /// caller's text, and the field; the input's own shape is refused + /// as the other operators refuse theirs. + #[test] + fn is_refused_with_its_reason_and_offset() { + for (predicate, vars, reason, at) in [ + ("$.age + 1 > 2", json!({}), "json_predicate_unsupported", 6), + ("$.age >", json!({}), "json_predicate_syntax", 7), + ( + "$.age > $n", + json!({}), + "json_predicate_unknown_variable", + 8, + ), + ( + "$.age > $n", + json!({"n": [1]}), + "json_predicate_value_type", + 8, + ), + ( + "$.tags[*] != \"a\"", + json!({}), + "json_predicate_not_equal_at_wildcard", + 10, + ), + ("$.tags[0] == \"a\"", json!({}), "json_path_position", 6), + ] { + let error = check(&input(predicate, vars)).unwrap_err(); + assert_eq!( + error.reason().map(Reason::as_str), + Some(reason), + "{predicate}" + ); + assert_eq!(error.field(), Some("profile"), "{predicate}"); + let payload = stack_encrypt::ErrorPayload::payload(&error); + assert_eq!(payload["at"], json!(at), "{predicate}: {payload:?}"); + assert!(payload["construct"].is_string(), "{predicate}: {payload:?}"); + } + for (input, reason) in [ + (json!({"predicate": "$.a == 1", "vars": "[]"}), "json_input"), + (json!({"predicate": "$.a == 1", "vars": "{"}), "json_syntax"), + ( + json!({"predicate": "$.a == 1", "wildcard_bound": 0}), + "json_wildcard_bound_zero", + ), + (json!({"predicate": "$.a == 1", "other": 1}), "json_input"), + (json!({"vars": "{}"}), "json_input"), + ] { + let error = check(&input).unwrap_err(); + assert_eq!(error.reason().map(Reason::as_str), Some(reason), "{input}"); + } + let error = check(&input("$.height > 1", json!({}))).unwrap_err(); + assert!( + matches!( + error, + dynamic::Error::Cipher(stack_encrypt::Error::Term( + stack_encrypt::sem::TermError::JsonPathNotOrdered + )) + ), + "an ordering at a path the field does not order: {error:?}" + ); + assert!( + check(&json!({"predicate": "$.a == 1"})).is_ok(), + "vars may be absent" + ); + } + } + + mod given_the_fields_type { + use super::*; + + /// A resolver whose `JsonSearch` names no compiled predicate + /// domain: what a host presents that has not been taught the + /// operand. + struct Unnamed; + + impl TargetResolver for Unnamed { + fn targets(&self) -> Vec { + let mut targets = Documents.targets(); + targets[0].predicate_query_sql_domain = None; + targets + } + fn encrypt<'a, K: 'static>( + &self, + name: &str, + options: Option<&TargetOptions>, + keyset: &'a KeysetCipher<'_, K>, + label: &Label, + plaintext: ValueFrame, + ) -> Result, K>, TargetError> { + Documents.encrypt(name, options, keyset, label, plaintext) + } + fn decrypt<'a, K: 'static>( + &self, + name: &str, + cipher: &'a StackCipher, + label: &Label, + bytes: &[u8], + ) -> Result, TargetError> { + Documents.decrypt(name, cipher, label, bytes) + } + fn query<'a, K: 'static>( + &self, + name: &str, + index: &IndexSpec, + keyset: &'a KeysetCipher<'_, K>, + label: &Label, + plaintext: ValueFrame, + ) -> Result, K>, TargetError> { + Documents.query(name, index, keyset, label, plaintext) + } + } + + fn names(plan: &record::Plan, resolver: &impl TargetResolver) -> Vec<&'static str> { + dynamic::field_operators(plan, "profile", resolver) + .unwrap() + .into_iter() + .map(|(op, _)| op.name()) + .collect() + } + + /// Answered through the JSON index whenever the type names the + /// domain, an ordered path or none, as the last operator; its input + /// is `predicate` and it is exact. + #[test] + fn lists_it_when_the_domain_is_named() { + for plan in [people(), plan_of(json!("JsonSearch"))] { + assert_eq!(names(&plan, &Documents).last(), Some(&"match")); + } + assert_eq!( + OpSpec::parse("match"), + Some(OpSpec::Match), + "the wire name parses" + ); + assert_eq!(OpSpec::Match.input(), "predicate", "its input shape"); + assert_eq!(OpSpec::Match.fidelity(), dynamic::Fidelity::Exact); + let (_, index) = dynamic::field_operators(&people(), "profile", &Documents) + .unwrap() + .into_iter() + .find(|(op, _)| *op == OpSpec::Match) + .unwrap(); + assert!(matches!(index, IndexSpec::Json(_)), "{index:?}"); + } + + #[test] + fn does_not_answer_it_when_the_type_names_no_domain() { + let wire = json!({ + "profile": {"context": ["users", "profile"], "type": "json", "null": "sql", + "target": "JsonSearch"} + }); + let plan = record::plan_with(&dynamic::frame::serde::to_vec(&wire).unwrap(), &Unnamed) + .unwrap(); + let names = names(&plan, &Unnamed); + assert!(names.contains(&"eq_at"), "the others still: {names:?}"); + assert!(!names.contains(&"match"), "{names:?}"); + let error = dynamic::check_query( + &plan, + "profile", + OpSpec::Match, + json_frame(&input("$.a == 1", json!({}))), + &Unnamed, + ) + .unwrap_err(); + assert_eq!( + error.reason(), + Some(Reason::OperatorNotAnswered), + "{error:?}" + ); + } + + /// A type's own list, which a host reads from its descriptors, has + /// `match` only when the type names the domain: a host that has not + /// been taught the operand sees no new operator. + #[test] + fn a_types_operators_name_it_only_with_the_domain() { + assert!( + Documents.targets()[0].operators().contains(&OpSpec::Match), + "named" + ); + let unnamed = Unnamed.targets()[0].operators(); + assert!(!unnamed.contains(&OpSpec::Match), "{unnamed:?}"); + assert!(unnamed.contains(&OpSpec::EqAt), "{unnamed:?}"); + } + } +}