diff --git a/.changeset/eql-json-tree-match.md b/.changeset/eql-json-tree-match.md new file mode 100644 index 000000000..5e9eb11d9 --- /dev/null +++ b/.changeset/eql-json-tree-match.md @@ -0,0 +1,9 @@ +--- +"@cipherstash/eql": minor +--- + +EQL v4 `json_tree`: a whole filter as one parameter, `eql_v4.json_tree_match(col, $1::eql_v4.query_json_predicate)` and `col @@ $1::eql_v4.query_json_predicate` (#1441). + +An application that compiles a filter language into SQL targets `jsonb_path_match(col, $1, $2)` on a plaintext column: one predicate, one set of values. On a `public.eql_v4_json_search` column each question is its own operator, so the same filter had to be assembled by hand, one operand per leaf. The new domain `eql_v4.query_json_predicate` is a boolean tree (`and`, `or`, `not`) whose leaves are the operands those operators already take (a `contains` needle, an `exists` selector, `cmp` ordering bounds and an `elements` predicate), each checked by its own domain's rule, and `eql_v4.json_tree_match` evaluates it with SQL's three-valued logic, the answer the hand-written WHERE gives. Stack Encrypt's `match` query operator compiles a jsonpath predicate in the subset the terms can answer into the tree. The tree is evaluated inside a function, so it filters a scan and engages no index; the operators written out per leaf are the fast path. The operand must carry its type: an untyped operand on the right of `@@` is refused as ambiguous, and `@@` with a `jsonpath` is still blocked. + +`eql_v4.json_tree_element_match` now reads an opened document through `eql_v4_internal.json_tree_element_match_opened`, which `json_tree_match` shares; its answers are unchanged. diff --git a/languages/golang/encrypt/eql/eql_gen.go b/languages/golang/encrypt/eql/eql_gen.go index 5094e8087..d29569c59 100644 --- a/languages/golang/encrypt/eql/eql_gen.go +++ b/languages/golang/encrypt/eql/eql_gen.go @@ -2252,6 +2252,29 @@ func (v *JSONSearchElementsQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } +// JSONSearchPredicateQuery is the json_tree family's compiled predicate, a boolean tree of the other operands, which eql_v4.json_tree_match and `@@` evaluate: the value eql_v4.query_json_predicate holds. +type JSONSearchPredicateQuery []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v JSONSearchPredicateQuery) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a eql_v4.query_json_predicate column. +func (v *JSONSearchPredicateQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "JSONSearchPredicateQuery") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v JSONSearchPredicateQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "JSONSearchPredicateQuery") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *JSONSearchPredicateQuery) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} + // JSONEntry is the json_tree family's entry: one node of a document, as `->` and the walks return it: the value public.eql_v4_json_entry holds. type JSONEntry []byte diff --git a/languages/golang/encrypt/guest/src/targets.rs b/languages/golang/encrypt/guest/src/targets.rs index 0754a9908..4dc74b805 100644 --- a/languages/golang/encrypt/guest/src/targets.rs +++ b/languages/golang/encrypt/guest/src/targets.rs @@ -268,6 +268,35 @@ mod tests { )); } + /// The JSON document's query domains this guest names, and so the JSON + /// operators the Go SDK is offered: the leaf operand, for the orderings + /// at a path, and not the element-scoped or compiled predicate, whose + /// operators (`element_match`, `match`) the Go side does not parse yet. + /// Naming one is a change to the Go SDK's operator list: update this + /// test with the Go side that takes the operator. + #[cfg(feature = "eql")] + #[test] + fn a_document_names_the_leaf_domain_and_not_the_predicate_domains() { + let targets = resolver().targets(); + let document = targets + .iter() + .find(|t| t.name == "JsonSearch") + .expect("JsonSearch"); + assert_eq!( + document.leaf_query_sql_domain.as_deref(), + Some("eql_v4.query_json_leaf") + ); + assert_eq!(document.elements_query_sql_domain, None); + assert_eq!(document.predicate_query_sql_domain, None); + let operators = document.operators(); + for absent in [ + stack_encrypt::dynamic::OpSpec::ElementMatch, + stack_encrypt::dynamic::OpSpec::Match, + ] { + assert!(!operators.contains(&absent), "{absent:?}: {operators:?}"); + } + } + #[cfg(feature = "eql")] #[test] fn eql_refusals_convert_to_the_engines_with_the_field_left_to_the_lowering() { diff --git a/packages/eql/AGENTS.md b/packages/eql/AGENTS.md index b742b9d9c..bed9617df 100644 --- a/packages/eql/AGENTS.md +++ b/packages/eql/AGENTS.md @@ -60,7 +60,7 @@ Where the two versions' SQL must differ, the source holds both forms in **versio #### Files confined to one EQL version -A source file belongs to every version unless it carries the line `-- ONLY EQL VERSION: N` (the payload version, not the schema name, which the source may not write). Three surfaces do today. The generated range surface: `src/v3/scalars//query__range_types.sql` and `query__range_functions.sql` hold the range query domains (`eql_v4.query__range_ore` / `_range_ope`), their `lo` / `hi` accessors and `eql_v4.between(column, range)` (ADR-0009 decision 7). And the hand-written JSON tree family under `src/v3/json_tree/` (`types.sql`, `functions.sql`, `operators.sql`, `elements.sql`, `blockers.sql`): `public.eql_v4_json_search`, `public.eql_v4_json_entry`, `eql_v4.query_json`, `eql_v4.query_json_leaf` and `eql_v4.query_json_elements`, the document Stack Encrypt's JSON index writes, under the names v3 gives its searchable JSON document but not a port of the SteVec surface (design: `docs/plans/2026-10-17-json-tree-index.md` at the repository root; bindings: `crates/eql-bindings/src/v4/json_tree.rs`, hand-written like the SteVec structs; catalog: `eql_domains::DOCUMENT_FAMILIES`, whose stored type is the target `JsonSearch`). Both are `-- ONLY EQL VERSION: 4`. The third surface goes the other way: the SteVec SQL under `src/v3/json/` and the generated `json_entry_` cross files under `src/v3/scalars//` are `-- ONLY EQL VERSION: 3`, because in v4 their domain names belong to the JSON tree family. EQL v3 is what cipherstash-client writes, nothing there produces a range or a JSON tree document, and its SQL must not change: the v3 bundle differs from an unconfined build only by those directive comment lines. `tasks/build.sh` runs the install order through `eql-codegen select <3|4>` before rendering each version, `eql-codegen render` refuses to render a confined file as the other version, `verify_installer_complete.sh` filters its disk side the same way with its own `grep`, the symbol-order checker runs on the v4 order too (on a copy mapped back to `eql_v3` names), the Doxygen filter reads a confined file from its own version's rendered tree, and `docs:validate:documented-sql` runs the whole v4 install order (`build/render/eql_v4/deps-ordered.txt`) in one rolled-back transaction, so every statement of the confined files runs after what it requires, besides running each file of the v3 render on its own. `eql-codegen order` still orders every file, so `src/deps-ordered-v3.txt` lists the files confined to either version; the per-version lists are `build/render/eql_v/deps-ordered.txt`. +A source file belongs to every version unless it carries the line `-- ONLY EQL VERSION: N` (the payload version, not the schema name, which the source may not write). Three surfaces do today. The generated range surface: `src/v3/scalars//query__range_types.sql` and `query__range_functions.sql` hold the range query domains (`eql_v4.query__range_ore` / `_range_ope`), their `lo` / `hi` accessors and `eql_v4.between(column, range)` (ADR-0009 decision 7). And the hand-written JSON tree family under `src/v3/json_tree/` (`types.sql`, `functions.sql`, `operators.sql`, `elements.sql`, `predicate.sql`, `blockers.sql`): `public.eql_v4_json_search`, `public.eql_v4_json_entry`, `eql_v4.query_json`, `eql_v4.query_json_leaf`, `eql_v4.query_json_elements` and `eql_v4.query_json_predicate`, the document Stack Encrypt's JSON index writes, under the names v3 gives its searchable JSON document but not a port of the SteVec surface (design: `docs/plans/2026-10-17-json-tree-index.md` at the repository root; bindings: `crates/eql-bindings/src/v4/json_tree.rs`, hand-written like the SteVec structs; catalog: `eql_domains::DOCUMENT_FAMILIES`, whose stored type is the target `JsonSearch`). Both are `-- ONLY EQL VERSION: 4`. The third surface goes the other way: the SteVec SQL under `src/v3/json/` and the generated `json_entry_` cross files under `src/v3/scalars//` are `-- ONLY EQL VERSION: 3`, because in v4 their domain names belong to the JSON tree family. EQL v3 is what cipherstash-client writes, nothing there produces a range or a JSON tree document, and its SQL must not change: the v3 bundle differs from an unconfined build only by those directive comment lines. `tasks/build.sh` runs the install order through `eql-codegen select <3|4>` before rendering each version, `eql-codegen render` refuses to render a confined file as the other version, `verify_installer_complete.sh` filters its disk side the same way with its own `grep`, the symbol-order checker runs on the v4 order too (on a copy mapped back to `eql_v3` names), the Doxygen filter reads a confined file from its own version's rendered tree, and `docs:validate:documented-sql` runs the whole v4 install order (`build/render/eql_v4/deps-ordered.txt`) in one rolled-back transaction, so every statement of the confined files runs after what it requires, besides running each file of the v3 render on its own. `eql-codegen order` still orders every file, so `src/deps-ordered-v3.txt` lists the files confined to either version; the per-version lists are `build/render/eql_v/deps-ordered.txt`. Anything that needs the source as real SQL reads the rendered tree, not `src/`: `docs:validate:documented-sql` runs psql over `build/render/eql_v3/src`, and the Doxygen input filter (`tasks/docs/doxygen-filter.sh`) reads each file's rendered copy from `build/render/eql_v3/`. A new tool that parses `src/v3` must do the same, and depend on `build`; do not substitute the placeholders a second way. diff --git a/packages/eql/crates/eql-bindings/src/encryption/targets.rs b/packages/eql/crates/eql-bindings/src/encryption/targets.rs index aed81c9d9..a1cb06544 100644 --- a/packages/eql/crates/eql-bindings/src/encryption/targets.rs +++ b/packages/eql/crates/eql-bindings/src/encryption/targets.rs @@ -164,6 +164,18 @@ impl Target { .map(|_| crate::v4::json_tree::JsonTreeElementsQuery::sql_domain_static()) } + /// The PostgreSQL domain of the type's compiled predicate, for a type + /// that stores a JSON document (`JsonSearch`: + /// `eql_v4.query_json_predicate`); `None` for every other type. Stack + /// Encrypt binds `match` as it + /// (`TargetDescriptor::with_predicate_query_sql_domain`), and answers + /// `match` only on a type that names it. + pub fn predicate_query_sql_domain(&self) -> Option<&'static str> { + use crate::v4::DomainType; + document(self.name) + .map(|_| crate::v4::json_tree::JsonTreePredicateQuery::sql_domain_static()) + } + /// The options this type's match index is built under, for a type that /// carries one (`TextMatch`): [`EqlMatch`](super::EqlMatch)'s, the one /// match configuration every EQL type's filter is built under. `None` diff --git a/packages/eql/crates/eql-bindings/src/v4/inventory.rs b/packages/eql/crates/eql-bindings/src/v4/inventory.rs index 613a60377..fc1ba1cfa 100644 --- a/packages/eql/crates/eql-bindings/src/v4/inventory.rs +++ b/packages/eql/crates/eql-bindings/src/v4/inventory.rs @@ -107,6 +107,7 @@ pub fn all_query() -> Vec> { Box::new(PhantomData::), Box::new(PhantomData::), Box::new(PhantomData::), + Box::new(PhantomData::), ] } /// Every v4 RANGE query operand (`eql_v4.query__range_`), diff --git a/packages/eql/crates/eql-bindings/src/v4/json_tree.rs b/packages/eql/crates/eql-bindings/src/v4/json_tree.rs index 1c2d56c2a..b2673d234 100644 --- a/packages/eql/crates/eql-bindings/src/v4/json_tree.rs +++ b/packages/eql/crates/eql-bindings/src/v4/json_tree.rs @@ -219,6 +219,57 @@ pub struct JsonTreeElementsQuery { pub m: Vec, } +/// The bounds of an ordering at one path: one or two ordering operands at +/// the path's selector, `gt` or `gte` the lower and `lt` or `lte` the upper. +/// The `cmp` leaf of a [`JsonTreePredicateQuery`]. The domain's CHECK also +/// holds what this type does not: at least one bound, at most one of each +/// side, and every bound at one selector. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct JsonTreeBounds { + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub gt: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub gte: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub lt: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub lte: Option, +} + +/// `eql_v4.query_json_predicate`: a compiled predicate, a boolean tree +/// whose leaves are the family's other query operands, which +/// `eql_v4.json_tree_match` and `col @@ $1` evaluate. What Stack Encrypt's +/// `match` operator compiles a jsonpath predicate into. Each node is an +/// object of one key naming its kind: `{"and": [node, …]}`, `{"or": […]}`, +/// `{"not": node}`, `{"contains": needle}`, `{"exists": selector}`, +/// `{"cmp": bounds}` and `{"elements": predicate}`. The domain's CHECK also +/// holds what this type does not: at least one child under `and` and `or`. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] +#[serde(rename_all = "snake_case", deny_unknown_fields)] +pub enum JsonTreePredicateQuery { + /// True when every child is; false when one is false; else NULL. + And(Vec), + /// True when one child is; false when every child is false; else NULL. + Or(Vec), + /// The child's negation, NULL of NULL. + Not(Box), + /// The document contains the needle, `col @> needle`. + Contains(JsonTreeQuery), + /// The document has a node at the path, `json_tree_path_exists`. + Exists(JsonTreeSelector), + /// The first entry at the bounds' selector against each bound, + /// `(col -> s) > gt AND (col -> s) < lt`. + Cmp(JsonTreeBounds), + /// One element of an array satisfies every member, + /// `json_tree_element_match`. + Elements(JsonTreeElementsQuery), +} + /// The entry `col -> selector` returned, with the selector that asked for /// it: opens to the scalar the entry holds, or to the whole document for a /// sealed root asked for at `$`. The only constructor, @@ -330,6 +381,7 @@ json_tree_domain_type!(JsonTreeEntry, "public.eql_v4_json_entry"); json_tree_domain_type!(JsonTreeQuery, "eql_v4.query_json"); json_tree_domain_type!(JsonTreeLeafQuery, "eql_v4.query_json_leaf"); json_tree_domain_type!(JsonTreeElementsQuery, "eql_v4.query_json_elements"); +json_tree_domain_type!(JsonTreePredicateQuery, "eql_v4.query_json_predicate"); // ============================================================================= // Conversions to and from the engine's types, through their one wire form @@ -594,6 +646,47 @@ mod tests { JsonTreeElementsQuery::sql_domain_static(), "eql_v4.query_json_elements" ); + assert_eq!( + JsonTreePredicateQuery::sql_domain_static(), + "eql_v4.query_json_predicate" + ); + } + + #[test] + fn a_predicate_query_is_a_tree_of_one_key_nodes() { + let leaf = serde_json::json!({"s": b64(0xaa), "op": "AQI="}); + let needle = serde_json::json!({"sv": [{"s": b64(1), "vs": b64(2)}], "el": []}); + let elements = serde_json::json!({"m": [{"t": [b64(1)], "gt": leaf}]}); + let query = serde_json::json!({"and": [ + {"exists": b64(0xaa)}, + {"not": {"cmp": {"gte": leaf, "lt": leaf}}}, + {"or": [{"contains": needle}, {"elements": elements}]}, + ]}); + let parsed: JsonTreePredicateQuery = serde_json::from_value(query.clone()).unwrap(); + let JsonTreePredicateQuery::And(children) = &parsed else { + panic!("the root is an and: {parsed:?}") + }; + assert!( + matches!(children[0], JsonTreePredicateQuery::Exists(_)), + "{children:?}" + ); + assert_eq!( + serde_json::to_value(&parsed).unwrap(), + query, + "a tree reads and writes as itself" + ); + for wrong in [ + serde_json::json!({"exists": b64(1), "not": {"exists": b64(1)}}), + serde_json::json!({"xor": []}), + serde_json::json!({"cmp": {"eq": leaf}}), + serde_json::json!({"exists": 1}), + serde_json::json!([]), + ] { + assert!( + serde_json::from_value::(wrong.clone()).is_err(), + "{wrong} should not deserialize" + ); + } } #[test] diff --git a/packages/eql/crates/eql-codegen/src/bindings.rs b/packages/eql/crates/eql-codegen/src/bindings.rs index 627be79f5..69caee2d2 100644 --- a/packages/eql/crates/eql-codegen/src/bindings.rs +++ b/packages/eql/crates/eql-codegen/src/bindings.rs @@ -1084,6 +1084,7 @@ pub fn render_v4_inventory_rs() -> String { eql_domains::DocumentRole::Query, eql_domains::DocumentRole::LeafQuery, eql_domains::DocumentRole::ElementsQuery, + eql_domains::DocumentRole::PredicateQuery, ]); quote! { #query_entries #documents } }; diff --git a/packages/eql/crates/eql-codegen/src/go_eql.rs b/packages/eql/crates/eql-codegen/src/go_eql.rs index 9dbc4636b..ef1771d76 100644 --- a/packages/eql/crates/eql-codegen/src/go_eql.rs +++ b/packages/eql/crates/eql-codegen/src/go_eql.rs @@ -189,6 +189,9 @@ fn document_types(out: &mut String) { DocumentRole::ElementsQuery => { "element-scoped predicate, which eql_v4.json_tree_element_match tests" } + DocumentRole::PredicateQuery => { + "compiled predicate, a boolean tree of the other operands, which eql_v4.json_tree_match and `@@` evaluate" + } DocumentRole::Entry => { "entry: one node of a document, as `->` and the walks return it" } @@ -275,6 +278,7 @@ mod tests { "JSONSearchQuery", "JSONSearchLeafQuery", "JSONSearchElementsQuery", + "JSONSearchPredicateQuery", "JSONEntry", ] { assert_eq!( diff --git a/packages/eql/crates/eql-domains/src/lib.rs b/packages/eql/crates/eql-domains/src/lib.rs index 1e7738eaa..7c431f0c6 100644 --- a/packages/eql/crates/eql-domains/src/lib.rs +++ b/packages/eql/crates/eql-domains/src/lib.rs @@ -567,6 +567,9 @@ pub enum DocumentRole { /// The element-scoped predicate: fields of one array element, each /// with its read tokens and its bounds. ElementsQuery, + /// The compiled predicate: a boolean tree whose leaves are the other + /// query operands. + PredicateQuery, /// One node, as `->` and the walks return it. Entry, } @@ -625,8 +628,10 @@ impl DocumentFamily { /// `JsonSearch` over `public.eql_v4_json_search`, its needle /// `JsonSearchQuery` (`eql_v4.query_json`), its ordering operand /// `JsonSearchLeafQuery` (`eql_v4.query_json_leaf`), its element-scoped -/// predicate `JsonSearchElementsQuery` (`eql_v4.query_json_elements`) and -/// its entry `JsonEntry` (`public.eql_v4_json_entry`). The storage-only `Json` +/// predicate `JsonSearchElementsQuery` (`eql_v4.query_json_elements`), its +/// compiled predicate `JsonSearchPredicateQuery` +/// (`eql_v4.query_json_predicate`) and its entry `JsonEntry` +/// (`public.eql_v4_json_entry`). The storage-only `Json` /// (`public.eql_v4_json`) is the `json` family's. The Rust structs keep the /// implementation's name, as v3's `SteVecDocument` does. pub const JSON_TREE: DocumentFamily = DocumentFamily { @@ -658,6 +663,12 @@ pub const JSON_TREE: DocumentFamily = DocumentFamily { rust: "JsonTreeElementsQuery", sql_domain: "eql_v4.query_json_elements", }, + DocumentDomain { + role: DocumentRole::PredicateQuery, + name: "JsonSearchPredicateQuery", + rust: "JsonTreePredicateQuery", + sql_domain: "eql_v4.query_json_predicate", + }, DocumentDomain { role: DocumentRole::Entry, name: "JsonEntry", diff --git a/packages/eql/crates/eql-domains/src/tests.rs b/packages/eql/crates/eql-domains/src/tests.rs index 103414893..9547707b8 100644 --- a/packages/eql/crates/eql-domains/src/tests.rs +++ b/packages/eql/crates/eql-domains/src/tests.rs @@ -1634,6 +1634,7 @@ mod document_families { DocumentRole::Query, DocumentRole::LeafQuery, DocumentRole::ElementsQuery, + DocumentRole::PredicateQuery, DocumentRole::Entry, ] { assert_eq!( @@ -1646,9 +1647,10 @@ mod document_families { for domain in family.domains { let schema = match domain.role { DocumentRole::Stored | DocumentRole::Entry => "public.eql_v4_", - DocumentRole::Query | DocumentRole::LeafQuery | DocumentRole::ElementsQuery => { - "eql_v4.query_" - } + DocumentRole::Query + | DocumentRole::LeafQuery + | DocumentRole::ElementsQuery + | DocumentRole::PredicateQuery => "eql_v4.query_", }; assert!( domain.sql_domain.starts_with(schema), @@ -1685,6 +1687,11 @@ mod document_families { "JsonSearchElementsQuery", "eql_v4.query_json_elements", ), + ( + DocumentRole::PredicateQuery, + "JsonSearchPredicateQuery", + "eql_v4.query_json_predicate", + ), (DocumentRole::Entry, "JsonEntry", "public.eql_v4_json_entry"), ] { let domain = JSON_TREE.domain(role).unwrap(); diff --git a/packages/eql/docs/reference/eql-functions.md b/packages/eql/docs/reference/eql-functions.md index 7eaf74266..103ecffbf 100644 --- a/packages/eql/docs/reference/eql-functions.md +++ b/packages/eql/docs/reference/eql-functions.md @@ -12,6 +12,7 @@ A reference for the functions and operators EQL exposes for querying encrypted d - [Encrypted JSON (`public.eql_v3_json_search`)](#encrypted-json-publiceql_v3_json_search) - [Range queries in EQL v4: `eql_v4.between`](#range-queries-in-eql-v4-eql_v4between) - [One array element in EQL v4: `eql_v4.json_tree_element_match`](#one-array-element-in-eql-v4-eql_v4json_tree_element_match) +- [A compiled predicate in EQL v4: `eql_v4.json_tree_match`](#a-compiled-predicate-in-eql-v4-eql_v4json_tree_match) - [Aggregate Functions](#aggregate-functions) - [Diagnostics](#diagnostics) @@ -219,6 +220,40 @@ That form does not check the probe: on a row whose array is longer than the boun --- +## A compiled predicate in EQL v4: `eql_v4.json_tree_match` + +On plaintext `jsonb`, one function answers a whole filter: `jsonb_path_match(col, $1, $2)`, a jsonpath predicate and its variables. An application that compiles a filter language into SQL can target that shape with one template and two parameters. On an encrypted JSON document (`public.eql_v4_json_search`) the database cannot evaluate a path or a value, so `@@` with a `jsonpath` is blocked, `jsonb_path_match` matches nothing ([Native jsonb functions read the envelope](json-tree-leakage.md#native-jsonb-functions-read-the-envelope)), and each question is its own operator. `eql_v4.json_tree_match` takes the whole filter as one parameter again: Stack Encrypt's `match` query operator compiles the jsonpath predicate on the client into a tree of those operators' operands, and the function evaluates the tree. + +```sql +eql_v4.json_tree_match(e public.eql_v4_json_search, p eql_v4.query_json_predicate) RETURNS boolean + +SELECT id FROM people WHERE eql_v4.json_tree_match(profile, $1::eql_v4.query_json_predicate); +SELECT id FROM people WHERE profile @@ $1::eql_v4.query_json_predicate; +``` + +The operand `eql_v4.query_json_predicate` is a boolean tree, serialised as JSON. Each node is an object with one key, which names its kind: + +| node | true when | the same question, written out | +|---|---|---| +| `{"and": [node, …]}`, `{"or": [node, …]}` | every child is, or one child is | `… AND …`, `… OR …` | +| `{"not": node}` | the child is false | `NOT …` | +| `{"contains": needle}` | the document contains the needle, an `eql_v4.query_json` | `col @> needle` | +| `{"exists": selector}` | the document has a node at the path | `eql_v4.json_tree_path_exists(col, selector)` | +| `{"cmp": {"gt" \| "gte": leaf, "lt" \| "lte": leaf}}` | the first entry at the bounds' selector is inside every bound, each an `eql_v4.query_json_leaf` | `(col -> s) > gt AND (col -> s) < lt` | +| `{"elements": predicate}` | one element of an array satisfies every member, an `eql_v4.query_json_elements` | `eql_v4.json_tree_element_match(col, predicate)` | + +Each leaf is checked by the CHECK of its own domain, and `and` and `or` take at least one child. + +- **The answer is the hand-written WHERE's, NULL included.** `and`, `or` and `not` follow SQL's three-valued logic, and a `cmp` leaf is NULL where the path is absent or its node has no ordering term, as `(col -> s) < $1` is. Stack Encrypt compiles a jsonpath comparison so that the tree is true of the rows `jsonb_path_match` is true of over the plaintext: `$.age < $n` is a node at `$.age` whose leaf compares; `$.age != $n` is a node there that is not the value. So an absent path is false under every comparison, as jsonpath has it, and an ordering of a boolean is unknown (NULL), as jsonpath's comparison of two kinds is. jsonpath also answers false of an ordering over a JSON `null`. The tree matches that only under an odd number of `!`, where it changes the rows: there `$.age < $n` is also not a `null`, a `null` needle beside the bound. Elsewhere the ordering over a `null` is NULL, which selects the same rows as false, so a `SELECT` of the boolean itself can show NULL where `jsonb_path_match` shows false. +- **Where the tree and jsonpath part.** In four places, each a property of the stored document or of the ordering terms. A path is read as written, so an array where the predicate names no `[*]` is not unwrapped as jsonpath's lax mode unwraps it (`$.tags == "a"` over `{"tags": ["a"]}` is true to jsonpath, false here). Nor is a value that is not an array wrapped where the predicate does name `[*]`: `$.tags[*] == "a"` over `{"tags": "a"}`, `exists($.tags[*])` over a scalar `tags` and `exists($.items[*] ? (@.sku == "x"))` over `{"items": {"sku": "x"}}` are true to jsonpath and false here, since `$.tags[*]` and `$.tags` have different selectors. A comparison of a string with a number, which jsonpath answers unknown, is false under `==` and true under `!=`, since the terms do not record a value's kind. An ordering compares the ordering terms, so text is case- and accent-folded, as every ordering at a path is; so does each condition of a filter with more than one, which compiles to an element leaf. +- **Cast the operand.** `col @@ $1::eql_v4.query_json_predicate`. An untyped operand on the right of `@@` matches both this operator and the blocked `jsonpath` one, and PostgreSQL refuses the statement with SQLSTATE 42725 (`operator is not unique`). A `jsonpath` operand still raises `operator @@ is not supported`. +- **Element leaves raise as their function does.** An array longer than the bound the predicate was compiled to raises SQLSTATE `22023`. Children are read in order and the read stops at the first child that decides the answer, so an element leaf the answer does not depend on is not read and does not raise. +- **The document is opened once.** It is decompressed once per row, and indexed once when the tree has an element leaf, however many leaves read it. +- **No index serves it.** The tree is evaluated inside a PL/pgSQL function, which the planner cannot see into: `json_tree_match` and `@@` filter the rows they are given, a sequential scan unless something else narrows them. The same predicate written out per operator (the right-hand column above) engages the GIN index on `eql_v4.json_tree_query(col)` and the functional btrees on `eql_v4.ord_term(col -> s)`, and is the fast path. Use `match` where one template matters more than the plan, and put an indexed predicate beside it where you can. +- **What it reveals** is what its leaves reveal sent one by one, and the boolean shape of the filter. See [What a query shows](./json-tree-leakage.md#what-a-query-shows). + +--- + ## Aggregate Functions ### `eql_v3.min()` / `eql_v3.max()` (per-domain) diff --git a/packages/eql/docs/reference/json-tree-leakage.md b/packages/eql/docs/reference/json-tree-leakage.md index 9bf763e10..4389f2a8a 100644 --- a/packages/eql/docs/reference/json-tree-leakage.md +++ b/packages/eql/docs/reference/json-tree-leakage.md @@ -52,6 +52,7 @@ A query hands the database a term derived from the plaintext it asks about, and - **The root.** A read of `$` sends the root's token, and a read of anything below `$` does not. With the sealed root on (the default) that token covers no document's structure: it finds the root entry, whose ciphertext is the document, and nothing beneath it, since the top-level fields chain from nothing. With it off, the root's token covers every row's whole document. - **`json_tree_children` and `json_tree_child_count`** take one token, of any node a path names, and send nothing else. Under the sealed root the root's token enumerates nothing (its count is NULL), for the same reason. - **An element-scoped predicate (`json_tree_element_match`).** For each field it tests, the tokens of a wildcard read of that field's path, probes included, and the bounds' ordering terms, as a subtree read and an ordering send them. So the database learns what those reads and orderings learn, and one thing more, which the answer itself is: in which rows one element holds every field inside its bounds. It does not open a value. Equality there is the ordering term's, of folded values (see `op` above). +- **A compiled predicate (`json_tree_match`, `@@` with an `eql_v4.query_json_predicate`).** What its leaves reveal sent separately (the selectors of the paths it tests, the (path, value) selectors of its equalities, the bounds' ordering terms, and the element leaves' tokens), plus the boolean shape of the filter: which leaves are joined by `and`, `or` and `not`. An ordering compiled from jsonpath under an odd number of `!` carries a `null` needle at its path beside its bound, so the database also learns which rows hold `null` there, as `== null` sent alone shows it. An ordering under no `!`, or an even number, carries none. The database holds every leaf and can evaluate each on every row, so it can learn each leaf's answer per row, as it can of the leaves sent separately. ## Native jsonb functions read the envelope @@ -60,7 +61,7 @@ A query hands the database a term derived from the plaintext it asks about, and - **Operators are blocked.** `?`, `?|`, `?&`, `@?`, `@@`, `#>`, `#>>`, `-`, `#-`, `||`, `->` and `->>` by position, root-document comparisons, and containment against a plain `jsonb` operand raise `operator … is not supported for public.eql_v4_json_search`. `col @@ '…'` with an untyped right operand is refused as ambiguous (`operator is not unique`). - **Functions are not blocked, and fail silently.** `jsonb_path_match(col, '$.score == 3')` returns false for every row, with no error, and so does `jsonb_path_exists(col, '$.score')`, whether the path is a literal, cast, or bound as a parameter. `jsonb_extract_path(col, 'score')` returns NULL. A key of the envelope returns the envelope's own field instead: `jsonb_extract_path(col, 'v')` is `4`, and `jsonb_object_keys(col)` lists `h`, `i`, `q`, `sv` and `v`. The jsonpath functions find nothing on an entry extracted with `->` either. Only `jsonb_array_elements`, `jsonb_array_elements_text` and `jsonb_array_length` raise, because the envelope and an entry are objects. -Query the document with EQL instead: `@>` with an `eql_v4.query_json` needle for equality, `col -> selector` compared with an `eql_v4.query_json_leaf` for ranges, and `col -> selector`, `eql_v4.json_tree_path_query` or `eql_v4.json_tree_subtree` to extract, then decrypt in the client. +Query the document with EQL instead: `@>` with an `eql_v4.query_json` needle for equality, `col -> selector` compared with an `eql_v4.query_json_leaf` for ranges, `col @@ $1::eql_v4.query_json_predicate` (`eql_v4.json_tree_match`) for a whole jsonpath filter compiled by Stack Encrypt, and `col -> selector`, `eql_v4.json_tree_path_query` or `eql_v4.json_tree_subtree` to extract, then decrypt in the client. EQL cannot make these functions raise. An overload taking the domain as its first argument would be a second candidate for every call whose first argument has no type, such as `jsonb_typeof($1)` from a driver that sends untyped parameters, so installing EQL would make those calls ambiguous on plain `jsonb` too, in the whole database. And it would still lose to the native function whenever any other argument is untyped, as the path is in `jsonb_path_match(col, '$.a == 1')`. A type of its own that `jsonb` functions do not accept would close this, but PostgreSQL lets only a superuser create one, and EQL installs without superuser. diff --git a/packages/eql/src/v3/json_tree/elements.sql b/packages/eql/src/v3/json_tree/elements.sql index 0b9856222..a9bd6ed1f 100644 --- a/packages/eql/src/v3/json_tree/elements.sql +++ b/packages/eql/src/v3/json_tree/elements.sql @@ -8,6 +8,94 @@ --! @brief Element-scoped predicates on public.{{prefix}}_json_search: tests --! that one element of an array satisfies them all. +--! @brief Whether an entry is inside every bound of an ordering at one +--! path: what an element predicate's member and a compiled +--! predicate's `cmp` leaf each test of the entry they find. +--! @internal +--! +--! `gt` by `>`, `gte` by `>=`, `lt` by `<` and `lte` by `<=`, the ordering +--! operators an entry has against a {{prefix}}.query_json_leaf, joined by +--! SQL `AND`; a bound `bounds` does not carry is no test. So it is NULL +--! where a bound's comparison is: an entry that is NULL (no node at the +--! path) or has no ordering term. +--! +--! @param entry public.{{prefix}}_json_entry The entry. +--! @param bounds jsonb Bounds, as {{prefix}}_internal.json_tree_is_valid_bounds +--! validates them; other keys (a member's `t`) are not read. +--! @return boolean Whether the entry satisfies every bound. +--! @note LANGUAGE sql and not STRICT, so it inlines into its callers. +CREATE FUNCTION {{prefix}}_internal.json_tree_within_bounds(entry public.{{prefix}}_json_entry, bounds jsonb) + RETURNS boolean + LANGUAGE sql IMMUTABLE PARALLEL SAFE +AS $$ + SELECT (NOT bounds ? 'gt' OR {{prefix}}.gt(entry, (bounds -> 'gt')::{{prefix}}.query_json_leaf)) + AND (NOT bounds ? 'gte' OR {{prefix}}.gte(entry, (bounds -> 'gte')::{{prefix}}.query_json_leaf)) + AND (NOT bounds ? 'lt' OR {{prefix}}.lt(entry, (bounds -> 'lt')::{{prefix}}.query_json_leaf)) + AND (NOT bounds ? 'lte' OR {{prefix}}.lte(entry, (bounds -> 'lte')::{{prefix}}.query_json_leaf)) +$$; + +--! @brief {{prefix}}.json_tree_element_match over a document already opened +--! and indexed: what that function and {{prefix}}.json_tree_match +--! each call, so a predicate with several element-scoped leaves +--! opens and indexes its row once. +--! @internal +--! @param iv bytea The document's `iv`, which salts its self tags. +--! @param sv jsonb The document's entries. +--! @param tags jsonb The document's self tags, each mapped to the entry's +--! ordinal in `sv`: the `nodes` {{prefix}}_internal.json_tree_index +--! builds. +--! @param p jsonb The predicate, a valid {{prefix}}.query_json_elements. +--! @return boolean As {{prefix}}.json_tree_element_match. +--! @throws invalid_parameter_value As {{prefix}}.json_tree_element_match. +CREATE FUNCTION {{prefix}}_internal.json_tree_element_match_opened(iv bytea, sv jsonb, tags jsonb, p jsonb) + RETURNS boolean + LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE +AS $$ +DECLARE + truncated boolean; + matched boolean; +BEGIN + WITH + member AS MATERIALIZED ( + SELECT m.i, + m.v, + -- The selector of the member's field: every bound carries it. + COALESCE(m.v -> 'gt', m.v -> 'gte', m.v -> 'lt', m.v -> 'lte') ->> 's' AS s + FROM jsonb_array_elements(p -> 'm') WITH ORDINALITY AS m(v, i) + ), + -- The node each member's `k`-th token names, by its self tag; a token + -- that names none yields no row. + node AS MATERIALIZED ( + SELECT member.i, member.v, member.s, t.k, sv -> found.j AS entry + FROM member + CROSS JOIN LATERAL jsonb_array_elements_text(member.v -> 't') WITH ORDINALITY AS t(token, k) + CROSS JOIN LATERAL ( + SELECT (tags ->> {{prefix}}_internal.json_tree_self_tag(iv, decode(t.token, 'base64')))::integer AS j + ) AS found + WHERE found.j IS NOT NULL + ), + satisfied AS ( + SELECT node.k + FROM node + CROSS JOIN LATERAL (SELECT node.entry::public.{{prefix}}_json_entry AS entry) AS typed + WHERE node.entry ->> 's' = node.s + AND {{prefix}}_internal.json_tree_within_bounds(typed.entry, node.v) + GROUP BY node.k + HAVING count(DISTINCT node.i) = (SELECT count(*) FROM member) + ) + SELECT EXISTS (SELECT 1 FROM node WHERE node.entry ->> 's' IS DISTINCT FROM node.s), + EXISTS (SELECT 1 FROM satisfied) + INTO truncated, matched; + + IF truncated THEN + RAISE EXCEPTION 'an array the element predicate reads is longer than the bound it was derived to, so the predicate would answer over part of it' + USING ERRCODE = 'invalid_parameter_value', + HINT = 'Derive the predicate (element_match) with a wildcard_bound past the longest array.'; + END IF; + RETURN matched; +END; +$$; + --! @brief Whether some one element of an array in the document satisfies --! every member of an element-scoped predicate. --! @@ -53,68 +141,18 @@ --! @note plpgsql, to raise, and so opaque to the planner: the call filters --! a scan, as a subtree read in a predicate does. Narrow the rows with --! an indexed predicate first, such as `@>` on the same values. +--! @see {{prefix}}_internal.json_tree_element_match_opened CREATE FUNCTION {{prefix}}.json_tree_element_match(e public.{{prefix}}_json_search, p {{prefix}}.query_json_elements) RETURNS boolean LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE AS $$ DECLARE - truncated boolean; - matched boolean; + -- The document, decompressed once. + d jsonb := e::jsonb #> '{}'::text[]; + tags jsonb; BEGIN - WITH - -- The document, decompressed and indexed once for every member. - raw AS MATERIALIZED (SELECT e::jsonb #> '{}'::text[] AS d), - doc AS MATERIALIZED ( - SELECT {{prefix}}_internal.json_tree_iv(raw.d ->> 'h') AS iv, - raw.d -> 'sv' AS sv, - idx.nodes - FROM raw - CROSS JOIN LATERAL {{prefix}}_internal.json_tree_index(raw.d) AS idx - ), - member AS MATERIALIZED ( - SELECT m.i, - m.v, - -- The selector of the member's field: every bound carries it. - COALESCE(m.v -> 'gt', m.v -> 'gte', m.v -> 'lt', m.v -> 'lte') ->> 's' AS s - FROM jsonb_array_elements(p::jsonb -> 'm') WITH ORDINALITY AS m(v, i) - ), - -- The node each member's `k`-th token names, by its self tag; a token - -- that names none yields no row. - node AS MATERIALIZED ( - SELECT member.i, member.v, member.s, t.k, doc.sv -> found.j AS entry - FROM member - CROSS JOIN doc - CROSS JOIN LATERAL jsonb_array_elements_text(member.v -> 't') WITH ORDINALITY AS t(token, k) - CROSS JOIN LATERAL ( - SELECT (doc.nodes ->> {{prefix}}_internal.json_tree_self_tag(doc.iv, decode(t.token, 'base64')))::integer AS j - ) AS found - WHERE found.j IS NOT NULL - ), - satisfied AS ( - SELECT node.k - FROM node - CROSS JOIN LATERAL (SELECT node.entry::public.{{prefix}}_json_entry AS entry) AS typed - WHERE node.entry ->> 's' = node.s - AND (NOT node.v ? 'gt' - OR {{prefix}}.gt(typed.entry, (node.v -> 'gt')::{{prefix}}.query_json_leaf)) - AND (NOT node.v ? 'gte' - OR {{prefix}}.gte(typed.entry, (node.v -> 'gte')::{{prefix}}.query_json_leaf)) - AND (NOT node.v ? 'lt' - OR {{prefix}}.lt(typed.entry, (node.v -> 'lt')::{{prefix}}.query_json_leaf)) - AND (NOT node.v ? 'lte' - OR {{prefix}}.lte(typed.entry, (node.v -> 'lte')::{{prefix}}.query_json_leaf)) - GROUP BY node.k - HAVING count(DISTINCT node.i) = (SELECT count(*) FROM member) - ) - SELECT EXISTS (SELECT 1 FROM node WHERE node.entry ->> 's' IS DISTINCT FROM node.s), - EXISTS (SELECT 1 FROM satisfied) - INTO truncated, matched; - - IF truncated THEN - RAISE EXCEPTION 'an array the element predicate reads is longer than the bound it was derived to, so the predicate would answer over part of it' - USING ERRCODE = 'invalid_parameter_value', - HINT = 'Derive the predicate (element_match) with a wildcard_bound past the longest array.'; - END IF; - RETURN matched; + SELECT idx.nodes INTO tags FROM {{prefix}}_internal.json_tree_index(d) AS idx; + RETURN {{prefix}}_internal.json_tree_element_match_opened( + {{prefix}}_internal.json_tree_iv(d ->> 'h'), d -> 'sv', tags, p::jsonb); END; $$; diff --git a/packages/eql/src/v3/json_tree/predicate.sql b/packages/eql/src/v3/json_tree/predicate.sql new file mode 100644 index 000000000..d28caed8f --- /dev/null +++ b/packages/eql/src/v3/json_tree/predicate.sql @@ -0,0 +1,294 @@ +-- ONLY EQL VERSION: 4 +-- REQUIRE: src/v3/schema.sql +-- REQUIRE: src/v3/json_tree/types.sql +-- REQUIRE: src/v3/json_tree/functions.sql +-- REQUIRE: src/v3/json_tree/operators.sql +-- REQUIRE: src/v3/json_tree/elements.sql + +--! @file v3/json_tree/predicate.sql +--! @brief A compiled predicate on public.{{prefix}}_json_search: a boolean +--! tree of the family's query operands, bound as one parameter and +--! evaluated by {{prefix}}.json_tree_match or `@@`. +--! +--! A filter compiler that targets plaintext `jsonb_path_match(col, $1, $2)` +--! binds one predicate and its values. The database cannot evaluate a +--! jsonpath over ciphertext, so Stack Encrypt's `match` operator compiles +--! the jsonpath on the client into this tree, whose leaves are the operands +--! the family's operators already take, and the server evaluates the tree. +--! +--! @note The tree is evaluated inside a function, which is opaque to the +--! planner: `col @@ $1` filters a scan, and engages no index. The +--! operators the leaves stand for, written out in the WHERE clause +--! (`col @> $1`, `(col -> $2) < $3`), engage the GIN and btree indexes +--! and are the fast path. + +--! @brief Validate a compiled predicate, recursively. +--! @internal +--! +--! A node is an object of exactly one key, which names its kind: +--! - `{"and": [node, …]}`, `{"or": [node, …]}`: at least one node; +--! - `{"not": node}`; +--! - `{"contains": needle}`: a containment needle, as +--! public.{{prefix}}_is_valid_json_tree_query_payload validates the +--! {{prefix}}.query_json CHECK; +--! - `{"exists": "…"}`: a path selector, base64; +--! - `{"cmp": bounds}`: one or two ordering operands at one selector, as +--! {{prefix}}_internal.json_tree_is_valid_bounds validates them; +--! - `{"elements": predicate}`: an element-scoped predicate, as +--! {{prefix}}_internal.json_tree_is_valid_elements_payload validates the +--! {{prefix}}.query_json_elements CHECK. +--! +--! @param val jsonb Candidate node. +--! @return boolean True when `val` is a node. +--! @note plpgsql: the recursion and the subqueries over the children cannot +--! be written as a CHECK expression. +CREATE OR REPLACE FUNCTION {{prefix}}_internal.json_tree_is_valid_predicate_payload(val jsonb) + RETURNS boolean + LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE +AS $$ +DECLARE + kind text; +BEGIN + IF jsonb_typeof(val) IS DISTINCT FROM 'object' + OR (SELECT count(*) FROM jsonb_object_keys(val)) <> 1 THEN + RETURN false; + END IF; + SELECT key INTO kind FROM jsonb_object_keys(val) AS key; + RETURN COALESCE( + CASE kind + WHEN 'and' THEN + jsonb_typeof(val -> 'and') = 'array' + AND jsonb_array_length(val -> 'and') >= 1 + AND NOT EXISTS ( + SELECT 1 FROM jsonb_array_elements(val -> 'and') AS child + WHERE NOT {{prefix}}_internal.json_tree_is_valid_predicate_payload(child) + ) + WHEN 'or' THEN + jsonb_typeof(val -> 'or') = 'array' + AND jsonb_array_length(val -> 'or') >= 1 + AND NOT EXISTS ( + SELECT 1 FROM jsonb_array_elements(val -> 'or') AS child + WHERE NOT {{prefix}}_internal.json_tree_is_valid_predicate_payload(child) + ) + WHEN 'not' THEN + {{prefix}}_internal.json_tree_is_valid_predicate_payload(val -> 'not') + WHEN 'contains' THEN + public.{{prefix}}_is_valid_json_tree_query_payload(val -> 'contains') + WHEN 'exists' THEN + jsonb_typeof(val -> 'exists') = 'string' + WHEN 'cmp' THEN + {{prefix}}_internal.json_tree_is_valid_bounds(val -> 'cmp') + WHEN 'elements' THEN + {{prefix}}_internal.json_tree_is_valid_elements_payload(val -> 'elements') + ELSE false + END, + false + ); +END; +$$; + +--! @brief Domain type for a compiled predicate on a JSON tree document. +--! +--! A boolean tree, serialised as JSON, whose leaves are the family's query +--! operands, as Stack Encrypt's `match` operator compiles a jsonpath +--! predicate into it: +--! - `{"and": [node, …]}`, `{"or": [node, …]}`, `{"not": node}`; +--! - `{"contains": {{prefix}}.query_json}`: the document contains the +--! needle, `col @> needle`; +--! - `{"exists": selector}`: the document has a node at the path, +--! `{{prefix}}.json_tree_path_exists(col, selector)`; +--! - `{"cmp": {"gt" | "gte": leaf, "lt" | "lte": leaf}}`: the first entry +--! at the bounds' selector against each bound, each leaf a +--! {{prefix}}.query_json_leaf, `(col -> s) > gt AND (col -> s) < lt`; +--! - `{"elements": {{prefix}}.query_json_elements}`: one element of an +--! array satisfies every member, +--! `{{prefix}}.json_tree_element_match(col, elements)`. +--! {{prefix}}.json_tree_match reads it. +--! +--! @note Construct from inline JSON via the DOMAIN cast: +--! `'{"and":[{"exists":"…"},{"contains":{"sv":[…],"el":[]}}]}'::{{prefix}}.query_json_predicate`. +--! An untyped literal on the right of `@@` does not resolve to the +--! domain: write the cast, `col @@ $1::{{prefix}}.query_json_predicate`. +--! @see {{prefix}}.json_tree_match +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_type + WHERE typname = 'query_json_predicate' AND typnamespace = '{{prefix}}'::regnamespace + ) THEN + CREATE DOMAIN {{prefix}}.query_json_predicate AS jsonb + CHECK ( + {{prefix}}_internal.json_tree_is_valid_predicate_payload(VALUE) + ); + END IF; + + COMMENT ON DOMAIN {{prefix}}.query_json_predicate IS 'EQL JSON tree query operand (compiled predicate)'; +END +$$; + +--! @brief One node of a compiled predicate, over a document already opened. +--! @internal +--! +--! `and`, `or` and `not` follow SQL's three-valued logic, as the operators +--! the leaves stand for would combine in a hand-written WHERE: `and` is +--! false when a child is false, else NULL when a child is NULL, else true; +--! `or` the dual; `not` NULL of NULL. Children are read in order and the +--! read stops at the first child that decides the answer, so an element +--! leaf the answer does not depend on is not read and cannot raise. +--! +--! Each leaf answers as its operator does on the column: +--! - `contains`: native `jsonb @>` of the document's projection over the +--! needle, and over its `alt` when it carries one, the rule +--! {{prefix}}.json_tree_contains applies (`col @> needle`), here over +--! the projection the caller read once; +--! - `exists`: {{prefix}}.json_tree_path_exists; +--! - `cmp`: the first entry at the selector, +--! {{prefix}}.json_tree_path_query_first (what `col -> s` returns), +--! against its bounds by {{prefix}}_internal.json_tree_within_bounds, +--! as an element member's entry is: NULL when the document has no node +--! at the path or the node has no ordering term; +--! - `elements`: {{prefix}}_internal.json_tree_element_match_opened over +--! the index the caller built once. +--! +--! @param d jsonb The document, decompressed. +--! @param sv jsonb Its entries, or NULL when the predicate has no element +--! leaf. +--! @param q jsonb Its containment projection. +--! @param iv bytea Its `iv`, or NULL when the predicate has no element +--! leaf. +--! @param tags jsonb Its self tags, as {{prefix}}_internal.json_tree_index +--! builds them, or NULL when the predicate has no element leaf. +--! @param node jsonb The node, valid. +--! @return boolean The node's answer: true, false or NULL. +--! @throws invalid_parameter_value As {{prefix}}.json_tree_element_match, +--! from an element leaf that is read. +--! @note Not STRICT: the parts a predicate does not need are NULL. +CREATE FUNCTION {{prefix}}_internal.json_tree_match_node(d jsonb, sv jsonb, q jsonb, iv bytea, tags jsonb, node jsonb) + RETURNS boolean + LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE +AS $$ +DECLARE + child jsonb; + answer boolean; + result boolean; + needle jsonb; + bounds jsonb; + entry public.{{prefix}}_json_entry; +BEGIN + IF node ? 'and' THEN + result := true; + FOR child IN SELECT jsonb_array_elements(node -> 'and') LOOP + answer := {{prefix}}_internal.json_tree_match_node(d, sv, q, iv, tags, child); + IF answer IS FALSE THEN + RETURN false; + ELSIF answer IS NULL THEN + result := NULL; + END IF; + END LOOP; + RETURN result; + ELSIF node ? 'or' THEN + result := false; + FOR child IN SELECT jsonb_array_elements(node -> 'or') LOOP + answer := {{prefix}}_internal.json_tree_match_node(d, sv, q, iv, tags, child); + IF answer IS TRUE THEN + RETURN true; + ELSIF answer IS NULL THEN + result := NULL; + END IF; + END LOOP; + RETURN result; + ELSIF node ? 'not' THEN + RETURN NOT {{prefix}}_internal.json_tree_match_node(d, sv, q, iv, tags, node -> 'not'); + ELSIF node ? 'contains' THEN + needle := node -> 'contains'; + RETURN q @> (needle - 'alt') + OR ((needle ? 'alt') AND q @> (needle -> 'alt')); + ELSIF node ? 'exists' THEN + RETURN {{prefix}}.json_tree_path_exists(d, node ->> 'exists'); + ELSIF node ? 'cmp' THEN + bounds := node -> 'cmp'; + entry := {{prefix}}.json_tree_path_query_first( + d, COALESCE(bounds -> 'gt', bounds -> 'gte', bounds -> 'lt', bounds -> 'lte') ->> 's'); + RETURN {{prefix}}_internal.json_tree_within_bounds(entry, bounds); + ELSE + RETURN {{prefix}}_internal.json_tree_element_match_opened(iv, sv, tags, node -> 'elements'); + END IF; +END; +$$; + +--! @brief Whether a document satisfies a compiled predicate. +--! +--! The one-parameter form of a filter over a JSON tree column, for a +--! compiler that targets plaintext `jsonb_path_match(col, $1, $2)`: Stack +--! Encrypt's `match` operator compiles the jsonpath predicate and its +--! values into `p`, and the tree is evaluated here, each leaf as its +--! operator answers on the column (see +--! {{prefix}}_internal.json_tree_match_node). The answer is the +--! hand-written WHERE's: `{"and": [{"exists": s}, {"cmp": {"lt": l}}]}` is +--! `{{prefix}}.json_tree_path_exists(col, s) AND (col -> s) < l`, NULL +--! where that is NULL. +--! +--! The document is decompressed once, and indexed once when the predicate +--! has an element leaf, however many leaves read it. The `exists` and +--! `cmp` leaves each scan the decompressed entries, as their functions do. +--! Whether the predicate has an element leaf is one walk of the predicate, +--! which is small, per row: the projection `q` is read without asking, as +--! one key of the decompressed document costs less than the walk. +--! +--! @param e public.{{prefix}}_json_search The stored document. +--! @param p {{prefix}}.query_json_predicate The predicate. +--! @return boolean The predicate's answer: true, false, or NULL as SQL's +--! three-valued logic gives it; NULL when either argument is NULL. +--! @throws invalid_parameter_value As {{prefix}}.json_tree_element_match, +--! from an element leaf that is read. +--! @note plpgsql, and so opaque to the planner: the call filters a scan, +--! and engages no index. Written out per operator, the same predicate +--! engages the GIN index on {{prefix}}.json_tree_query(col) and the +--! functional btrees on {{prefix}}.ord_term(col -> s), so narrow the +--! rows with an indexed predicate beside it, or write the operators. +--! @see {{prefix}}.query_json_predicate +CREATE FUNCTION {{prefix}}.json_tree_match(e public.{{prefix}}_json_search, p {{prefix}}.query_json_predicate) + RETURNS boolean + LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE +AS $$ +DECLARE + -- The document, decompressed once. + d jsonb := e::jsonb #> '{}'::text[]; + node jsonb := p::jsonb; + sv jsonb; + q jsonb; + iv bytea; + tags jsonb; +BEGIN + q := d -> 'q'; + IF node @? '$.**.elements' THEN + sv := d -> 'sv'; + iv := {{prefix}}_internal.json_tree_iv(d ->> 'h'); + SELECT idx.nodes INTO tags FROM {{prefix}}_internal.json_tree_index(d) AS idx; + END IF; + RETURN {{prefix}}_internal.json_tree_match_node(d, sv, q, iv, tags, node); +END; +$$; + +--! @brief @@ operator with a compiled predicate: {{prefix}}.json_tree_match. +--! +--! `col @@ $1::{{prefix}}.query_json_predicate`. The operand MUST carry the +--! domain's type, as a typed parameter or an explicit cast; `@@` with a +--! `jsonpath` operand is blocked (blockers.sql), since the database cannot +--! evaluate a path over ciphertext. +--! +--! @param a public.{{prefix}}_json_search The stored document. +--! @param b {{prefix}}.query_json_predicate The predicate. +--! @return boolean As {{prefix}}.json_tree_match. +CREATE FUNCTION {{prefix}}."@@"(a public.{{prefix}}_json_search, b {{prefix}}.query_json_predicate) + RETURNS boolean + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT {{prefix}}.json_tree_match(a, b) +$$; + +CREATE OPERATOR @@( + FUNCTION={{prefix}}."@@", + LEFTARG=public.{{prefix}}_json_search, + RIGHTARG={{prefix}}.query_json_predicate +); diff --git a/packages/eql/src/v3/json_tree/types.sql b/packages/eql/src/v3/json_tree/types.sql index e1cb324c0..7ba6d6d1a 100644 --- a/packages/eql/src/v3/json_tree/types.sql +++ b/packages/eql/src/v3/json_tree/types.sql @@ -20,6 +20,9 @@ --! - {{prefix}}.query_json_elements — an element-scoped predicate: per --! member of an array's elements, the --! member's read tokens and its bounds. +--! A sixth, {{prefix}}.query_json_predicate, a boolean tree whose leaves +--! are these operands, is in predicate.sql beside the function that reads +--! it. --! --! The design is `docs/plans/2026-10-17-json-tree-index.md` --! at the repository root. The document's per-entry fields: @@ -301,21 +304,56 @@ BEGIN END $$; +--! @brief Validate the bounds of an ordering at one path. +--! @internal +--! +--! Bounds are `{"gt" | "gte": leaf, "lt" | "lte": leaf}`: one or two +--! ordering operands of the shape `{{prefix}}.query_json_leaf` holds +--! (`{"s", "op"}`). At most one lower bound (`gt` or `gte`) and at most one +--! upper (`lt` or `lte`), at least one in all, and every bound at the one +--! selector: the path's. A member of an element-scoped predicate is bounds +--! with its tokens beside them, and the `cmp` leaf of a +--! `{{prefix}}.query_json_predicate` is bounds alone. +--! +--! @param val jsonb Candidate bounds. +--! @return boolean True when `val` is bounds. +--! @note plpgsql, for the subqueries over the bounds, which a CHECK +--! expression cannot hold. +CREATE OR REPLACE FUNCTION {{prefix}}_internal.json_tree_is_valid_bounds(val jsonb) + RETURNS boolean + LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE +AS $$ +BEGIN + RETURN COALESCE( + jsonb_typeof(val) = 'object' + AND val - ARRAY['gt', 'gte', 'lt', 'lte']::text[] = '{}'::jsonb + AND val ?| ARRAY['gt', 'gte', 'lt', 'lte']::text[] + AND NOT (val ? 'gt' AND val ? 'gte') + AND NOT (val ? 'lt' AND val ? 'lte') + -- Each bound by the `{{prefix}}.query_json_leaf` CHECK's own rule. + AND NOT EXISTS ( + SELECT 1 FROM jsonb_each(val) AS bound + WHERE NOT {{prefix}}_internal.json_tree_is_valid_leaf_query_payload(bound.value) + ) + AND (SELECT count(DISTINCT bound.value ->> 's') FROM jsonb_each(val) AS bound) = 1, + false + ); +END; +$$; + --! @brief Validate one member of an element-scoped predicate. --! @internal --! --! A member is `{"t": ["…", …], "gt" | "gte": leaf, "lt" | "lte": leaf}`: --! the tokens of a read of the member's path, one per element below the ---! wildcard bound in element order and then the probe, and one or two ---! bounds, each an ordering operand of the shape `{{prefix}}.query_json_leaf` ---! holds (`{"s", "op"}`). At most one lower bound (`gt` or `gte`) and at most ---! one upper (`lt` or `lte`), at least one in all, and every bound of the ---! member at the one selector: the member's path's. +--! wildcard bound in element order and then the probe, and the bounds of +--! the member's path beside them, as +--! {{prefix}}_internal.json_tree_is_valid_bounds validates them. --! --! @param val jsonb Candidate member. --! @return boolean True when `val` is a member. ---! @note plpgsql, for the subqueries over the tokens and bounds, which a ---! CHECK expression cannot hold. +--! @note plpgsql, for the subquery over the tokens, which a CHECK +--! expression cannot hold. CREATE OR REPLACE FUNCTION {{prefix}}_internal.json_tree_is_valid_elements_member(val jsonb) RETURNS boolean LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE @@ -323,22 +361,13 @@ AS $$ BEGIN RETURN COALESCE( jsonb_typeof(val) = 'object' - AND val - ARRAY['t', 'gt', 'gte', 'lt', 'lte']::text[] = '{}'::jsonb AND jsonb_typeof(val -> 't') = 'array' AND jsonb_array_length(val -> 't') >= 1 AND NOT EXISTS ( SELECT 1 FROM jsonb_array_elements(val -> 't') AS token WHERE jsonb_typeof(token) <> 'string' ) - AND val ?| ARRAY['gt', 'gte', 'lt', 'lte']::text[] - AND NOT (val ? 'gt' AND val ? 'gte') - AND NOT (val ? 'lt' AND val ? 'lte') - -- Each bound by the `{{prefix}}.query_json_leaf` CHECK's own rule. - AND NOT EXISTS ( - SELECT 1 FROM jsonb_each(val - 't') AS bound - WHERE NOT {{prefix}}_internal.json_tree_is_valid_leaf_query_payload(bound.value) - ) - AND (SELECT count(DISTINCT bound.value ->> 's') FROM jsonb_each(val - 't') AS bound) = 1, + AND {{prefix}}_internal.json_tree_is_valid_bounds(val - 't'), false ); END; diff --git a/packages/eql/tests/encryption/tests/json_tree.rs b/packages/eql/tests/encryption/tests/json_tree.rs index a1d506cf5..e3c69c9a8 100644 --- a/packages/eql/tests/encryption/tests/json_tree.rs +++ b/packages/eql/tests/encryption/tests/json_tree.rs @@ -520,6 +520,11 @@ mod native_functions; #[path = "json_tree/elements.rs"] mod elements; +/// The compiled predicate, through a data plan, against the column and +/// against `jsonb_path_match` over the plaintexts. +#[path = "json_tree/predicate.rs"] +mod predicate; + /// Needs an EQL v4 installation. Extraction by selector opens to the leaf; /// the subtree walk by a client-derived token returns exactly the nodes /// under it and reassembles; the array functions count and list elements; diff --git a/packages/eql/tests/encryption/tests/json_tree/elements.rs b/packages/eql/tests/encryption/tests/json_tree/elements.rs index 3b9c01abe..5e4ea4359 100644 --- a/packages/eql/tests/encryption/tests/json_tree/elements.rs +++ b/packages/eql/tests/encryption/tests/json_tree/elements.rs @@ -1,7 +1,12 @@ //! The element-scoped predicate end to end: documents sealed by Stack //! Encrypt's JSON index, the `element_match` operand derived through a data -//! plan over eql-bindings' target table (the path the Go guest takes), and -//! `eql_v4.json_tree_element_match` over the stored column. +//! plan over eql-bindings' target table, and +//! `eql_v4.json_tree_element_match` over the stored column. The resolver +//! here ([`Eql`]) names the document's leaf, element-scoped and compiled +//! predicate domains, so the engine answers `element_match` and `match` +//! through it. The Go guest's resolver names the leaf domain alone (its +//! `targets` tests pin that), so the Go SDK is not offered either operator +//! until the Go side takes them. //! //! These tests pin: //! @@ -26,10 +31,11 @@ use stack_encrypt::dynamic::{ }; use stack_encrypt::{KeysetCipher, Label, Pending, StackCipher}; -/// eql-bindings' table as the engine's resolver, naming each type's leaf -/// and element predicate domains, as a host that answers `element_match` -/// describes it. -struct Eql; +/// eql-bindings' table as the engine's resolver, naming each type's leaf, +/// element predicate and compiled predicate domains: a host that answers +/// `element_match` and `match`. Not the Go guest's resolver, which names +/// the leaf domain alone. +pub(super) struct Eql; fn describe(target: &Target) -> TargetDescriptor { let mut descriptor = TargetDescriptor::new( @@ -50,6 +56,9 @@ fn describe(target: &Target) -> TargetDescriptor { if let Some(domain) = target.elements_query_sql_domain() { descriptor = descriptor.with_elements_query_sql_domain(domain); } + if let Some(domain) = target.predicate_query_sql_domain() { + descriptor = descriptor.with_predicate_query_sql_domain(domain); + } descriptor } @@ -116,13 +125,13 @@ fn contact_plan() -> record::Plan { record::plan_with(&dynamic::frame::serde::to_vec(&wire).unwrap(), &Eql).unwrap() } -fn frame_of(value: &Value) -> ValueFrame { +pub(super) fn frame_of(value: &Value) -> ValueFrame { ValueFrame::from_bytes(&dynamic::frame::serde::to_vec(value).unwrap()).unwrap() } /// The operand of `operator` on `profile` for `input`, as the SQL binds /// it: an EQL operand's JSON text, or a list of tokens. -fn derive( +pub(super) fn derive( runtime: &tokio::runtime::Runtime, keyset: &KeysetCipher<'_, common::Observed>, plan: &record::Plan, @@ -145,7 +154,7 @@ fn derive( term.operand().clone() } -fn eql_text(operand: &Operand) -> String { +pub(super) fn eql_text(operand: &Operand) -> String { match operand { Operand::One(bytes) => String::from_utf8(bytes.clone()).unwrap(), other => panic!("one EQL operand, got {other:?}"), diff --git a/packages/eql/tests/encryption/tests/json_tree/predicate.rs b/packages/eql/tests/encryption/tests/json_tree/predicate.rs new file mode 100644 index 000000000..202937053 --- /dev/null +++ b/packages/eql/tests/encryption/tests/json_tree/predicate.rs @@ -0,0 +1,1006 @@ +//! The compiled predicate end to end: documents sealed by Stack Encrypt's +//! JSON index, jsonpath predicates compiled by the `match` operator through +//! a data plan over eql-bindings' target table, with a resolver that names +//! the compiled predicate domain ([`Eql`]; the Go guest's does not yet), +//! and `eql_v4.json_tree_match` and `@@` over the stored column, +//! against PostgreSQL's own `jsonb_path_match` over the plaintexts and +//! against the WHERE a hand writes from the operators the leaves stand for. + +use super::elements::{derive, eql_text, frame_of, Eql}; +use super::*; + +use stack_encrypt::dynamic::{self, record, OpSpec, Operand}; +use stack_encrypt::KeysetCipher; + +/// The paths the documents order: what the predicates' ranges ask at. +const ORDERED: [&str; 4] = [ + "$.age", + "$.name", + "$.contacts[*].calls", + "$.contacts[*].relationship", +]; + +fn people_options() -> JsonOptions { + JsonOptions::default().ordering(ORDERED).unwrap() +} + +/// `profile` under `users/profile`, the column the documents are sealed +/// under, naming `JsonSearch` with [`people_options`]. +fn people_plan() -> record::Plan { + let wire = json!({ + "profile": { + "context": ["users", "profile"], + "type": "json", + "null": "sql", + "target": {"name": "JsonSearch", "options": {"ordered": ORDERED}} + } + }); + record::plan_with(&dynamic::frame::serde::to_vec(&wire).unwrap(), &Eql).unwrap() +} + +/// Documents with every field absent, `null` or a value of the one type its +/// path holds, and arrays only where the predicates name `[*]`. +fn people() -> Vec { + vec![ + json!({"age": 34, "name": "ann", "tags": ["admin", "ops"], + "contacts": [{"relationship": "sibling", "calls": 3}, + {"relationship": "friend", "calls": 7}]}), + json!({"age": 17, "name": "bob", + "contacts": [{"relationship": "friend", "calls": 9}, + {"relationship": "sibling", "calls": 7}]}), + json!({"age": null, "name": "cat", "tags": []}), + json!({"name": null, "contacts": []}), + json!({}), + json!({"age": 60, "tags": ["ops"], "contacts": [{"relationship": "sibling"}]}), + json!({"age": 18, "name": "dan", "ok": true}), + ] +} + +/// Each row's answer under `sql`, which takes `params`, NULL as `None`. +fn answers( + db: &mut postgres::Client, + sql: &str, + params: &[&(dyn postgres::types::ToSql + Sync)], +) -> Vec<(i32, Option)> { + db.query(sql, params) + .unwrap_or_else(|error| panic!("{sql}: {error}")) + .iter() + .map(|row| (row.get(0), row.get(1))) + .collect() +} + +/// `match`'s operand for `predicate` under `vars`. +fn compiled( + runtime: &tokio::runtime::Runtime, + keyset: &KeysetCipher<'_, common::Observed>, + plan: &record::Plan, + predicate: &str, + vars: &Value, +) -> String { + eql_text(&derive( + runtime, + keyset, + plan, + "match", + json!({"predicate": predicate, "vars": vars.to_string(), "wildcard_bound": 4}), + )) +} + +/// Needs an EQL v4 installation. For each predicate, `json_tree_match` and +/// `@@` over the encrypted column are true of the rows `jsonb_path_match` +/// is true of, for the same predicate and variables over the plaintexts, +/// and answer as it does row by row, save NULL for false where an ordering +/// over a `null` sends no `null` needle ([`assert_jsonpath_rows`]): +/// equality, `!=` and the orderings with absent paths and JSON `null`s, +/// wildcards, filters, and `&&`, `||` and `!` over them. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_a_compiled_predicate_answers_as_jsonb_path_match() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = people_plan(); + let mut db = connect(); + store( + &mut db, + &runtime, + &cipher, + "json_tree_people", + &people_options(), + &people(), + ); + let vars = json!({"min": 18, "who": "ann", "r": "sibling", "n": 5, "tag": "ops"}); + for predicate in [ + "$.age >= $min", + "!($.age >= $min)", + "$.age < 18", + "$.age == null", + "$.age != null", + "$.age != 34", + "$.name == $who", + "$.name != $who", + "$.name > \"bob\"", + "!($.name <= \"bob\")", + "$.ok == true", + "exists($.tags)", + "exists($.tags[*])", + "$.tags[*] == $tag", + "exists($.tags[*] ? (@ == $tag))", + "$.contacts[*].calls > $n", + "$.contacts[*].relationship == $r", + "exists($.contacts[*] ? (@.relationship == $r))", + "exists($.contacts[*] ? (@.relationship == $r && @.calls > $n))", + "!exists($.contacts[*] ? (@.relationship == $r && @.calls > $n))", + "$.age >= $min && exists($.contacts[*] ? (@.relationship == $r && @.calls > $n))", + "$.name == $who || $.tags[*] == $tag", + "!($.age < $min || $.name == null) && exists($.name)", + "$.age > 20 && $.age < 50", + ] { + let operand = compiled(&runtime, &keyset, &plan, predicate, &vars); + let matched = answers( + &mut db, + "SELECT id, eql_v4.json_tree_match(doc, $1::text::jsonb::eql_v4.query_json_predicate) \ + FROM json_tree_people ORDER BY id", + &[&operand], + ); + let by_operator = answers( + &mut db, + "SELECT id, doc @@ $1::text::jsonb::eql_v4.query_json_predicate \ + FROM json_tree_people ORDER BY id", + &[&operand], + ); + let plaintext = answers( + &mut db, + "SELECT id, jsonb_path_match(plain, $1::text::jsonpath, $2::text::jsonb) \ + FROM json_tree_people ORDER BY id", + &[&predicate, &vars.to_string()], + ); + assert_jsonpath_rows(predicate, &matched, &plaintext); + assert_eq!(by_operator, matched, "{predicate}: @@ is json_tree_match"); + } +} + +/// Needs an EQL v4 installation. A compiled predicate answers what the +/// hand-written WHERE answers, NULL included, each leaf by the operand its +/// own operator derives: `json_tree_path_exists` for the path, `@>` for +/// the `null` needle and the equalities, `->` and `>=` for the ordering, +/// and `json_tree_element_match` for the filter. And what `match` refuses +/// it refuses before anything is derived, at the offset of the construct. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_a_compiled_predicate_answers_as_its_operators() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = people_plan(); + let mut db = connect(); + store( + &mut db, + &runtime, + &cipher, + "json_tree_operators", + &people_options(), + &people(), + ); + let one = + |operator: &str, input: Value| eql_text(&derive(&runtime, &keyset, &plan, operator, input)); + let Operand::Text(age) = derive( + &runtime, + &keyset, + &plan, + "selector", + json!({"path": "$.age"}), + ) else { + panic!("a selector is text") + }; + let null_age = one("eq_at", json!({"path": "$.age", "json": "null"})); + let at_least = one("gte_at", json!({"path": "$.age", "json": "18"})); + let sibling_over_five = one( + "element_match", + json!({"path": "$.contacts[*]", "wildcard_bound": 4, "predicates": [ + {"member": "relationship", "op": "eq", "json": "\"sibling\""}, + {"member": "calls", "op": "gt", "json": "5"}, + ]}), + ); + let ann = one("eq_at", json!({"path": "$.name", "json": "\"ann\""})); + + let predicate = + "!($.age >= 18 && exists($.contacts[*] ? (@.relationship == \"sibling\" && @.calls > 5))) \ + || $.name == \"ann\""; + let operand = compiled(&runtime, &keyset, &plan, predicate, &json!({})); + let matched = answers( + &mut db, + "SELECT id, eql_v4.json_tree_match(doc, $1::text::jsonb::eql_v4.query_json_predicate) \ + FROM json_tree_operators ORDER BY id", + &[&operand], + ); + let written = answers( + &mut db, + "SELECT id, (NOT (eql_v4.json_tree_path_exists(doc, $1::text) \ + AND NOT (doc @> $2::text::jsonb::eql_v4.query_json) \ + AND (doc -> $1::text) >= $3::text::jsonb::eql_v4.query_json_leaf \ + AND eql_v4.json_tree_element_match(doc, $4::text::jsonb::eql_v4.query_json_elements))) \ + OR doc @> $5::text::jsonb::eql_v4.query_json \ + FROM json_tree_operators ORDER BY id", + &[&age, &null_age, &at_least, &sibling_over_five, &ann], + ); + assert_eq!( + matched, written, + "the tree against the operators written out" + ); + let plaintext = answers( + &mut db, + "SELECT id, jsonb_path_match(plain, $1::text::jsonpath) FROM json_tree_operators ORDER BY id", + &[&predicate], + ); + assert_eq!(matched, plaintext, "and against jsonb_path_match"); + + for (text, reason, at) in [ + ("$.age + 1 > 18", "json_predicate_unsupported", 6), + ("$.name like_regex \"^a\"", "json_predicate_unsupported", 7), + ("$.age.size() > 1", "json_predicate_unsupported", 5), + ( + "$.tags[*] != \"ops\"", + "json_predicate_not_equal_at_wildcard", + 10, + ), + ("$.age > $missing", "json_predicate_unknown_variable", 8), + ] { + let error = dynamic::check_query( + &plan, + "profile", + OpSpec::Match, + frame_of(&json!({"predicate": text})), + &Eql, + ) + .unwrap_err(); + let payload = stack_encrypt::ErrorPayload::payload(&error); + assert_eq!(payload["reason"], json!(reason), "{text}: {payload:?}"); + assert_eq!(payload["at"], json!(at), "{text}: {payload:?}"); + } + let error = dynamic::check_query( + &plan, + "profile", + OpSpec::Match, + frame_of(&json!({"predicate": "$.height > 1"})), + &Eql, + ) + .unwrap_err(); + assert!( + matches!( + error, + dynamic::Error::Cipher(stack_encrypt::Error::Term( + stack_encrypt::sem::TermError::JsonPathNotOrdered + )) + ), + "a range at a path the column does not order: {error:?}" + ); +} + +/// SQLSTATE `check_violation`, a value its domain's CHECK refuses. +const CHECK_VIOLATION: &str = "23514"; +/// SQLSTATE `raise_exception`, which an operator blocker raises. +const RAISE_EXCEPTION: &str = "P0001"; +/// SQLSTATE `ambiguous_function`, an operator PostgreSQL cannot choose. +const AMBIGUOUS_FUNCTION: &str = "42725"; +/// SQLSTATE `invalid_parameter_value`, which a probe's hit raises with. +const INVALID_PARAMETER_VALUE: &str = "22023"; + +/// The SQLSTATE a statement fails with, or `None` if it succeeds. +fn sqlstate( + db: &mut postgres::Client, + sql: &str, + params: &[&(dyn postgres::types::ToSql + Sync)], +) -> Option { + match db.query(sql, params) { + Ok(_) => None, + Err(error) => match error.as_db_error() { + Some(e) => Some(e.code().code().to_owned()), + None => panic!("{sql}: not a database error: {error}"), + }, + } +} + +/// A JSON value as an SQL string literal. +fn literal(value: &Value) -> String { + let text = match value { + Value::String(text) => text.clone(), + other => other.to_string(), + }; + format!("'{}'", text.replace('\'', "''")) +} + +/// The WHERE a hand writes from the operators a compiled tree's leaves +/// stand for, over the column `doc`, each leaf's operand inlined: `@>` +/// for `contains`, `json_tree_path_exists` for `exists`, `->` and the +/// orderings for `cmp`, and `json_tree_element_match` for `elements`. +fn written(tree: &Value) -> String { + let node = tree.as_object().expect("a node is an object"); + assert_eq!(node.len(), 1, "a node has one key: {tree}"); + let (kind, operand) = node.iter().next().unwrap(); + let list = |join: &str| { + let parts: Vec = operand.as_array().unwrap().iter().map(written).collect(); + format!("({})", parts.join(join)) + }; + match kind.as_str() { + "and" => list(" AND "), + "or" => list(" OR "), + "not" => format!("(NOT {})", written(operand)), + "contains" => format!("(doc @> {}::jsonb::eql_v4.query_json)", literal(operand)), + "exists" => format!( + "eql_v4.json_tree_path_exists(doc, {}::text)", + literal(operand) + ), + "cmp" => { + let parts: Vec = operand + .as_object() + .unwrap() + .iter() + .map(|(bound, leaf)| { + let op = match bound.as_str() { + "gt" => ">", + "gte" => ">=", + "lt" => "<", + "lte" => "<=", + other => panic!("a bound named {other}"), + }; + format!( + "((doc -> {}::text) {op} {}::jsonb::eql_v4.query_json_leaf)", + literal(&leaf["s"]), + literal(leaf) + ) + }) + .collect(); + format!("({})", parts.join(" AND ")) + } + "elements" => format!( + "eql_v4.json_tree_element_match(doc, {}::jsonb::eql_v4.query_json_elements)", + literal(operand) + ), + other => panic!("a node of kind {other}"), + } +} + +/// The first node of `kind` in `tree`, depth first. +fn find<'a>(tree: &'a Value, kind: &str) -> Option<&'a Value> { + let (key, operand) = tree.as_object()?.iter().next()?; + if key == kind { + return Some(tree); + } + match operand { + Value::Array(children) => children.iter().find_map(|child| find(child, kind)), + Value::Object(_) if key == "not" => find(operand, kind), + _ => None, + } +} + +/// Each row's answer under the compiled `operand` by `json_tree_match`, +/// by `@@`, and under the WHERE written from it, in `table`. +fn three_ways( + db: &mut postgres::Client, + table: &str, + operand: &str, +) -> [Vec<(i32, Option)>; 3] { + let tree: Value = serde_json::from_str(operand).unwrap(); + [ + answers( + db, + &format!( + "SELECT id, eql_v4.json_tree_match(doc, $1::text::jsonb::eql_v4.query_json_predicate) \ + FROM {table} ORDER BY id" + ), + &[&operand], + ), + answers( + db, + &format!( + "SELECT id, doc @@ $1::text::jsonb::eql_v4.query_json_predicate FROM {table} ORDER BY id" + ), + &[&operand], + ), + answers( + db, + &format!("SELECT id, {} FROM {table} ORDER BY id", written(&tree)), + &[], + ), + ] +} + +/// Each row's answer under `jsonb_path_match` of `predicate` over the +/// plaintexts in `table`. +/// Assert that a compiled predicate's answers are true of exactly the rows +/// `jsonb_path_match` is true of, and are its answers save NULL where it +/// says false: an ordering over a `null` under an even number of `!` sends +/// no `null` needle, and is NULL there. +fn assert_jsonpath_rows( + predicate: &str, + compiled: &[(i32, Option)], + plaintext: &[(i32, Option)], +) { + let rows = |answers: &[(i32, Option)]| -> Vec { + answers + .iter() + .filter(|(_, answer)| *answer == Some(true)) + .map(|(id, _)| *id) + .collect() + }; + assert_eq!( + rows(compiled), + rows(plaintext), + "{predicate}: jsonb_path_match's rows" + ); + assert_eq!( + compiled.len(), + plaintext.len(), + "{predicate}: one answer a row" + ); + for ((id, got), (_, want)) in compiled.iter().zip(plaintext) { + assert!( + got == want || (*want == Some(false) && got.is_none()), + "{predicate}: row {id} answers {got:?}, jsonb_path_match {want:?}" + ); + } +} + +fn plaintext(db: &mut postgres::Client, table: &str, predicate: &str) -> Vec<(i32, Option)> { + answers( + db, + &format!("SELECT id, jsonb_path_match(plain, $1::text::jsonpath) FROM {table} ORDER BY id"), + &[&predicate], + ) +} + +/// Needs an EQL v4 installation. The predicate domain's CHECK takes +/// one-key nodes, each kind's operand by its own domain's rule, and refuses +/// everything else. The values are operands the engine derived and their +/// parts: selectors, a needle, a root scalar's needle with its `alt`, leaf +/// operands at two paths, an element predicate and one of its tokens. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_the_predicate_domain_checks_each_node() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = people_plan(); + let mut db = connect(); + let one = |operator: &str, input: Value| -> Value { + serde_json::from_str(&eql_text(&derive( + &runtime, &keyset, &plan, operator, input, + ))) + .unwrap() + }; + let Operand::Text(selector) = derive( + &runtime, + &keyset, + &plan, + "selector", + json!({"path": "$.age"}), + ) else { + panic!("a selector is text") + }; + let needle = one("eq_at", json!({"path": "$.name", "json": "\"ann\""})); + let root_one = one("contains", json!({"json": "1"})); + assert!( + root_one.get("alt").is_some(), + "a root scalar's needle has an alt" + ); + let leaf = one("gt_at", json!({"path": "$.age", "json": "18"})); + let other = one("gt_at", json!({"path": "$.name", "json": "\"a\""})); + let elements = one( + "element_match", + json!({"path": "$.contacts[*]", "wildcard_bound": 1, "predicates": [ + {"member": "calls", "op": "gt", "json": "5"} + ]}), + ); + let token = &elements["m"][0]["t"][0]; + let compiled_tree: Value = serde_json::from_str(&compiled( + &runtime, + &keyset, + &plan, + "$.age >= 18 && (exists($.tags) || !($.name == \"ann\")) \ + && exists($.contacts[*] ? (@.calls > 5))", + &json!({}), + )) + .unwrap(); + let without = |value: &Value, key: &str| { + let mut value = value.clone(); + let _ = value.as_object_mut().unwrap().remove(key); + value + }; + + let valid = [ + compiled_tree, + json!({"exists": selector}), + json!({"contains": needle}), + json!({"contains": root_one}), + json!({"cmp": {"gt": leaf}}), + json!({"cmp": {"gte": leaf, "lt": leaf}}), + json!({"elements": elements}), + json!({"not": {"exists": selector}}), + json!({"and": [{"exists": selector}, {"or": [{"cmp": {"lte": leaf}}]}]}), + ]; + let invalid = [ + ("not an object", json!([])), + ("no key", json!({})), + ( + "two keys", + json!({"exists": selector, "not": {"exists": selector}}), + ), + ("an unknown kind", json!({"xor": []})), + ("an empty and", json!({"and": []})), + ("an empty or", json!({"or": []})), + ("an and not a list", json!({"and": {"exists": selector}})), + ("an invalid child", json!({"or": [{"exists": 1}]})), + ( + "a deep invalid child", + json!({"not": {"not": {"and": [{}]}}}), + ), + ("not of a list", json!({"not": [{"exists": selector}]})), + ("exists not text", json!({"exists": [selector]})), + ( + "a needle missing el", + json!({"contains": without(&needle, "el")}), + ), + ( + "a needle with a bad pair", + json!({"contains": {"sv": [{"s": 1}], "el": []}}), + ), + ("no bound", json!({"cmp": {}})), + ("a bound by another name", json!({"cmp": {"eq": leaf}})), + ( + "two lower bounds", + json!({"cmp": {"gt": leaf, "gte": leaf}}), + ), + ( + "bounds at two selectors", + json!({"cmp": {"gt": leaf, "lt": other}}), + ), + ( + "a bound with no term", + json!({"cmp": {"gt": without(&leaf, "op")}}), + ), + ( + "cmp with tokens", + json!({"cmp": {"t": [token], "gt": leaf}}), + ), + ("elements with no members", json!({"elements": {"m": []}})), + ( + "elements of bounds alone", + json!({"elements": {"m": [{"gt": leaf}]}}), + ), + ]; + let cast = "SELECT $1::text::jsonb::eql_v4.query_json_predicate"; + for value in &valid { + assert_eq!( + sqlstate(&mut db, cast, &[&value.to_string()]), + None, + "valid: {value}" + ); + } + for (why, value) in &invalid { + assert_eq!( + sqlstate(&mut db, cast, &[&value.to_string()]).as_deref(), + Some(CHECK_VIOLATION), + "{why}: {value}" + ); + } + // The element member's CHECK, through the bounds validator the two + // domains share, refuses what it refused. + let mut member_and_more = elements["m"][0].clone(); + let _ = member_and_more + .as_object_mut() + .unwrap() + .insert("x".into(), json!(1)); + for (why, value) in [ + ("no tokens", json!({"m": [{"gt": leaf}]})), + ("no bound", json!({"m": [{"t": [token]}]})), + ("another key", json!({"m": [member_and_more]})), + ] { + assert_eq!( + sqlstate( + &mut db, + "SELECT $1::text::jsonb::eql_v4.query_json_elements", + &[&value.to_string()] + ) + .as_deref(), + Some(CHECK_VIOLATION), + "{why}: {value}" + ); + } +} + +/// Needs an EQL v4 installation. For every predicate over every document: +/// `json_tree_match` and `@@` answer what the WHERE written from the +/// compiled tree's leaves answers, NULL included, and all three are true +/// of exactly the rows `jsonb_path_match` is true of over the plaintext the +/// document was sealed from. Their answer is `jsonb_path_match`'s, save +/// that it is NULL where jsonpath says false of an ordering over a `null` +/// under an even number of `!`, which sends no `null` needle and picks the +/// same rows. Every field is absent, `null` or a value of the one type its +/// path holds. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_a_compiled_predicate_answers_as_the_operators_written_out() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = people_plan(); + let mut db = connect(); + store( + &mut db, + &runtime, + &cipher, + "json_tree_written", + &people_options(), + &[ + json!({"age": 1, "name": "ann", "tags": ["a", "b"]}), + json!({"age": 3, "name": "bob"}), + json!({"age": 5, "tags": []}), + json!({"age": null, "name": null}), + json!({}), + json!({"name": "dan", "tags": ["c"]}), + json!({"age": 3, "ok": true, "tags": ["a"]}), + ], + ); + for predicate in [ + "$.age == 3", + "$.age != 3", + "$.age == null", + "$.age != null", + "$.age < 3", + "!($.age < 3)", + "$.age >= 3", + "$.name > \"ann\"", + "$.name != \"ann\"", + "$.ok == true", + "$.ok != true", + "exists($.tags)", + "exists($.tags[*])", + "$.tags[*] == \"a\"", + "exists($.tags[*] ? (@ == \"c\"))", + "$.age > 1 && $.name == \"bob\" || !exists($.tags)", + "$.age >= 1 && $.age <= 3", + "!($.age >= 2 || $.name == \"dan\")", + "!(!($.age < 3))", + "!($.age < 3) && exists($.name)", + "!(!($.age > 1) && exists($.age))", + ] { + let operand = compiled(&runtime, &keyset, &plan, predicate, &json!({})); + let [matched, by_operator, by_hand] = three_ways(&mut db, "json_tree_written", &operand); + assert_eq!(matched, by_hand, "{predicate}: against the WHERE"); + assert_eq!(by_operator, matched, "{predicate}: @@ is json_tree_match"); + let plain = plaintext(&mut db, "json_tree_written", predicate); + assert_jsonpath_rows(predicate, &matched, &plain); + } +} + +/// Needs an EQL v4 installation. A NULL leaf follows SQL's three-valued +/// logic: an ordering of a node with no ordering term (a boolean at an +/// ordered path) is NULL, and so is `not` of it; `and` with a false child +/// is false and `or` with a true one true. jsonpath answers each the same: +/// a boolean against a number is unknown. The ordering leaf alone, without +/// the guards `match` puts beside it, is NULL also where the path is +/// absent, as `(col -> s) < $1` is. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_a_null_leaf_is_unknown_under_and_or_and_not() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = people_plan(); + let mut db = connect(); + store( + &mut db, + &runtime, + &cipher, + "json_tree_unknown", + &people_options(), + &[json!({"age": true, "name": "eve"})], + ); + for (predicate, want) in [ + ("$.age < 3", None), + ("!($.age < 3)", None), + ("$.age < 3 && exists($.name)", None), + ("$.age < 3 && $.name == \"x\"", Some(false)), + ("$.age < 3 || exists($.name)", Some(true)), + ("$.age < 3 || $.name == \"x\"", None), + ] { + let operand = compiled(&runtime, &keyset, &plan, predicate, &json!({})); + let [matched, by_operator, by_hand] = three_ways(&mut db, "json_tree_unknown", &operand); + assert_eq!(matched, [(1, want)], "{predicate}"); + assert_eq!(by_operator, matched, "{predicate}: @@ is json_tree_match"); + assert_eq!(by_hand, matched, "{predicate}: against the WHERE"); + assert_eq!( + plaintext(&mut db, "json_tree_unknown", predicate), + matched, + "{predicate}: against jsonb_path_match" + ); + } + + let compiled_lt: Value = + serde_json::from_str(&compiled(&runtime, &keyset, &plan, "$.age < 3", &json!({}))).unwrap(); + let cmp = find(&compiled_lt, "cmp") + .expect("an ordering compiles to a cmp leaf") + .to_string(); + store( + &mut db, + &runtime, + &cipher, + "json_tree_cmp_alone", + &people_options(), + &[ + json!({}), + json!({"age": null}), + json!({"age": 1}), + json!({"age": 4}), + ], + ); + assert_eq!( + answers( + &mut db, + "SELECT id, eql_v4.json_tree_match(doc, $1::text::jsonb::eql_v4.query_json_predicate) \ + FROM json_tree_cmp_alone ORDER BY id", + &[&cmp], + ), + [(1, None), (2, None), (3, Some(true)), (4, Some(false))], + "the cmp leaf alone" + ); +} + +/// Needs an EQL v4 installation. A `contains` leaf is `@>`, the needle's +/// `alt` included: a root scalar matches a root array that holds it. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_a_contains_leaf_is_the_containment_operator() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = people_plan(); + let mut db = connect(); + store( + &mut db, + &runtime, + &cipher, + "json_tree_roots", + &people_options(), + &[json!([1, 2]), json!(1), json!({"a": [1]}), json!([[1]])], + ); + let needle = |json: &str| { + eql_text(&derive( + &runtime, + &keyset, + &plan, + "contains", + json!({"json": json}), + )) + }; + let root_one = needle("1"); + for json in ["1", "[1]", "{\"a\": [1]}", "[[1]]", "[]"] { + let held = needle(json); + let tree = format!(r#"{{"contains": {held}}}"#); + let matched = answers( + &mut db, + "SELECT id, eql_v4.json_tree_match(doc, $1::text::jsonb::eql_v4.query_json_predicate) \ + FROM json_tree_roots ORDER BY id", + &[&tree], + ); + let contained = answers( + &mut db, + "SELECT id, doc @> $1::text::jsonb::eql_v4.query_json FROM json_tree_roots ORDER BY id", + &[&held], + ); + assert_eq!(matched, contained, "{json}"); + } + assert_eq!( + answers( + &mut db, + "SELECT id, eql_v4.json_tree_match(doc, $1::text::jsonb::eql_v4.query_json_predicate) \ + FROM json_tree_roots ORDER BY id", + &[&format!(r#"{{"contains": {root_one}}}"#)], + ), + [ + (1, Some(true)), + (2, Some(true)), + (3, Some(false)), + (4, Some(false)) + ], + "the root scalar, and a root array holding it through `alt`" + ); +} + +/// Needs an EQL v4 installation. An `elements` leaf is +/// `json_tree_element_match` over the document the tree opened once; a leaf +/// the answer does not depend on is not read, so it cannot raise, and one +/// the answer depends on raises as the function does. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_an_elements_leaf_is_the_element_match() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = people_plan(); + let mut db = connect(); + store( + &mut db, + &runtime, + &cipher, + "json_tree_calls", + &people_options(), + &[ + json!({"contacts": [{"calls": 3}, {"calls": 7}]}), + json!({"contacts": [{"calls": 1}]}), + json!({"contacts": []}), + ], + ); + let calls_over = |n: i64, bound: u32| { + eql_text(&derive( + &runtime, + &keyset, + &plan, + "element_match", + json!({"path": "$.contacts[*]", "wildcard_bound": bound, "predicates": [ + {"member": "calls", "op": "gt", "json": n.to_string()} + ]}), + )) + }; + let selector = + |path: &str| match derive(&runtime, &keyset, &plan, "selector", json!({"path": path})) { + Operand::Text(selector) => selector, + other => panic!("a selector is text, got {other:?}"), + }; + let over_five = calls_over(5, 2); + let tree_match = |db: &mut postgres::Client, tree: &str| { + answers( + db, + "SELECT id, eql_v4.json_tree_match(doc, $1::text::jsonb::eql_v4.query_json_predicate) \ + FROM json_tree_calls ORDER BY id", + &[&tree], + ) + }; + let matched = tree_match(&mut db, &format!(r#"{{"elements": {over_five}}}"#)); + let element = answers( + &mut db, + "SELECT id, eql_v4.json_tree_element_match(doc, $1::text::jsonb::eql_v4.query_json_elements) \ + FROM json_tree_calls ORDER BY id", + &[&over_five], + ); + assert_eq!(matched, element, "the element match itself"); + assert_eq!( + matched, + [(1, Some(true)), (2, Some(false)), (3, Some(false))] + ); + let two_leaves = format!( + r#"{{"and": [{{"elements": {over_five}}}, {{"not": {{"elements": {}}}}}]}}"#, + calls_over(7, 2) + ); + assert_eq!( + tree_match(&mut db, &two_leaves), + [(1, Some(true)), (2, Some(false)), (3, Some(false))], + "two element leaves over one opened document" + ); + + // A bound of one: the two-contact row is longer than the bound. + let raising = format!(r#"{{"elements": {}}}"#, calls_over(5, 1)); + let present = json!({"exists": selector("$.contacts")}).to_string(); + let absent = json!({"exists": selector("$.none")}).to_string(); + let on_row_one = + "SELECT eql_v4.json_tree_match(doc, $1::text::jsonb::eql_v4.query_json_predicate) \ + FROM json_tree_calls WHERE id = 1"; + for (tree, want) in [ + (format!(r#"{{"and": [{absent}, {raising}]}}"#), None), + (format!(r#"{{"or": [{present}, {raising}]}}"#), None), + ( + format!(r#"{{"and": [{present}, {raising}]}}"#), + Some(INVALID_PARAMETER_VALUE), + ), + (raising.clone(), Some(INVALID_PARAMETER_VALUE)), + ] { + assert_eq!( + sqlstate(&mut db, on_row_one, &[&tree]).as_deref(), + want, + "{tree}" + ); + } +} + +/// Needs an EQL v4 installation. `@@` takes a typed predicate; with a +/// `jsonpath` it is still blocked; an untyped operand resolves to neither +/// silently; and a NULL argument is NULL. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_the_operator_takes_a_typed_predicate_and_jsonpath_stays_blocked() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (cipher, _) = wait(&runtime, common::cipher()); + let keyset = cipher.default_keyset(); + let plan = people_plan(); + let mut db = connect(); + store( + &mut db, + &runtime, + &cipher, + "json_tree_typed", + &people_options(), + &[json!({"age": 3})], + ); + let exists = + |path: &str| match derive(&runtime, &keyset, &plan, "selector", json!({"path": path})) { + Operand::Text(selector) => json!({"exists": selector}).to_string(), + other => panic!("a selector is text, got {other:?}"), + }; + let (age, none) = (exists("$.age"), exists("$.none")); + let answer: Option = db + .query_one( + "SELECT doc @@ $1::text::jsonb::eql_v4.query_json_predicate FROM json_tree_typed", + &[&age], + ) + .unwrap() + .get(0); + assert_eq!(answer, Some(true)); + let bound: Option = db + .query_typed( + "SELECT doc @@ $1::eql_v4.query_json_predicate FROM json_tree_typed", + &[(&none, postgres::types::Type::TEXT)], + ) + .unwrap()[0] + .get(0); + assert_eq!(bound, Some(false), "a bound parameter cast to the domain"); + assert_eq!( + sqlstate( + &mut db, + "SELECT doc @@ '$.age == 3'::jsonpath FROM json_tree_typed", + &[] + ) + .as_deref(), + Some(RAISE_EXCEPTION), + "a jsonpath is blocked" + ); + for untyped in [ + format!( + "SELECT doc @@ {} FROM json_tree_typed", + literal(&json!(none)) + ), + "SELECT doc @@ '$.age == 3' FROM json_tree_typed".to_owned(), + ] { + assert_eq!( + sqlstate(&mut db, &untyped, &[]).as_deref(), + Some(AMBIGUOUS_FUNCTION), + "an untyped operand is refused, not read as native jsonb: {untyped}" + ); + } + let predicate = format!( + "{}::jsonb::eql_v4.query_json_predicate", + literal(&json!(none)) + ); + for sql in [ + format!("SELECT eql_v4.json_tree_match(NULL, {predicate})"), + "SELECT eql_v4.json_tree_match(doc, NULL) FROM json_tree_typed".to_owned(), + format!("SELECT NULL::public.eql_v4_json_search @@ {predicate}"), + ] { + let answer: Option = db.query_one(&sql, &[]).unwrap().get(0); + assert_eq!(answer, None, "{sql}"); + } +} diff --git a/packages/stack-encrypt/src/codes.rs b/packages/stack-encrypt/src/codes.rs index c1cb4a3de..77c45aa03 100644 --- a/packages/stack-encrypt/src/codes.rs +++ b/packages/stack-encrypt/src/codes.rs @@ -212,6 +212,12 @@ mod tests { Error::UntypedIndex { .. } => Error::UntypedIndex { field: name() }, Error::Source { .. } => Error::bad_source(Reason::FieldMissing), Error::Record { .. } => Error::bad_record(Reason::NoCiphertextNode), + Error::Predicate { .. } => Error::Predicate { + field: Some(name()), + reason: Reason::JsonPredicateUnsupported, + construct: "`like_regex`", + at: 4, + }, Error::Internal => Error::Internal, Error::Target(_) => Error::Target(TargetError::NoTargets { name: target() }), Error::Cipher(_) => Error::Cipher(crate::Error::Aead), @@ -596,6 +602,23 @@ mod tests { Box::new(Error::UntypedIndex { field: name() }), json!({ "field": "email" }), ), + ( + Box::new( + Error::Predicate { + field: None, + reason: Reason::JsonPredicateUnsupported, + construct: "`like_regex`", + at: 4, + } + .in_field("email"), + ), + json!({ + "field": "email", + "reason": "json_predicate_unsupported", + "construct": "`like_regex`", + "at": 4 + }), + ), (Box::new(Error::Internal), json!({})), ( Box::new(TargetError::NoTargets { name: target() }), diff --git a/packages/stack-encrypt/src/dynamic/json_predicate.rs b/packages/stack-encrypt/src/dynamic/json_predicate.rs new file mode 100644 index 000000000..75f410dce --- /dev/null +++ b/packages/stack-encrypt/src/dynamic/json_predicate.rs @@ -0,0 +1,1847 @@ +//! `match`'s predicate: the SQL/JSON path predicates the JSON index can +//! answer, parsed from the caller's text and compiled onto the inputs of +//! the other JSON operators. +//! +//! The text is PostgreSQL's SQL/JSON path language, as `jsonb_path_match` +//! takes it ([SQL/JSON path language](https://www.postgresql.org/docs/current/functions-json.html#FUNCTIONS-SQLJSON-PATH)), +//! restricted to predicates the terms can answer: +//! +//! ```text +//! $.age >= $min a comparison with a variable +//! $.name == "ann" || $."first name" != null literals and a quoted key +//! !($.age < 18) && exists($.email) negation and presence +//! $.tags[*] == $tag any element equals +//! $.orders[*].total > 100 any element's field orders +//! exists($.contacts[*] ? (@.relationship == $r && @.calls > 5)) +//! one element satisfies both +//! ``` +//! +//! `OpSpec::Match`'s documentation runs one through the data plan. The +//! grammar and the compilation are documented with the operator, in +//! the [`json_query`](super::json_query) module docs. This module holds the +//! parser ([`parse`]), which reads the text into an [`Ast`] and refuses +//! what the grammar does not read at the byte where it stops, and the +//! compiler ([`compile`]), which binds the values and maps each comparison +//! onto the inputs `eq_at`, `selector`, the orderings and `element_match` +//! take, so that every leaf is derived by that operator's own code +//! ([`derive`]). +//! +//! A refusal names a construct in the grammar's words and an offset, never +//! a slice of the text: a key name or a value in it can be customer data. +//! The text itself is the caller's buffer, wiped when it drops; the tree +//! holds offsets into it, the key names in [`JsonPath`]s, which wipe them, +//! and the values in [`Parsed`], which wipes them. + +use serde_json::Value; +use zeroize::Zeroizing; + +use super::json_query::{ + derive_input, eql, json_reason, leaf, member_of, ordered_path_of, Bounds, Compare, Input, + Parsed, Predicate, +}; +use super::query::{OpSpec, Operand}; +use super::{Error, Reason}; +use crate::json::{parse_text, JsonOptions, JsonPath, Segment, MAX_DEPTH}; +use crate::{KeysetCipher, Label, Pending}; + +/// How deeply a predicate may nest parentheses, `!` and `exists`. The +/// parser descends once per level, so the bound keeps a hostile predicate +/// from overflowing the stack; the SQL evaluates the compiled tree by +/// recursion too. +pub(super) const MAX_NESTING: usize = 32; + +// The constructs a refusal names, in the grammar's own words. +const C_TEXT: &str = "text the grammar does not read"; +const C_TRAILING: &str = "text after the predicate"; +const C_STRING: &str = "a string literal"; +const C_NUMBER: &str = "a numeric literal"; +const C_MODE: &str = "a mode (`strict`, `lax`)"; +const C_ARITHMETIC: &str = "arithmetic (`+`, `-`, `*`, `/`, `%`)"; +const C_LIKE_REGEX: &str = "`like_regex`"; +const C_STARTS_WITH: &str = "`starts with`"; +const C_IS_UNKNOWN: &str = "`is unknown`"; +const C_METHOD: &str = "a method (`.name()`)"; +const C_ANY_MEMBER: &str = "`.*`"; +const C_ANY_DEPTH: &str = "`.**`"; +const C_SLICE: &str = "an array slice or list (`[a to b]`, `[a, b]`)"; +const C_LAST: &str = "`last`"; +const C_SUBSCRIPT: &str = "a subscript other than `[*]`"; +const C_POSITION: &str = "a position (`[2]`)"; +const C_AT_OUTSIDE: &str = "`@` outside a filter"; +const C_DOLLAR_IN_FILTER: &str = "a `$` path in a filter"; +const C_VARIABLE_PATH: &str = "an accessor on a variable"; +const C_TWO_PATHS: &str = "a comparison of two paths"; +const C_TWO_VALUES: &str = "a comparison of two values"; +const C_ALONE: &str = "an operand with no comparison"; +const C_FILTER_OUTSIDE: &str = "a filter outside `exists`"; +const C_AFTER_FILTER: &str = "an accessor after a filter"; +const C_IN_EXISTS: &str = "a comparison inside `exists`"; +const C_OR_IN_FILTER: &str = "`||` in a filter"; +const C_NOT_IN_FILTER: &str = "`!` in a filter"; +const C_NE_IN_FILTER: &str = "`!=` in a filter"; +const C_EXISTS_IN_FILTER: &str = "`exists` in a filter"; +const C_ELEMENTS_THEMSELVES: &str = "an ordering of an array's elements themselves"; +const C_WHOLE_ELEMENT: &str = "`@` beside another condition"; +const C_FILTER_WILDCARD: &str = "a filter on a path with no wildcard"; +const C_MEMBER_WILDCARD: &str = "a wildcard in a filter's member"; +const C_NOT_EQUAL: &str = "`!=` at a path with a wildcard"; +const C_VALUE: &str = "a value"; +const C_VARIABLE: &str = "a variable"; +const C_NESTING: &str = "nesting"; + +/// A refusal at a byte of the predicate's text. +#[derive(Debug, PartialEq, Eq)] +struct Refusal { + reason: Reason, + construct: &'static str, + at: usize, +} + +impl Refusal { + fn syntax(construct: &'static str, at: usize) -> Self { + Self { + reason: Reason::JsonPredicateSyntax, + construct, + at, + } + } + + fn unsupported(construct: &'static str, at: usize) -> Self { + Self { + reason: Reason::JsonPredicateUnsupported, + construct, + at, + } + } +} + +impl From for Error { + fn from(refusal: Refusal) -> Self { + Error::Predicate { + field: None, + reason: refusal.reason, + construct: refusal.construct, + at: refusal.at, + } + } +} + +/// A comparison operator, as the text spells it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Op { + Eq, + Ne, + Lt, + Lte, + Gt, + Gte, +} + +impl Op { + /// The operator with its operands swapped: `1 < $.a` is `$.a > 1`. + fn flipped(self) -> Self { + match self { + Op::Lt => Op::Gt, + Op::Lte => Op::Gte, + Op::Gt => Op::Lt, + Op::Gte => Op::Lte, + same => same, + } + } + + /// How an element-scoped filter's condition compares; `None` for + /// `!=`, which a filter does not take. + fn compare(self) -> Option { + match self { + Op::Eq => Some(Compare::Eq), + Op::Lt => Some(Compare::Lt), + Op::Lte => Some(Compare::Lte), + Op::Gt => Some(Compare::Gt), + Op::Gte => Some(Compare::Gte), + Op::Ne => None, + } + } +} + +/// A path in the text: `$` or `@` and the steps after it, and where it +/// starts. +#[derive(Debug)] +struct PathAt { + path: JsonPath, + at: usize, +} + +/// A value in the text: a literal, by its bytes, or a variable, by name. +#[derive(Debug, PartialEq, Eq)] +enum ValueAt { + /// `"…"`, a number, `true`, `false` or `null`: the literal's bytes, + /// `start..end`, which are JSON text. + Literal { start: usize, end: usize }, + /// `$name` or `$"name"`. + Variable { name: String, at: usize }, +} + +impl ValueAt { + fn at(&self) -> usize { + match self { + ValueAt::Literal { start, .. } => *start, + ValueAt::Variable { at, .. } => *at, + } + } +} + +/// One side of a comparison. +#[derive(Debug)] +enum Side { + /// A path from `$`. + Root(PathAt), + /// A path from `@`, inside a filter. + Current(PathAt), + Value(ValueAt), +} + +/// A comparison of a path with a value, the path on the left. +#[derive(Debug)] +struct Comparison { + path: PathAt, + op: Op, + value: ValueAt, + /// Where the operator is. + at: usize, +} + +/// A predicate as the text writes it. +#[derive(Debug)] +enum Ast { + And(Vec), + Or(Vec), + Not(Box), + Compare(Comparison), + /// `exists(path)`, or `exists(path ? (conditions))`, each condition a + /// comparison of a path from `@`. + Exists { + path: PathAt, + filter: Option>, + }, +} + +/// The characters that end a name and are not a name's: what +/// PostgreSQL's jsonpath scanner reads as punctuation. +fn special(c: char) -> bool { + c.is_whitespace() || "?%$.[]{}()|&!=<>@#,*:-+/\\\"'".contains(c) +} + +/// A recursive descent over the text, holding the offset it has read to. +struct Parser<'t> { + text: &'t str, + pos: usize, + depth: usize, +} + +impl<'t> Parser<'t> { + fn rest(&self) -> &'t str { + &self.text[self.pos..] + } + + /// Skip whitespace and `/* … */` comments. + fn space(&mut self) -> Result<(), Refusal> { + loop { + let rest = self.rest(); + let trimmed = rest.trim_start(); + self.pos += rest.len() - trimmed.len(); + if let Some(comment) = trimmed.strip_prefix("/*") { + let Some(end) = comment.find("*/") else { + return Err(Refusal::syntax(C_TEXT, self.pos)); + }; + self.pos += 2 + end + 2; + } else { + return Ok(()); + } + } + } + + /// The next character after any space, unread. + fn peek(&mut self) -> Result, Refusal> { + self.space()?; + Ok(self.rest().chars().next()) + } + + /// Read `token` if it is next. + fn eat(&mut self, token: &str) -> Result { + self.space()?; + if self.rest().starts_with(token) { + self.pos += token.len(); + Ok(true) + } else { + Ok(false) + } + } + + fn expect(&mut self, token: &str) -> Result<(), Refusal> { + if self.eat(token)? { + Ok(()) + } else { + Err(Refusal::syntax(C_TEXT, self.pos)) + } + } + + /// The name at the cursor, unread: a run of characters that are not + /// punctuation, not starting with a digit. + fn word(&mut self) -> Result, Refusal> { + self.space()?; + let rest = self.rest(); + if rest.starts_with(|c: char| c.is_ascii_digit()) { + return Ok(None); + } + let end = rest.find(special).unwrap_or(rest.len()); + Ok((end > 0).then(|| &rest[..end])) + } + + /// One more level of nesting, refused past [`MAX_NESTING`]. + fn descend(&mut self) -> Result<(), Refusal> { + self.depth += 1; + if self.depth > MAX_NESTING { + return Err(Refusal { + reason: Reason::JsonPredicateTooDeep, + construct: C_NESTING, + at: self.pos, + }); + } + Ok(()) + } + + fn ascend(&mut self) { + self.depth -= 1; + } + + /// `or := and ('||' and)*`. + fn or(&mut self) -> Result { + let mut terms = vec![self.and()?]; + while self.eat("||")? { + terms.push(self.and()?); + } + Ok(if terms.len() == 1 { + terms.remove(0) + } else { + Ast::Or(terms) + }) + } + + /// `and := unary ('&&' unary)*`. + fn and(&mut self) -> Result { + let mut terms = vec![self.unary()?]; + while self.eat("&&")? { + terms.push(self.unary()?); + } + Ok(if terms.len() == 1 { + terms.remove(0) + } else { + Ast::And(terms) + }) + } + + /// `unary := '!' primary | primary`. As in PostgreSQL, `!` takes a + /// parenthesised predicate or `exists`, not a bare comparison. + fn unary(&mut self) -> Result { + self.space()?; + if self.rest().starts_with('!') && !self.rest().starts_with("!=") { + self.pos += 1; + self.descend()?; + let at = self.pos; + let negated = match self.peek()? { + Some('(') => self.primary()?, + _ if self.keyword("exists")? => self.primary()?, + _ => return Err(Refusal::syntax(C_TEXT, at)), + }; + self.ascend(); + return Ok(Ast::Not(Box::new(negated))); + } + self.primary() + } + + /// Whether the word at the cursor is `keyword`, in any case, as + /// PostgreSQL reads `exists`. + fn keyword(&mut self, keyword: &str) -> Result { + Ok(self + .word()? + .is_some_and(|word| word.eq_ignore_ascii_case(keyword))) + } + + /// `primary := '(' or ')' | 'exists' '(' path filter? ')' | comparison`. + fn primary(&mut self) -> Result { + if self.eat("(")? { + self.descend()?; + let inner = self.or()?; + self.expect(")")?; + self.ascend(); + return Ok(inner); + } + if self.keyword("exists")? { + self.pos += "exists".len(); + if self.eat("(")? { + return self.exists(); + } + return Err(Refusal::syntax(C_TEXT, self.pos)); + } + let comparison = self.comparison(false)?; + Ok(Ast::Compare(comparison)) + } + + /// The inside of `exists(…)`, the `(` read. + fn exists(&mut self) -> Result { + self.descend()?; + self.space()?; + let at = self.pos; + let path = match self.operand(false)? { + Side::Root(path) => path, + Side::Current(_) => return Err(Refusal::unsupported(C_AT_OUTSIDE, at)), + Side::Value(_) => return Err(Refusal::syntax(C_TEXT, at)), + }; + let filter = if self.eat("?")? { + self.expect("(")?; + self.descend()?; + let conditions = self.conditions()?; + self.expect(")")?; + self.ascend(); + let at = self.pos; + match self.peek()? { + Some('.' | '[') => return Err(Refusal::unsupported(C_AFTER_FILTER, at)), + Some('?') => return Err(Refusal::unsupported(C_FILTER_OUTSIDE, at)), + _ => {} + } + Some(conditions) + } else { + None + }; + if let Some(refusal) = self.after_operand()? { + return Err(refusal); + } + self.space()?; + if self.op()?.is_some() { + return Err(Refusal::unsupported(C_IN_EXISTS, at)); + } + self.expect(")")?; + self.ascend(); + Ok(Ast::Exists { path, filter }) + } + + /// A filter's conditions: comparisons of a path from `@`, joined by + /// `&&`, with parentheses. + fn conditions(&mut self) -> Result, Refusal> { + let mut conditions = Vec::new(); + loop { + self.space()?; + let at = self.pos; + if self.rest().starts_with('!') && !self.rest().starts_with("!=") { + return Err(Refusal::unsupported(C_NOT_IN_FILTER, at)); + } + if self.keyword("exists")? { + return Err(Refusal::unsupported(C_EXISTS_IN_FILTER, at)); + } + if self.eat("(")? { + self.descend()?; + conditions.extend(self.conditions()?); + self.expect(")")?; + self.ascend(); + } else { + conditions.push(self.comparison(true)?); + } + self.space()?; + if self.rest().starts_with("||") { + return Err(Refusal::unsupported(C_OR_IN_FILTER, self.pos)); + } + if !self.eat("&&")? { + return Ok(conditions); + } + } + } + + /// A comparison operator at the cursor, read, and where it was. + fn op(&mut self) -> Result, Refusal> { + self.space()?; + let at = self.pos; + let rest = self.rest(); + let (op, len) = if rest.starts_with("==") { + (Op::Eq, 2) + } else if rest.starts_with("!=") || rest.starts_with("<>") { + (Op::Ne, 2) + } else if rest.starts_with("<=") { + (Op::Lte, 2) + } else if rest.starts_with(">=") { + (Op::Gte, 2) + } else if rest.starts_with('<') { + (Op::Lt, 1) + } else if rest.starts_with('>') { + (Op::Gt, 1) + } else { + return Ok(None); + }; + self.pos += len; + Ok(Some((op, at))) + } + + /// The construct at the cursor that follows an operand and that the + /// index cannot answer, if one is there. + fn after_operand(&mut self) -> Result, Refusal> { + self.space()?; + let at = self.pos; + let rest = self.rest(); + if rest.starts_with(['+', '-', '*', '/', '%']) { + return Ok(Some(Refusal::unsupported(C_ARITHMETIC, at))); + } + if rest.starts_with('?') { + return Ok(Some(Refusal::unsupported(C_FILTER_OUTSIDE, at))); + } + let word = self.word()?.unwrap_or_default(); + Ok(if word.eq_ignore_ascii_case("like_regex") { + Some(Refusal::unsupported(C_LIKE_REGEX, at)) + } else if word.eq_ignore_ascii_case("starts") { + Some(Refusal::unsupported(C_STARTS_WITH, at)) + } else if word.eq_ignore_ascii_case("is") { + Some(Refusal::unsupported(C_IS_UNKNOWN, at)) + } else { + None + }) + } + + /// `comparison := operand op operand`, one side a path and the other a + /// value; in a filter the path is from `@`. + fn comparison(&mut self, in_filter: bool) -> Result { + self.space()?; + let left = self.operand(in_filter)?; + if let Some(refusal) = self.after_operand()? { + return Err(refusal); + } + let Some((op, at)) = self.op()? else { + self.space()?; + let at = self.pos; + let ends = self.rest().is_empty() || self.rest().starts_with([')', '&', '|']); + return Err(if ends { + Refusal::unsupported(C_ALONE, at) + } else { + Refusal::syntax(C_TEXT, at) + }); + }; + if in_filter && op == Op::Ne { + return Err(Refusal::unsupported(C_NE_IN_FILTER, at)); + } + self.space()?; + let right = self.operand(in_filter)?; + if let Some(refusal) = self.after_operand()? { + return Err(refusal); + } + let (path, op, value) = match (left, right) { + (Side::Root(path) | Side::Current(path), Side::Value(value)) => (path, op, value), + (Side::Value(value), Side::Root(path) | Side::Current(path)) => { + (path, op.flipped(), value) + } + (Side::Value(_), Side::Value(_)) => return Err(Refusal::unsupported(C_TWO_VALUES, at)), + _ => return Err(Refusal::unsupported(C_TWO_PATHS, at)), + }; + Ok(Comparison { + path, + op, + value, + at, + }) + } + + /// A path, a variable or a literal. In a filter a path is from `@`; + /// outside one, from `$`. + fn operand(&mut self, in_filter: bool) -> Result { + self.space()?; + let at = self.pos; + let rest = self.rest(); + let Some(first) = rest.chars().next() else { + return Err(Refusal::syntax(C_TEXT, at)); + }; + match first { + '$' => { + let after = &rest[1..]; + if after.starts_with('"') { + let (mut name, len) = self.string_at(at + 1)?; + self.pos = at + 1 + len; + return self.variable(std::mem::take(&mut *name), at); + } + if after.starts_with(|c: char| !special(c)) { + self.pos += 1; + let Some(name) = self.word()? else { + return Err(Refusal::syntax(C_TEXT, self.pos)); + }; + self.pos += name.len(); + return self.variable(name.to_owned(), at); + } + if in_filter { + return Err(Refusal::unsupported(C_DOLLAR_IN_FILTER, at)); + } + self.pos += 1; + Ok(Side::Root(self.steps(at)?)) + } + '@' => { + if !in_filter { + return Err(Refusal::unsupported(C_AT_OUTSIDE, at)); + } + self.pos += 1; + Ok(Side::Current(self.steps(at)?)) + } + '"' => { + let (_, len) = self.string_at(at)?; + self.pos += len; + Ok(Side::Value(ValueAt::Literal { + start: at, + end: at + len, + })) + } + '-' | '0'..='9' => { + let len = number_len(rest).ok_or(Refusal::syntax(C_NUMBER, at))?; + self.pos += len; + Ok(Side::Value(ValueAt::Literal { + start: at, + end: at + len, + })) + } + '+' => Err(Refusal::unsupported(C_ARITHMETIC, at)), + _ => match self.word()? { + Some(word @ ("true" | "false" | "null")) => { + self.pos += word.len(); + Ok(Side::Value(ValueAt::Literal { + start: at, + end: at + word.len(), + })) + } + Some(word) if word.eq_ignore_ascii_case("last") => { + Err(Refusal::unsupported(C_LAST, at)) + } + Some(word) if is_mode(word) => Err(Refusal::unsupported(C_MODE, at)), + _ => Err(Refusal::syntax(C_TEXT, at)), + }, + } + } + + /// A variable named `name`, which takes no accessor. + fn variable(&mut self, name: String, at: usize) -> Result { + self.space()?; + if self.rest().starts_with(['.', '[']) { + return Err(Refusal::unsupported(C_VARIABLE_PATH, self.pos)); + } + Ok(Side::Value(ValueAt::Variable { name, at })) + } + + /// The JSON string at `at`, decoded, and its length in the text. Its + /// escapes are JSON's. The decoded text is wiped when it drops: a + /// literal's is a value, read only to check it, and a key's is moved + /// into its path, which wipes it. + fn string_at(&self, at: usize) -> Result<(Zeroizing, usize), Refusal> { + let text = &self.text[at..]; + let mut escaped = false; + for (end, byte) in text.bytes().enumerate().skip(1) { + match (escaped, byte) { + (true, _) => escaped = false, + (false, b'\\') => escaped = true, + (false, b'"') => { + let decoded = serde_json::from_str(&text[..=end]) + .map(Zeroizing::new) + .map_err(|_| Refusal::syntax(C_STRING, at))?; + return Ok((decoded, end + 1)); + } + (false, _) => {} + } + } + Err(Refusal::syntax(C_STRING, at)) + } + + /// The steps after `$` or `@`: `.name`, `."name"`, `[*]`, and the + /// accessors refused by name. A step past a document's depth is + /// refused before it is read, so the path held is never deeper than + /// [`MAX_DEPTH`]. + fn steps(&mut self, start: usize) -> Result { + let mut segments = Vec::new(); + loop { + self.space()?; + let at = self.pos; + let rest = self.rest(); + if !rest.starts_with(['.', '[']) { + break; + } + if segments.len() >= MAX_DEPTH { + return Err(Refusal { + reason: Reason::JsonPredicateTooDeep, + construct: C_NESTING, + at: start, + }); + } + if let Some(after) = rest.strip_prefix('.') { + let after_space = after.trim_start(); + let gap = after.len() - after_space.len(); + if after_space.starts_with("**") { + return Err(Refusal::unsupported(C_ANY_DEPTH, at)); + } + if after_space.starts_with('*') { + return Err(Refusal::unsupported(C_ANY_MEMBER, at)); + } + if after_space.starts_with('"') { + let (mut key, len) = self.string_at(at + 1 + gap)?; + self.pos = at + 1 + gap + len; + segments.push(Segment::Key(std::mem::take(&mut *key))); + continue; + } + self.pos = at + 1; + let Some(name) = self.word()? else { + return Err(Refusal::syntax(C_TEXT, self.pos)); + }; + self.pos += name.len(); + if self.peek()? == Some('(') { + return Err(Refusal::unsupported(C_METHOD, at)); + } + segments.push(Segment::Key(name.to_owned())); + } else { + self.pos += 1; + self.space()?; + if self.eat("*")? { + self.expect("]")?; + segments.push(Segment::Wildcard); + } else if self.rest().starts_with(|c: char| c.is_ascii_digit()) { + let digits = self + .rest() + .find(|c: char| !c.is_ascii_digit()) + .unwrap_or(self.rest().len()); + self.pos += digits; + self.space()?; + if self.rest().starts_with(']') { + return Err(Refusal { + reason: Reason::JsonPathPosition, + construct: C_POSITION, + at, + }); + } + return Err(Refusal::unsupported(C_SLICE, at)); + } else if self.keyword("last")? { + return Err(Refusal::unsupported(C_LAST, at)); + } else { + return Err(Refusal::unsupported(C_SUBSCRIPT, at)); + } + } + } + Ok(PathAt { + path: JsonPath::of(segments), + at: start, + }) + } +} + +/// The length of the JSON number at the front of `text`: +/// `-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)?`, followed by +/// something that does not continue it. `None` for text that is not one. +fn number_len(text: &str) -> Option { + let bytes = text.as_bytes(); + let mut i = usize::from(bytes.first() == Some(&b'-')); + let digits = |from: usize| { + bytes[from..] + .iter() + .take_while(|b| b.is_ascii_digit()) + .count() + }; + match bytes.get(i) { + Some(b'0') => i += 1, + Some(b'1'..=b'9') => i += digits(i), + _ => return None, + } + if bytes.get(i) == Some(&b'.') { + let fraction = digits(i + 1); + if fraction == 0 { + return None; + } + i += 1 + fraction; + } + if matches!(bytes.get(i), Some(b'e' | b'E')) { + let mut j = i + 1; + if matches!(bytes.get(j), Some(b'+' | b'-')) { + j += 1; + } + let exponent = digits(j); + if exponent == 0 { + return None; + } + i = j + exponent; + } + match text[i..].chars().next() { + Some(c) if !special(c) || c == '.' => None, + _ => Some(i), + } +} + +/// Whether `word` is a mode word, `strict` or `lax`, in any case. +fn is_mode(word: &str) -> bool { + word.eq_ignore_ascii_case("strict") || word.eq_ignore_ascii_case("lax") +} + +/// Parse a predicate's text. +fn parse(text: &str) -> Result { + let mut parser = Parser { + text, + pos: 0, + depth: 0, + }; + parser.space()?; + let at = parser.pos; + if parser.word()?.is_some_and(is_mode) { + return Err(Refusal::unsupported(C_MODE, at)); + } + let ast = parser.or()?; + parser.space()?; + if !parser.rest().is_empty() { + return Err(Refusal::syntax(C_TRAILING, parser.pos)); + } + Ok(ast) +} + +/// A predicate compiled: a tree of the inputs of the operators whose +/// operands are its leaves. +#[derive(PartialEq)] +pub(super) enum Node { + And(Vec), + Or(Vec), + Not(Box), + /// One leaf: an operator and the input it derives from, checked. + Leaf(OpSpec, Input), +} + +impl std::fmt::Debug for Node { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Node::And(children) => f.debug_tuple("And").field(children).finish(), + Node::Or(children) => f.debug_tuple("Or").field(children).finish(), + Node::Not(child) => f.debug_tuple("Not").field(child).finish(), + // The input holds the predicate's values: the operator alone. + Node::Leaf(op, _) => f.debug_tuple("Leaf").field(op).finish(), + } + } +} + +/// What a compilation reads besides the tree: the text the tree's +/// offsets are into, the variables, the field's options and the bound an +/// element leaf is read to. +struct Compiler<'c> { + text: &'c str, + vars: &'c serde_json::Map, + options: &'c JsonOptions, + bound: u32, +} + +/// Compile a predicate's text under `vars`, a JSON object, for a field +/// under `options`, its element leaves read to `bound`. +/// +/// # Errors +/// +/// [`Error::Predicate`] for text the grammar does not read or a construct +/// it reads and the index cannot answer, at its offset; and the refusals +/// the leaves' own operators make of their inputs, such as +/// [`TermError::JsonPathNotOrdered`](crate::sem::TermError::JsonPathNotOrdered) for an ordering at a path the field +/// does not order. +pub(super) fn compile( + text: &str, + vars: &Value, + options: &JsonOptions, + bound: u32, +) -> Result { + let empty = serde_json::Map::new(); + let compiler = Compiler { + text, + vars: vars.as_object().unwrap_or(&empty), + options, + bound, + }; + let ast = parse(text)?; + Ok(simplify(compiler.node(ast, false)?)) +} + +impl Compiler<'_> { + /// The node for `ast`, which sits under an odd number of `!` when + /// `negated`. `and` and `or` are monotone in each child, so only a `!` + /// turns a child's NULL into a different row set than its false; see + /// [`Compiler::comparison`]. + fn node(&self, ast: Ast, negated: bool) -> Result { + Ok(match ast { + Ast::And(terms) => Node::And( + terms + .into_iter() + .map(|t| self.node(t, negated)) + .collect::>()?, + ), + Ast::Or(terms) => Node::Or( + terms + .into_iter() + .map(|t| self.node(t, negated)) + .collect::>()?, + ), + Ast::Not(inner) => Node::Not(Box::new(self.node(*inner, !negated)?)), + Ast::Compare(comparison) => self.comparison(comparison, negated)?, + Ast::Exists { path, filter: None } => { + Node::Leaf(OpSpec::Selector, Input::Path(path.path)) + } + Ast::Exists { + path, + filter: Some(conditions), + } => self.filter(path, conditions)?, + }) + } + + /// The value a literal or a variable holds. + fn value(&self, value: &ValueAt) -> Result { + match value { + ValueAt::Literal { start, end } => parse_text(&self.text.as_bytes()[*start..*end]) + .map(Parsed) + .map_err(|error| { + Refusal { + reason: json_reason(error), + construct: C_VALUE, + at: *start, + } + .into() + }), + ValueAt::Variable { name, at } => self + .vars + .get(name) + .map(|value| Parsed(value.clone())) + .ok_or_else(|| { + Refusal { + reason: Reason::JsonPredicateUnknownVariable, + construct: C_VARIABLE, + at: *at, + } + .into() + }), + } + } + + /// The value of `at`, a scalar: what equality takes. + fn scalar(&self, at: &ValueAt) -> Result { + let value = self.value(at)?; + if matches!(value.0, Value::Object(_) | Value::Array(_)) { + return Err(value_type(at)); + } + Ok(value) + } + + /// The value of `at`, a number or a string: what an ordering takes. + fn ordered(&self, at: &ValueAt) -> Result { + let value = self.value(at)?; + leaf(&value.0).map_err(|_| value_type(at))?; + Ok(value) + } + + /// `path op value`, under an odd number of `!` when `negated`. + fn comparison(&self, comparison: Comparison, negated: bool) -> Result { + let Comparison { + path, + op, + value, + at, + } = comparison; + let wild = !path.path.is_positional(); + let (compare, operator) = match op { + // `==`: containment of the value at the path, exact; at a + // wildcard, of any element. + Op::Eq => { + return Ok(Node::Leaf( + OpSpec::EqAt, + Input::At(path.path, self.scalar(&value)?), + )) + } + // `!=` over several elements is true when one differs, which + // containment cannot say. + Op::Ne if wild => { + return Err(Refusal { + reason: Reason::JsonPredicateNotEqualAtWildcard, + construct: C_NOT_EQUAL, + at, + } + .into()) + } + // `!=` at one node: a node there that is not the value. + Op::Ne => { + let value = self.scalar(&value)?; + return Ok(Node::And(vec![ + Node::Leaf(OpSpec::Selector, Input::Path(path.path.clone())), + Node::Not(Box::new(Node::Leaf( + OpSpec::EqAt, + Input::At(path.path, value), + ))), + ])); + } + Op::Lt => (Compare::Lt, OpSpec::LtAt), + Op::Lte => (Compare::Lte, OpSpec::LteAt), + Op::Gt => (Compare::Gt, OpSpec::GtAt), + Op::Gte => (Compare::Gte, OpSpec::GteAt), + }; + Ok(if wild { + // An ordering at a wildcard: some element's field, which is + // `element_match` on the array through the last wildcard. + let value = self.ordered(&value)?; + let mut segments = path.path.segments().to_vec(); + let last = segments + .iter() + .rposition(|segment| *segment == Segment::Wildcard) + .unwrap_or_default(); + let member = segments.split_off(last + 1); + if member.is_empty() { + return Err(Refusal::unsupported(C_ELEMENTS_THEMSELVES, at).into()); + } + let base = JsonPath::of(segments); + let predicate = self.member(&base, JsonPath::of(member), compare, value)?; + Node::Leaf( + OpSpec::ElementMatch, + Input::Elements(self.bound, vec![predicate]), + ) + } else { + // An ordering at one node: a node there whose leaf compares. + // jsonpath answers false of an absent path, which the selector + // gives, and false of a JSON null, where the ordering is NULL + // (a null has no ordering term). Under an even number of `!` + // the two pick the same rows, since `and` and `or` cannot turn + // a child's NULL into true; under an odd number `!` turns false + // into true and NULL into NULL, so the tree also asks that the + // node is not null. That `null` needle tells the database which + // rows hold null at the path, so it is sent only there. + let value = self.ordered(&value)?; + let path = ordered_path_of(self.options, path.path)?; + let mut leaves = vec![Node::Leaf(OpSpec::Selector, Input::Path(path.clone()))]; + if negated { + leaves.push(Node::Not(Box::new(Node::Leaf( + OpSpec::EqAt, + Input::At(path.clone(), Parsed(Value::Null)), + )))); + } + leaves.push(Node::Leaf(operator, Input::At(path, value))); + Node::And(leaves) + }) + } + + /// One field of an element-scoped predicate: its full path, which the + /// field orders, and its bound. + fn member( + &self, + base: &JsonPath, + member: JsonPath, + compare: Compare, + value: Parsed, + ) -> Result { + Ok(Predicate { + path: member_of(self.options, base, member)?, + bounds: Bounds::One(compare, value), + }) + } + + /// `exists(path ? (conditions))`: some element at the path satisfies + /// every condition. One equality is containment at the condition's + /// path, exact; anything else is `element_match`, every condition on a + /// field of one element. + fn filter(&self, path: PathAt, conditions: Vec) -> Result { + if path.path.is_positional() { + return Err(Refusal { + reason: Reason::JsonPathNoWildcard, + construct: C_FILTER_WILDCARD, + at: path.at, + } + .into()); + } + let base = path.path; + for condition in &conditions { + if base.segments().len() + condition.path.path.segments().len() > MAX_DEPTH { + return Err(Refusal { + reason: Reason::JsonPredicateTooDeep, + construct: C_NESTING, + at: path.at, + } + .into()); + } + if !condition.path.path.is_positional() { + return Err(Refusal { + reason: Reason::JsonPathWildcard, + construct: C_MEMBER_WILDCARD, + at: condition.path.at, + } + .into()); + } + } + if let [only] = conditions.as_slice() { + if only.op == Op::Eq { + let full = only.path.path.segments().iter().fold( + base.segments().to_vec(), + |mut at, segment| { + at.push(segment.clone()); + at + }, + ); + return Ok(Node::Leaf( + OpSpec::EqAt, + Input::At(JsonPath::of(full), self.scalar(&only.value)?), + )); + } + } + let mut predicates = Vec::with_capacity(conditions.len()); + for condition in conditions { + let Some(compare) = condition.op.compare() else { + return Err(Refusal::unsupported(C_NE_IN_FILTER, condition.at).into()); + }; + if condition.path.path.segments().is_empty() { + let construct = if compare == Compare::Eq { + C_WHOLE_ELEMENT + } else { + C_ELEMENTS_THEMSELVES + }; + return Err(Refusal::unsupported(construct, condition.path.at).into()); + } + let value = self.ordered(&condition.value)?; + predicates.push(self.member(&base, condition.path.path, compare, value)?); + } + Ok(Node::Leaf( + OpSpec::ElementMatch, + Input::Elements(self.bound, predicates), + )) + } +} + +fn value_type(at: &ValueAt) -> Error { + Refusal { + reason: Reason::JsonPredicateValueType, + construct: C_VALUE, + at: at.at(), + } + .into() +} + +/// The tree with every `and` in an `and` and every `or` in an `or` lifted +/// into its parent, a child equal to an earlier sibling dropped, a list of +/// one replaced by its child, and `not` of `not` by its child: each +/// preserves SQL's three-valued answer. +fn simplify(node: Node) -> Node { + fn gather(children: Vec, lift: fn(Node) -> Result, Node>) -> Vec { + let mut out: Vec = Vec::with_capacity(children.len()); + for child in children { + let lifted = match lift(simplify(child)) { + Ok(grandchildren) => grandchildren, + Err(child) => vec![child], + }; + for child in lifted { + if !out.contains(&child) { + out.push(child); + } + } + } + out + } + let one_or = |mut children: Vec, wrap: fn(Vec) -> Node| { + if children.len() == 1 { + children.remove(0) + } else { + wrap(children) + } + }; + match node { + Node::And(children) => one_or( + gather(children, |node| match node { + Node::And(children) => Ok(children), + other => Err(other), + }), + Node::And, + ), + Node::Or(children) => one_or( + gather(children, |node| match node { + Node::Or(children) => Ok(children), + other => Err(other), + }), + Node::Or, + ), + Node::Not(child) => match simplify(*child) { + Node::Not(inner) => *inner, + child => Node::Not(Box::new(child)), + }, + leaf => leaf, + } +} + +/// The shape of a compiled tree with its leaves taken out, in the order +/// they were: what the leaves' operands are put back into. +enum Shape { + And(Vec), + Or(Vec), + Not(Box), + /// A leaf's key in the tree (`contains`, `exists`, `cmp`, `elements`) + /// and, for `cmp`, its bound's. + Leaf(&'static str, Option<&'static str>), +} + +impl Shape { + /// The shape of `node`, its leaves pushed to `leaves` in order. + fn of(node: Node, leaves: &mut Vec<(OpSpec, Input)>) -> Result { + Ok(match node { + Node::And(children) => Shape::And( + children + .into_iter() + .map(|c| Self::of(c, leaves)) + .collect::>()?, + ), + Node::Or(children) => Shape::Or( + children + .into_iter() + .map(|c| Self::of(c, leaves)) + .collect::>()?, + ), + Node::Not(child) => Shape::Not(Box::new(Self::of(*child, leaves)?)), + Node::Leaf(op, input) => { + let shape = match op { + OpSpec::EqAt => Shape::Leaf("contains", None), + OpSpec::Selector => Shape::Leaf("exists", None), + OpSpec::LtAt => Shape::Leaf("cmp", Some("lt")), + OpSpec::LteAt => Shape::Leaf("cmp", Some("lte")), + OpSpec::GtAt => Shape::Leaf("cmp", Some("gt")), + OpSpec::GteAt => Shape::Leaf("cmp", Some("gte")), + OpSpec::ElementMatch => Shape::Leaf("elements", None), + _ => return Err(Error::Internal), + }; + leaves.push((op, input)); + shape + } + }) + } + + /// The tree, each leaf's operand taken from `operands` in order: the + /// `eql_v4.query_json_predicate` operand. + fn fill(self, operands: &mut std::vec::IntoIter) -> Result { + let missing = || crate::Error::Other("a compiled predicate lost a leaf".into()); + let list = |shapes: Vec, operands: &mut std::vec::IntoIter| { + shapes + .into_iter() + .map(|shape| shape.fill(operands)) + .collect::, _>>() + }; + Ok(match self { + Shape::And(children) => serde_json::json!({ "and": list(children, operands)? }), + Shape::Or(children) => serde_json::json!({ "or": list(children, operands)? }), + Shape::Not(child) => serde_json::json!({ "not": child.fill(operands)? }), + Shape::Leaf(key, None) => { + serde_json::json!({ key: operands.next().ok_or_else(missing)? }) + } + Shape::Leaf(key, Some(bound)) => { + serde_json::json!({ key: { bound: operands.next().ok_or_else(missing)? } }) + } + }) + } +} + +/// A leaf's operand as JSON: the EQL value's, or a selector's text. +fn operand_value(operand: Operand) -> Result { + match operand { + Operand::One(bytes) => { + serde_json::from_slice(&bytes).map_err(|error| crate::Error::Other(Box::new(error))) + } + Operand::Text(text) => Ok(Value::String(text)), + _ => Err(crate::Error::Other( + "a predicate leaf derives one operand".into(), + )), + } +} + +/// Derive a compiled predicate's operand: each leaf by its own operator's +/// derivation ([`derive_input`]), all in one pending, put back into the +/// tree. +pub(super) fn derive<'a, K: 'static>( + keyset: &'a KeysetCipher<'_, K>, + label: &Label, + options: &JsonOptions, + node: Node, +) -> Result, Error> { + let mut leaves = Vec::new(); + let shape = Shape::of(node, &mut leaves)?; + let operands = leaves + .into_iter() + .map(|(op, input)| { + derive_input(keyset, label, options, op, input).map(|p| p.try_map(operand_value)) + }) + .collect::, _>>()?; + Ok(Pending::all(keyset, operands) + .try_map(move |operands| eql(&shape.fill(&mut operands.into_iter())?).map(Operand::One))) +} + +#[cfg(test)] +mod tests { + use super::*; + + use crate::sem::TermError; + + use serde_json::json; + + /// A field ordering `$.age`, `$.name` and every contact's `calls` and + /// `relationship`. + fn options() -> JsonOptions { + JsonOptions::new() + .ordering([ + "$.age", + "$.name", + "$.contacts[*].calls", + "$.contacts[*].relationship", + ]) + .unwrap() + } + + fn compiled(text: &str, vars: Value) -> Result { + compile(text, &vars, &options(), 4) + } + + /// A compiled tree written out: each leaf its operator and its input. + fn written(node: &Node) -> String { + let list = |children: &[Node]| children.iter().map(written).collect::>().join(", "); + match node { + Node::And(children) => format!("and({})", list(children)), + Node::Or(children) => format!("or({})", list(children)), + Node::Not(child) => format!("not({})", written(child)), + Node::Leaf(op, Input::At(path, value)) => format!("{}({path}, {})", op.name(), value.0), + Node::Leaf(op, Input::Path(path)) => format!("{}({path})", op.name()), + Node::Leaf(op, Input::Elements(bound, predicates)) => { + let members: Vec = predicates + .iter() + .map(|p| match &p.bounds { + Bounds::One(compare, value) => { + format!("{} {compare:?} {}", p.path, value.0) + } + Bounds::Between(low, high) => { + format!("{} in {}..{}", p.path, low.0, high.0) + } + }) + .collect(); + format!("{}({bound}, [{}])", op.name(), members.join("; ")) + } + Node::Leaf(op, _) => op.name().to_owned(), + } + } + + fn compiles_to(text: &str, vars: Value, want: &str) { + let node = compiled(text, vars).unwrap_or_else(|error| panic!("{text}: {error}")); + assert_eq!(written(&node), want, "{text}"); + } + + /// The refusal `text` meets: its reason, construct and offset. + fn refusal(text: &str, vars: Value) -> (Reason, &'static str, usize) { + match compiled(text, vars) { + Err(Error::Predicate { + reason, + construct, + at, + .. + }) => (reason, construct, at), + Err(other) => panic!("{text}: not a predicate refusal: {other:?}"), + Ok(node) => panic!("{text}: compiled to {}", written(&node)), + } + } + + mod given_the_grammar_it_reads { + use super::*; + + /// Each comparison compiles onto the operators whose operands + /// answer it as jsonpath does: `==` containment; `!=` a node that is + /// not the value; an ordering a node, not null, whose leaf compares; + /// and at a wildcard, an element's field. + #[test] + fn each_comparison_compiles_onto_its_operators() { + for (text, want) in [ + (r#"$.name == "Ann""#, r#"eq_at($.name, "Ann")"#), + ("$.age == 3", "eq_at($.age, 3)"), + ("$.ok == true", "eq_at($.ok, true)"), + ("$.gone == null", "eq_at($.gone, null)"), + ("$.age != 3", "and(selector($.age), not(eq_at($.age, 3)))"), + ( + "$.age != null", + "and(selector($.age), not(eq_at($.age, null)))", + ), + ("$.age < 3", "and(selector($.age), lt_at($.age, 3))"), + ("$.age <= 3", "and(selector($.age), lte_at($.age, 3))"), + ("$.age > 3", "and(selector($.age), gt_at($.age, 3))"), + ("$.age >= 3", "and(selector($.age), gte_at($.age, 3))"), + ("$.tags[*] == \"x\"", r#"eq_at($.tags[*], "x")"#), + ( + "$.contacts[*].calls > 5", + "element_match(4, [$.contacts[*].calls Gt 5])", + ), + ("exists($.a.b)", "selector($.a.b)"), + ("exists($.tags[*])", "selector($.tags[*])"), + ] { + compiles_to(text, json!({}), want); + } + } + + /// A value on the left is the comparison turned round, `<>` is + /// `!=`, and a variable is its value in `vars`, of any JSON type a + /// comparison takes. + #[test] + fn operands_are_read_in_either_order_and_by_variable() { + for (text, want) in [ + ("3 < $.age", "and(selector($.age), gt_at($.age, 3))"), + ("3 >= $.age", "and(selector($.age), lte_at($.age, 3))"), + ("3 == $.age", "eq_at($.age, 3)"), + ("$.age <> 3", "and(selector($.age), not(eq_at($.age, 3)))"), + ("$.name == $who", r#"eq_at($.name, "Ann")"#), + (r#"$.name == $"who""#, r#"eq_at($.name, "Ann")"#), + ("$.age < $n", "and(selector($.age), lt_at($.age, 7))"), + ("$.ok == $yes", "eq_at($.ok, true)"), + ("$.gone == $nothing", "eq_at($.gone, null)"), + ] { + compiles_to( + text, + json!({"who": "Ann", "n": 7, "yes": true, "nothing": null}), + want, + ); + } + } + + /// Paths read as PostgreSQL reads them: quoted keys with JSON's + /// escapes, keys that are words of the grammar, and space between + /// steps; and literals as JSON reads them. + #[test] + fn paths_and_literals_read_as_postgres_reads_them() { + for (text, want) in [ + (r#"$."first name" == 1"#, "eq_at($.first name, 1)"), + (r#"$. "a b".c == 1"#, "eq_at($.a b.c, 1)"), + (r#"$."a\"b" == 1"#, r#"eq_at($["a\"b"], 1)"#), + (r#"$."a" == 1"#, "eq_at($.a, 1)"), + ("$.exists == 1", "eq_at($.exists, 1)"), + ("$.size == 1", "eq_at($.size, 1)"), + ("$.true == 1", "eq_at($.true, 1)"), + ("$.é == 1", "eq_at($.é, 1)"), + ("$ . a [ * ] . b == 1", "eq_at($.a[*].b, 1)"), + (" $.a==1 ", "eq_at($.a, 1)"), + ("$.a == 1 /* why */", "eq_at($.a, 1)"), + ("$.a == -1.5e3", "eq_at($.a, -1500.0)"), + ("$.a == 0", "eq_at($.a, 0)"), + ("$.a == 1.25", "eq_at($.a, 1.25)"), + ("$.a == 10.125E+2", "eq_at($.a, 1012.5)"), + ("$.a == 2e-1", "eq_at($.a, 0.2)"), + (r#"$.a == "x\ty""#, r#"eq_at($.a, "x\ty")"#), + ("$ == 1", "eq_at($, 1)"), + ] { + compiles_to(text, json!({}), want); + } + } + + /// `&&` binds tighter than `||`, parentheses group, `!` takes a + /// group or `exists`, and the tree is flattened: an `and` in an + /// `and` lifted, a repeated child dropped, `not` of `not` removed. + #[test] + fn the_boolean_structure_is_kept_and_flattened() { + for (text, want) in [ + ( + "$.a == 1 && $.b == 2 || $.c == 3", + "or(and(eq_at($.a, 1), eq_at($.b, 2)), eq_at($.c, 3))", + ), + ( + "$.a == 1 && ($.b == 2 || $.c == 3)", + "and(eq_at($.a, 1), or(eq_at($.b, 2), eq_at($.c, 3)))", + ), + ("!($.a == 1)", "not(eq_at($.a, 1))"), + ("! exists($.a)", "not(selector($.a))"), + ("!(!($.a == 1))", "eq_at($.a, 1)"), + ("((($.a == 1)))", "eq_at($.a, 1)"), + ( + "$.a == 1 && ($.b == 2 && $.c == 3)", + "and(eq_at($.a, 1), eq_at($.b, 2), eq_at($.c, 3))", + ), + ( + "$.a == 1 || ($.b == 2 || $.c == 3)", + "or(eq_at($.a, 1), eq_at($.b, 2), eq_at($.c, 3))", + ), + ("$.a == 1 || $.a == 1", "eq_at($.a, 1)"), + ("$.a == 1 && $.a == 1", "eq_at($.a, 1)"), + ( + "$.age > 1 && $.age < 9", + "and(selector($.age), gt_at($.age, 1), lt_at($.age, 9))", + ), + ( + "EXISTS($.a) && Exists($.b)", + "and(selector($.a), selector($.b))", + ), + ] { + compiles_to(text, json!({}), want); + } + } + + /// An ordering at one node carries the `null` needle only under an + /// odd number of `!`, where it changes which rows match; elsewhere + /// the selector and the ordering pick jsonpath's rows without it. + #[test] + fn an_ordering_sends_the_null_needle_only_under_an_odd_number_of_negations() { + let needle = "not(eq_at($.age, null))"; + for (text, want) in [ + ("$.age < 3", "and(selector($.age), lt_at($.age, 3))"), + ( + "!($.age < 3)", + "not(and(selector($.age), not(eq_at($.age, null)), lt_at($.age, 3)))", + ), + ("!(!($.age < 3))", "and(selector($.age), lt_at($.age, 3))"), + ( + "!($.b == 1 || $.age >= 3)", + "not(or(eq_at($.b, 1), and(selector($.age), not(eq_at($.age, null)), gte_at($.age, 3))))", + ), + ( + "!(!($.age > 3) && $.b == 1)", + "not(and(not(and(selector($.age), gt_at($.age, 3))), eq_at($.b, 1)))", + ), + ( + "$.b == 1 && !($.age <= 3) || $.age > 9", + "or(and(eq_at($.b, 1), not(and(selector($.age), not(eq_at($.age, null)), lte_at($.age, 3)))), and(selector($.age), gt_at($.age, 9)))", + ), + ] { + compiles_to(text, json!({}), want); + } + // A filter's conditions and an ordering at a wildcard are + // element leaves, which carry no needle under any `!`. + for text in [ + "!($.contacts[*].calls > 5)", + "!(exists($.contacts[*] ? (@.calls > 5 && @.calls < 9)))", + ] { + let tree = written(&compiled(text, json!({})).unwrap()); + assert!(tree.starts_with("not(element_match("), "{text}: {tree}"); + assert!(!tree.contains(needle), "{text}: {tree}"); + } + } + + /// A compiled tree prints its structure and operators, never the + /// values or paths it holds. + #[test] + fn a_compiled_tree_prints_no_value() { + let node = compiled( + r#"!($."zq-secret" == "zq-secret") || exists($.age)"#, + json!({}), + ) + .unwrap(); + let shown = format!("{node:?}"); + assert_eq!(shown, "Or([Not(Leaf(EqAt)), Leaf(Selector)])"); + } + + /// `exists(path ? (…))`: one equality is containment at its field, + /// exact; anything else is `element_match`, every condition on one + /// element, read to the input's bound. + #[test] + fn a_filter_compiles_to_containment_or_an_element_match() { + for (text, want) in [ + ( + "exists($.contacts[*] ? (@.relationship == $r))", + r#"eq_at($.contacts[*].relationship, "Sibling")"#, + ), + ( + "exists($.tags[*] ? (@ == $r))", + r#"eq_at($.tags[*], "Sibling")"#, + ), + ("exists($.tags[*] ? (1 == @))", "eq_at($.tags[*], 1)"), + ( + "exists($.contacts[*] ? (@.relationship == $r && @.calls > 5))", + r#"element_match(4, [$.contacts[*].relationship Eq "Sibling"; $.contacts[*].calls Gt 5])"#, + ), + ( + "exists($.contacts[*] ? ((@.calls >= 1) && (@.calls < 9)))", + "element_match(4, [$.contacts[*].calls Gte 1; $.contacts[*].calls Lt 9])", + ), + ( + "exists($.contacts[*] ? (@.calls < 2))", + "element_match(4, [$.contacts[*].calls Lt 2])", + ), + ] { + compiles_to(text, json!({"r": "Sibling"}), want); + } + } + } + + mod given_text_it_refuses { + use super::*; + + use Reason::{ + JsonPathNoWildcard, JsonPathPosition, JsonPathWildcard, + JsonPredicateNotEqualAtWildcard, JsonPredicateSyntax, JsonPredicateTooDeep, + JsonPredicateUnknownVariable, JsonPredicateUnsupported, JsonPredicateValueType, + JsonSyntax, + }; + + /// Each construct the index cannot answer is refused by name, at + /// the byte in the caller's text where it starts. + #[test] + fn names_each_unsupported_construct_at_its_offset() { + for (text, construct, at) in [ + ("strict $.a == 1", C_MODE, 0), + ("STRICT $.a == 1", C_MODE, 0), + ("$.a == Lax", C_MODE, 7), + (r#"$.a LIKE_REGEX "x""#, C_LIKE_REGEX, 4), + (r#"$.a Starts with "x""#, C_STARTS_WITH, 4), + ("$.a IS unknown", C_IS_UNKNOWN, 4), + ("$.a[LAST] == 1", C_LAST, 3), + ("$.a == LAST", C_LAST, 7), + (r#"exists($.a like_regex "x")"#, C_LIKE_REGEX, 11), + ("exists($.a + 1)", C_ARITHMETIC, 11), + ( + "exists($.a[*] ? (@ == 1) starts with \"x\")", + C_STARTS_WITH, + 25, + ), + (" lax $.a == 1", C_MODE, 2), + ("$.a + 1 == 2", C_ARITHMETIC, 4), + ("$.a == 1 - 2", C_ARITHMETIC, 9), + ("$.a == +1", C_ARITHMETIC, 7), + (r#"$.a like_regex "x""#, C_LIKE_REGEX, 4), + (r#"$.a starts with "x""#, C_STARTS_WITH, 4), + ("$.a is unknown", C_IS_UNKNOWN, 4), + ("$.a.size() == 1", C_METHOD, 3), + ("$.a.type () == 1", C_METHOD, 3), + ("$.* == 1", C_ANY_MEMBER, 1), + ("$.** == 1", C_ANY_DEPTH, 1), + ("$.a[1 to 2] == 1", C_SLICE, 3), + ("$.a[1, 2] == 1", C_SLICE, 3), + ("$.a[last] == 1", C_LAST, 3), + ("$.a == last", C_LAST, 7), + ("$.a == lax", C_MODE, 7), + ("$.a[$i] == 1", C_SUBSCRIPT, 3), + ("@ == 1", C_AT_OUTSIDE, 0), + ("$.a == $.b", C_TWO_PATHS, 4), + ("1 == 1", C_TWO_VALUES, 2), + ("$x == $y", C_TWO_VALUES, 3), + ("$.a", C_ALONE, 3), + ("$.a && $.b == 1", C_ALONE, 4), + ("true", C_ALONE, 4), + ("$.a[*] ? (@ == 1) == 1", C_FILTER_OUTSIDE, 7), + ("exists($.a[*] ? (@ == 1).b)", C_AFTER_FILTER, 24), + ("exists($.a[*] ? (@ == 1) ? (@ == 2))", C_FILTER_OUTSIDE, 24), + ("exists($.a == 1)", C_IN_EXISTS, 7), + ("exists(@)", C_AT_OUTSIDE, 7), + ( + "exists($.a[*] ? (@.b == 1 || @.c == 2))", + C_OR_IN_FILTER, + 26, + ), + ("exists($.a[*] ? (!(@.b == 1)))", C_NOT_IN_FILTER, 17), + ("exists($.a[*] ? (@.b != 1))", C_NE_IN_FILTER, 21), + ("exists($.a[*] ? (exists(@.b)))", C_EXISTS_IN_FILTER, 17), + ("exists($.a[*] ? ($.b == 1))", C_DOLLAR_IN_FILTER, 17), + ("$.v == $x.y", C_VARIABLE_PATH, 9), + ("$.tags[*] < 1", C_ELEMENTS_THEMSELVES, 10), + ( + "exists($.contacts[*] ? (@ > 1 && @.calls < 2))", + C_ELEMENTS_THEMSELVES, + 24, + ), + ( + "exists($.contacts[*] ? (@ == 1 && @.calls < 2))", + C_WHOLE_ELEMENT, + 24, + ), + ] { + assert_eq!( + refusal(text, json!({"x": 1, "y": 2})), + (JsonPredicateUnsupported, construct, at), + "{text}" + ); + } + } + + /// Text that is not a predicate is refused where reading stops. + #[test] + fn refuses_text_that_is_not_a_predicate_where_reading_stops() { + for (text, construct, at) in [ + ("", C_TEXT, 0), + ("$.a == 1 &&", C_TEXT, 11), + ("$.a = 1", C_TEXT, 4), + ("$.a === 1", C_TEXT, 6), + ("$.a > 1 > 2", C_TRAILING, 8), + ("$.a == 1 2", C_TRAILING, 9), + ("($.a == 1) is unknown", C_TRAILING, 11), + ("!$.a == 1", C_TEXT, 1), + ("!!($.a == 1)", C_TEXT, 1), + ("($.a == 1", C_TEXT, 9), + ("$.a == 01", C_NUMBER, 7), + ("$.a == 1.", C_NUMBER, 7), + ("$.a == 1e", C_NUMBER, 7), + ("$.a == .5", C_TEXT, 7), + ("$.a == 'x'", C_TEXT, 7), + (r#"$.a == "x"#, C_STRING, 7), + (r#"$."a == 1"#, C_STRING, 2), + ("$.a == TRUE", C_TEXT, 7), + ("$.1a == 1", C_TEXT, 2), + ("$. == 1", C_TEXT, 3), + ("$.a == 1 /* open", C_TEXT, 9), + ("exists $.a", C_TEXT, 7), + ("$.a == $1", C_TEXT, 8), + ] { + assert_eq!( + refusal(text, json!({})), + (JsonPredicateSyntax, construct, at), + "{text}" + ); + } + } + + /// A value is refused at its offset when no variable holds it, its + /// type is not one the comparison takes, or its text is not JSON. + #[test] + fn refuses_a_value_the_comparison_cannot_take_at_its_offset() { + let vars = json!({"doc": {"a": 1}, "list": [1], "n": 1, "flag": true, "none": null}); + for (text, reason, at) in [ + ("$.a == $missing", JsonPredicateUnknownVariable, 7), + ("$missing < $.age", JsonPredicateUnknownVariable, 0), + ("$.a == $doc", JsonPredicateValueType, 7), + ("$.a != $list", JsonPredicateValueType, 7), + ("$.age < true", JsonPredicateValueType, 8), + ("$.age < $flag", JsonPredicateValueType, 8), + ("$.age >= null", JsonPredicateValueType, 9), + ("$.age >= $none", JsonPredicateValueType, 9), + ( + "exists($.contacts[*] ? (@.calls == $n && @.relationship == $flag))", + JsonPredicateValueType, + 59, + ), + (r#"$.a == "\x41""#, JsonPredicateSyntax, 7), + (r#"$.a == 1e999"#, JsonSyntax, 7), + ] { + let (got, _, offset) = refusal(text, vars.clone()); + assert_eq!((got, offset), (reason, at), "{text}"); + } + } + + /// What the index's paths cannot name: a position, `!=` at a + /// wildcard, a filter on a path with no wildcard, a wildcard in a + /// filter's member, and nesting or paths past the limits. + #[test] + fn refuses_paths_the_index_cannot_name() { + let deep = format!( + "{} $.a == 1{}", + "(".repeat(MAX_NESTING + 1), + ")".repeat(MAX_NESTING + 1) + ); + let long = format!("$.{} == 1", vec!["a"; MAX_DEPTH + 1].join(".")); + for (text, want) in [ + ("$.a[2] == 1", (JsonPathPosition, C_POSITION, 3)), + ("exists($.a[ 0 ])", (JsonPathPosition, C_POSITION, 10)), + ( + "$.a[*] != 1", + (JsonPredicateNotEqualAtWildcard, C_NOT_EQUAL, 7), + ), + ( + "$.a[*].b != null", + (JsonPredicateNotEqualAtWildcard, C_NOT_EQUAL, 9), + ), + ( + "exists($.a ? (@ == 1))", + (JsonPathNoWildcard, C_FILTER_WILDCARD, 7), + ), + ( + "exists($.a[*] ? (@.b[*] == 1))", + (JsonPathWildcard, C_MEMBER_WILDCARD, 17), + ), + ( + deep.as_str(), + (JsonPredicateTooDeep, C_NESTING, MAX_NESTING + 1), + ), + (long.as_str(), (JsonPredicateTooDeep, C_NESTING, 0)), + ] { + assert_eq!(refusal(text, json!({})), want, "{text}"); + } + let at_limit = format!( + "{}$.a == 1{}", + "(".repeat(MAX_NESTING), + ")".repeat(MAX_NESTING) + ); + assert!(compiled(&at_limit, json!({})).is_ok(), "the limit is read"); + // A filter's equality is at the base and the member joined, as + // deep as a document at most. + let member = + |steps: usize| format!("exists($.a[*] ? (@.{} == 1))", vec!["b"; steps].join(".")); + assert!( + compiled(&member(MAX_DEPTH - 2), json!({})).is_ok(), + "a filter's full path as deep as a document" + ); + assert_eq!( + refusal(&member(MAX_DEPTH - 1), json!({})), + (JsonPredicateTooDeep, C_NESTING, 7), + "one equality: a filter's full path deeper than a document" + ); + let members = |steps: usize| { + format!( + "exists($.contacts[*] ? (@.calls > 1 && @.{} < 2))", + vec!["b"; steps].join(".") + ) + }; + assert_eq!( + refusal(&members(MAX_DEPTH - 1), json!({})), + (JsonPredicateTooDeep, C_NESTING, 7), + "two conditions: a member's full path deeper than a document" + ); + assert!( + matches!( + compiled(&members(MAX_DEPTH - 2), json!({})), + Err(Error::Cipher(crate::Error::Term( + TermError::JsonPathNotOrdered + ))) + ), + "as deep as a document is read, and refused only as unordered" + ); + let deepest = format!("$.{} == 1", vec!["a"; MAX_DEPTH].join(".")); + assert!( + compiled(&deepest, json!({})).is_ok(), + "a path as deep as a document" + ); + // A step past a document's depth is refused before it is read. + let past = format!("$.{}.** == 1", vec!["a"; MAX_DEPTH].join(".")); + assert_eq!( + refusal(&past, json!({})), + (JsonPredicateTooDeep, C_NESTING, 0), + "the step past the depth is not read" + ); + // Nesting is depth, not count: siblings past the limit, each + // in its own group or under its own `!` or `exists`, are read. + for sibling in ["($.a == 1)", "!($.a == 1)", "exists($.a)"] { + let wide = vec![sibling; MAX_NESTING + 1].join(" && "); + assert!(compiled(&wide, json!({})).is_ok(), "{sibling} side by side"); + } + let filters = + vec!["exists($.contacts[*] ? ((@.calls > 1)))"; MAX_NESTING + 1].join(" || "); + assert!( + compiled(&filters, json!({})).is_ok(), + "filters side by side" + ); + } + + /// The index's own refusals of the leaves' inputs come through as + /// the operators give them: an ordering at a path the field does not + /// order, here or at an element's field. + #[test] + fn refuses_an_ordering_at_a_path_the_field_does_not_order() { + for text in [ + "$.height > 1", + "$.contacts[*].note > 1", + "exists($.contacts[*] ? (@.note > 1 && @.calls < 2))", + ] { + assert!( + matches!( + compiled(text, json!({})), + Err(Error::Cipher(crate::Error::Term( + TermError::JsonPathNotOrdered + ))) + ), + "{text}" + ); + } + } + + /// A refusal carries the construct and the offset, never text from + /// the predicate. + #[test] + fn a_refusal_quotes_no_text_of_the_predicate() { + let secret = "zq-secret-key-name"; + for text in [ + format!(r#"$."{secret}" like_regex "x""#), + format!(r#"$.{secret} == "{secret}" + 1"#), + format!(r#"$.{secret} == ${secret}"#), + format!(r#"$.{secret} == "{secret}" {secret}"#), + ] { + let error = compiled(&text, json!({})).unwrap_err(); + let shown = format!("{error} {:?}", crate::ErrorPayload::payload(&error)); + assert!(!shown.contains(secret), "{shown}"); + } + } + } +} diff --git a/packages/stack-encrypt/src/dynamic/json_query.rs b/packages/stack-encrypt/src/dynamic/json_query.rs index fcf4e9874..09ffaa1f7 100644 --- a/packages/stack-encrypt/src/dynamic/json_query.rs +++ b/packages/stack-encrypt/src/dynamic/json_query.rs @@ -17,6 +17,7 @@ //! | `subtree` | `{"path", "wildcard_bound"?}` | text array: the tokens `json_tree_subtree` takes | //! | `children`, `child_count` | `{"path"}`: a path with no wildcard | text: the node's token | //! | `element_match` | `{"path", "wildcard_bound"?, "predicates"}`: an array's elements, by a path with a wildcard, and predicates on fields of an element at ordered paths | EQL: per predicate, the field's read tokens and its bounds (`eql_v4.query_json_elements`), compared as the orderings compare, so its `eq` is of folded values | +//! | `match` | `{"predicate", "vars"?, "wildcard_bound"?}`: a jsonpath predicate in the subset below, and its variables as one JSON object | EQL: the predicate as a boolean tree of the operands above (`eql_v4.query_json_predicate`), which `json_tree_match` and `@@` evaluate | //! //! Every JSON value in an input is UTF-8 JSON text, parsed by //! [`parse_text`](crate::json::parse_text)'s rules. A path is JSONPath text @@ -79,6 +80,105 @@ //! subtree read of it opens to a refusal: each member's tokens end with //! the probe, and `json_tree_element_match` raises when it names a node. //! +//! # Compiled predicates +//! +//! `match` takes what plaintext `jsonb_path_match(col, $1, $2)` takes: a +//! jsonpath predicate and its variables, `vars` the JSON text of one object +//! (`{}` when absent). It compiles the predicate into one operand, a tree +//! of `and`, `or` and `not` over the operands the operators above derive, +//! each leaf derived by its operator's own code: +//! +//! ```json +//! {"predicate": "$.age >= $min && exists($.contacts[*] ? (@.relationship == $r && @.calls > 5))", +//! "vars": "{\"min\": 18, \"r\": \"Sibling\"}", "wildcard_bound": 16} +//! ``` +//! +//! The grammar is the subset of SQL/JSON path predicates the terms answer, +//! as PostgreSQL reads it: a path is `$` and steps `.key`, `."quoted key"` +//! (JSON's escapes) and `[*]`; a value is a variable (`$v`, `$"v"`) or a +//! JSON literal (a string, a number, `true`, `false`, `null`); predicates +//! are comparisons of a path with a value (`==`, `!=` or `<>`, `<`, `<=`, +//! `>`, `>=`, either side first), `exists(path)` and `exists(path ? (…))` +//! with a filter of comparisons on `@` and `@.member` joined by `&&`, and +//! `&&`, `||`, `!(…)` and parentheses over them. Each maps onto operands +//! this way: +//! +//! | jsonpath | operand | +//! |---|---| +//! | `P == v`, `P == null` | `eq_at`'s containment needle | +//! | `P != v`, `P != null` | `selector` and not that needle: a node there that is not the value | +//! | `P < v` … `P >= v` | `selector` and the ordering's leaf: a node there whose leaf compares; under an odd number of `!`, also not the `null` needle | +//! | `P < v` … `P >= v`, `P` with a wildcard | `element_match` on the array through `P`'s last wildcard, the rest of `P` its member | +//! | `exists(P)` | `selector` | +//! | `exists(P ? (@.m == v))`, `exists(P ? (@ == v))` | `eq_at` at `P.m` or `P` | +//! | `exists(P ? (@.m op v && …))` | `element_match` on `P`'s elements, one predicate per condition | +//! +//! So the tree is true of exactly the rows `jsonb_path_match` is true of +//! over the plaintext, and answers as it does, true, false or NULL +//! (unknown) as SQL's three-valued logic combines them. An absent path is +//! false under every comparison, `!=` is true of a `null`, and an ordering +//! is unknown of a node with no ordering term (a boolean, a container), as +//! jsonpath's comparison of two kinds is. A JSON `null` is false under an +//! ordering in jsonpath. The tree says so only where it changes the rows, +//! under an odd number of `!`, by sending the `null` needle, which tells +//! the database which rows hold `null` at the path. Elsewhere it sends no +//! needle and an ordering of a `null` is NULL, which `and` and `or` cannot +//! turn into true, so the rows are the same. The tree's answer +//! and jsonpath's part in four places, each a property of the stored +//! document or of the ordering terms rather than of the predicate: +//! +//! - A path is read as written: lax mode unwraps an array the path names +//! no `[*]` for (`$.tags == "a"` over `{"tags": ["a"]}` is true), and +//! the tree does not. Write `[*]`. +//! - Lax mode also wraps a value that is not an array where the path names +//! `[*]`: `$.tags[*] == "a"` over `{"tags": "a"}`, `exists($.tags[*])` +//! over a scalar `tags` and `exists($.items[*] ? (@.sku == "x"))` over +//! `{"items": {"sku": "x"}}` are all true to jsonpath. The tree finds +//! only array elements at `[*]`, since `$.tags[*]` and `$.tags` have +//! different selectors, so each is false here. Keep an array an array. +//! - A comparison of a string with a number, or of a boolean with a +//! string or a number, is unknown to jsonpath. The terms do not record +//! a value's kind, so the tree answers it false under `==` (and in a +//! filter, as jsonpath's filter does), true under `!=`, and by the +//! ordering terms under an ordering. Keep one kind of value at a path. +//! - An ordering compares the ordering terms, as the orderings do: text +//! case- and accent-folded, numbers as float64. So does every condition +//! of a filter of more than one, `element_match`'s, whose equality is +//! therefore of folded values; `==` elsewhere is exact. +//! +//! An ordering at a path the field does not order is refused, as the +//! orderings refuse it. +//! +//! Refused, each at the byte of the caller's text where the construct +//! starts and naming the construct by the grammar's words and never the +//! text ([`Error::Predicate`]): text the grammar does not read +//! ([`Reason::JsonPredicateSyntax`]); arithmetic, `like_regex`, `starts +//! with`, `is unknown`, a method (`.size()`), a mode word (`strict`, +//! `lax`), `.*`, `.**`, a slice, `last`, a comparison of two paths or two +//! values, a filter outside `exists`, and `||`, `!`, `!=` or `exists` +//! inside a filter ([`Reason::JsonPredicateUnsupported`]); a variable +//! `vars` does not hold ([`Reason::JsonPredicateUnknownVariable`]); a +//! value of a type its comparison cannot take, an object or array +//! anywhere, and under an ordering or in a filter of more than one +//! condition anything but a number or a string +//! ([`Reason::JsonPredicateValueType`]); `!=` at a path with a wildcard +//! ([`Reason::JsonPredicateNotEqualAtWildcard`]); nesting past 32 levels or +//! a path deeper than a document ([`Reason::JsonPredicateTooDeep`]); a +//! position (`[2]`, [`Reason::JsonPathPosition`]); a filter on a path with +//! no wildcard ([`Reason::JsonPathNoWildcard`]); and a wildcard in a +//! filter's member ([`Reason::JsonPathWildcard`]). The leaves' operators +//! refuse their own inputs as they would alone: an ordering at a path the +//! field does not order ([`TermError::JsonPathNotOrdered`]). A +//! `wildcard_bound` of `0` is [`Reason::JsonWildcardBoundZero`], and `vars` +//! that are not an object [`Reason::JsonInput`]. +//! +//! The operand reveals what its leaves reveal when sent one by one, and +//! the boolean shape of the filter; an ordering under an odd number of `!` +//! sends the `null` needle too, one leaf more than the ordering alone +//! sends. `json_tree_match` evaluates the +//! tree inside a function, which engages no index, so a predicate written +//! out per operator is the fast path. +//! //! # Reads //! //! [`json_open`] opens the rows a read returned, against the request that @@ -125,6 +225,8 @@ pub(super) enum Binds { TextArray, /// The target's element predicate domain (`eql_v4.query_json_elements`). Elements, + /// The target's compiled predicate domain (`eql_v4.query_json_predicate`). + Predicate, } /// What a JSON operator's operand binds as, or `None` for an operator @@ -140,6 +242,7 @@ pub(super) fn binds(op: OpSpec) -> Option { OpSpec::Selector | OpSpec::Children | OpSpec::ChildCount => Some(Binds::Text), OpSpec::Subtree => Some(Binds::TextArray), OpSpec::ElementMatch => Some(Binds::Elements), + OpSpec::Match => Some(Binds::Predicate), OpSpec::Eq | OpSpec::Matches | OpSpec::Lt @@ -159,12 +262,14 @@ pub(super) fn domain(target: &super::TargetDescriptor, binds: Binds) -> Option Some("text".to_owned()), Binds::TextArray => Some("text[]".to_owned()), Binds::Elements => target.elements_query_sql_domain.clone(), + Binds::Predicate => target.predicate_query_sql_domain.clone(), } } /// UTF-8 JSON text in an input: a text or bytes leaf, held in a buffer -/// wiped on drop. -struct JsonText(Zeroizing>); +/// wiped on drop. Also how `match` holds its predicate's text, whose +/// literals are values. +pub(super) struct JsonText(pub(super) Zeroizing>); impl<'de> Deserialize<'de> for JsonText { fn deserialize>(d: D) -> Result { @@ -186,7 +291,8 @@ impl<'de> Deserialize<'de> for JsonText { } /// A document parsed from an input, wiped when it drops. -struct Parsed(Value); +#[derive(PartialEq)] +pub(super) struct Parsed(pub(super) Value); impl Drop for Parsed { fn drop(&mut self) { @@ -195,7 +301,7 @@ impl Drop for Parsed { } impl JsonText { - fn parse(&self) -> Result { + pub(super) fn parse(&self) -> Result { parse_text(&self.0) .map(Parsed) .map_err(|error| Error::Source { @@ -206,7 +312,7 @@ impl JsonText { } /// The reason a JSON text refusal is reported with. -fn json_reason(error: crate::json::JsonTextError) -> Reason { +pub(super) fn json_reason(error: crate::json::JsonTextError) -> Reason { use crate::json::JsonTextError as E; match error { E::Syntax { .. } => Reason::JsonSyntax, @@ -272,9 +378,19 @@ struct PredicateInput { high: Option, } +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct MatchInput { + predicate: JsonText, + #[serde(default)] + vars: Option, + #[serde(default)] + wildcard_bound: Option, +} + /// How a predicate of `element_match` compares its field with one value. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Compare { +pub(super) enum Compare { Eq, Lt, Lte, @@ -283,7 +399,8 @@ enum Compare { } /// A predicate's bounds, as its input gives them. -enum Bounds { +#[derive(PartialEq)] +pub(super) enum Bounds { /// One value, compared by the predicate's `op`. One(Compare, Parsed), /// `between`: the low bound then the high, both inclusive. @@ -292,20 +409,24 @@ enum Bounds { /// One predicate of `element_match`, checked: the full path of the /// element's field, which the field orders, and its bounds. -struct Predicate { - path: JsonPath, - bounds: Bounds, +#[derive(PartialEq)] +pub(super) struct Predicate { + pub(super) path: JsonPath, + pub(super) bounds: Bounds, } /// An operator's input, read from its frame and checked: everything -/// [`derive`] would refuse before it derives anything. -enum Input { +/// [`derive`] would refuse before it derives anything. `match`'s is its +/// predicate compiled to a tree of the other operators' inputs. +#[derive(PartialEq)] +pub(super) enum Input { Document(Parsed), At(JsonPath, Parsed), Between(JsonPath, Parsed, Parsed), Path(JsonPath), Read(JsonPath, u32), Elements(u32, Vec), + Predicate(Box), } fn read_frame Deserialize<'de>>(input: &ValueFrame) -> Result { @@ -380,10 +501,41 @@ fn read_input(op: OpSpec, options: &JsonOptions, input: &ValueFrame) -> Result read_elements(options, input), + OpSpec::Match => read_match(options, input), _ => Err(Error::Internal), } } +/// The wildcard bound an input names, or the default: never `0`, which +/// would send the probe alone, so an element predicate could never be true. +fn wildcard_bound(bound: Option) -> Result { + match bound.unwrap_or(JsonSubtreeKey::DEFAULT_BOUND) { + 0 => Err(Error::bad_source(Reason::JsonWildcardBoundZero)), + bound => Ok(bound), + } +} + +/// `match`'s input: a jsonpath predicate, its variables as one JSON object +/// (the `vars` jsonb `jsonb_path_match` takes, `{}` when absent) and the +/// bound its element-scoped leaves are read to. The predicate is compiled +/// here, so every refusal it can meet is met before anything is derived. +fn read_match(options: &JsonOptions, input: &ValueFrame) -> Result { + let input: MatchInput = read_frame(input)?; + let bound = wildcard_bound(input.wildcard_bound)?; + let vars = match &input.vars { + Some(vars) => vars.parse()?, + None => Parsed(Value::Object(serde_json::Map::new())), + }; + if !vars.0.is_object() { + return Err(Error::bad_source(Reason::JsonInput)); + } + let text = std::str::from_utf8(&input.predicate.0) + .map_err(|_| Error::bad_source(Reason::JsonInput))?; + Ok(Input::Predicate(Box::new(super::json_predicate::compile( + text, &vars.0, options, bound, + )?))) +} + /// `element_match`'s input: an array's elements, by a path with a wildcard /// and no position, and one or more predicates on their fields. fn read_elements(options: &JsonOptions, input: &ValueFrame) -> Result { @@ -395,12 +547,7 @@ fn read_elements(options: &JsonOptions, input: &ValueFrame) -> Result Result Result { if !relative.is_positional() { return Err(Error::bad_source(Reason::JsonPathWildcard)); } @@ -514,7 +672,7 @@ fn ordered_path(options: &JsonOptions, text: &str) -> Result { } /// [`ordered_path`] of a path already parsed. -fn ordered_path_of(options: &JsonOptions, path: JsonPath) -> Result { +pub(super) fn ordered_path_of(options: &JsonOptions, path: JsonPath) -> Result { if path.names_a_position() { Err(term(TermError::JsonOrderingAtPosition)) } else if options.orders(&path) { @@ -525,7 +683,7 @@ fn ordered_path_of(options: &JsonOptions, path: JsonPath) -> Result Result<(), Error> { +pub(super) fn leaf(value: &Value) -> Result<(), Error> { match value { Value::Number(_) | Value::String(_) => Ok(()), _ => Err(term(TermError::JsonLeafNotOrdered)), @@ -558,7 +716,7 @@ fn terms<'a, K: 'static>( ) } -fn eql(value: &T) -> Result, crate::Error> { +pub(super) fn eql(value: &T) -> Result, crate::Error> { serde_json::to_vec(value).map_err(|error| crate::Error::Other(Box::new(error))) } @@ -570,9 +728,22 @@ pub(super) fn derive<'a, K: 'static>( options: &JsonOptions, op: OpSpec, input: &ValueFrame, +) -> Result, Error> { + derive_input(keyset, label, options, op, read_input(op, options, input)?) +} + +/// Derive an operand for an input already read and checked: what +/// [`derive`] does past reading, and what `match` does for each of its +/// leaves. +pub(super) fn derive_input<'a, K: 'static>( + keyset: &'a KeysetCipher<'_, K>, + label: &Label, + options: &JsonOptions, + op: OpSpec, + input: Input, ) -> Result, Error> { let column = || NonEmpty::from(label.clone()); - Ok(match read_input(op, options, input)? { + Ok(match input { Input::Document(document) => terms(keyset, options, label, document.0.clone()) .try_map(|terms| eql(&JsonQuery::of_terms(terms)).map(Operand::One)), Input::At(path, value) if op == OpSpec::EqAt => { @@ -612,6 +783,7 @@ pub(super) fn derive<'a, K: 'static>( Pending::ready(keyset, Ok(Operand::TextList(key.tokens()))) } Input::Elements(bound, predicates) => elements(keyset, label, options, bound, &predicates)?, + Input::Predicate(node) => super::json_predicate::derive(keyset, label, options, *node)?, }) } diff --git a/packages/stack-encrypt/src/dynamic/mod.rs b/packages/stack-encrypt/src/dynamic/mod.rs index e1f8b2f61..7a9a7b633 100644 --- a/packages/stack-encrypt/src/dynamic/mod.rs +++ b/packages/stack-encrypt/src/dynamic/mod.rs @@ -89,6 +89,7 @@ //! reason, given on the type. mod context; mod json; +mod json_predicate; mod json_query; mod kind; mod query; @@ -314,6 +315,33 @@ pub enum Error { reason: Reason, }, + /// A `match` predicate is refused at a place in its text: text the + /// grammar does not read, a construct of SQL/JSON path the JSON index + /// cannot answer, a variable its `vars` do not hold, or a value of a + /// type its comparison cannot take. The error names the construct in + /// the grammar's own words and the byte offset in the caller's + /// predicate text, and nothing of the text itself: a key name or a + /// value in it can be customer data. + #[error( + "the predicate is refused at byte {at}: {reason} ({construct}){}", + in_field_text(field) + )] + #[diagnostic( + code(stack_encrypt::dynamic_predicate), + help("A predicate compares a path (`$.a.b`, `$.a[*].b`, `$.\"a b\"`) with a variable (`$v`) or a literal by `==`, `!=`, `<`, `<=`, `>` or `>=`, tests `exists($.path)` or `exists($.array[*] ? (@.x == $v && @.y > $w))`, and combines them with `&&`, `||`, `!` and parentheses.") + )] + Predicate { + /// The plan field the predicate is for, if one is named yet. + field: Option, + /// What was wrong. + reason: Reason, + /// The construct refused, in the grammar's words: never text from + /// the predicate. + construct: &'static str, + /// The byte offset in the predicate text the construct starts at. + at: usize, + }, + /// An invariant this module maintains did not hold — a slot count that /// did not line up, a re-proof that should not have been able to fail. /// Always a bug here, never a statement about the caller's data. @@ -391,6 +419,7 @@ impl Error { | Self::Plan { field, .. } | Self::Source { field, .. } | Self::Record { field, .. } + | Self::Predicate { field, .. } if field.is_none() => { *field = Some(name.to_owned()); @@ -407,19 +436,22 @@ impl Error { | Self::Term { field, .. } | Self::Plan { field, .. } | Self::Source { field, .. } - | Self::Record { field, .. } => field.as_deref(), + | Self::Record { field, .. } + | Self::Predicate { field, .. } => field.as_deref(), Self::UntypedIndex { field } => Some(field), Self::Internal | Self::Target(_) | Self::Cipher(_) => None, } } - /// What was wrong, for a context, plan, source or record error. + /// What was wrong, for a context, plan, source, record or predicate + /// error. pub fn reason(&self) -> Option { match self { Self::Context { reason, .. } | Self::Plan { reason, .. } | Self::Source { reason, .. } - | Self::Record { reason, .. } => Some(*reason), + | Self::Record { reason, .. } + | Self::Predicate { reason, .. } => Some(*reason), Self::Term { .. } | Self::UntypedIndex { .. } | Self::Internal @@ -435,6 +467,10 @@ impl crate::ErrorPayload for Error { Self::Target(error) => return error.payload(), Self::Cipher(error) => return error.payload(), Self::Term { kind, .. } => crate::diagnostic::payload([("index", kind.key().into())]), + Self::Predicate { construct, at, .. } => crate::diagnostic::payload([ + ("construct", (*construct).into()), + ("at", (*at as u64).into()), + ]), _ => serde_json::Map::new(), }; if let Some(field) = self.field() { @@ -669,6 +705,32 @@ reasons! { /// A predicate of an element predicate has an `op` that is not /// `eq`, `lt`, `lte`, `gt`, `gte` or `between`. JsonUnknownPredicateOp => "json_unknown_predicate_op", r#"a predicate's "op" is not eq, lt, lte, gt, gte or between"#; + /// A `match` predicate is not text the SQL/JSON path predicate + /// grammar reads: an unexpected character, an unterminated string, + /// a literal that is not JSON, or text after the predicate. + JsonPredicateSyntax => "json_predicate_syntax", "the predicate is not SQL/JSON path predicate text"; + /// A `match` predicate uses a construct of SQL/JSON path the JSON + /// index cannot answer: arithmetic, `like_regex`, `starts with`, + /// `is unknown`, a method, a `strict` or `lax` mode, `.*`, `.**`, + /// a slice, `last`, a comparison of two paths or of two values, a + /// filter outside `exists`, or `||`, `!` or `!=` inside a filter. + JsonPredicateUnsupported => "json_predicate_unsupported", "the predicate uses a construct the JSON index cannot answer"; + /// A `match` predicate names a variable (`$v`) its `vars` do not + /// hold. + JsonPredicateUnknownVariable => "json_predicate_unknown_variable", "the predicate names a variable its vars do not hold"; + /// A value in a `match` predicate, a literal or a variable, is of a + /// JSON type its comparison cannot take: an object or array + /// anywhere, and anything but a number or a string beside `<`, + /// `<=`, `>`, `>=` or in an element-scoped filter of more than one + /// equality. + JsonPredicateValueType => "json_predicate_value_type", "a value in the predicate is of a type its comparison cannot take"; + /// `!=` at a path with a wildcard: jsonpath answers it true when + /// some element differs, which no term of the index can say. + JsonPredicateNotEqualAtWildcard => "json_predicate_not_equal_at_wildcard", "!= at a path with a wildcard asks whether some element differs, which the index cannot answer"; + /// A `match` predicate nests parentheses, `!` and `exists` deeper + /// than the engine reads, or names a path with more steps than a + /// document has levels. + JsonPredicateTooDeep => "json_predicate_too_deep", "the predicate nests too deep"; } } diff --git a/packages/stack-encrypt/src/dynamic/query.rs b/packages/stack-encrypt/src/dynamic/query.rs index c6161a570..e678d0a63 100644 --- a/packages/stack-encrypt/src/dynamic/query.rs +++ b/packages/stack-encrypt/src/dynamic/query.rs @@ -159,12 +159,91 @@ pub enum OpSpec { /// which `json_tree_element_match` takes. True of a row where some one /// element satisfies every member. ElementMatch, + /// `"match"`: a jsonpath predicate and its variables, compiled to a + /// tree of the other operators' operands, which `json_tree_match` and + /// `@@` take: the one-parameter form of `jsonb_path_match`. Its input + /// is `{"predicate", "vars"?, "wildcard_bound"?}`, `vars` the JSON text + /// of the object `jsonb_path_match` takes. It reads the subset of + /// SQL/JSON path predicates the index's terms answer: comparisons of a + /// path (`$.a.b`, `$.a[*].b`, `$."a b"`) with a variable or a literal, + /// `exists(path)`, `exists(path[*] ? (@.x == $v && @.y > $w))`, `&&`, + /// `||`, `!` and parentheses; and refuses anything else at the byte + /// it starts ([`Error::Predicate`](super::Error::Predicate)). The text + /// is PostgreSQL's [SQL/JSON path language](https://www.postgresql.org/docs/current/functions-json.html#FUNCTIONS-SQLJSON-PATH). + /// It is true of jsonpath's rows, and its answer is jsonpath's, + /// three-valued, save that an ordering of a JSON `null` is unknown + /// rather than false except under an odd number of `!` (which picks + /// the same rows, and keeps a `null` needle out of the operand), a + /// path is read as written (lax mode's unwrapping of an array the path + /// names no `[*]` for, and its wrapping of a value that is not an + /// array where it does, are not applied), a comparison of two kinds + /// of value is decided by the terms where jsonpath answers unknown, + /// and an ordering or a filter of more than one condition compares by + /// the ordering terms, as `element_match` does. + /// + /// ``` + /// use serde_json::json; + /// use stack_encrypt::dynamic::{check_query, frame, record, OpSpec, Reason, ValueFrame}; + /// # use stack_encrypt::dynamic::{TargetDescriptor, TargetError, TargetOptions, TargetResolver, ValueKind}; + /// # use stack_encrypt::target::IndexSpec; + /// # use stack_encrypt::{KeysetCipher, Label, Pending, StackCipher}; + /// # /// A host whose `JsonSearch` names the compiled predicate's domain. + /// # struct Documents; + /// # impl TargetResolver for Documents { + /// # fn targets(&self) -> Vec { + /// # vec![TargetDescriptor::new( + /// # "JsonSearch", "json", "", Some(ValueKind::Json), "public.eql_v4_json_search", + /// # vec!["json".to_owned()], Some("JsonSearchQuery".to_owned()), + /// # Some("eql_v4.query_json".to_owned()), true, None, + /// # ) + /// # .with_predicate_query_sql_domain("eql_v4.query_json_predicate")] + /// # } + /// # fn encrypt<'a, K: 'static>(&self, _: &str, _: Option<&TargetOptions>, + /// # _: &'a KeysetCipher<'_, K>, _: &Label, _: ValueFrame, + /// # ) -> Result, K>, TargetError> { unreachable!() } + /// # fn decrypt<'a, K: 'static>(&self, _: &str, _: &'a StackCipher, _: &Label, _: &[u8], + /// # ) -> Result, TargetError> { unreachable!() } + /// # fn query<'a, K: 'static>(&self, _: &str, _: &IndexSpec, _: &'a KeysetCipher<'_, K>, + /// # _: &Label, _: ValueFrame, + /// # ) -> Result, K>, TargetError> { unreachable!() } + /// # } + /// // A document field that orders `$.age`. + /// let plan = record::plan_with( + /// &frame::serde::to_vec(&json!({"profile": { + /// "context": ["users", "profile"], "type": "json", "null": "sql", + /// "target": {"name": "JsonSearch", "options": {"ordered": ["$.age"]}} + /// }}))?, + /// &Documents, + /// )?; + /// // What `jsonb_path_match(col, $1, $2)` takes: the predicate, and its + /// // variables as the JSON text of one object. + /// let input = |predicate: &str| -> ValueFrame { + /// let input = json!({"predicate": predicate, "vars": r#"{"min": 18, "r": "sibling"}"#}); + /// ValueFrame::from_bytes(&frame::serde::to_vec(&input).unwrap()).unwrap() + /// }; + /// let check = |predicate: &str| { + /// check_query(&plan, "profile", OpSpec::Match, input(predicate), &Documents) + /// }; + /// + /// check("$.age >= $min && !(exists($.retired))")?; + /// check(r#"$.name == "ann" || $."first name" != null"#)?; + /// check("exists($.contacts[*] ? (@.relationship == $r))")?; + /// + /// // Refused at the byte the construct starts, by name, never quoting + /// // the text. + /// let error = check(r#"$.email like_regex "@""#).unwrap_err(); + /// assert_eq!(error.reason(), Some(Reason::JsonPredicateUnsupported)); + /// let error = check("$.age + 1 > $min").unwrap_err(); + /// assert!(error.to_string().contains("at byte 6"), "{error}"); + /// # Ok::<(), Box>(()) + /// ``` + Match, } impl OpSpec { /// Every operator, in declaration order: the order a catalog lists them /// in. - pub const ALL: [OpSpec; 20] = [ + pub const ALL: [OpSpec; 21] = [ OpSpec::Eq, OpSpec::Lt, OpSpec::Lte, @@ -185,6 +264,7 @@ impl OpSpec { OpSpec::Children, OpSpec::ChildCount, OpSpec::ElementMatch, + OpSpec::Match, ]; /// The operator a name names, or `None` for a name that is not one. @@ -216,6 +296,7 @@ impl OpSpec { OpSpec::Children => "children", OpSpec::ChildCount => "child_count", OpSpec::ElementMatch => "element_match", + OpSpec::Match => "match", } } @@ -230,8 +311,9 @@ impl OpSpec { /// (one value of the field's kind), `pair` (two, low then high), /// `json` (`{"json"}`), `path_json` (`{"path", "json"}`), `path_pair` /// (`{"path", "low", "high"}`), `path` (`{"path"}`), `path_bound` - /// (`{"path", "wildcard_bound"?}`) or `path_predicates` (`{"path", - /// "wildcard_bound"?, "predicates"}`). + /// (`{"path", "wildcard_bound"?}`), `path_predicates` (`{"path", + /// "wildcard_bound"?, "predicates"}`) or `predicate` (`{"predicate", + /// "vars"?, "wildcard_bound"?}`). pub fn input(self) -> &'static str { match self { OpSpec::Between => "pair", @@ -243,6 +325,7 @@ impl OpSpec { OpSpec::Selector | OpSpec::Children | OpSpec::ChildCount => "path", OpSpec::Subtree => "path_bound", OpSpec::ElementMatch => "path_predicates", + OpSpec::Match => "predicate", OpSpec::Eq | OpSpec::Lt | OpSpec::Lte | OpSpec::Gt | OpSpec::Gte | OpSpec::Matches => { "value" } @@ -265,6 +348,11 @@ impl OpSpec { /// two values the plaintext tells apart can compare equal, and /// `element_match`'s `eq`, which is `gte` and `lte` at one value, is an /// equality of folded values, where `eq_at` is exact on the value. + /// `match` is exact on the terms its leaves are: its equalities are + /// `eq_at`'s save inside an element-scoped filter of more than one + /// condition, which is `element_match`'s, and its orderings are the + /// orderings' ([`OpSpec::Match`] says where its answer and jsonpath's + /// part). pub fn fidelity(self) -> Fidelity { match self { OpSpec::Matches => Fidelity::Probabilistic, @@ -341,7 +429,8 @@ impl OpSpec { | OpSpec::Subtree | OpSpec::Children | OpSpec::ChildCount - | OpSpec::ElementMatch => None, + | OpSpec::ElementMatch + | OpSpec::Match => None, } .filter(|index| admits(kind, index)) } @@ -411,7 +500,8 @@ impl Fidelity { /// lists them: none for a type the engine does not produce or that has no /// query twin, and otherwise every operator of one value that one of the /// type's indexes answers for its plaintext kind, `matches` only when the -/// type's match options are known. The match index answers under those +/// type's match options are known, and a JSON operator only when the type +/// names the domain its operand binds as. The match index answers under those /// options. See the [module docs](self#which-operators-a-field-answers). pub(super) fn target_operators(target: &TargetDescriptor) -> Vec<(OpSpec, IndexSpec)> { let Some(kind) = target @@ -432,10 +522,16 @@ pub(super) fn target_operators(target: &TargetDescriptor) -> Vec<(OpSpec, IndexS ) .collect(); // `between` through an EQL type is not answered yet; `between_at` on a - // document is (its bounds are two leaf operands). + // document is (its bounds are two leaf operands). A JSON operator is + // listed only when the type names the SQL type its operand binds as, + // as a field's list has it. OpSpec::ALL .into_iter() .filter(|op| !op.takes_pair()) + .filter(|op| { + super::json_query::binds(*op) + .is_none_or(|binds| super::json_query::domain(target, binds).is_some()) + }) .filter_map(|op| { op.answered_by(kind, &indexes) .map(|index| (op, index.clone())) @@ -3198,7 +3294,7 @@ mod wire_bytes { assert_eq!(written, PINNED); } - const PINNED: [&str; 163] = [ + const PINNED: [&str; 170] = [ "0a020000006571", "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0500000076616c7565", "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0500000076616c7565", @@ -3220,6 +3316,7 @@ mod wire_bytes { "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0d000000656c656d656e745f6d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0f000000706174685f70726564696361746573", + "f105000000080000006f70657261746f720a050000006d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a09000000707265646963617465", "0a050000006d61746368", "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", @@ -3241,6 +3338,7 @@ mod wire_bytes { "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0d000000656c656d656e745f6d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0f000000706174685f70726564696361746573", + "f105000000080000006f70657261746f720a050000006d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000707265646963617465", "f101000000050000006d61746368f10400000009000000746f6b656e697a65720a080000007374616e6461726408000000646f776e6361736502010000006b070400000000000000010000006d070002000000000000", "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", @@ -3262,6 +3360,7 @@ mod wire_bytes { "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0d000000656c656d656e745f6d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0f000000706174685f70726564696361746573", + "f105000000080000006f70657261746f720a050000006d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000707265646963617465", "f101000000050000006d61746368f10400000009000000746f6b656e697a6572f101000000050000006e6772616d07040000000000000008000000646f776e6361736503010000006b070300000000000000010000006d070001000000000000", "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", @@ -3283,6 +3382,7 @@ mod wire_bytes { "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0d000000656c656d656e745f6d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0f000000706174685f70726564696361746573", + "f105000000080000006f70657261746f720a050000006d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000707265646963617465", "0a030000006f7265", "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0500000076616c7565", "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0500000076616c7565", @@ -3304,6 +3404,7 @@ mod wire_bytes { "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0d000000656c656d656e745f6d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0f000000706174685f70726564696361746573", + "f105000000080000006f70657261746f720a050000006d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a09000000707265646963617465", "0a030000006f7065", "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0500000076616c7565", "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0500000076616c7565", @@ -3325,6 +3426,7 @@ mod wire_bytes { "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0d000000656c656d656e745f6d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0f000000706174685f70726564696361746573", + "f105000000080000006f70657261746f720a050000006d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a09000000707265646963617465", "0a040000006a736f6e", "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0500000076616c7565", "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0500000076616c7565", @@ -3346,6 +3448,7 @@ mod wire_bytes { "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a0d000000656c656d656e745f6d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0f000000706174685f70726564696361746573", + "f105000000080000006f70657261746f720a050000006d6174636808000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a09000000707265646963617465", "f106000000080000006f70657261746f720a02000000657108000000666964656c6974790a05000000657861637405000000696e6465780a02000000657106000000646f6d61696e0008000000656e636f64696e670a050000006279746573040000007465726d0b03000000010203", "f106000000080000006f70657261746f720a070000006265747765656e08000000666964656c6974790a05000000657861637405000000696e6465780a030000006f706506000000646f6d61696e0a1000000065716c5f76342e71756572795f696e7408000000656e636f64696e670a0300000065716c040000007465726df0020000000b01000000040b020000000506", "f106000000080000006f70657261746f720a070000006d61746368657308000000666964656c6974790a0d00000070726f626162696c697374696305000000696e6465780a050000006d6174636806000000646f6d61696e0008000000656e636f64696e670a050000006279746573040000007465726d0b00000000", diff --git a/packages/stack-encrypt/src/dynamic/target.rs b/packages/stack-encrypt/src/dynamic/target.rs index 77555ec42..85db274bc 100644 --- a/packages/stack-encrypt/src/dynamic/target.rs +++ b/packages/stack-encrypt/src/dynamic/target.rs @@ -117,6 +117,12 @@ pub struct TargetDescriptor { /// [`leaf_query_sql_domain`](Self::leaf_query_sql_domain) is not; a type /// that names none does not answer `element_match`. pub elements_query_sql_domain: Option, + /// The PostgreSQL domain of the type's compiled predicate, for a type + /// that stores a JSON document + /// ([`with_predicate_query_sql_domain`](Self::with_predicate_query_sql_domain)): + /// what `match` binds as. Not on the wire; a type that names none does + /// not answer `match`. + pub predicate_query_sql_domain: Option, } /// The descriptor in its wire form: the object a guest's `se_targets` @@ -186,6 +192,7 @@ impl TargetDescriptor { match_options: None, leaf_query_sql_domain: None, elements_query_sql_domain: None, + predicate_query_sql_domain: None, } } @@ -205,6 +212,14 @@ impl TargetDescriptor { self } + /// The descriptor of a type that stores a JSON document, whose compiled + /// predicate is the domain `domain`. + #[must_use] + pub fn with_predicate_query_sql_domain(mut self, domain: impl Into) -> Self { + self.predicate_query_sql_domain = Some(domain.into()); + self + } + /// The descriptor of a type whose match index is built under `options`: /// such a type answers `matches` through its query twin /// ([`operators`](Self::operators)). A type with a match index and no @@ -229,8 +244,11 @@ impl TargetDescriptor { /// in [`OpSpec::ALL`]'s order: none for a type it does not produce or /// that has no query twin, and otherwise each operator of one value /// that one of the type's indexes answers for its plaintext kind - /// ([`OpSpec::answered_by`]): the exact ones, and `matches` when the - /// type's match options are known ([`with_match_options`](Self::with_match_options)). + /// ([`OpSpec::answered_by`]): the exact ones, `matches` when the + /// type's match options are known ([`with_match_options`](Self::with_match_options)), + /// and a JSON operator when the type names the SQL type its operand + /// binds as (`match` with + /// [`with_predicate_query_sql_domain`](Self::with_predicate_query_sql_domain)). /// The query for each is the type's query twin. `between` is not /// answered through a type yet: see [`query_term`](super::query_term)'s /// module. diff --git a/packages/stack-encrypt/src/json/path.rs b/packages/stack-encrypt/src/json/path.rs index 95b327e89..0e0d06108 100644 --- a/packages/stack-encrypt/src/json/path.rs +++ b/packages/stack-encrypt/src/json/path.rs @@ -264,6 +264,12 @@ impl JsonPath { current } + /// The path of these steps, from the root down. + #[cfg(feature = "dynamic")] + pub(crate) fn of(segments: Vec) -> Self { + Self { segments } + } + /// This path with one more step. pub(crate) fn then(&self, segment: Segment) -> Self { let mut segments = Vec::with_capacity(self.segments.len() + 1); diff --git a/packages/stack-encrypt/tests/heap_hygiene.rs b/packages/stack-encrypt/tests/heap_hygiene.rs index 91c226c93..0b59145c6 100644 --- a/packages/stack-encrypt/tests/heap_hygiene.rs +++ b/packages/stack-encrypt/tests/heap_hygiene.rs @@ -426,7 +426,8 @@ impl stack_encrypt::dynamic::TargetResolver for Documents { None, ) .with_leaf_query_sql_domain("eql_v4.query_json_leaf") - .with_elements_query_sql_domain("eql_v4.query_json_elements")] + .with_elements_query_sql_domain("eql_v4.query_json_elements") + .with_predicate_query_sql_domain("eql_v4.query_json_predicate")] } fn encrypt<'a, K: 'static>( &self, @@ -845,6 +846,12 @@ mod json_index { let mark = marked("operand"); let quoted = format!("{mark:?}"); let document = format!(r#"{{{quoted}: [{quoted}], "name": {quoted}}}"#); + // Every leaf kind, with a literal and a variable carrying the mark. + let predicate = format!( + "$.name == {quoted} && $.name != $v && $.age > {quoted} && \ + exists($.visits[*] ? (@.at == $v && @.at < {quoted})) && $.visits[*].at >= $v" + ); + let vars = format!(r#"{{"v": {quoted}}}"#); let inputs = [ ( OpSpec::Contains, @@ -894,6 +901,13 @@ mod json_index { ), ]), ), + ( + OpSpec::Match, + object(vec![ + ("predicate", frame(predicate.as_str())), + ("vars", frame(vars.as_str())), + ]), + ), ]; for (operator, input) in &inputs { let term = watched(&format!("query_term ({operator:?})"), || { diff --git a/packages/stack-encrypt/tests/json_operators.rs b/packages/stack-encrypt/tests/json_operators.rs index 4bfc7e870..63e913121 100644 --- a/packages/stack-encrypt/tests/json_operators.rs +++ b/packages/stack-encrypt/tests/json_operators.rs @@ -1,7 +1,7 @@ //! A document field's JSON operators and reads through the data plan, with //! the engine's own JSON index behind the target: what a binding names by //! operator (`contains`, `eq_at`, the orderings at a path, `selector`, -//! `subtree`, `children`, `child_count`, `element_match`), the operand each derives and the +//! `subtree`, `children`, `child_count`, `element_match`, `match`), the operand each derives and the //! type it binds as, the refusals made before anything is derived, and //! reads opened against the request that prepared them. //! @@ -52,7 +52,8 @@ impl TargetResolver for Documents { None, ) .with_leaf_query_sql_domain("eql_v4.query_json_leaf") - .with_elements_query_sql_domain("eql_v4.query_json_elements")] + .with_elements_query_sql_domain("eql_v4.query_json_elements") + .with_predicate_query_sql_domain("eql_v4.query_json_predicate")] } fn encrypt<'a, K: 'static>( @@ -400,7 +401,8 @@ fn the_orderings_follow_the_resolved_options() { "selector", "subtree", "children", - "child_count" + "child_count", + "match" ] ); let error = dynamic::check_query( @@ -1011,7 +1013,7 @@ mod element_match { /// member's selector and inside its bounds, compared as the SQL /// compares ordering terms (bytes); or `None` where a token names a /// node at another selector, a probe's hit, which the SQL raises on. - fn element_match(doc: &JsonDocument, operand: &Value) -> Option { + pub(super) fn element_match(doc: &JsonDocument, operand: &Value) -> Option { let members = operand["m"].as_array().unwrap(); let width = members[0]["t"].as_array().unwrap().len(); let term = |leaf: &Value| Base64::decode_vec(leaf["op"].as_str().unwrap()).unwrap(); @@ -1534,10 +1536,9 @@ mod element_match { #[test] fn lists_it_when_a_path_is_ordered_and_the_domain_named() { let plan = contacts(); - assert_eq!( - names(&plan, &Documents).last(), - Some(&"element_match"), - "the last operator, in OpSpec::ALL's order" + assert!( + names(&plan, &Documents).contains(&"element_match"), + "listed" ); let (_, index) = dynamic::field_operators(&plan, "profile", &Documents) .unwrap() @@ -1603,3 +1604,768 @@ mod element_match { } } } + +/// `match`: a jsonpath predicate compiled to one operand, a tree of the +/// other operators' operands. `json_tree_match` is modelled over sealed +/// documents (`evaluate`), as the element predicate's tests model +/// `json_tree_element_match`, and held to an oracle of jsonpath's own +/// semantics over the plaintext; `packages/eql/tests/encryption` runs the +/// SQL against PostgreSQL's `jsonb_path_match`. +mod compiled_match { + use super::*; + + use proptest::prelude::*; + use proptest::test_runner::{Config, TestCaseError, TestRunner}; + use stack_encrypt::json::{JsonPath, JsonQuery}; + + /// `profile`, ordering `$.age`, `$.name` and every contact's `calls` + /// and `relationship`. + fn people() -> record::Plan { + plan_of(json!({ + "name": "JsonSearch", + "options": {"ordered": [ + "$.age", + "$.name", + "$.contacts[*].calls", + "$.contacts[*].relationship", + ]} + })) + } + + /// The bound every test reads its element leaves to. + const BOUND: u32 = 4; + + fn input(predicate: &str, vars: Value) -> Value { + json!({"predicate": predicate, "vars": vars.to_string(), "wildcard_bound": BOUND}) + } + + /// The operand's JSON, `eql_v4.query_json_predicate`. + async fn operand( + keyset: &KeysetCipher<'_, K>, + plan: &record::Plan, + predicate: &str, + vars: Value, + ) -> Value { + let term = term(keyset, plan, "match", input(predicate, vars)).await; + assert_eq!(term.encoding(), "eql", "an EQL operand"); + assert_eq!( + term.domain(), + Some("eql_v4.query_json_predicate"), + "bound as the target's compiled predicate domain" + ); + match term.operand() { + Operand::One(bytes) => serde_json::from_slice(bytes).unwrap(), + other => panic!("one operand, got {other:?}"), + } + } + + /// The one operand `operator` derives for `input`, as JSON. + async fn one( + keyset: &KeysetCipher<'_, K>, + plan: &record::Plan, + operator: &str, + input: Value, + ) -> Value { + match term(keyset, plan, operator, input).await.operand() { + Operand::One(bytes) => serde_json::from_slice(bytes).unwrap(), + Operand::Text(text) => json!(text), + other => panic!("{operator}: {other:?}"), + } + } + + fn needle(value: &Value) -> JsonQuery { + serde_json::from_value(value.clone()).unwrap() + } + + /// What `json_tree_match(doc, operand)` answers, as the SQL evaluates + /// each node: `and`, `or` and `not` in three-valued logic; `contains` + /// as `@>` over the stored projection; `exists` as some entry at the + /// selector; `cmp` as the first entry at the selector against each + /// bound, compared as the SQL compares ordering terms (bytes), NULL with + /// no entry or no term; `elements` as `json_tree_element_match`. + fn evaluate(doc: &JsonDocument, node: &Value) -> Option { + let (kind, argument) = node.as_object().unwrap().iter().next().unwrap(); + let children = || argument.as_array().unwrap().iter(); + match kind.as_str() { + "and" => children() + .try_fold(Some(true), |answer, child| { + match (answer, evaluate(doc, child)) { + (_, Some(false)) => Err(()), + (Some(true), Some(true)) => Ok(Some(true)), + _ => Ok(None), + } + }) + .unwrap_or(Some(false)), + "or" => children() + .try_fold(Some(false), |answer, child| { + match (answer, evaluate(doc, child)) { + (_, Some(true)) => Err(()), + (Some(false), Some(false)) => Ok(Some(false)), + _ => Ok(None), + } + }) + .unwrap_or(Some(true)), + "not" => evaluate(doc, argument).map(|answer| !answer), + "contains" => Some(needle(argument).matches(doc.selector_set())), + "exists" => { + let selector = tag(argument.as_str().unwrap()); + Some(doc.entries().iter().any(|e| e.selector() == &selector)) + } + "cmp" => { + let bounds = argument.as_object().unwrap(); + let selector = tag(bounds.values().next().unwrap()["s"].as_str().unwrap()); + let entry = doc.entries().iter().find(|e| e.selector() == &selector)?; + let stored = entry.ordering_term()?; + Some(bounds.iter().all(|(key, leaf)| { + let bound = Base64::decode_vec(leaf["op"].as_str().unwrap()).unwrap(); + match key.as_str() { + "gt" => stored > bound.as_slice(), + "gte" => stored >= bound.as_slice(), + "lt" => stored < bound.as_slice(), + _ => stored <= bound.as_slice(), + } + })) + } + "elements" => Some( + super::element_match::element_match(doc, argument) + .expect("every array within the bound"), + ), + other => panic!("not a node: {other}"), + } + } + + mod given_a_predicate { + use super::*; + + /// Each leaf is the operand its operator derives for the same path + /// and value: `exists` the selector, `contains` `eq_at`'s needle, + /// `cmp` the ordering's leaf operand, `elements` `element_match`'s + /// operand; and an ordering carries its guard, a node there, and + /// under a `!` the `null` needle too. + #[tokio::test] + async fn each_leaf_is_its_operators_operand() { + let cipher = common::stack_cipher().await; + let keyset = cipher.default_keyset(); + let plan = people(); + let tree = operand( + &keyset, + &plan, + "$.age >= $min && $.name == \"ann\" \ + && exists($.contacts[*] ? (@.relationship == $r && @.calls > 5))", + json!({"min": 18, "r": "sibling"}), + ) + .await; + let leaves = tree["and"].as_array().unwrap(); + assert_eq!(leaves.len(), 4, "{tree}"); + let selector = one(&keyset, &plan, "selector", json!({"path": "$.age"})).await; + assert_eq!(leaves[0], json!({"exists": selector}), "the node is there"); + let gte = json!({"cmp": {"gte": one(&keyset, &plan, "gte_at", json!({"path": "$.age", "json": "18"})).await}}); + assert_eq!(leaves[1], gte, "the ordering's own leaf operand"); + let ann = one( + &keyset, + &plan, + "eq_at", + json!({"path": "$.name", "json": "\"ann\""}), + ) + .await; + assert!( + needle(&leaves[2]["contains"]) == needle(&ann), + "{}", + leaves[2] + ); + let elements = one( + &keyset, + &plan, + "element_match", + json!({"path": "$.contacts[*]", "wildcard_bound": BOUND, "predicates": [ + {"member": "relationship", "op": "eq", "json": "\"sibling\""}, + {"member": "calls", "op": "gt", "json": "5"}, + ]}), + ) + .await; + assert_eq!( + leaves[3], + json!({"elements": elements}), + "element_match's operand" + ); + + // Under a `!`, the ordering is also not a `null`: eq_at's own + // needle for `null` at the path. + let negated = operand(&keyset, &plan, "!($.age >= $min)", json!({"min": 18})).await; + let guarded = negated["not"]["and"].as_array().unwrap(); + assert_eq!(guarded.len(), 3, "{negated}"); + assert_eq!(guarded[0], json!({"exists": selector}), "the node is there"); + let null = one( + &keyset, + &plan, + "eq_at", + json!({"path": "$.age", "json": "null"}), + ) + .await; + assert!( + needle(&guarded[1]["not"]["contains"]) == needle(&null), + "and is not a null: {}", + guarded[1] + ); + assert_eq!(guarded[2], gte, "the ordering's own leaf operand"); + } + + /// `||`, `!` and the operators' guards keep their structure, and an + /// ordering at a wildcard is an element predicate of one member. + #[tokio::test] + async fn the_boolean_structure_is_the_predicates() { + let cipher = common::stack_cipher().await; + let keyset = cipher.default_keyset(); + let plan = people(); + let tree = operand( + &keyset, + &plan, + "!(exists($.name)) || $.contacts[*].calls < 2", + json!({}), + ) + .await; + let either = tree["or"].as_array().unwrap(); + assert!(either[0]["not"]["exists"].is_string(), "{tree}"); + let m = either[1]["elements"]["m"].as_array().unwrap(); + assert_eq!(m.len(), 1, "one member: {tree}"); + assert!(m[0]["lt"].is_object(), "its bound: {tree}"); + assert_eq!( + m[0]["t"].as_array().unwrap().len(), + usize::try_from(BOUND).unwrap() + 1, + "read to the input's bound, then the probe" + ); + } + } + + mod given_documents { + use super::*; + + /// A document generated to the shape the predicates read: each + /// field absent, `null` or a value of the one type its path holds. + fn document() -> impl Strategy { + let names = prop::sample::select(vec!["ann", "bob", "cat"]); + let field = |values: BoxedStrategy| { + prop_oneof![ + 1 => Just(None), + 1 => Just(Some(Value::Null)), + 3 => values.prop_map(Some), + ] + }; + let contact = ( + proptest::option::of(names.clone()), + proptest::option::of(0u64..6), + ) + .prop_map(|(relationship, calls)| { + let mut contact = serde_json::Map::new(); + if let Some(relationship) = relationship { + contact.insert("relationship".into(), json!(relationship)); + } + if let Some(calls) = calls { + contact.insert("calls".into(), json!(calls)); + } + Value::Object(contact) + }); + ( + field((0u64..6).prop_map(|n| json!(n)).boxed()), + field(names.clone().prop_map(|n| json!(n)).boxed()), + field(any::().prop_map(|b| json!(b)).boxed()), + proptest::option::of(prop::collection::vec(names, 0..3)), + proptest::option::of(prop::collection::vec(contact, 0..=3)), + ) + .prop_map(|(age, name, ok, tags, contacts)| { + let mut document = serde_json::Map::new(); + for (key, value) in [("age", age), ("name", name), ("ok", ok)] { + if let Some(value) = value { + document.insert(key.into(), value); + } + } + if let Some(tags) = tags { + document.insert("tags".into(), json!(tags)); + } + if let Some(contacts) = contacts { + document.insert("contacts".into(), json!(contacts)); + } + Value::Object(document) + }) + } + + /// A jsonpath predicate in the subset `match` reads, as text and as + /// what jsonpath answers of it over a plaintext document. + #[derive(Clone, Debug)] + enum Plain { + /// `path op value`. + Compare(&'static str, &'static str, Value), + /// `exists(path)`. + Exists(&'static str), + /// `exists(base ? (@member op value && …))`. + Filter(&'static str, Vec<(&'static str, &'static str, Value)>), + And(Box, Box), + Or(Box, Box), + Not(Box), + } + + impl Plain { + fn text(&self) -> String { + match self { + Plain::Compare(path, op, value) => format!("{path} {op} {value}"), + Plain::Exists(path) => format!("exists({path})"), + Plain::Filter(base, conditions) => { + let conditions: Vec = conditions + .iter() + .map(|(member, op, value)| format!("@{member} {op} {value}")) + .collect(); + format!("exists({base} ? ({}))", conditions.join(" && ")) + } + Plain::And(a, b) => format!("({} && {})", a.text(), b.text()), + Plain::Or(a, b) => format!("({} || {})", a.text(), b.text()), + Plain::Not(a) => format!("!({})", a.text()), + } + } + + /// What jsonpath answers, in lax mode over a document with no + /// array where a path names none: `None` for unknown. + fn answer(&self, document: &Value) -> Option { + match self { + Plain::Compare(path, op, value) => { + let items = JsonPath::parse(path).unwrap(); + some_item( + items + .select(document) + .into_iter() + .map(|item| compare(item, op, value)), + ) + } + Plain::Exists(path) => { + Some(!JsonPath::parse(path).unwrap().select(document).is_empty()) + } + Plain::Filter(base, conditions) => Some( + JsonPath::parse(base) + .unwrap() + .select(document) + .into_iter() + .any(|element| { + conditions.iter().all(|(member, op, value)| { + let item = match member.strip_prefix('.') { + Some(key) => element.get(key), + None => Some(element), + }; + item.and_then(|item| compare(item, op, value)) == Some(true) + }) + }), + ), + Plain::And(a, b) => match (a.answer(document), b.answer(document)) { + (Some(false), _) | (_, Some(false)) => Some(false), + (Some(true), Some(true)) => Some(true), + _ => None, + }, + Plain::Or(a, b) => match (a.answer(document), b.answer(document)) { + (Some(true), _) | (_, Some(true)) => Some(true), + (Some(false), Some(false)) => Some(false), + _ => None, + }, + Plain::Not(a) => a.answer(document).map(|answer| !answer), + } + } + } + + /// jsonpath's `any`: true when one comparison is, else unknown + /// when one is, else false (and false over no items). + fn some_item(answers: impl Iterator>) -> Option { + let mut answer = Some(false); + for one in answers { + match one { + Some(true) => return Some(true), + None => answer = None, + Some(false) => {} + } + } + answer + } + + /// One jsonpath comparison: `null` equals only `null` and orders + /// below nothing; two numbers, two strings or two booleans compare; + /// anything else is unknown. + fn compare(item: &Value, op: &str, value: &Value) -> Option { + use std::cmp::Ordering; + let order = match (item, value) { + (Value::Null, Value::Null) => Some(Ordering::Equal), + (Value::Null, _) | (_, Value::Null) => { + return Some(match op { + "==" => false, + "!=" => true, + _ => false, + }) + } + (Value::Number(a), Value::Number(b)) => { + a.as_f64().unwrap().partial_cmp(&b.as_f64().unwrap()) + } + (Value::String(a), Value::String(b)) => Some(a.cmp(b)), + (Value::Bool(a), Value::Bool(b)) if matches!(op, "==" | "!=") => Some(a.cmp(b)), + _ => return None, + }?; + Some(match op { + "==" => order.is_eq(), + "!=" => order.is_ne(), + "<" => order.is_lt(), + "<=" => order.is_le(), + ">" => order.is_gt(), + _ => order.is_ge(), + }) + } + + /// The predicates: each comparison `match` compiles, on paths of + /// every shape the documents hold, and `&&`, `||` and `!` over them. + fn predicate() -> impl Strategy { + let ordering = prop::sample::select(vec!["<", "<=", ">", ">="]); + let any_op = prop::sample::select(vec!["==", "!=", "<", "<=", ">", ">="]); + let equality = prop::sample::select(vec!["==", "!="]); + let name = prop::sample::select(vec!["ann", "bob", "cat"]).prop_map(|n| json!(n)); + let number = (0u64..6).prop_map(|n| json!(n)); + let leaf = prop_oneof![ + (any_op.clone(), number.clone()).prop_map(|(op, n)| Plain::Compare("$.age", op, n)), + (any_op, name.clone()).prop_map(|(op, n)| Plain::Compare("$.name", op, n)), + (equality.clone(), any::()).prop_map(|(op, b)| Plain::Compare( + "$.ok", + op, + json!(b) + )), + ( + equality, + prop::sample::select(vec!["$.age", "$.name", "$.ok"]) + ) + .prop_map(|(op, path)| Plain::Compare(path, op, Value::Null)), + prop::sample::select(vec!["$.age", "$.name", "$.tags", "$.tags[*]", "$.contacts"]) + .prop_map(Plain::Exists), + name.clone() + .prop_map(|n| Plain::Compare("$.tags[*]", "==", n)), + ( + prop::sample::select(vec!["==", "<", "<=", ">", ">="]), + number.clone() + ) + .prop_map(|(op, n)| Plain::Compare( + "$.contacts[*].calls", + op, + n + )), + name.clone() + .prop_map(|n| Plain::Filter("$.tags[*]", vec![("", "==", n)])), + name.clone() + .prop_map(|n| Plain::Filter("$.contacts[*]", vec![(".relationship", "==", n)])), + (name, ordering, number).prop_map(|(n, op, m)| Plain::Filter( + "$.contacts[*]", + vec![(".relationship", "==", n), (".calls", op, m)] + )), + ]; + leaf.prop_recursive(3, 12, 2, |inner| { + prop_oneof![ + (inner.clone(), inner.clone()) + .prop_map(|(a, b)| Plain::And(Box::new(a), Box::new(b))), + (inner.clone(), inner.clone()) + .prop_map(|(a, b)| Plain::Or(Box::new(a), Box::new(b))), + inner.prop_map(|a| Plain::Not(Box::new(a))), + ] + }) + } + + /// Over random documents and predicates, the compiled operand is + /// true of exactly the documents jsonpath is true of over the + /// plaintext, so a WHERE returns jsonpath's rows. Its answer is + /// jsonpath's (true, false or unknown: `None`, SQL NULL), save + /// that it may be unknown where jsonpath says false: an ordering + /// over a JSON `null` under an even number of `!` sends no `null` + /// needle, and is NULL there, which picks the same rows. + #[test] + fn picks_jsonpaths_rows_over_the_plaintext() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let cipher = runtime.block_on(common::stack_cipher()); + let keyset = cipher.default_keyset(); + let plan = people(); + let mut runner = TestRunner::new(Config { + cases: 256, + ..Config::default() + }); + runner + .run(&(document(), predicate()), |(document, predicate)| { + let text = predicate.text(); + let (doc, tree) = runtime.block_on(async { + ( + seal(&keyset, &plan, &document).await, + operand(&keyset, &plan, &text, json!({})).await, + ) + }); + let (got, want) = (evaluate(&doc, &tree), predicate.answer(&document)); + let same_rows = (got == Some(true)) == (want == Some(true)); + let same_answer = got == want || (want == Some(false) && got.is_none()); + if !(same_rows && same_answer) { + return Err(TestCaseError::fail(format!( + "{text} over {document}: got {got:?}, want {want:?}" + ))); + } + Ok(()) + }) + .unwrap(); + } + + /// The three-valued cases a hand-written WHERE has: an ordering is + /// unknown of a node with no ordering term (a boolean), and so is + /// `!` of it, where an absent path is false; an ordering of a + /// `null` is false under an odd number of `!`, where the `null` + /// needle is sent, and unknown elsewhere, which matches no row as + /// false does; `!=` is true of a `null`; and `== null` finds only a + /// `null`. + #[tokio::test] + async fn unknown_null_and_absent_answer_as_jsonpath_does() { + let cipher = common::stack_cipher().await; + let keyset = cipher.default_keyset(); + let plan = people(); + for (document, predicate, want) in [ + (json!({"age": true}), "$.age < 3", None), + (json!({"age": true}), "!($.age < 3)", None), + ( + json!({"age": true}), + "!($.age < 3) || exists($.age)", + Some(true), + ), + ( + json!({"age": true}), + "!($.age < 3) && $.age == 1", + Some(false), + ), + (json!({"age": null}), "$.age < 3", None), + (json!({"age": null}), "!(!($.age < 3))", None), + (json!({"age": null}), "$.age < 3 || $.age == 1", None), + (json!({"age": null}), "!($.age < 3)", Some(true)), + ( + json!({"age": null}), + "!($.age < 3 || $.age == 1)", + Some(true), + ), + ( + json!({"age": null}), + "!($.age < 3) && exists($.age)", + Some(true), + ), + ( + json!({"age": null}), + "!(!($.age < 3) && exists($.age))", + None, + ), + (json!({}), "$.age < 3", Some(false)), + (json!({}), "!($.age < 3)", Some(true)), + (json!({"age": null}), "$.age != 3", Some(true)), + (json!({}), "$.age != 3", Some(false)), + (json!({"age": null}), "$.age != null", Some(false)), + (json!({"age": 1}), "$.age != null", Some(true)), + (json!({"age": null}), "$.age == null", Some(true)), + (json!({}), "$.age == null", Some(false)), + (json!({"age": 1}), "$.age == null", Some(false)), + ] { + let doc = seal(&keyset, &plan, &document).await; + let tree = operand(&keyset, &plan, predicate, json!({})).await; + assert_eq!(evaluate(&doc, &tree), want, "{predicate} over {document}"); + } + } + } + + mod given_an_input_it_cannot_answer { + use super::*; + + fn check(input: &Value) -> Result<(), dynamic::Error> { + dynamic::check_query( + &people(), + "profile", + OpSpec::Match, + json_frame(input), + &Documents, + ) + } + + /// A refusal names its reason, the construct and its offset in the + /// caller's text, and the field; the input's own shape is refused + /// as the other operators refuse theirs. + #[test] + fn is_refused_with_its_reason_and_offset() { + for (predicate, vars, reason, at) in [ + ("$.age + 1 > 2", json!({}), "json_predicate_unsupported", 6), + ("$.age >", json!({}), "json_predicate_syntax", 7), + ( + "$.age > $n", + json!({}), + "json_predicate_unknown_variable", + 8, + ), + ( + "$.age > $n", + json!({"n": [1]}), + "json_predicate_value_type", + 8, + ), + ( + "$.tags[*] != \"a\"", + json!({}), + "json_predicate_not_equal_at_wildcard", + 10, + ), + ("$.tags[0] == \"a\"", json!({}), "json_path_position", 6), + ] { + let error = check(&input(predicate, vars)).unwrap_err(); + assert_eq!( + error.reason().map(Reason::as_str), + Some(reason), + "{predicate}" + ); + assert_eq!(error.field(), Some("profile"), "{predicate}"); + let payload = stack_encrypt::ErrorPayload::payload(&error); + assert_eq!(payload["at"], json!(at), "{predicate}: {payload:?}"); + assert!(payload["construct"].is_string(), "{predicate}: {payload:?}"); + } + for (input, reason) in [ + (json!({"predicate": "$.a == 1", "vars": "[]"}), "json_input"), + (json!({"predicate": "$.a == 1", "vars": "{"}), "json_syntax"), + ( + json!({"predicate": "$.a == 1", "wildcard_bound": 0}), + "json_wildcard_bound_zero", + ), + (json!({"predicate": "$.a == 1", "other": 1}), "json_input"), + (json!({"vars": "{}"}), "json_input"), + ] { + let error = check(&input).unwrap_err(); + assert_eq!(error.reason().map(Reason::as_str), Some(reason), "{input}"); + } + let error = check(&input("$.height > 1", json!({}))).unwrap_err(); + assert!( + matches!( + error, + dynamic::Error::Cipher(stack_encrypt::Error::Term( + stack_encrypt::sem::TermError::JsonPathNotOrdered + )) + ), + "an ordering at a path the field does not order: {error:?}" + ); + assert!( + check(&json!({"predicate": "$.a == 1"})).is_ok(), + "vars may be absent" + ); + } + } + + mod given_the_fields_type { + use super::*; + + /// A resolver whose `JsonSearch` names no compiled predicate + /// domain: what a host presents that has not been taught the + /// operand. + struct Unnamed; + + impl TargetResolver for Unnamed { + fn targets(&self) -> Vec { + let mut targets = Documents.targets(); + targets[0].predicate_query_sql_domain = None; + targets + } + fn encrypt<'a, K: 'static>( + &self, + name: &str, + options: Option<&TargetOptions>, + keyset: &'a KeysetCipher<'_, K>, + label: &Label, + plaintext: ValueFrame, + ) -> Result, K>, TargetError> { + Documents.encrypt(name, options, keyset, label, plaintext) + } + fn decrypt<'a, K: 'static>( + &self, + name: &str, + cipher: &'a StackCipher, + label: &Label, + bytes: &[u8], + ) -> Result, TargetError> { + Documents.decrypt(name, cipher, label, bytes) + } + fn query<'a, K: 'static>( + &self, + name: &str, + index: &IndexSpec, + keyset: &'a KeysetCipher<'_, K>, + label: &Label, + plaintext: ValueFrame, + ) -> Result, K>, TargetError> { + Documents.query(name, index, keyset, label, plaintext) + } + } + + fn names(plan: &record::Plan, resolver: &impl TargetResolver) -> Vec<&'static str> { + dynamic::field_operators(plan, "profile", resolver) + .unwrap() + .into_iter() + .map(|(op, _)| op.name()) + .collect() + } + + /// Answered through the JSON index whenever the type names the + /// domain, an ordered path or none, as the last operator; its input + /// is `predicate` and it is exact. + #[test] + fn lists_it_when_the_domain_is_named() { + for plan in [people(), plan_of(json!("JsonSearch"))] { + assert_eq!(names(&plan, &Documents).last(), Some(&"match")); + } + assert_eq!( + OpSpec::parse("match"), + Some(OpSpec::Match), + "the wire name parses" + ); + assert_eq!(OpSpec::Match.input(), "predicate", "its input shape"); + assert_eq!(OpSpec::Match.fidelity(), dynamic::Fidelity::Exact); + let (_, index) = dynamic::field_operators(&people(), "profile", &Documents) + .unwrap() + .into_iter() + .find(|(op, _)| *op == OpSpec::Match) + .unwrap(); + assert!(matches!(index, IndexSpec::Json(_)), "{index:?}"); + } + + #[test] + fn does_not_answer_it_when_the_type_names_no_domain() { + let wire = json!({ + "profile": {"context": ["users", "profile"], "type": "json", "null": "sql", + "target": "JsonSearch"} + }); + let plan = record::plan_with(&dynamic::frame::serde::to_vec(&wire).unwrap(), &Unnamed) + .unwrap(); + let names = names(&plan, &Unnamed); + assert!(names.contains(&"eq_at"), "the others still: {names:?}"); + assert!(!names.contains(&"match"), "{names:?}"); + let error = dynamic::check_query( + &plan, + "profile", + OpSpec::Match, + json_frame(&input("$.a == 1", json!({}))), + &Unnamed, + ) + .unwrap_err(); + assert_eq!( + error.reason(), + Some(Reason::OperatorNotAnswered), + "{error:?}" + ); + } + + /// A type's own list, which a host reads from its descriptors, has + /// `match` only when the type names the domain: a host that has not + /// been taught the operand sees no new operator. + #[test] + fn a_types_operators_name_it_only_with_the_domain() { + assert!( + Documents.targets()[0].operators().contains(&OpSpec::Match), + "named" + ); + let unnamed = Unnamed.targets()[0].operators(); + assert!(!unnamed.contains(&OpSpec::Match), "{unnamed:?}"); + assert!(unnamed.contains(&OpSpec::EqAt), "{unnamed:?}"); + } + } +}