From b476bb0a559ebe2ead8c4759c7d77a14c4aeb3d0 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 30 Oct 2026 13:10:51 +1100 Subject: [PATCH 1/2] feat(go): test several members of one array element of a JsonSearch field A filter such as "a contact who is a sibling and was called more than five times" had no Go method. An EqAt and an ordering at the contacts' paths can each be answered by a different contact, so the application had to read the array back and re-apply the predicate itself. The engine and EQL already answer it: element_match derives one eql_v4.query_json_elements operand, and json_tree_element_match is true of a row where one element passes every predicate. This wires it into the Go SDK. The EQL guest's resolver now names the elements domain, so the engine lists element_match (input path_predicates) on a field that orders a path, and the checker and stashgen read the operator. The compiled predicate domain stays unnamed: match is #1443's. stashgen groups the tag's ordered paths by the array their last wildcard names (record.TargetOptions.ElementArrays) and writes one accessor per array under Elements, named as At names a path, with Match(ctx, cipher, opts, predicates...). An array with no ordered member beneath it gets none, and a root array ($[*]) is Root. The members are compile-time names. stashgen writes Paths (NPaths with -name), a tree nested as the JSON nests it, of encrypt.Path[E, T] values: Paths.Profile.Visits.Place. E is a type stashgen writes for each array, and T is the member's Go type, read from the document's struct as encoding/json reads it, or any where the type names none (a json.RawMessage, say). encrypt.AtPath(path) gives Eq, Lt, Lte, Gt, Gte and Between, each taking a T and returning an encrypt.ElementPredicate[E], and an array's Match takes only its own E. So a value of another type, or a predicate on another array's member, does not compile. Paths holds fields and no methods, so no JSON key is reserved. For a filter built at run time, the array accessor's Member("place") gives the same comparisons with any values. An array gets no accessor, and a notice naming its ordered paths, when its keys give no accessor name, when another array's keys give the same one, or when Paths cannot name its members: a key that gives no Go name on its own, or one name that would stand for two paths. A tag that generated before still generates, with its At unchanged. Match writes each value with gensupport.JSONText as the orderings do, and returns an eql.JSONSearchElementsQuery. The member check runs in the SDK, against the members the declaration orders beneath the array, before the guest is called: the engine's json_path_not_ordered names no member, so the first error a caller sees names the member, the array and the members it orders. It wraps ErrTerm, as the engine's refusal does; no predicates, a zero predicate, a zero Path and a member with a wildcard wrap ErrEncoding. Every other refusal is the engine's Diagnostic. A predicate prints and logs its member and comparison, never its values, and copies the bytes of a json.RawMessage or []byte value. Tests: the hermetic suite pins the operand against the subtree reads and the orderings of the same members, the run-time form against the typed one, every pre-engine refusal, and each engine refusal's sentinel and code through the guest. A type-checking test loads the generated testusers package with a probe file and holds the two compile errors, a value of another type and another array's predicate; the stashgen test does the same against a generated module, and pins the member types and every case that gets no accessor. The PostgreSQL suite (go:test:postgres) finds the row where one contact passes both predicates, by path and by name, and not the row where two contacts each pass one, which the EqAt and ordering superset returns, and raises SQLSTATE 22023 past the bound. The guest test now pins the elements domain named and match not listed. Closes #1442 Claude-Session: https://claude.ai/code/session_01MrxG73fnX1p7oiizh3GRG5 --- languages/golang/cmd/stashgen/main.go | 2 +- languages/golang/cmd/stashgen/main_test.go | 192 ++++++++ languages/golang/encrypt/checker.go | 6 +- languages/golang/encrypt/gensupport/json.go | 96 ++++ languages/golang/encrypt/guest/src/targets.rs | 58 ++- .../internal/testusers/customer_stash.go | 90 +++- .../encrypt/internal/testusers/entities.go | 32 ++ .../internal/testusers/entity_stash.go | 350 ++++++++++++++ .../internal/testusers/subscriber_stash.go | 91 +++- languages/golang/encrypt/json_elements.go | 165 +++++++ .../encrypt/json_elements_postgres_test.go | 162 +++++++ .../golang/encrypt/json_elements_test.go | 457 ++++++++++++++++++ languages/golang/encrypt/json_tree_test.go | 2 +- languages/golang/encrypt/targets_test.go | 2 +- languages/golang/internal/record/record.go | 55 ++- .../golang/internal/record/record_test.go | 28 ++ languages/golang/stashgen/doc.go | 3 +- languages/golang/stashgen/emit.go | 362 +++++++++++++- languages/golang/stashgen/read.go | 170 ++++++- .../testdata/stubsdk/encrypt/encrypt.go | 23 + .../testdata/stubsdk/encrypt/eql/eql.go | 15 + .../stubsdk/encrypt/gensupport/gensupport.go | 5 + mise.toml | 4 +- 23 files changed, 2324 insertions(+), 46 deletions(-) create mode 100644 languages/golang/encrypt/internal/testusers/entities.go create mode 100644 languages/golang/encrypt/internal/testusers/entity_stash.go create mode 100644 languages/golang/encrypt/json_elements.go create mode 100644 languages/golang/encrypt/json_elements_postgres_test.go create mode 100644 languages/golang/encrypt/json_elements_test.go diff --git a/languages/golang/cmd/stashgen/main.go b/languages/golang/cmd/stashgen/main.go index 7d1c30040..fcaa74fe4 100644 --- a/languages/golang/cmd/stashgen/main.go +++ b/languages/golang/cmd/stashgen/main.go @@ -59,7 +59,7 @@ func run(args []string, dir string, stdout, stderr io.Writer, newEngine func(con req := stashgen.Request{Dir: dir} var models modelFlags fs.StringVar(&req.Type, "type", "", "the struct that carries the stash tags (required)") - fs.StringVar(&req.Name, "name", "", "write EncryptN, DecryptN and NFields instead of Encrypt, Decrypt and Fields") + fs.StringVar(&req.Name, "name", "", "write EncryptN, DecryptN, NFields and NPaths instead of Encrypt, Decrypt, Fields and Paths") fs.StringVar(&req.For, "for", "", "P.F: -type declares the tags for F, a type in package P") fs.Var(&models, "model", "Name=R or Name=R:D: a model R for separate columns; writes EncryptName and DecryptName (repeatable)") fs.BoolVar(&req.Redact, "redact", false, "write String, GoString and LogValue methods on the -type struct") diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index 3aa884c5d..39c65ef0c 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "os" + "os/exec" "path/filepath" "strings" "testing" @@ -442,6 +443,117 @@ func TestRunGeneratesJSONSearchFields(t *testing.T) { if !strings.Contains(written, "func (p ProfileFieldPath) Selector(ctx context.Context, c *encrypt.Cipher) (encrypt.JSONSelection[eql.JSONEntry], error)") { t.Error("an ordered path has no Selector of its own") } + // The array with an ordered member gets an accessor under Elements, + // named as At names its path, whose Match derives the element match. + for _, want := range []string{ + "Elements ProfileFieldElements", + "Visits ProfileFieldArray[ProfileFieldElementsVisits] // $.visits[*]: at", + `Visits: ProfileFieldArray[ProfileFieldElementsVisits]{field: f, path: "$.visits[*]"},`, + "type ProfileFieldElementsVisits struct{}", + "type ProfileFieldArray[E any] struct {", + "func (a ProfileFieldArray[E]) Match(ctx context.Context, c *encrypt.Cipher, opts encrypt.JSONReadOptions, predicates ...encrypt.ElementPredicate[E]) (eql.JSONSearchElementsQuery, error)", + "gensupport.JSONElementMatch[eql.JSONSearchElementsQuery](ctx, a.field, c, a.path, opts, predicates)", + "func (a ProfileFieldArray[E]) Member(member string) encrypt.MemberTest[E, any]", + "return encrypt.AtPath(encrypt.NewPath[E, any](a.path, member))", + // Paths names the array's members, nested as the JSON nests them; + // a json.RawMessage document names no Go type, so each is any. + "var Paths = func() (paths struct {", + "At encrypt.Path[ProfileFieldElementsVisits, any] // $.visits[*].at", + `paths.Profile.Visits.At = encrypt.NewPath[ProfileFieldElementsVisits, any]("$.visits[*]", "at")`, + } { + if !strings.Contains(written, want) { + t.Errorf("the generated file lacks %q", want) + } + } + if strings.Contains(written, "PrefsFieldElements") { + t.Error("Prefs orders no member, and got Elements") + } + // The members of one array, named as At names their paths; an ordered + // path that ends at its wildcard, or has none, is no array's member. + written, stderr, code = generate("\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.extra.emergency_contacts[*].relationship;$.extra.emergency_contacts[*].previous_call_count;$.tags[*];$.age)\"`") + if code != 0 { + t.Fatalf("exit %d\n%s", code, stderr) + } + // gofmt aligns the struct fields: read each line's words. + words := strings.Join(strings.Fields(written), " ") + for _, want := range []string{ + "ExtraEmergencyContactsRelationship ProfileFieldPath // $.extra.emergency_contacts[*].relationship", + "ExtraEmergencyContacts ProfileFieldArray[ProfileFieldElementsExtraEmergencyContacts] // $.extra.emergency_contacts[*]: relationship, previous_call_count", + "Profile struct { Extra struct { EmergencyContacts struct { Relationship encrypt.Path[ProfileFieldElementsExtraEmergencyContacts, any] // $.extra.emergency_contacts[*].relationship PreviousCallCount encrypt.Path[ProfileFieldElementsExtraEmergencyContacts, any] // $.extra.emergency_contacts[*].previous_call_count } } }", + `paths.Profile.Extra.EmergencyContacts.PreviousCallCount = encrypt.NewPath[ProfileFieldElementsExtraEmergencyContacts, any]("$.extra.emergency_contacts[*]", "previous_call_count")`, + } { + if !strings.Contains(words, want) { + t.Errorf("the generated file lacks %q", want) + } + } + if strings.Contains(written, "Tags ProfileFieldArray") { + t.Error("$.tags[*] orders the elements themselves, no member, and got an accessor under Elements") + } + // A field that orders no member beneath an array has no Elements. + written, stderr, code = generate("\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.age;$.tags[*])\"`") + if code != 0 { + t.Fatalf("exit %d\n%s", code, stderr) + } + if strings.Contains(written, "Elements") || strings.Contains(written, "JSONElementMatch") || strings.Contains(written, "Paths") { + t.Error("a field with no ordered member beneath an array got Elements or Paths") + } + // A root array's accessor is Root, beside its paths' At accessors. + written, stderr, code = generate("\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$[*].name;$[*].age)\"`") + if code != 0 { + t.Fatalf("a root array: exit %d\n%s", code, stderr) + } + words = strings.Join(strings.Fields(written), " ") + for _, want := range []string{ + "Name ProfileFieldPath // $[*].name", + "Root ProfileFieldArray[ProfileFieldElementsRoot] // $[*]: name, age", + `Root: ProfileFieldArray[ProfileFieldElementsRoot]{field: f, path: "$[*]"},`, + // A root array's members are the field's own in Paths. + "Profile struct { Name encrypt.Path[ProfileFieldElementsRoot, any] // $[*].name Age encrypt.Path[ProfileFieldElementsRoot, any] // $[*].age }", + } { + if !strings.Contains(words, want) { + t.Errorf("a root array: the generated file lacks %q", want) + } + } + // Arrays whose keys give one accessor name, or none, get no accessor: + // the tag still generates, At is as it is without them, and a notice + // in the file and on stderr names the paths as the tag writes them. + for _, c := range []struct{ name, ordered, at, notice string }{ + {"two arrays, one name", "$.a_b[*].x;$.aB[*].y;$.c[*].z", "ABX ProfileFieldPath // $.a_b[*].x", "no Elements accessor for the elements of $.a_b[*] (ordered $.a_b[*].x): another array's keys give the same accessor name, Elements.AB"}, + {"an array with no name", "$.__[*].x;$.c[*].z", "X ProfileFieldPath // $.__[*].x", "no Elements accessor for the elements of $.__[*] (ordered $.__[*].x): its keys give no Go accessor name"}, + // Paths names each key on its own, so a key of digits, which At + // joins to the keys before it, names nothing there. + {"a key with no name under Paths", "$.a.1[*].x;$.c[*].z", "A1X ProfileFieldPath // $.a.1[*].x", `no Elements accessor for the elements of $.a.1[*] (ordered $.a.1[*].x): the key "1" of $.a.1[*].x gives no Go name under Paths that starts with a letter`}, + // Paths nests: a member that is also the object or array above + // another member, or two keys that give one name at one place, + // would make one name stand for two paths. + {"a member above a member", "$.a[*].b;$.a[*].b.c;$.c[*].z", "ABC ProfileFieldPath // $.a[*].b.c", "no Elements accessor for the elements of $.a[*] (ordered $.a[*].b, $.a[*].b.c): Paths.Profile.A.B would name both $.a[*].b and $.a[*].b.c, and can name one of them only"}, + {"a member above an array", "$.a[*].b;$.a[*].b[*].d;$.c[*].z", "ABD ProfileFieldPath // $.a[*].b[*].d", "no Elements accessor for the elements of $.a[*].b[*] (ordered $.a[*].b[*].d): Paths.Profile.A.B would name both $.a[*].b and $.a[*].b[*].d, and can name one of them only"}, + {"two keys, one name in Paths", "$.x.a_b[*].m;$.x.aB.n[*].o;$.c[*].z", "XABM ProfileFieldPath // $.x.a_b[*].m", "no Elements accessor for the elements of $.x.aB.n[*] (ordered $.x.aB.n[*].o): Paths.Profile.X.AB would name both $.x.a_b[*].m and $.x.aB.n[*].o, and can name one of them only"}, + } { + written, stderr, code := generate("\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=" + c.ordered + ")\"`") + if code != 0 { + t.Fatalf("%s: exit %d\n%s", c.name, code, stderr) + } + // A wrapped comment reads as one line. + words := strings.Join(strings.Fields(strings.ReplaceAll(written, "\n// ", " ")), " ") + for _, want := range []string{c.at, "C ProfileFieldArray[ProfileFieldElementsC] // $.c[*]: z", "Z encrypt.Path[ProfileFieldElementsC, any] // $.c[*].z", "Profile: " + c.notice} { + if !strings.Contains(words, want) { + t.Errorf("%s: the generated file lacks %q", c.name, want) + } + } + if !strings.Contains(stderr, "stashgen: Customer.Profile: "+c.notice) { + t.Errorf("%s: stderr = %q", c.name, stderr) + } + // $.c[*] alone has an accessor: its field, its initializer and + // its Elements struct field. + if n := strings.Count(written, "ProfileFieldArray[ProfileFieldElements"); n != 2 { + t.Errorf("%s: %d accessors written, want $.c[*]'s alone", c.name, n) + } + } + written, _, code = generate("\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.a_b[*].x;$.aB[*].y)\"`") + if code != 0 || strings.Contains(written, "ProfileFieldElements") || strings.Contains(written, "Paths") || !strings.Contains(written, "ABY ProfileFieldPath // $.aB[*].y") { + t.Errorf("two colliding arrays and no other: exit %d, Elements or Paths written, or At changed", code) + } // A json.RawMessage sealed any other way is bytes, as it was before // documents; an object key of digits names its accessor. written, stderr, code = generate("\tData json.RawMessage `stash:\"data,encrypt,index=equality\"`\n\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.a.1)\"`") @@ -473,6 +585,19 @@ func TestRunGeneratesJSONSearchFields(t *testing.T) { if _, err := os.Stat(filepath.Join(dir, "customer_stash.go")); !os.IsNotExist(err) { t.Fatal("a file was written after the refusal") } + // The same for an array whose elements have an ordered member, and an + // engine that answers no element_match. + dir = writeModule(t, map[string]string{"model.go": source("\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.visits[*].at)\"`")}) + stderrBuf.Reset() + if code := run([]string{"-type", "Customer"}, dir, &bytes.Buffer{}, &stderrBuf, withoutOperator("element_match")); code != 1 { + t.Fatalf("exit %d, want 1\n%s", code, stderrBuf.String()) + } + if got := stderrBuf.String(); !strings.Contains(got, "Customer.Profile") || !strings.Contains(got, "the engine answers no element_match") { + t.Fatalf("stderr = %q", got) + } + if _, err := os.Stat(filepath.Join(dir, "customer_stash.go")); !os.IsNotExist(err) { + t.Fatal("a file was written after the refusal") + } for _, c := range []struct{ name, fields, want string }{ {"a sealed null", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=JsonSearch,null=sealed\"`", "Customer.Prefs: a nil has an equality term and no other, so another index's column would show which values are absent; write null=sql"}, {"a nil document that does not say null=sql", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=JsonSearch\"`", "null=sql"}, @@ -499,6 +624,73 @@ func TestRunGeneratesJSONSearchFields(t *testing.T) { } } +// A document of a Go type names each member's Go type in Paths, read as +// encoding/json reads the document: through pointers, a promoted field, a +// map and a slice, with any where the type names none. The compiler then +// refuses a value of another type, and a predicate on another array's +// member, in a call to Match. +func TestRunTypesTheElementMatch(t *testing.T) { + checker, err := encrypt.NewChecker(context.Background()) + if err != nil { + t.Skip(err) + } + _ = checker.Close() + model := "package users\n\nimport \"time\"\n\ntype Place string\n\ntype Stamp struct {\n\tWhen time.Time `json:\"when\"`\n}\n\ntype Visit struct {\n\tStamp\n\tAt *int64 `json:\"at\"`\n\tPlace Place `json:\"place\"`\n\tNote string\n}\n\ntype Tag struct {\n\tName string `json:\"name\"`\n}\n\ntype Profile struct {\n\tVisits []Visit `json:\"visits\"`\n\tTags map[string][]Tag `json:\"tags\"`\n}\n\ntype Customer struct {\n\t_ struct{} `stash:\"context=customers\"`\n\tID int64 `stash:\"id,passthrough\"`\n\tProfile Profile `stash:\"profile,encrypt_into=JsonSearch(ordered=$.visits[*].at;$.visits[*].place;$.visits[*].when;$.visits[*].Note;$.visits[*].missing;$.tags.home[*].name)\"`\n}\n" + dir := writeModule(t, map[string]string{"model.go": model}) + var stdout, stderr bytes.Buffer + if code := run([]string{"-type", "Customer"}, dir, &stdout, &stderr, stashgen.GuestEngine); code != 0 { + t.Fatalf("exit %d\n%s", code, stderr.String()) + } + data, err := os.ReadFile(filepath.Join(dir, "customer_stash.go")) + if err != nil { + t.Fatal(err) + } + words := strings.Join(strings.Fields(string(data)), " ") + for _, want := range []string{ + "At encrypt.Path[ProfileFieldElementsVisits, int64] // $.visits[*].at", + "Place encrypt.Path[ProfileFieldElementsVisits, Place] // $.visits[*].place", + "When encrypt.Path[ProfileFieldElementsVisits, time.Time] // $.visits[*].when", + "Note encrypt.Path[ProfileFieldElementsVisits, string] // $.visits[*].Note", + "Missing encrypt.Path[ProfileFieldElementsVisits, any] // $.visits[*].missing", + "Tags struct { Home struct { Name encrypt.Path[ProfileFieldElementsTagsHome, string] // $.tags.home[*].name } }", + `paths.Profile.Visits.At = encrypt.NewPath[ProfileFieldElementsVisits, int64]("$.visits[*]", "at")`, + `paths.Profile.Tags.Home.Name = encrypt.NewPath[ProfileFieldElementsTagsHome, string]("$.tags.home[*]", "name")`, + } { + if !strings.Contains(words, want) { + t.Errorf("the generated file lacks %q", want) + } + } + // What a program writes compiles, and each mistake the types exist to + // catch does not. + build := func(probe string) string { + t.Helper() + src := "package users\n\nimport (\n\t\"context\"\n\n\t\"github.com/cipherstash/stack/languages/golang/encrypt\"\n)\n\nfunc probe(ctx context.Context, c *encrypt.Cipher) {\n" + probe + "}\n" + if err := os.WriteFile(filepath.Join(dir, "probe.go"), []byte(src), 0o600); err != nil { + t.Fatal(err) + } + cmd := exec.Command("go", "build", "./...") + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GOPROXY=off", "GOWORK=off", "GOFLAGS=-mod=mod", "CGO_ENABLED=0") + out, err := cmd.CombinedOutput() + if err == nil { + return "" + } + return string(out) + } + if out := build("\t_, _ = Fields.Profile.Elements.Visits.Match(ctx, c, encrypt.JSONReadOptions{},\n\t\tencrypt.AtPath(Paths.Profile.Visits.Place).Eq(\"home\"),\n\t\tencrypt.AtPath(Paths.Profile.Visits.At).Between(1, 2),\n\t\tFields.Profile.Elements.Visits.Member(\"at\").Gt(\"any value\"),\n\t)\n"); out != "" { + t.Fatalf("the element match as a program writes it does not build:\n%s", out) + } + out := build("\t_, _ = Fields.Profile.Elements.Visits.Match(ctx, c, encrypt.JSONReadOptions{}, encrypt.AtPath(Paths.Profile.Visits.At).Gt(\"15\"))\n\t_, _ = Fields.Profile.Elements.Visits.Match(ctx, c, encrypt.JSONReadOptions{}, encrypt.AtPath(Paths.Profile.Tags.Home.Name).Eq(\"x\"))\n") + for _, want := range []string{ + `cannot use "15" (untyped string constant) as int64 value`, + "encrypt.ElementPredicate[ProfileFieldElementsTagsHome]) as encrypt.ElementPredicate[ProfileFieldElementsVisits] value", + } { + if !strings.Contains(out, want) { + t.Errorf("the build does not refuse with %q:\n%s", want, out) + } + } +} + // withoutOperator is the embedded engine with op removed from every field's // answer: an engine that disagrees with the generator. func withoutOperator(op string) func(context.Context) (stashgen.Engine, error) { diff --git a/languages/golang/encrypt/checker.go b/languages/golang/encrypt/checker.go index f0cab5a94..832b63426 100644 --- a/languages/golang/encrypt/checker.go +++ b/languages/golang/encrypt/checker.go @@ -258,7 +258,7 @@ func parseOperators(value any) ([]record.Operator, error) { op.Index, err = parseOperatorIndex(s) case "input": switch s { - case "value", "pair", "json", "path_json", "path_pair", "path", "path_bound": + case "value", "pair", "json", "path_json", "path_pair", "path", "path_bound", "path_predicates": op.Input = s default: return nil, fmt.Errorf("operator %d: unknown input %q", i, s) @@ -282,14 +282,14 @@ func parseOperators(value any) ([]record.Operator, error) { // of the names the engine's OpSpec::parse accepts, the scalar operators // (eq, lt, lte, gt, gte, between, matches) and a document's (contains, // contained_by, eq_at, the orderings at a path, between_at, selector, -// subtree, children, child_count). The catalog (se_targets) and se_query_term are +// subtree, children, child_count, element_match). The catalog (se_targets) and se_query_term are // read through this, parseFidelity and parseOperatorIndex, so the two // readers refuse the same values. func parseOperatorName(s string) (string, error) { switch s { case "eq", "lt", "lte", "gt", "gte", "between", "matches", "contains", "contained_by", "eq_at", "lt_at", "lte_at", "gt_at", "gte_at", "between_at", - "selector", "subtree", "children", "child_count": + "selector", "subtree", "children", "child_count", "element_match": return s, nil } return "", fmt.Errorf("unknown operator %q", s) diff --git a/languages/golang/encrypt/gensupport/json.go b/languages/golang/encrypt/gensupport/json.go index aa70702ab..e5c0f5190 100644 --- a/languages/golang/encrypt/gensupport/json.go +++ b/languages/golang/encrypt/gensupport/json.go @@ -8,6 +8,8 @@ import ( "errors" "fmt" "reflect" + "slices" + "strings" "github.com/cipherstash/stack/languages/golang/encrypt" "github.com/cipherstash/stack/languages/golang/internal/record" @@ -241,6 +243,100 @@ func JSONBetweenAt[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt return encrypt.JSONRange[Q]{Low: Q(term.Terms[0]), High: Q(term.Terms[1])}, nil } +// JSONElementMatch derives element_match on a JSON field: the one operand +// eql_v4.json_tree_element_match takes, which is true of a row where some +// element of the array at path passes every predicate. path is an array +// through its last wildcard ($.contacts[*]) that the field orders members +// beneath ([record.TargetOptions.ElementArrays]). +// +// Refused before the engine is asked, each wrapping the sentinel the +// engine's own refusal would: no predicates, a zero ElementPredicate, or a +// predicate on the zero Path, which names no member +// ([encrypt.ErrEncoding]); a member with a wildcard ([encrypt.ErrEncoding]); +// a member whose full path the field does not order ([encrypt.ErrTerm]), +// naming the member, the array and the members it orders. A member read +// from the generated Paths is one the field orders; a member named at run +// time, through the array accessor's Member, may not be. A value is +// written as [JSONText] writes it. What the engine refuses past these, a +// value that is not a number or a string say, is its [encrypt.Diagnostic]. +// opts.WildcardBound 0 sends no bound, and the engine's default, 64, +// applies. +func JSONElementMatch[Q ~[]byte, T, E any](ctx context.Context, f Field[T], c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions, predicates []encrypt.ElementPredicate[E]) (Q, error) { + if f.err != nil { + return nil, f.err + } + if len(predicates) == 0 { + return nil, fmt.Errorf("%w: gensupport: %s: Match at %s needs at least one predicate: with none it is true of every row with an element", encrypt.ErrEncoding, f.name, path) + } + members := f.elementMembers(path) + list := make([]any, len(predicates)) + for i, p := range predicates { + member, op, values := p.Member(), p.Op(), p.Values() + if op == "" { + return nil, fmt.Errorf("%w: gensupport: %s: Match at %s: predicate %d is a zero ElementPredicate; make one with encrypt.AtPath and a path from the generated Paths, or with the array's Member", encrypt.ErrEncoding, f.name, path, i) + } + if member == "" { + return nil, fmt.Errorf("%w: gensupport: %s: Match at %s: predicate %d names no member, as a zero encrypt.Path does; read the path from the generated Paths (%s)", encrypt.ErrEncoding, f.name, path, i, orderedMembers(members)) + } + if strings.Contains(member, "[*]") { + return nil, fmt.Errorf("%w: gensupport: %s: Match at %s: the member %q has a wildcard; a member names one field of each element (%s)", encrypt.ErrEncoding, f.name, path, member, orderedMembers(members)) + } + if !slices.Contains(members, member) { + return nil, fmt.Errorf("%w: gensupport: %s: Match at %s: the member %q is not ordered, so no element carries a term to compare (%s); order its full path in the field's tag, ordered=", encrypt.ErrTerm, f.name, path, member, orderedMembers(members)) + } + entry := vcvalue.Object{{Key: "member", Value: member}, {Key: "op", Value: op}} + // Each comparison gives its op its values: between the low and + // the high, every other op one. + keys := []string{"json"} + if op == "between" { + keys = []string{"low", "high"} + } + for j, key := range keys { + text, err := f.jsonOperand(values[j]) + if err != nil { + return nil, err + } + entry = append(entry, vcvalue.Field{Key: key, Value: text}) + } + list[i] = entry + } + input := vcvalue.Object{{Key: "path", Value: path}} + if opts.WildcardBound != 0 { + input = append(input, vcvalue.Field{Key: "wildcard_bound", Value: opts.WildcardBound}) + } + input = append(input, vcvalue.Field{Key: "predicates", Value: list}) + term, err := f.jsonTerm(ctx, c, "element_match", "eql", 1, input) + if err != nil { + return nil, err + } + return Q(term.Terms[0]), nil +} + +// orderedMembers says which members of an array's elements the field +// orders. +func orderedMembers(members []string) string { + if len(members) == 0 { + return "the field orders no member beneath this array" + } + return "the field orders " + strings.Join(members, ", ") + " beneath this array" +} + +// elementMembers are the members the field orders beneath the array at +// path, as its declaration's ordered paths give them. +func (f Field[T]) elementMembers(path string) []string { + if f.plan == nil { + return nil + } + if pf := f.plan.Field(f.name); pf != nil { + for _, a := range pf.TargetOptions.ElementArrays() { + if a.Path == path { + return a.Members + } + } + } + return nil +} + // JSONSelect prepares `->` at a path on a JSON field: the selector to // bind, and what opens the entries it returns. E is the field's entry type. func JSONSelect[E ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, path string) (encrypt.JSONSelection[E], error) { diff --git a/languages/golang/encrypt/guest/src/targets.rs b/languages/golang/encrypt/guest/src/targets.rs index 4dc74b805..065982246 100644 --- a/languages/golang/encrypt/guest/src/targets.rs +++ b/languages/golang/encrypt/guest/src/targets.rs @@ -114,10 +114,17 @@ fn describe(target: &eql_bindings::encryption::targets::Target) -> TargetDescrip Some(options) => descriptor.with_match_options(options), None => descriptor, }; - // A document's orderings at a path bind as its leaf operand. - match target.leaf_query_sql_domain() { + // A document's orderings at a path bind as its leaf operand, and its + // element predicate (`element_match`) as its elements operand. Its + // compiled predicate (`match`) is not named: the Go SDK does not take + // that operator yet. + let descriptor = match target.leaf_query_sql_domain() { Some(domain) => descriptor.with_leaf_query_sql_domain(domain), None => descriptor, + }; + match target.elements_query_sql_domain() { + Some(domain) => descriptor.with_elements_query_sql_domain(domain), + None => descriptor, } } @@ -270,13 +277,14 @@ mod tests { /// The JSON document's query domains this guest names, and so the JSON /// operators the Go SDK is offered: the leaf operand, for the orderings - /// at a path, and not the element-scoped or compiled predicate, whose - /// operators (`element_match`, `match`) the Go side does not parse yet. - /// Naming one is a change to the Go SDK's operator list: update this - /// test with the Go side that takes the operator. + /// at a path, and the element predicate, for `element_match`; not the + /// compiled predicate, whose operator (`match`) the Go side does not + /// parse yet. Naming it is a change to the Go SDK's operator list: + /// update this test with the Go side that takes the operator. #[cfg(feature = "eql")] #[test] - fn a_document_names_the_leaf_domain_and_not_the_predicate_domains() { + fn a_document_names_the_leaf_and_element_domains_and_not_the_predicate_domain() { + use stack_encrypt::dynamic::{self, record, OpSpec}; let targets = resolver().targets(); let document = targets .iter() @@ -286,14 +294,36 @@ mod tests { document.leaf_query_sql_domain.as_deref(), Some("eql_v4.query_json_leaf") ); - assert_eq!(document.elements_query_sql_domain, None); + assert_eq!( + document.elements_query_sql_domain.as_deref(), + Some("eql_v4.query_json_elements") + ); assert_eq!(document.predicate_query_sql_domain, None); - let operators = document.operators(); - for absent in [ - stack_encrypt::dynamic::OpSpec::ElementMatch, - stack_encrypt::dynamic::OpSpec::Match, - ] { - assert!(!operators.contains(&absent), "{absent:?}: {operators:?}"); + // The operators a field answers: the orderings and `element_match` + // only where it orders a path, and `match` nowhere. + let field_operators = |ordered: &[&str]| { + let wire = serde_json::json!({ + "profile": { + "context": ["users", "profile"], + "type": "json", + "null": "sql", + "target": {"name": "JsonSearch", "options": {"ordered": ordered}} + } + }); + let frame = dynamic::frame::serde::to_vec(&wire).expect("plan frame"); + let plan = record::plan_with(&frame, &resolver()).expect("plan"); + dynamic::field_operators(&plan, "profile", &resolver()) + .expect("operators") + .into_iter() + .map(|(op, _)| op) + .collect::>() + }; + let ordered = field_operators(&["$.contacts[*].relationship"]); + assert!(ordered.contains(&OpSpec::ElementMatch), "{ordered:?}"); + assert!(!ordered.contains(&OpSpec::Match), "{ordered:?}"); + let unordered = field_operators(&[]); + for absent in [OpSpec::ElementMatch, OpSpec::Match] { + assert!(!unordered.contains(&absent), "{absent:?}: {unordered:?}"); } } diff --git a/languages/golang/encrypt/internal/testusers/customer_stash.go b/languages/golang/encrypt/internal/testusers/customer_stash.go index e8ebb07db..853b5aec5 100644 --- a/languages/golang/encrypt/internal/testusers/customer_stash.go +++ b/languages/golang/encrypt/internal/testusers/customer_stash.go @@ -129,13 +129,22 @@ type CustomerProfileField struct { // At has one accessor per path the field orders, whose methods // are the orderings there. At CustomerProfileFieldAt + // Elements has one accessor per array whose elements have ordered + // members, whose Match tests the members of one element together. + Elements CustomerProfileFieldElements } func newCustomerProfileField(f gensupport.Field[json.RawMessage]) CustomerProfileField { - return CustomerProfileField{field: f, At: CustomerProfileFieldAt{ - Age: CustomerProfileFieldPath{field: f, path: "$.age"}, - VisitsAt: CustomerProfileFieldPath{field: f, path: "$.visits[*].at"}, - }} + return CustomerProfileField{ + field: f, + At: CustomerProfileFieldAt{ + Age: CustomerProfileFieldPath{field: f, path: "$.age"}, + VisitsAt: CustomerProfileFieldPath{field: f, path: "$.visits[*].at"}, + }, + Elements: CustomerProfileFieldElements{ + Visits: CustomerProfileFieldArray[CustomerProfileFieldElementsVisits]{field: f, path: "$.visits[*]"}, + }, + } } // Encrypt seals one Profile document for an update of its stored column. @@ -204,6 +213,62 @@ func (f CustomerProfileField) ChildCount(ctx context.Context, c *encrypt.Cipher, return gensupport.JSONNodeToken(ctx, f.field, c, "child_count", path) } +// CustomerProfileFieldElements names the arrays of Profile whose elements have +// ordered members. Each accessor's Match tests the members of one element +// together. +type CustomerProfileFieldElements struct { + Visits CustomerProfileFieldArray[CustomerProfileFieldElementsVisits] // $.visits[*]: at +} + +// CustomerProfileFieldElementsVisits is the array $.visits[*] of Profile, as a +// type: Elements.Visits.Match takes an +// encrypt.ElementPredicate[CustomerProfileFieldElementsVisits], which +// encrypt.AtPath makes from a member under CustomerPaths.Profile.Visits. +type CustomerProfileFieldElementsVisits struct{} + +// CustomerProfileFieldArray is one array of Profile whose elements have ordered +// members. E is the array's own type, which the predicates its Match takes +// carry. +type CustomerProfileFieldArray[E any] struct { + field gensupport.Field[json.RawMessage] + path string +} + +// Match derives the operand that finds a row where one element of this array +// passes every predicate: eql_v4.json_tree_element_match(col, +// $1::eql_v4.query_json_elements). An EqAt and an ordering at the array's paths +// can each be answered by a different element; Match is answered by one. Make +// each predicate with encrypt.AtPath and a member of this array under +// CustomerPaths, so a member of another array, or a value of another type, does +// not compile; or with Member, for a member named at run time. No predicates, +// and a zero ElementPredicate, are refused before the engine is asked. +// +// Each comparison is of the member's ordering term, so Eq is an equality of +// folded values: text compares case- and accent-folded ("sibling" equals +// "Sibling"), and integers past 2^53 can tie. EqAt at a member's full path is +// exact, of any element. +// +// The operand reads each array to opts.WildcardBound elements, 64 when it is 0. +// A row whose array is longer fails the query with SQLSTATE 22023 +// (invalid_parameter_value) rather than answer over part of it: raise the bound +// past the longest array. The function filters the rows a scan gives it, so +// narrow them with an indexed predicate first, such as @> on the same values. +func (a CustomerProfileFieldArray[E]) Match(ctx context.Context, c *encrypt.Cipher, opts encrypt.JSONReadOptions, predicates ...encrypt.ElementPredicate[E]) (eql.JSONSearchElementsQuery, error) { + return gensupport.JSONElementMatch[eql.JSONSearchElementsQuery](ctx, a.field, c, a.path, opts, predicates) +} + +// Member is the comparisons of the member of this array's elements that member +// names, for a filter whose members are known only at run time: relationship, +// or address.city, written as the member is written below the array in the +// field's ordered paths. Its values are not checked by the compiler, and Match +// refuses a member the field does not order beneath this array, naming it, the +// array and the members the field orders there, before the engine is asked. +// Where the member is known when the program is written, use encrypt.AtPath and +// CustomerPaths. +func (a CustomerProfileFieldArray[E]) Member(member string) encrypt.MemberTest[E, any] { + return encrypt.AtPath(encrypt.NewPath[E, any](a.path, member)) +} + // CustomerProfileFieldAt names the declared ordered paths of Profile. Use an // accessor's Selector with its ordering query. type CustomerProfileFieldAt struct { @@ -345,3 +410,20 @@ func (f CustomerPrefsField) Children(ctx context.Context, c *encrypt.Cipher, pat func (f CustomerPrefsField) ChildCount(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONToken, error) { return gensupport.JSONNodeToken(ctx, f.field, c, "child_count", path) } + +// CustomerPaths names the ordered members of the arrays of Customer's +// documents, nested as the JSON nests them, for the element match: +// encrypt.AtPath(CustomerPaths.Profile.Visits.At) makes the predicates +// CustomerFields.Profile.Elements.Visits.Match takes. Each path's type names +// its array and the member's Go type, so a predicate on another array's member, +// or with a value of another type, does not compile. +var CustomerPaths = func() (paths struct { + Profile struct { + Visits struct { + At encrypt.Path[CustomerProfileFieldElementsVisits, any] // $.visits[*].at + } + } +}) { + paths.Profile.Visits.At = encrypt.NewPath[CustomerProfileFieldElementsVisits, any]("$.visits[*]", "at") + return paths +}() diff --git a/languages/golang/encrypt/internal/testusers/entities.go b/languages/golang/encrypt/internal/testusers/entities.go new file mode 100644 index 000000000..447c5ca41 --- /dev/null +++ b/languages/golang/encrypt/internal/testusers/entities.go @@ -0,0 +1,32 @@ +package testusers + +//go:generate go run github.com/cipherstash/stack/languages/golang/cmd/stashgen -type Entity -name Entity + +// Entity holds its custom field values as an encrypted JSON document (the +// EQL type JsonSearch). Each emergency contact's relationship, call count +// and age order, so the generated EntityFields.CustomFieldValues has +// Elements.ExtraEmergencyContacts, whose Match tests them on one contact +// together. A contact's name is not ordered. +type Entity struct { + _ struct{} `stash:"context=entities"` + ID int64 `stash:"id,passthrough"` + CustomFieldValues CustomFieldValues `stash:"custom_field_values,encrypt_into=JsonSearch(ordered=$.extra.emergency_contacts[*].relationship;$.extra.emergency_contacts[*].previous_call_count;$.extra.emergency_contacts[*].age)"` +} + +// CustomFieldValues is an entity's custom field values document. +type CustomFieldValues struct { + Extra CustomFieldExtra `json:"extra"` +} + +// CustomFieldExtra holds an entity's emergency contacts. +type CustomFieldExtra struct { + EmergencyContacts []EmergencyContact `json:"emergency_contacts"` +} + +// EmergencyContact is one emergency contact of an entity. +type EmergencyContact struct { + Name string `json:"name"` + Relationship string `json:"relationship"` + PreviousCallCount int64 `json:"previous_call_count"` + Age int64 `json:"age"` +} diff --git a/languages/golang/encrypt/internal/testusers/entity_stash.go b/languages/golang/encrypt/internal/testusers/entity_stash.go new file mode 100644 index 000000000..effa24d6c --- /dev/null +++ b/languages/golang/encrypt/internal/testusers/entity_stash.go @@ -0,0 +1,350 @@ +// Code generated by stashgen. DO NOT EDIT. + +package testusers + +import ( + "context" + "log/slog" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/eql" + "github.com/cipherstash/stack/languages/golang/encrypt/gensupport" +) + +// Stops compiling when the library does not accept this version of generated file. +const _ = gensupport.GeneratedVersion2 + +// Entity prints its sealed fields in the clear: it has no String or LogValue +// method. Write them, or run stashgen with -redact. + +type EncryptedEntity struct { + ID int64 + CustomFieldValues eql.JSONSearch +} + +func (e EncryptedEntity) String() string { + return gensupport.Redacted("EncryptedEntity", map[string]any{"ID": e.ID}, "CustomFieldValues") +} + +func (e EncryptedEntity) LogValue() slog.Value { + return gensupport.RedactedLog(map[string]any{"ID": e.ID}, "CustomFieldValues") +} + +// Stops compiling when Entity gains, loses, reorders or retypes a field. +var _ = entityShape(Entity{}) + +type entityShape struct { + _ struct{} + ID int64 + CustomFieldValues CustomFieldValues +} + +var entityDeclaration = gensupport.Declare("entities"). + Passthrough("id"). + EncryptInto("custom_field_values", gensupport.JSON, "JsonSearch"). + Options("custom_field_values", gensupport.TargetOptions{Ordered: []string{"$.extra.emergency_contacts[*].relationship", "$.extra.emergency_contacts[*].previous_call_count", "$.extra.emergency_contacts[*].age"}}) + +var entityCodec = gensupport.New(gensupport.Generated[Entity, EncryptedEntity]{ + TypeName: "Entity", + Declaration: entityDeclaration, + PrintsPlaintext: true, + Source: func(v Entity) gensupport.Values { + return gensupport.Values{ + "id": v.ID, + "custom_field_values": v.CustomFieldValues, + } + }, + Seal: func(rec gensupport.Record) (EncryptedEntity, error) { + var e EncryptedEntity + var err error + if e.ID, err = gensupport.Passthrough[int64](rec, "id"); err != nil { + return EncryptedEntity{}, err + } + e.CustomFieldValues = eql.JSONSearch(rec["custom_field_values"].EQL) + return e, nil + }, + Open: func(e EncryptedEntity) gensupport.Record { + return gensupport.Record{ + "id": {Value: e.ID}, + "custom_field_values": {EQL: e.CustomFieldValues}, + } + }, + Value: func(e EncryptedEntity, vals gensupport.Values) (Entity, error) { + var v Entity + var err error + if v.ID, err = gensupport.Get[int64](vals, "id"); err != nil { + return Entity{}, err + } + if v.CustomFieldValues, err = gensupport.GetDocument[CustomFieldValues](vals, "custom_field_values"); err != nil { + return Entity{}, err + } + return v, nil + }, +}) + +// EncryptEntity seals each Entity, with one ZeroKMS request for each 500 sealed +// values, plus one the first time a keyset is used. The result has one element +// for each input, in the same order. +func EncryptEntity(ctx context.Context, cipher *encrypt.Cipher, values []Entity) ([]EncryptedEntity, error) { + return entityCodec.Encrypt(ctx, cipher, values) +} + +// DecryptEntity opens each EncryptedEntity, with one ZeroKMS request for each +// 500 sealed values, plus one the first time a keyset is used. +func DecryptEntity(ctx context.Context, d encrypt.Decrypter, encrypted []EncryptedEntity) ([]Entity, error) { + return entityCodec.Decrypt(ctx, d, encrypted) +} + +// EntityFields provides operations for the sealed fields of Entity. Each entry +// binds queries and encrypts updates using its field's declaration. +var EntityFields = struct { + CustomFieldValues EntityCustomFieldValuesField +}{ + CustomFieldValues: newEntityCustomFieldValuesField(gensupport.NewDocumentField[CustomFieldValues](entityDeclaration, "custom_field_values")), +} + +// EntityCustomFieldValuesField provides document queries and reads for the +// CustomFieldValues field. +type EntityCustomFieldValuesField struct { + field gensupport.Field[CustomFieldValues] + // At has one accessor per path the field orders, whose methods + // are the orderings there. + At EntityCustomFieldValuesFieldAt + // Elements has one accessor per array whose elements have ordered + // members, whose Match tests the members of one element together. + Elements EntityCustomFieldValuesFieldElements +} + +func newEntityCustomFieldValuesField(f gensupport.Field[CustomFieldValues]) EntityCustomFieldValuesField { + return EntityCustomFieldValuesField{ + field: f, + At: EntityCustomFieldValuesFieldAt{ + ExtraEmergencyContactsRelationship: EntityCustomFieldValuesFieldPath{field: f, path: "$.extra.emergency_contacts[*].relationship"}, + ExtraEmergencyContactsPreviousCallCount: EntityCustomFieldValuesFieldPath{field: f, path: "$.extra.emergency_contacts[*].previous_call_count"}, + ExtraEmergencyContactsAge: EntityCustomFieldValuesFieldPath{field: f, path: "$.extra.emergency_contacts[*].age"}, + }, + Elements: EntityCustomFieldValuesFieldElements{ + ExtraEmergencyContacts: EntityCustomFieldValuesFieldArray[EntityCustomFieldValuesFieldElementsExtraEmergencyContacts]{field: f, path: "$.extra.emergency_contacts[*]"}, + }, + } +} + +// Encrypt seals one CustomFieldValues document for an update of its stored +// column. +func (f EntityCustomFieldValuesField) Encrypt(ctx context.Context, c *encrypt.Cipher, v CustomFieldValues) (eql.JSONSearch, error) { + out, err := f.field.Encrypt(ctx, c, v) + return eql.JSONSearch(out.EQL), err +} + +// Contains derives the query for documents that contain needle: col @> query. +// needle is a json.RawMessage of JSON text, or a map, struct or slice that +// encoding/json writes; a Go string or a pointer to one is refused, as one JSON +// string is not JSON text, and so is a needle that writes JSON null (nil, or a +// nil pointer, map or slice). Each member it writes must be in the document, so +// a zero-valued struct field narrows the query to documents holding that zero: +// give a needle struct pointer fields, or omitempty, or use a map. Each array +// element of the needle must sit inside one element of the document's array. A +// needle of the field's own type, CustomFieldValues, is refused for that +// reason, and so is a pointer to it: write a needle type whose fields are +// pointers with omitempty, or use a map. +func (f EntityCustomFieldValuesField) Contains(ctx context.Context, c *encrypt.Cipher, needle any) (eql.JSONSearchQuery, error) { + return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contains", needle) +} + +// ContainedBy derives the query for documents contained by needle: col <@ +// query. needle is a json.RawMessage of JSON text, or a map, struct or slice +// that encoding/json writes (not a Go string or a pointer to one, nor a needle +// that writes JSON null); here a zero-valued field widens the query, admitting +// documents that hold that zero. +func (f EntityCustomFieldValuesField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle any) (eql.JSONSearchQuery, error) { + return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contained_by", needle) +} + +// EqAt derives the query for documents holding the scalar v at path: col @> +// query. v is a Go string, number or bool, nil for JSON null, or a +// json.RawMessage of JSON text; a Go string is the JSON string it holds, never +// parsed as JSON. A wildcard step matches when any element holds it; a position +// ([2]) is refused, since the stored selector reads every position as the +// wildcard. +func (f EntityCustomFieldValuesField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v any) (eql.JSONSearchQuery, error) { + return gensupport.JSONQueryAt[eql.JSONSearchQuery](ctx, f.field, c, "eq_at", path, v) +} + +// Selector prepares `->` at path: bind Selector() as col -> $1::text, and open +// the entries it returns with Open; encrypt.DecodeJSON reads each value into a +// Go type. A position ([2]) is refused; read one with Subtree. +func (f EntityCustomFieldValuesField) Selector(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONSelection[eql.JSONEntry], error) { + return gensupport.JSONSelect[eql.JSONEntry](ctx, f.field, c, path) +} + +// Subtree prepares a read of the value at path: bind Tokens() as +// eql_v4.json_tree_subtree(col, $1::text[], budget), and open the rows it +// returns with Open or OpenOne; encrypt.DecodeJSON reads each value into a Go +// type. +func (f EntityCustomFieldValuesField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.JSONEntry], error) { + return gensupport.JSONSubtree[eql.JSONEntry](ctx, f.field, c, path, opts) +} + +// Children derives the token of the node at path, a path with no wildcard, +// which eql_v4.json_tree_children(col, $1::text) takes. Its rows are encrypted +// entries one level down. +func (f EntityCustomFieldValuesField) Children(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONToken, error) { + return gensupport.JSONNodeToken(ctx, f.field, c, "children", path) +} + +// ChildCount derives the token of the node at path, which +// eql_v4.json_tree_child_count(col, $1::text) takes: zero for a scalar or an +// empty container, NULL where the row has no node there or for the sealed root. +func (f EntityCustomFieldValuesField) ChildCount(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONToken, error) { + return gensupport.JSONNodeToken(ctx, f.field, c, "child_count", path) +} + +// EntityCustomFieldValuesFieldElements names the arrays of CustomFieldValues +// whose elements have ordered members. Each accessor's Match tests the members +// of one element together. +type EntityCustomFieldValuesFieldElements struct { + ExtraEmergencyContacts EntityCustomFieldValuesFieldArray[EntityCustomFieldValuesFieldElementsExtraEmergencyContacts] // $.extra.emergency_contacts[*]: relationship, previous_call_count, age +} + +// EntityCustomFieldValuesFieldElementsExtraEmergencyContacts is the array +// $.extra.emergency_contacts[*] of CustomFieldValues, as a type: +// Elements.ExtraEmergencyContacts.Match takes an +// encrypt.ElementPredicate[EntityCustomFieldValuesFieldElementsExtraEmergencyContacts], +// which encrypt.AtPath makes from a member under +// EntityPaths.CustomFieldValues.Extra.EmergencyContacts. +type EntityCustomFieldValuesFieldElementsExtraEmergencyContacts struct{} + +// EntityCustomFieldValuesFieldArray is one array of CustomFieldValues whose +// elements have ordered members. E is the array's own type, which the +// predicates its Match takes carry. +type EntityCustomFieldValuesFieldArray[E any] struct { + field gensupport.Field[CustomFieldValues] + path string +} + +// Match derives the operand that finds a row where one element of this array +// passes every predicate: eql_v4.json_tree_element_match(col, +// $1::eql_v4.query_json_elements). An EqAt and an ordering at the array's paths +// can each be answered by a different element; Match is answered by one. Make +// each predicate with encrypt.AtPath and a member of this array under +// EntityPaths, so a member of another array, or a value of another type, does +// not compile; or with Member, for a member named at run time. No predicates, +// and a zero ElementPredicate, are refused before the engine is asked. +// +// Each comparison is of the member's ordering term, so Eq is an equality of +// folded values: text compares case- and accent-folded ("sibling" equals +// "Sibling"), and integers past 2^53 can tie. EqAt at a member's full path is +// exact, of any element. +// +// The operand reads each array to opts.WildcardBound elements, 64 when it is 0. +// A row whose array is longer fails the query with SQLSTATE 22023 +// (invalid_parameter_value) rather than answer over part of it: raise the bound +// past the longest array. The function filters the rows a scan gives it, so +// narrow them with an indexed predicate first, such as @> on the same values. +func (a EntityCustomFieldValuesFieldArray[E]) Match(ctx context.Context, c *encrypt.Cipher, opts encrypt.JSONReadOptions, predicates ...encrypt.ElementPredicate[E]) (eql.JSONSearchElementsQuery, error) { + return gensupport.JSONElementMatch[eql.JSONSearchElementsQuery](ctx, a.field, c, a.path, opts, predicates) +} + +// Member is the comparisons of the member of this array's elements that member +// names, for a filter whose members are known only at run time: relationship, +// or address.city, written as the member is written below the array in the +// field's ordered paths. Its values are not checked by the compiler, and Match +// refuses a member the field does not order beneath this array, naming it, the +// array and the members the field orders there, before the engine is asked. +// Where the member is known when the program is written, use encrypt.AtPath and +// EntityPaths. +func (a EntityCustomFieldValuesFieldArray[E]) Member(member string) encrypt.MemberTest[E, any] { + return encrypt.AtPath(encrypt.NewPath[E, any](a.path, member)) +} + +// EntityCustomFieldValuesFieldAt names the declared ordered paths of +// CustomFieldValues. Use an accessor's Selector with its ordering query. +type EntityCustomFieldValuesFieldAt struct { + ExtraEmergencyContactsRelationship EntityCustomFieldValuesFieldPath // $.extra.emergency_contacts[*].relationship + ExtraEmergencyContactsPreviousCallCount EntityCustomFieldValuesFieldPath // $.extra.emergency_contacts[*].previous_call_count + ExtraEmergencyContactsAge EntityCustomFieldValuesFieldPath // $.extra.emergency_contacts[*].age +} + +// EntityCustomFieldValuesFieldPath is one ordered path of CustomFieldValues. +// Its orderings derive the leaf operand (eql_v4.query_json_leaf) a stored +// leaf's ordering term is compared with, and Selector the path's selector. At a +// path with no array step, compare col -> Selector() with the operand, and +// order by eql_v4.ord_term(col -> Selector()). At a wildcard path ([*]) col -> +// Selector() is an unspecified element of each row, so write the comparison +// over every match, EXISTS (SELECT 1 FROM eql_v4.json_tree_path_query(col, +// $sel) e WHERE e < $1::eql_v4.query_json_leaf), and order by the least or +// greatest match's term: (SELECT eql_v4.ord_term(e) FROM +// eql_v4.json_tree_path_query(col, $sel) e ORDER BY 1 LIMIT 1). Each operand is +// a Go number or string, or a json.RawMessage of JSON text. Numbers order as +// float64, so integers past 2^53 can tie; text orders case- and accent-folded. +type EntityCustomFieldValuesFieldPath struct { + field gensupport.Field[CustomFieldValues] + path string +} + +// Lt derives the operand that finds a leaf at this path less than v: (col -> +// $sel) < $1::eql_v4.query_json_leaf, or at a wildcard path the same comparison +// over every match (see EntityCustomFieldValuesFieldPath). +func (p EntityCustomFieldValuesFieldPath) Lt(ctx context.Context, c *encrypt.Cipher, v any) (eql.JSONSearchLeafQuery, error) { + return gensupport.JSONQueryAt[eql.JSONSearchLeafQuery](ctx, p.field, c, "lt_at", p.path, v) +} + +// Lte derives the operand that finds a leaf at this path at most v: (col -> +// $sel) <= $1::eql_v4.query_json_leaf, or at a wildcard path the same +// comparison over every match (see EntityCustomFieldValuesFieldPath). +func (p EntityCustomFieldValuesFieldPath) Lte(ctx context.Context, c *encrypt.Cipher, v any) (eql.JSONSearchLeafQuery, error) { + return gensupport.JSONQueryAt[eql.JSONSearchLeafQuery](ctx, p.field, c, "lte_at", p.path, v) +} + +// Gt derives the operand that finds a leaf at this path greater than v: (col -> +// $sel) > $1::eql_v4.query_json_leaf, or at a wildcard path the same comparison +// over every match (see EntityCustomFieldValuesFieldPath). +func (p EntityCustomFieldValuesFieldPath) Gt(ctx context.Context, c *encrypt.Cipher, v any) (eql.JSONSearchLeafQuery, error) { + return gensupport.JSONQueryAt[eql.JSONSearchLeafQuery](ctx, p.field, c, "gt_at", p.path, v) +} + +// Gte derives the operand that finds a leaf at this path at least v: (col -> +// $sel) >= $1::eql_v4.query_json_leaf, or at a wildcard path the same +// comparison over every match (see EntityCustomFieldValuesFieldPath). +func (p EntityCustomFieldValuesFieldPath) Gte(ctx context.Context, c *encrypt.Cipher, v any) (eql.JSONSearchLeafQuery, error) { + return gensupport.JSONQueryAt[eql.JSONSearchLeafQuery](ctx, p.field, c, "gte_at", p.path, v) +} + +// Between derives the two operands that find a leaf at this path from lo to hi, +// inclusive: (col -> $sel) >= Low AND (col -> $sel) <= High, or at a wildcard +// path the same over every match, with both bounds on one element (see +// EntityCustomFieldValuesFieldPath). +func (p EntityCustomFieldValuesFieldPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi any) (encrypt.JSONRange[eql.JSONSearchLeafQuery], error) { + return gensupport.JSONBetweenAt[eql.JSONSearchLeafQuery](ctx, p.field, c, p.path, lo, hi) +} + +// Selector prepares `->` at this path: bind Selector() beside the ordering's +// operand, from the same path, as col -> $sel, or at a wildcard path as +// eql_v4.json_tree_path_query(col, $sel), whose matches are every element. +func (p EntityCustomFieldValuesFieldPath) Selector(ctx context.Context, c *encrypt.Cipher) (encrypt.JSONSelection[eql.JSONEntry], error) { + return gensupport.JSONSelect[eql.JSONEntry](ctx, p.field, c, p.path) +} + +// EntityPaths names the ordered members of the arrays of Entity's documents, +// nested as the JSON nests them, for the element match: +// encrypt.AtPath(EntityPaths.CustomFieldValues.Extra.EmergencyContacts.Relationship) +// makes the predicates +// EntityFields.CustomFieldValues.Elements.ExtraEmergencyContacts.Match takes. +// Each path's type names its array and the member's Go type, so a predicate on +// another array's member, or with a value of another type, does not compile. +var EntityPaths = func() (paths struct { + CustomFieldValues struct { + Extra struct { + EmergencyContacts struct { + Relationship encrypt.Path[EntityCustomFieldValuesFieldElementsExtraEmergencyContacts, string] // $.extra.emergency_contacts[*].relationship + PreviousCallCount encrypt.Path[EntityCustomFieldValuesFieldElementsExtraEmergencyContacts, int64] // $.extra.emergency_contacts[*].previous_call_count + Age encrypt.Path[EntityCustomFieldValuesFieldElementsExtraEmergencyContacts, int64] // $.extra.emergency_contacts[*].age + } + } + } +}) { + paths.CustomFieldValues.Extra.EmergencyContacts.Relationship = encrypt.NewPath[EntityCustomFieldValuesFieldElementsExtraEmergencyContacts, string]("$.extra.emergency_contacts[*]", "relationship") + paths.CustomFieldValues.Extra.EmergencyContacts.PreviousCallCount = encrypt.NewPath[EntityCustomFieldValuesFieldElementsExtraEmergencyContacts, int64]("$.extra.emergency_contacts[*]", "previous_call_count") + paths.CustomFieldValues.Extra.EmergencyContacts.Age = encrypt.NewPath[EntityCustomFieldValuesFieldElementsExtraEmergencyContacts, int64]("$.extra.emergency_contacts[*]", "age") + return paths +}() diff --git a/languages/golang/encrypt/internal/testusers/subscriber_stash.go b/languages/golang/encrypt/internal/testusers/subscriber_stash.go index 486c74648..35c02e194 100644 --- a/languages/golang/encrypt/internal/testusers/subscriber_stash.go +++ b/languages/golang/encrypt/internal/testusers/subscriber_stash.go @@ -139,13 +139,22 @@ type SubscriberProfileField struct { // At has one accessor per path the field orders, whose methods // are the orderings there. At SubscriberProfileFieldAt + // Elements has one accessor per array whose elements have ordered + // members, whose Match tests the members of one element together. + Elements SubscriberProfileFieldElements } func newSubscriberProfileField(f gensupport.Field[SubscriberProfile]) SubscriberProfileField { - return SubscriberProfileField{field: f, At: SubscriberProfileFieldAt{ - Age: SubscriberProfileFieldPath{field: f, path: "$.age"}, - VisitsAt: SubscriberProfileFieldPath{field: f, path: "$.visits[*].at"}, - }} + return SubscriberProfileField{ + field: f, + At: SubscriberProfileFieldAt{ + Age: SubscriberProfileFieldPath{field: f, path: "$.age"}, + VisitsAt: SubscriberProfileFieldPath{field: f, path: "$.visits[*].at"}, + }, + Elements: SubscriberProfileFieldElements{ + Visits: SubscriberProfileFieldArray[SubscriberProfileFieldElementsVisits]{field: f, path: "$.visits[*]"}, + }, + } } // Encrypt seals one Profile document for an update of its stored column. @@ -217,6 +226,63 @@ func (f SubscriberProfileField) ChildCount(ctx context.Context, c *encrypt.Ciphe return gensupport.JSONNodeToken(ctx, f.field, c, "child_count", path) } +// SubscriberProfileFieldElements names the arrays of Profile whose elements +// have ordered members. Each accessor's Match tests the members of one element +// together. +type SubscriberProfileFieldElements struct { + Visits SubscriberProfileFieldArray[SubscriberProfileFieldElementsVisits] // $.visits[*]: at +} + +// SubscriberProfileFieldElementsVisits is the array $.visits[*] of Profile, as +// a type: Elements.Visits.Match takes an +// encrypt.ElementPredicate[SubscriberProfileFieldElementsVisits], which +// encrypt.AtPath makes from a member under SubscriberPaths.Profile.Visits. +type SubscriberProfileFieldElementsVisits struct{} + +// SubscriberProfileFieldArray is one array of Profile whose elements have +// ordered members. E is the array's own type, which the predicates its Match +// takes carry. +type SubscriberProfileFieldArray[E any] struct { + field gensupport.Field[SubscriberProfile] + path string +} + +// Match derives the operand that finds a row where one element of this array +// passes every predicate: eql_v4.json_tree_element_match(col, +// $1::eql_v4.query_json_elements). An EqAt and an ordering at the array's paths +// can each be answered by a different element; Match is answered by one. Make +// each predicate with encrypt.AtPath and a member of this array under +// SubscriberPaths, so a member of another array, or a value of another type, +// does not compile; or with Member, for a member named at run time. No +// predicates, and a zero ElementPredicate, are refused before the engine is +// asked. +// +// Each comparison is of the member's ordering term, so Eq is an equality of +// folded values: text compares case- and accent-folded ("sibling" equals +// "Sibling"), and integers past 2^53 can tie. EqAt at a member's full path is +// exact, of any element. +// +// The operand reads each array to opts.WildcardBound elements, 64 when it is 0. +// A row whose array is longer fails the query with SQLSTATE 22023 +// (invalid_parameter_value) rather than answer over part of it: raise the bound +// past the longest array. The function filters the rows a scan gives it, so +// narrow them with an indexed predicate first, such as @> on the same values. +func (a SubscriberProfileFieldArray[E]) Match(ctx context.Context, c *encrypt.Cipher, opts encrypt.JSONReadOptions, predicates ...encrypt.ElementPredicate[E]) (eql.JSONSearchElementsQuery, error) { + return gensupport.JSONElementMatch[eql.JSONSearchElementsQuery](ctx, a.field, c, a.path, opts, predicates) +} + +// Member is the comparisons of the member of this array's elements that member +// names, for a filter whose members are known only at run time: relationship, +// or address.city, written as the member is written below the array in the +// field's ordered paths. Its values are not checked by the compiler, and Match +// refuses a member the field does not order beneath this array, naming it, the +// array and the members the field orders there, before the engine is asked. +// Where the member is known when the program is written, use encrypt.AtPath and +// SubscriberPaths. +func (a SubscriberProfileFieldArray[E]) Member(member string) encrypt.MemberTest[E, any] { + return encrypt.AtPath(encrypt.NewPath[E, any](a.path, member)) +} + // SubscriberProfileFieldAt names the declared ordered paths of Profile. Use an // accessor's Selector with its ordering query. type SubscriberProfileFieldAt struct { @@ -437,3 +503,20 @@ func (f SubscriberLabelsField) Children(ctx context.Context, c *encrypt.Cipher, func (f SubscriberLabelsField) ChildCount(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONToken, error) { return gensupport.JSONNodeToken(ctx, f.field, c, "child_count", path) } + +// SubscriberPaths names the ordered members of the arrays of Subscriber's +// documents, nested as the JSON nests them, for the element match: +// encrypt.AtPath(SubscriberPaths.Profile.Visits.At) makes the predicates +// SubscriberFields.Profile.Elements.Visits.Match takes. Each path's type names +// its array and the member's Go type, so a predicate on another array's member, +// or with a value of another type, does not compile. +var SubscriberPaths = func() (paths struct { + Profile struct { + Visits struct { + At encrypt.Path[SubscriberProfileFieldElementsVisits, int64] // $.visits[*].at + } + } +}) { + paths.Profile.Visits.At = encrypt.NewPath[SubscriberProfileFieldElementsVisits, int64]("$.visits[*]", "at") + return paths +}() diff --git a/languages/golang/encrypt/json_elements.go b/languages/golang/encrypt/json_elements.go new file mode 100644 index 000000000..ab64cbc6d --- /dev/null +++ b/languages/golang/encrypt/json_elements.go @@ -0,0 +1,165 @@ +package encrypt + +import ( + "encoding/json" + "fmt" + "io" + "log/slog" + "slices" +) + +// Path is one ordered member of the elements of an array in a JSONSearch +// field, for the element match: the generated +// Fields..Elements..Match. Read one from the generated +// Paths, which names each ordered member as the JSON nests it: +// Paths.Profile.Visits.Place is the member place of each element of +// $.visits[*]. +// +// E is the array: a type the generator writes for each array, which the +// array's Match takes predicates of. T is the member's Go type in the +// document's struct, or any when the generator cannot name one (a +// json.RawMessage document, say). Pass a Path to [AtPath] for a predicate. +type Path[E, T any] struct { + array string + member string +} + +// NewPath is the Path of member below array: for generated code. A +// program reads its paths from the generated Paths, or names a member at +// run time with the generated array accessor's Member. +func NewPath[E, T any](array, member string) Path[E, T] { + return Path[E, T]{array: array, member: member} +} + +// String is the member's full path: $.visits[*].place. +func (p Path[E, T]) String() string { + if p.member == "" { + return p.array + } + return p.array + "." + p.member +} + +// AtPath is the comparisons of the member at p, for the element match. +// Each returns an [ElementPredicate] of p's array, so only that array's +// Match takes it, and each takes a value of the member's Go type: +// +// encrypt.AtPath(Paths.Profile.Visits.At).Gt(15) +// +// AtPath is a function, not a method of Path, because a Go method cannot +// take type parameters of its own. +func AtPath[E, T any](p Path[E, T]) MemberTest[E, T] { + return MemberTest[E, T]{member: p.member} +} + +// MemberTest makes the predicates on one member of an array's elements. +// Make one with [AtPath], or with the generated array accessor's Member for +// a member named at run time, whose T is any. +// +// Each comparison is of the member's ordering term, as the orderings at a +// path compare: numbers order as float64, so integers past 2^53 can tie, +// and text orders case- and accent-folded. Equality is both inclusive +// bounds at one value, so [MemberTest.Eq] is an equality of those folded +// values. +// +// A value is a Go number or string, or a json.RawMessage of JSON text, +// written to JSON as the orderings at a path write theirs. It is +// plaintext: an ElementPredicate prints and logs its member and comparison, +// and never its values. A predicate copies the bytes of a json.RawMessage +// or a []byte value, so changing the caller's bytes does not change it. A +// value of any other reference type, such as a map, a slice or a pointer, +// is not copied: changing what it refers to changes the predicate. +type MemberTest[E, T any] struct { + member string +} + +// Eq is the test that the member equals v, compared by its ordering term: +// text is case- and accent-folded, so "sibling" equals "Sibling", and +// integers past 2^53 can tie. For an exact equality of one member, use the +// field's EqAt at the member's full path, which any element can answer. +func (m MemberTest[E, T]) Eq(v T) ElementPredicate[E] { return m.predicate("eq", v) } + +// Lt is the test that the member is less than v. +func (m MemberTest[E, T]) Lt(v T) ElementPredicate[E] { return m.predicate("lt", v) } + +// Lte is the test that the member is at most v. +func (m MemberTest[E, T]) Lte(v T) ElementPredicate[E] { return m.predicate("lte", v) } + +// Gt is the test that the member is greater than v. +func (m MemberTest[E, T]) Gt(v T) ElementPredicate[E] { return m.predicate("gt", v) } + +// Gte is the test that the member is at least v. +func (m MemberTest[E, T]) Gte(v T) ElementPredicate[E] { return m.predicate("gte", v) } + +// Between is the test that the member is from lo to hi, inclusive. +func (m MemberTest[E, T]) Between(lo, hi T) ElementPredicate[E] { + return ElementPredicate[E]{member: m.member, op: "between", values: []any{detached(lo), detached(hi)}} +} + +func (m MemberTest[E, T]) predicate(op string, v T) ElementPredicate[E] { + return ElementPredicate[E]{member: m.member, op: op, values: []any{detached(v)}} +} + +// ElementPredicate is one test on a member of the elements of the array E, +// for that array's generated Match. Make one with [AtPath], or with the +// generated array accessor's Member. +// +// Its member names one field of each element, relative to the element: +// relationship, or address.city, written as the field's ordered path is +// written below the array. Its full path must be one the field orders. +type ElementPredicate[E any] struct { + member string + op string + values []any +} + +// Member is the member the predicate tests, relative to an element. +func (p ElementPredicate[E]) Member() string { return p.member } + +// Op is the comparison, as the engine names it: eq, lt, lte, gt, gte or +// between; "" for a zero ElementPredicate. +func (p ElementPredicate[E]) Op() string { return p.op } + +// Values are a copy of the predicate's values: one, or for between the +// low bound then the high. They are the plaintext the caller gave. The +// slice and the bytes of a json.RawMessage or []byte in it are copies, so +// changing them does not change the predicate; a value of another +// reference type is the caller's own. For generated code. +func (p ElementPredicate[E]) Values() []any { + values := slices.Clone(p.values) + for i, v := range values { + values[i] = detached(v) + } + return values +} + +// detached is v with the bytes of a json.RawMessage or a []byte copied, so +// the copy shares no bytes with v. Any other value is returned as it is. +func detached(v any) any { + switch b := v.(type) { + case json.RawMessage: + return slices.Clone(b) + case []byte: + return slices.Clone(b) + } + return v +} + +// String names the member and the comparison, and hides the values. +func (p ElementPredicate[E]) String() string { + if p.op == "" { + return "encrypt.ElementPredicate{}" + } + return fmt.Sprintf("encrypt.ElementPredicate{member: %q, op: %s, values: %s}", p.member, p.op, redacted) +} + +// GoString is what %#v prints: [ElementPredicate.String]. +func (p ElementPredicate[E]) GoString() string { return p.String() } + +// Format prints [ElementPredicate.String] for every verb fmt hands it. +func (p ElementPredicate[E]) Format(f fmt.State, _ rune) { _, _ = io.WriteString(f, p.String()) } + +// LogValue is what slog records: the member and the comparison, and +// [redacted] for the values. +func (p ElementPredicate[E]) LogValue() slog.Value { + return slog.GroupValue(slog.String("member", p.member), slog.String("op", p.op), slog.String("values", redacted)) +} diff --git a/languages/golang/encrypt/json_elements_postgres_test.go b/languages/golang/encrypt/json_elements_postgres_test.go new file mode 100644 index 000000000..112251bba --- /dev/null +++ b/languages/golang/encrypt/json_elements_postgres_test.go @@ -0,0 +1,162 @@ +package encrypt_test + +import ( + "context" + "errors" + "os" + "slices" + "testing" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/eql" + "github.com/cipherstash/stack/languages/golang/encrypt/internal/testusers" + "github.com/jackc/pgx/v5/pgconn" +) + +// entities hold emergency contacts: Ann's family has one contact who is a +// sibling and another called more than five times, and no one contact who +// is both; Ben's has one; Cy has none. +var entities = []testusers.Entity{ + {ID: 1, CustomFieldValues: testusers.CustomFieldValues{Extra: testusers.CustomFieldExtra{EmergencyContacts: []testusers.EmergencyContact{ + {Name: "Ann", Relationship: "Sibling", PreviousCallCount: 2, Age: 30}, + {Name: "Al", Relationship: "Friend", PreviousCallCount: 9, Age: 50}, + }}}}, + {ID: 2, CustomFieldValues: testusers.CustomFieldValues{Extra: testusers.CustomFieldExtra{EmergencyContacts: []testusers.EmergencyContact{ + {Name: "Bea", Relationship: "Friend", PreviousCallCount: 1, Age: 41}, + {Name: "Ben", Relationship: "Sibling", PreviousCallCount: 7, Age: 25}, + }}}}, + {ID: 3, CustomFieldValues: testusers.CustomFieldValues{Extra: testusers.CustomFieldExtra{EmergencyContacts: []testusers.EmergencyContact{}}}}, +} + +// The element match inside PostgreSQL with EQL v4 installed: Match finds +// the row where one contact passes both predicates, where an equality and +// an ordering at the array's paths find a row where each passes on a +// different contact too. Skipped unless DATABASE_URL is set. +func TestJSONElementMatchInPostgres(t *testing.T) { + url := os.Getenv("DATABASE_URL") + if url == "" { + t.Skip("DATABASE_URL is not set") + } + ctx := context.Background() + c := deterministicEQLClient(t) + cipher := c.DefaultKeyset() + conn := eqlPostgresConn(t, ctx, url) + if _, err := conn.ExecContext(ctx, `CREATE TEMP TABLE entities (id bigint PRIMARY KEY, custom_field_values public.eql_v4_json_search NOT NULL)`); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _, _ = conn.ExecContext(context.Background(), `DROP TABLE IF EXISTS entities`) }) + encrypted, err := testusers.EncryptEntity(ctx, cipher, entities) + if err != nil { + t.Fatal(err) + } + for _, e := range encrypted { + if _, err := conn.ExecContext(ctx, `INSERT INTO entities VALUES ($1, $2)`, e.ID, e.CustomFieldValues); err != nil { + t.Fatal(err) + } + } + ids := func(q string, args ...any) ([]int64, error) { + t.Helper() + rows, err := conn.QueryContext(ctx, q, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, err + } + out = append(out, id) + } + return out, rows.Err() + } + want := func(name string, got []int64, err error, want ...int64) { + t.Helper() + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if !slices.Equal(got, want) { + t.Errorf("%s: rows %v, want %v", name, got, want) + } + } + f := testusers.EntityFields.CustomFieldValues + elementMatch := `SELECT id FROM entities WHERE eql_v4.json_tree_element_match(custom_field_values, $1::eql_v4.query_json_elements) ORDER BY id` + contact := testusers.EntityPaths.CustomFieldValues.Extra.EmergencyContacts + relationship, callCount, age := encrypt.AtPath(contact.Relationship), encrypt.AtPath(contact.PreviousCallCount), encrypt.AtPath(contact.Age) + match := func(opts encrypt.JSONReadOptions, predicates ...encrypt.ElementPredicate[testusers.EntityCustomFieldValuesFieldElementsExtraEmergencyContacts]) eql.JSONSearchElementsQuery { + t.Helper() + q, err := f.Elements.ExtraEmergencyContacts.Match(ctx, cipher, opts, predicates...) + if err != nil { + t.Fatal(err) + } + return q + } + + // The superset: some contact is a sibling (EqAt), and some contact was + // called more than five times (an ordering over every match). Ann's + // family passes on two different contacts. + sibling, err := f.EqAt(ctx, cipher, "$.extra.emergency_contacts[*].relationship", "Sibling") + if err != nil { + t.Fatal(err) + } + calls, err := f.At.ExtraEmergencyContactsPreviousCallCount.Selector(ctx, cipher) + if err != nil { + t.Fatal(err) + } + moreThanFive, err := f.At.ExtraEmergencyContactsPreviousCallCount.Gt(ctx, cipher, 5) + if err != nil { + t.Fatal(err) + } + got, err := ids(`SELECT id FROM entities WHERE custom_field_values @> $1::eql_v4.query_json AND EXISTS (SELECT 1 FROM eql_v4.json_tree_path_query(custom_field_values, $2::text) AS e WHERE e > $3::eql_v4.query_json_leaf) ORDER BY id`, sibling, calls.Selector(), moreThanFive) + want("an equality and an ordering, any contacts", got, err, 1, 2) + + // One contact passing both: Ben's family alone. + got, err = ids(elementMatch, match(encrypt.JSONReadOptions{}, relationship.Eq("Sibling"), callCount.Gt(5))) + want("Match, one contact", got, err, 2) + // The same members named at run time find the same row. + named := f.Elements.ExtraEmergencyContacts.Member + got, err = ids(elementMatch, match(encrypt.JSONReadOptions{}, named("relationship").Eq("Sibling"), named("previous_call_count").Gt(5))) + want("Match, members named at run time", got, err, 2) + // Each half alone finds both families, as the superset's halves do. + got, err = ids(elementMatch, match(encrypt.JSONReadOptions{}, relationship.Eq("Sibling"))) + want("Match, a sibling", got, err, 1, 2) + got, err = ids(elementMatch, match(encrypt.JSONReadOptions{}, callCount.Gt(5))) + want("Match, more than five calls", got, err, 1, 2) + // Three members and a range, on one contact: Ann (30, 2 calls) and + // Ben (25, 7 calls) are 18 to 40, and only Ben was called more than + // five times. + got, err = ids(elementMatch, match(encrypt.JSONReadOptions{}, age.Between(18, 40), callCount.Gte(2), relationship.Eq("Sibling"))) + want("Match, a range", got, err, 1, 2) + got, err = ids(elementMatch, match(encrypt.JSONReadOptions{}, age.Between(18, 40), callCount.Gt(5))) + want("Match, a range and more than five calls", got, err, 2) + // Equality is of the ordering term, which folds case: "sibling" + // equals "Sibling". EqAt is exact. + got, err = ids(elementMatch, match(encrypt.JSONReadOptions{}, relationship.Eq("sibling"), age.Lt(26))) + want("Match, folded", got, err, 2) + exact, err := f.EqAt(ctx, cipher, "$.extra.emergency_contacts[*].relationship", "sibling") + if err != nil { + t.Fatal(err) + } + got, err = ids(`SELECT id FROM entities WHERE custom_field_values @> $1::eql_v4.query_json ORDER BY id`, exact) + want("EqAt, exact", got, err) + + // A bound shorter than an array fails the query, rather than answer + // over part of it: each family has two contacts. + _, err = ids(elementMatch, match(encrypt.JSONReadOptions{WildcardBound: 1}, relationship.Eq("Sibling"))) + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || pgErr.Code != "22023" { + t.Errorf("a bound of 1 over two contacts: %v, want SQLSTATE 22023", err) + } + // A bound that covers every array answers. + got, err = ids(elementMatch, match(encrypt.JSONReadOptions{WildcardBound: 2}, relationship.Eq("Sibling"), callCount.Gt(5))) + want("Match, a bound of 2", got, err, 2) + + // A member the field does not order is refused before the engine is + // asked, naming it: no Diagnostic, since the guest never saw it. + _, err = f.Elements.ExtraEmergencyContacts.Match(ctx, cipher, encrypt.JSONReadOptions{}, f.Elements.ExtraEmergencyContacts.Member("name").Eq("Ben")) + var d *encrypt.Diagnostic + if !errors.Is(err, encrypt.ErrTerm) || errors.As(err, &d) { + t.Errorf("an unordered member: %v", err) + } +} diff --git a/languages/golang/encrypt/json_elements_test.go b/languages/golang/encrypt/json_elements_test.go new file mode 100644 index 000000000..9d4655a5f --- /dev/null +++ b/languages/golang/encrypt/json_elements_test.go @@ -0,0 +1,457 @@ +package encrypt_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "path/filepath" + "reflect" + "slices" + "strings" + "testing" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/internal/testusers" + "github.com/cipherstash/stack/languages/golang/internal/record" + "github.com/cipherstash/vitaminc/bindings/go/vcvalue" + "golang.org/x/tools/go/packages" +) + +// The element match of a document field (element_match): the generated +// Elements..Match on testusers.Entity's custom field values, whose +// contacts order relationship, previous_call_count and age. + +const emergencyContacts = "$.extra.emergency_contacts[*]" + +// elementsOperand is eql_v4.query_json_elements as the engine writes it: +// per predicate, the tokens of a read of the member's full path and its +// bounds, each a leaf operand. +type elementsOperand struct { + M []map[string]json.RawMessage `json:"m"` +} + +func decodeElements(t *testing.T, q []byte) elementsOperand { + t.Helper() + var op elementsOperand + dec := json.NewDecoder(bytes.NewReader(q)) + dec.DisallowUnknownFields() + if err := dec.Decode(&op); err != nil { + t.Fatalf("the operand is not {\"m\": [...]}: %v", err) + } + return op +} + +// keys are a member's keys, sorted. +func keys(m map[string]json.RawMessage) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + slices.Sort(out) + return out +} + +func tokensOf(t *testing.T, raw json.RawMessage) []string { + t.Helper() + var tokens []string + if err := json.Unmarshal(raw, &tokens); err != nil { + t.Fatal(err) + } + return tokens +} + +// The operand Match derives is built of what the other operators derive on +// the same field: each member's tokens are a subtree read of its full path +// at the bound, the probe last, and its bounds are the leaf operands the +// orderings at that path derive, equality both inclusive bounds at one +// value. +func TestElementMatchIsTheReadsAndTheOrderingsOfEachMember(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + cipher := c.DefaultKeyset() + f := testusers.EntityFields.CustomFieldValues + contact := testusers.EntityPaths.CustomFieldValues.Extra.EmergencyContacts + opts := encrypt.JSONReadOptions{WildcardBound: 4} + q, err := f.Elements.ExtraEmergencyContacts.Match(ctx, cipher, opts, + encrypt.AtPath(contact.Relationship).Eq("Sibling"), + encrypt.AtPath(contact.PreviousCallCount).Gt(5), + encrypt.AtPath(contact.Age).Between(18, 40), + ) + if err != nil { + t.Fatal(err) + } + if v, err := q.Value(); err != nil || !strings.HasPrefix(fmt.Sprint(v), `{"m":`) { + t.Fatalf("Match binds as its JSON: %v %v", v, err) + } + op := decodeElements(t, q) + if len(op.M) != 3 { + t.Fatalf("%d members, want one per predicate", len(op.M)) + } + for i, want := range [][]string{{"gte", "lte", "t"}, {"gt", "t"}, {"gte", "lte", "t"}} { + if got := keys(op.M[i]); !slices.Equal(got, want) { + t.Errorf("member %d has %v, want %v", i, got, want) + } + } + // Each member's tokens are the subtree read of its full path at the + // same bound: bound elements, then the probe. + for i, member := range []string{"relationship", "previous_call_count", "age"} { + read, err := f.Subtree(ctx, cipher, emergencyContacts+"."+member, opts) + if err != nil { + t.Fatal(err) + } + tokens := tokensOf(t, op.M[i]["t"]) + if len(tokens) != int(opts.WildcardBound)+1 || !slices.Equal(tokens, []string(read.Tokens())) { + t.Errorf("%s: %d tokens, not the subtree read's %d", member, len(tokens), len(read.Tokens())) + } + } + // The bounds are the orderings' leaf operands at the member's path. + leaf := func(q []byte, err error) json.RawMessage { + t.Helper() + if err != nil { + t.Fatal(err) + } + return q + } + sameLeaf := func(name string, got, want json.RawMessage) { + t.Helper() + if !sameJSON(t, got, want) { + t.Errorf("%s: %s, want the ordering's %s", name, got, want) + } + } + eq := op.M[0] + sameLeaf("eq gte", eq["gte"], leaf(f.At.ExtraEmergencyContactsRelationship.Gte(ctx, cipher, "Sibling"))) + sameLeaf("eq lte", eq["lte"], eq["gte"]) + sameLeaf("gt", op.M[1]["gt"], leaf(f.At.ExtraEmergencyContactsPreviousCallCount.Gt(ctx, cipher, 5))) + rng, err := f.At.ExtraEmergencyContactsAge.Between(ctx, cipher, 18, 40) + if err != nil { + t.Fatal(err) + } + sameLeaf("between low", op.M[2]["gte"], json.RawMessage(rng.Low)) + sameLeaf("between high", op.M[2]["lte"], json.RawMessage(rng.High)) + // A member named at run time derives the operand its path does, and + // so does a value given as its JSON text. + contacts := f.Elements.ExtraEmergencyContacts + named, err := contacts.Match(ctx, cipher, opts, + contacts.Member("relationship").Eq("Sibling"), + contacts.Member("previous_call_count").Gt(5), + contacts.Member("age").Between(18, int64(40)), + ) + if err != nil || !sameJSON(t, named, q) { + t.Fatalf("a member named at run time derives another operand: %v", err) + } + raw, err := contacts.Match(ctx, cipher, opts, + contacts.Member("relationship").Eq(json.RawMessage(`"Sibling"`)), + contacts.Member("previous_call_count").Gt(json.RawMessage(`5`)), + contacts.Member("age").Between(json.RawMessage(`18`), json.RawMessage(`40`)), + ) + if err != nil || !sameJSON(t, raw, q) { + t.Fatalf("JSON text derives another operand: %v", err) + } + // The other comparisons, one bound each; the default bound is 64. + for _, c := range []struct { + p encrypt.ElementPredicate[testusers.EntityCustomFieldValuesFieldElementsExtraEmergencyContacts] + key string + }{ + {encrypt.AtPath(contact.Age).Lt(30), "lt"}, + {encrypt.AtPath(contact.Age).Lte(30), "lte"}, + {encrypt.AtPath(contact.Age).Gte(30), "gte"}, + } { + q, err := contacts.Match(ctx, cipher, encrypt.JSONReadOptions{}, c.p) + if err != nil { + t.Fatal(err) + } + m := decodeElements(t, q).M[0] + if got := keys(m); !slices.Equal(got, []string{c.key, "t"}) { + t.Errorf("%s: %v", c.key, got) + } + if n := len(tokensOf(t, m["t"])); n != 65 { + t.Errorf("%s: %d tokens at the default bound, want 65", c.key, n) + } + } + // A field whose arrays order no member has no Elements. + if _, ok := reflect.TypeOf(testusers.SubscriberFields.Prefs).FieldByName("Elements"); ok { + t.Fatal("Prefs orders no member, and has Elements") + } +} + +// What Match refuses before it asks the engine: each is the sentinel the +// engine's refusal would be, with no Diagnostic, since the guest was never +// called. An unordered member is named with the array and the members the +// field orders there. +func TestElementMatchRefusesBeforeTheEngine(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + cipher := c.DefaultKeyset() + contacts := testusers.EntityFields.CustomFieldValues.Elements.ExtraEmergencyContacts + member := contacts.Member + type predicate = encrypt.ElementPredicate[testusers.EntityCustomFieldValuesFieldElementsExtraEmergencyContacts] + var zeroPath encrypt.Path[testusers.EntityCustomFieldValuesFieldElementsExtraEmergencyContacts, int64] + for _, tc := range []struct { + name string + predicates []predicate + kind error + says []string + }{ + {"no predicates", nil, encrypt.ErrEncoding, []string{"at least one predicate", emergencyContacts}}, + {"a zero predicate", []predicate{{}}, encrypt.ErrEncoding, []string{"zero ElementPredicate", "encrypt.AtPath", "Member"}}, + {"a zero path", []predicate{encrypt.AtPath(zeroPath).Eq(1)}, encrypt.ErrEncoding, []string{"names no member", "zero encrypt.Path", "relationship, previous_call_count, age"}}, + {"a wildcard in a member", []predicate{member("calls[*].at").Eq(1)}, encrypt.ErrEncoding, []string{`"calls[*].at"`, "has a wildcard", emergencyContacts}}, + {"an unordered member", []predicate{member("relationship").Eq("secret value"), member("name").Eq("secret value")}, encrypt.ErrTerm, []string{`"name"`, "is not ordered", emergencyContacts, "relationship, previous_call_count, age"}}, + {"a member spelled another way", []predicate{member(`["age"]`).Gt(5)}, encrypt.ErrTerm, []string{`"[\"age\"]"`, "is not ordered"}}, + {"a []byte value", []predicate{member("relationship").Eq([]byte(`"Sibling"`))}, encrypt.ErrEncoding, []string{"base64"}}, + } { + _, err := contacts.Match(ctx, cipher, encrypt.JSONReadOptions{}, tc.predicates...) + var d *encrypt.Diagnostic + if !errors.Is(err, tc.kind) || errors.As(err, &d) { + t.Errorf("%s: %v, want %v from the SDK", tc.name, err, tc.kind) + continue + } + for _, want := range tc.says { + if !strings.Contains(err.Error(), want) { + t.Errorf("%s: %q does not say %q", tc.name, err, want) + } + } + if strings.Contains(err.Error(), "secret") || strings.Contains(err.Error(), "Sibling") { + t.Errorf("%s: the error holds a value: %v", tc.name, err) + } + } + // No cipher is a returned error, not a panic. + if _, err := contacts.Match(ctx, nil, encrypt.JSONReadOptions{}, member("age").Eq(3)); err == nil { + t.Fatal("Match with no cipher") + } +} + +// contactsPlan declares a document whose contacts order relationship, +// previous_call_count and age, for element_match through QueryTerm, which +// sends any input the engine reads. +func contactsPlan() *record.Plan { + return &record.Plan{Context: []string{"entities"}, Fields: []record.Field{{ + Name: "custom", Kind: record.JSON, Target: "JsonSearch", Null: record.SQLNull, + TargetOptions: &record.TargetOptions{Ordered: []string{ + emergencyContacts + ".relationship", emergencyContacts + ".previous_call_count", emergencyContacts + ".age", + }}, + }}} +} + +// What the engine refuses of an element_match input, through the guest: +// each is ErrEncoding (the input) or ErrTerm (the index), with the engine's +// reason or code, the field named, and no value in the text. +func TestElementMatchEngineRefusalsAreTypedErrors(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + cipher := c.DefaultKeyset() + type obj = vcvalue.Object + pred := func(member, op string, kv ...vcvalue.Field) obj { + return append(obj{{Key: "member", Value: member}, {Key: "op", Value: op}}, kv...) + } + eq := func(member, v string) obj { return pred(member, "eq", vcvalue.Field{Key: "json", Value: []byte(v)}) } + input := func(path string, preds ...any) obj { + return obj{{Key: "path", Value: path}, {Key: "predicates", Value: preds}} + } + deep := strings.Repeat("a.", 130) + "a" + for _, tc := range []struct { + name string + input obj + kind error + reason string // the data plan's reason + code string // or the index's code + }{ + {"a path with no wildcard", input("$.extra", eq("relationship", `"secret"`)), encrypt.ErrEncoding, "json_path_no_wildcard", ""}, + {"a bound of 0", append(input(emergencyContacts, eq("relationship", `"secret"`)), vcvalue.Field{Key: "wildcard_bound", Value: uint32(0)}), encrypt.ErrEncoding, "json_wildcard_bound_zero", ""}, + {"no predicates", input(emergencyContacts), encrypt.ErrEncoding, "json_no_predicates", ""}, + {"an unknown op", input(emergencyContacts, pred("age", "ne", vcvalue.Field{Key: "json", Value: []byte(`1`)})), encrypt.ErrEncoding, "json_unknown_predicate_op", ""}, + {"keys that do not fit the op", input(emergencyContacts, pred("age", "eq", vcvalue.Field{Key: "low", Value: []byte(`1`)}, vcvalue.Field{Key: "high", Value: []byte(`2`)})), encrypt.ErrEncoding, "json_input", ""}, + {"a wildcard in a member", input(emergencyContacts, eq("calls[*].at", `1`)), encrypt.ErrEncoding, "json_path_wildcard", ""}, + {"a position in the path", input("$.extra.emergency_contacts[*].x[2].y[*]", eq("age", `1`)), encrypt.ErrTerm, "", "stack_encrypt::json_ordering_at_position"}, + {"an unordered member", input(emergencyContacts, eq("name", `"secret"`)), encrypt.ErrTerm, "", "stack_encrypt::json_path_not_ordered"}, + {"a value no ordering takes", input(emergencyContacts, eq("relationship", `true`)), encrypt.ErrTerm, "", "stack_encrypt::json_leaf_not_ordered"}, + {"a member that is not a path", input(emergencyContacts, eq("relationship.", `"secret"`)), encrypt.ErrTerm, "", "stack_encrypt::json_path"}, + {"a member past the document's depth", input(emergencyContacts, eq(deep, `1`)), encrypt.ErrTerm, "", "stack_encrypt::json_too_deep"}, + } { + _, err := cipher.QueryTerm(ctx, contactsPlan(), "custom", "element_match", tc.input) + var d *encrypt.Diagnostic + if !errors.Is(err, tc.kind) || !errors.As(err, &d) { + t.Errorf("%s: %v, want %v with a Diagnostic", tc.name, err, tc.kind) + continue + } + if tc.reason != "" && (d.Reason() != tc.reason || d.Field() != "custom") { + t.Errorf("%s: reason %q on %q, want %q: %v", tc.name, d.Reason(), d.Field(), tc.reason, err) + } + if tc.code != "" && d.Code != tc.code { + t.Errorf("%s: code %q, want %q: %v", tc.name, d.Code, tc.code, err) + } + if strings.Contains(err.Error(), "secret") { + t.Errorf("%s: the error holds a value: %v", tc.name, err) + } + } + // A member that is not a path is refused at an offset into the + // member's own text: "relationship." breaks after its 13 bytes. + _, err := cipher.QueryTerm(ctx, contactsPlan(), "custom", "element_match", input(emergencyContacts, eq("relationship.", `"x"`))) + var d *encrypt.Diagnostic + if !errors.As(err, &d) || d.Fields["at"] != uint64(13) { + t.Errorf("the offset into the member: %+v", d) + } +} + +// visits stands for an array's type, as the generator writes one per array. +type visits struct{} + +// A path names its member, and a predicate made at it keeps the member +// relative to the array, with the comparison and its values. +func TestAtPathMakesThePredicatesOfItsMember(t *testing.T) { + place := encrypt.NewPath[visits, string]("$.visits[*]", "address.city") + if got := place.String(); got != "$.visits[*].address.city" { + t.Fatalf("String: %s", got) + } + for _, c := range []struct { + p encrypt.ElementPredicate[visits] + op string + values []any + }{ + {encrypt.AtPath(place).Eq("a"), "eq", []any{"a"}}, + {encrypt.AtPath(place).Lt("b"), "lt", []any{"b"}}, + {encrypt.AtPath(place).Lte("c"), "lte", []any{"c"}}, + {encrypt.AtPath(place).Gt("d"), "gt", []any{"d"}}, + {encrypt.AtPath(place).Gte("e"), "gte", []any{"e"}}, + {encrypt.AtPath(place).Between("f", "g"), "between", []any{"f", "g"}}, + } { + if c.p.Member() != "address.city" || c.p.Op() != c.op || !reflect.DeepEqual(c.p.Values(), c.values) { + t.Errorf("%s: %q %q %v", c.op, c.p.Member(), c.p.Op(), c.p.Values()) + } + } + if (encrypt.ElementPredicate[visits]{}).Op() != "" || (encrypt.Path[visits, int64]{}).String() != "" { + t.Fatal("a zero predicate has a comparison, or a zero path a member") + } +} + +// A predicate holds plaintext values: it prints and logs its member and +// comparison, never them. +func TestElementPredicatesPrintAndLogWithoutTheirValues(t *testing.T) { + contact := testusers.EntityPaths.CustomFieldValues.Extra.EmergencyContacts + predicates := []encrypt.ElementPredicate[testusers.EntityCustomFieldValuesFieldElementsExtraEmergencyContacts]{ + encrypt.AtPath(contact.Relationship).Eq("Sibling"), + encrypt.AtPath(contact.Age).Between(1234567, 7654321), + } + var logged bytes.Buffer + slog.New(slog.NewJSONHandler(&logged, nil)).Info("predicates", "p", predicates[0], "q", predicates[1]) + for _, verb := range []string{"%v", "%+v", "%#v", "%s", "%q", "%x", "%d"} { + out := fmt.Sprintf(verb, predicates) + for _, hidden := range []string{"Sibling", "1234567", "7654321"} { + if strings.Contains(out, hidden) || strings.Contains(logged.String(), hidden) { + t.Errorf("%s shows %q: %s %s", verb, hidden, out, logged.String()) + } + } + if !strings.Contains(out, "relationship") || !strings.Contains(out, "between") { + t.Errorf("%s names no member or comparison: %s", verb, out) + } + } + if p := predicates[1]; p.Member() != "age" || p.Op() != "between" || !reflect.DeepEqual(p.Values(), []any{int64(1234567), int64(7654321)}) { + t.Fatalf("accessors: %q %q %v", p.Member(), p.Op(), p.Values()) + } + // Values is a copy: changing it leaves the predicate as built. + predicates[1].Values()[0] = 0 + if got := predicates[1].Values(); !reflect.DeepEqual(got, []any{int64(1234567), int64(7654321)}) { + t.Fatalf("a predicate changed through Values: %v", got) + } + if s := (encrypt.ElementPredicate[visits]{}).String(); s != "encrypt.ElementPredicate{}" { + t.Fatalf("a zero predicate prints %s", s) + } +} + +// A predicate holds its own copy of a json.RawMessage or []byte value: +// changing the caller's bytes after the predicate is made, or the bytes +// Values returns, leaves the predicate as built. +func TestElementPredicatesCopyTheirByteValues(t *testing.T) { + raw := json.RawMessage(`"Sibling"`) + low, high := []byte("10"), []byte("20") + relationship := encrypt.AtPath(encrypt.NewPath[visits, json.RawMessage]("$.visits[*]", "relationship")) + age := encrypt.AtPath(encrypt.NewPath[visits, any]("$.visits[*]", "age")) + predicates := []encrypt.ElementPredicate[visits]{ + relationship.Eq(raw), + relationship.Lt(raw), + relationship.Lte(raw), + relationship.Gt(raw), + relationship.Gte(raw), + age.Between(low, high), + } + want := make([][]any, len(predicates)) + for i, p := range predicates { + want[i] = p.Values() + } + copy(raw, `"XXXXXXX"`) + copy(low, "99") + copy(high, "99") + for i, p := range predicates { + if got := p.Values(); !reflect.DeepEqual(got, want[i]) { + t.Errorf("%s changed with the caller's bytes: %s", p.Op(), got) + } + for _, v := range p.Values() { + switch b := v.(type) { + case json.RawMessage: + b[0] = 'X' + case []byte: + b[0] = 'X' + default: + t.Fatalf("%s holds a %T, not the type it was given", p.Op(), v) + } + } + if got := p.Values(); !reflect.DeepEqual(got, want[i]) { + t.Errorf("%s changed through the bytes Values returned: %s", p.Op(), got) + } + } + if got := predicates[0].Values()[0]; !reflect.DeepEqual(got, json.RawMessage(`"Sibling"`)) { + t.Errorf("Eq holds %s", got) + } + if got := predicates[5].Values(); !reflect.DeepEqual(got, []any{[]byte("10"), []byte("20")}) { + t.Errorf("Between holds %s", got) + } +} + +// The compiler refuses a predicate whose value is not the member's Go +// type, and one on a member of another array, as the generated code types +// them. Checked by loading testusers with a probe file beside it. +func TestElementMatchTypeChecks(t *testing.T) { + if testing.Short() { + t.Skip("loads and type-checks a package") + } + dir, err := filepath.Abs(filepath.Join("internal", "testusers")) + if err != nil { + t.Fatal(err) + } + check := func(body string) []string { + t.Helper() + src := "package testusers\n\nimport (\n\t\"context\"\n\n\t\"github.com/cipherstash/stack/languages/golang/encrypt\"\n)\n\nfunc probe(ctx context.Context, c *encrypt.Cipher) {\n\tcontacts := EntityFields.CustomFieldValues.Elements.ExtraEmergencyContacts\n\tcontact := EntityPaths.CustomFieldValues.Extra.EmergencyContacts\n\t_, _ = contacts, contact\n" + body + "}\n" + cfg := &packages.Config{Context: context.Background(), Dir: dir, Mode: packages.NeedName | packages.NeedTypes, Overlay: map[string][]byte{filepath.Join(dir, "zz_probe.go"): []byte(src)}} + pkgs, err := packages.Load(cfg, ".") + if err != nil || len(pkgs) != 1 { + t.Fatalf("load: %v", err) + } + var errs []string + for _, e := range pkgs[0].Errors { + errs = append(errs, e.Msg) + } + return errs + } + if errs := check("\t_, _ = contacts.Match(ctx, c, encrypt.JSONReadOptions{}, encrypt.AtPath(contact.Relationship).Eq(\"Sibling\"), encrypt.AtPath(contact.Age).Gt(15), contacts.Member(\"age\").Gt(\"15\"))\n"); len(errs) != 0 { + t.Fatalf("the element match as a program writes it does not compile: %v", errs) + } + for _, c := range []struct{ name, body, want string }{ + {"a value of another type", "\t_, _ = contacts.Match(ctx, c, encrypt.JSONReadOptions{}, encrypt.AtPath(contact.Age).Gt(\"15\"))\n", `cannot use "15" (untyped string constant) as int64 value`}, + {"a member of another array", "\t_, _ = contacts.Match(ctx, c, encrypt.JSONReadOptions{}, encrypt.AtPath(SubscriberPaths.Profile.Visits.At).Gt(15))\n", "encrypt.ElementPredicate[SubscriberProfileFieldElementsVisits]) as encrypt.ElementPredicate[EntityCustomFieldValuesFieldElementsExtraEmergencyContacts] value"}, + } { + // The loader reports the one mistake from go list and from the + // type checker: every error must be it. + errs := check(c.body) + if len(errs) == 0 { + t.Errorf("%s: compiles", c.name) + } + for _, e := range errs { + if !strings.Contains(e, c.want) { + t.Errorf("%s: %s, want only %q", c.name, e, c.want) + } + } + } +} diff --git a/languages/golang/encrypt/json_tree_test.go b/languages/golang/encrypt/json_tree_test.go index b55a6302f..dc6d57840 100644 --- a/languages/golang/encrypt/json_tree_test.go +++ b/languages/golang/encrypt/json_tree_test.go @@ -154,7 +154,7 @@ func TestJSONOperatorsDeriveTaggedTerms(t *testing.T) { "contains": "json", "contained_by": "json", "eq_at": "path_json", "lt_at": "path_json", "lte_at": "path_json", "gt_at": "path_json", "gte_at": "path_json", "between_at": "path_pair", "selector": "path", "subtree": "path_bound", - "children": "path", "child_count": "path", + "children": "path", "child_count": "path", "element_match": "path_predicates", } if !reflect.DeepEqual(inputs, want) { t.Fatalf("operators = %v", inputs) diff --git a/languages/golang/encrypt/targets_test.go b/languages/golang/encrypt/targets_test.go index 4cdb97784..3b48f8274 100644 --- a/languages/golang/encrypt/targets_test.go +++ b/languages/golang/encrypt/targets_test.go @@ -117,7 +117,7 @@ func TestParseQueryTermTakesOnlyWhatTheEngineWrites(t *testing.T) { } for _, name := range []string{"eq", "lt", "lte", "gt", "gte", "between", "matches", "contains", "contained_by", "eq_at", "lt_at", "lte_at", "gt_at", "gte_at", "between_at", - "selector", "subtree", "children", "child_count"} { + "selector", "subtree", "children", "child_count", "element_match"} { got, err := parseQueryTerm(term(name, "exact", "ore")) if err != nil || got.Operator != name || got.Fidelity != record.Exact || got.Index != record.Ore { t.Errorf("%s: %+v, %v", name, got, err) diff --git a/languages/golang/internal/record/record.go b/languages/golang/internal/record/record.go index 54e96794b..05992135d 100644 --- a/languages/golang/internal/record/record.go +++ b/languages/golang/internal/record/record.go @@ -140,6 +140,54 @@ type TargetOptions struct { Ordered []string } +// ElementArray is an array of a document whose elements have ordered +// members: the path through the array's elements ($.contacts[*]), and each +// ordered path beneath it as a member of one element (relationship, +// address.city). The element predicate (element_match) tests these members +// of one element together. +type ElementArray struct { + Path string + Members []string +} + +// ElementArrays groups the ordered paths by the array each one's last +// wildcard names: $.contacts[*].relationship and +// $.contacts[*].address.city give the array $.contacts[*] with the members +// relationship and address.city. A path with no wildcard, one that ends +// at its wildcard, or one whose member crosses another array has no +// element member. Only paths a stash tag spells are read ($, then .key, +// [n] and [*] steps): a path with a quoted key has none. Arrays and +// members are in the order the paths are given. +func (o *TargetOptions) ElementArrays() []ElementArray { + if o == nil { + return nil + } + var arrays []ElementArray + index := map[string]int{} + for _, path := range o.Ordered { + if strings.ContainsAny(path, `"'`) { + continue + } + at := strings.LastIndex(path, "[*]") + if at < 0 { + continue + } + base, rest := path[:at+len("[*]")], path[at+len("[*]"):] + member, ok := strings.CutPrefix(rest, ".") + if !ok || member == "" || strings.Contains(member, "[") { + continue + } + i, seen := index[base] + if !seen { + i = len(arrays) + index[base] = i + arrays = append(arrays, ElementArray{Path: base}) + } + arrays[i].Members = append(arrays[i].Members, member) + } + return arrays +} + // wire is the options object as the plan spells it: only the options set. func (o *TargetOptions) wire() vcvalue.Object { out := vcvalue.Object{} @@ -572,7 +620,8 @@ type Operator struct { // Name is the operator's name, as a query names it: eq, lt, lte, gt, // gte, between or matches; for a field that stores a JSON document, // contains, contained_by, eq_at, lt_at, lte_at, gt_at, gte_at, - // between_at, selector, subtree, children or child_count. + // between_at, selector, subtree, children, child_count or + // element_match. Name string // Fidelity is how far the server's answer can be trusted. Fidelity Fidelity @@ -583,7 +632,7 @@ type Operator struct { Index Output // Input is the shape of the operator's input, as the engine names it: // value (one value of the field's kind), pair (two, low then high), - // json, path_json, path_pair, path or path_bound (objects of named - // inputs, for a document's operators). + // json, path_json, path_pair, path, path_bound or path_predicates + // (objects of named inputs, for a document's operators). Input string } diff --git a/languages/golang/internal/record/record_test.go b/languages/golang/internal/record/record_test.go index ed28f8c51..21740a25b 100644 --- a/languages/golang/internal/record/record_test.go +++ b/languages/golang/internal/record/record_test.go @@ -220,3 +220,31 @@ func TestWireGivesATargetItsOptions(t *testing.T) { t.Fatal("json is a kind") } } + +// Ordered paths group by the array their last wildcard names, each one a +// member of an element of it; a path with no member below a wildcard, or +// one that crosses another array, is none. +func TestElementArraysGroupOrderedPathsByTheirLastWildcard(t *testing.T) { + o := &TargetOptions{Ordered: []string{ + "$.age", + "$.extra.contacts[*].relationship", + "$.tags[*]", + "$.extra.contacts[*].calls", + "$.orders[*].lines[*].sku", + "$.orders[*].lines[*].qty.units", + "$.visits[*].stops[2]", + "$.m[*][*].x", + `$["a b"][*].c`, + }} + want := []ElementArray{ + {Path: "$.extra.contacts[*]", Members: []string{"relationship", "calls"}}, + {Path: "$.orders[*].lines[*]", Members: []string{"sku", "qty.units"}}, + {Path: "$.m[*][*]", Members: []string{"x"}}, + } + if got := o.ElementArrays(); !reflect.DeepEqual(got, want) { + t.Fatalf("ElementArrays = %+v, want %+v", got, want) + } + if got := (*TargetOptions)(nil).ElementArrays(); got != nil { + t.Fatalf("nil options: %+v", got) + } +} diff --git a/languages/golang/stashgen/doc.go b/languages/golang/stashgen/doc.go index e62346085..d079d7a26 100644 --- a/languages/golang/stashgen/doc.go +++ b/languages/golang/stashgen/doc.go @@ -3,7 +3,8 @@ // A struct's stash tags declare how each field is encrypted. [FromTags] reads // those tags through go/packages and go/types, checks the declaration with an // [Engine], and returns the file that the command stashgen writes beside the -// struct: the encrypted type, Encrypt, Decrypt, Fields and the print methods. +// struct: the encrypted type, Encrypt, Decrypt, Fields, Paths (for a JSON +// document whose arrays have ordered members) and the print methods. // The command at languages/golang/cmd/stashgen is the go:generate front end // for this package. // diff --git a/languages/golang/stashgen/emit.go b/languages/golang/stashgen/emit.go index a537ab0c6..ed586b904 100644 --- a/languages/golang/stashgen/emit.go +++ b/languages/golang/stashgen/emit.go @@ -40,6 +40,7 @@ func emit(f *genFile) ([]byte, error) { w.codec(f) w.functions(f) w.fieldEntries(f) + w.paths(f) for i := range f.models { w.model(f, &f.models[i]) } @@ -703,7 +704,7 @@ func (w *writer) fieldEntries(f *genFile) { for _, g := range sealed { w.nl() if g.isJSON() { - w.jsonEntry(g) + w.jsonEntry(g, f.pathsVar) continue } w.p("type %s struct {", g.fieldType) @@ -1011,6 +1012,7 @@ var jsonOperators = map[string]bool{ "contains": true, "contained_by": true, "eq_at": true, "selector": true, "subtree": true, "children": true, "child_count": true, "lt_at": true, "lte_at": true, "gt_at": true, "gte_at": true, "between_at": true, + "element_match": true, } // planJSON checks a document field's operators against the methods the @@ -1054,6 +1056,66 @@ func planJSON(f *genFile, g *genField) error { if len(g.jsonPaths) > 0 && !answers(g, "lt_at") { return fieldErr(f.typeName, g.GoName, "the tag orders %d paths, and the engine answers no ordering (lt_at) on %s: the generator and its engine disagree, which is a bug in this stashgen", len(g.jsonPaths), g.EQLType) } + // An array whose elements have an ordered member gets an accessor + // under Elements, named as At names a path, whose Match tests members + // of one element together. An array with no ordered member gets none: + // Match there could test nothing. A root array ($[*]) is Root. An + // array whose keys give no name, or whose name another array's keys + // give too, gets no accessor and a notice naming its ordered paths: + // the tag stays valid, and At is written as it is for any other field. + // + // Each accessor's members are named under Paths too, nested as the + // JSON nests them: Paths..Visits.At for $.visits[*].at. A key + // that gives no Go name there, or a name that would stand for two + // paths (two keys that give one name, or a member that is also the + // object or array above another member), is the same case: the arrays + // it touches get no accessor and a notice. + g.jsonArrays = nil + arrays := g.TargetOptions.ElementArrays() + names := make([]string, len(arrays)) + count := map[string]int{} + for i, a := range arrays { + names[i] = jsonArrayName(a.Path) + if names[i] != "" { + count[names[i]]++ + } + } + why := make([]string, len(arrays)) + leaves := make([][]jsonLeaf, len(arrays)) + for i, a := range arrays { + switch { + case names[i] == "": + why[i] = "its keys give no Go accessor name" + case count[names[i]] > 1: + why[i] = "another array's keys give the same accessor name, Elements." + names[i] + default: + leaves[i], why[i] = arrayLeaves(a) + } + } + pathsWhy(f.pathsVar+"."+g.GoName, leaves, why) + for i, a := range arrays { + if why[i] == "" { + ja := jsonArray{name: names[i], path: a.Path, members: a.Members, marker: g.fieldType + "Elements" + names[i], leaves: leaves[i]} + for j := range ja.leaves { + ja.leaves[j].typeExpr = "any" + if g.orderedType != nil { + ja.leaves[j].typeExpr = g.orderedType(ja.leaves[j].path) + } + } + g.jsonArrays = append(g.jsonArrays, ja) + continue + } + paths := make([]string, len(a.Members)) + for j, m := range a.Members { + paths[j] = a.Path + "." + m + } + n := fmt.Sprintf("no Elements accessor for the elements of %s (ordered %s): %s. Their orderings under At are written, and Match on them is not.", a.Path, strings.Join(paths, ", "), why[i]) + f.notices = append(f.notices, g.GoName+": "+n) + f.unqueried = append(f.unqueried, fmt.Sprintf("stashgen: %s.%s: %s", f.typeName, g.GoName, n)) + } + if len(g.jsonArrays) > 0 && !answers(g, "element_match") { + return fieldErr(f.typeName, g.GoName, "the tag orders members of %d arrays' elements, and the engine answers no element_match on %s: the generator and its engine disagree, which is a bug in this stashgen", len(g.jsonArrays), g.EQLType) + } return nil } @@ -1093,16 +1155,158 @@ func jsonPathName(path string) (string, error) { return name, nil } +// jsonArrayName is an array's accessor name under Elements, named as +// jsonPathName names a path: $.visits[*] is Visits, and a root array, $[*] +// or $[*][*], is Root. "" for an array whose keys give no name. +func jsonArrayName(path string) string { + if strings.Trim(strings.TrimPrefix(path, "$"), "[*]") == "" { + return "Root" + } + name, err := jsonPathName(path) + if err != nil { + return "" + } + return name +} + +// jsonKeyName is one key's name in Paths, named as jsonPathName names a +// path: emergency_contacts is EmergencyContacts. "" for a key that gives +// no name starting with a letter. +func jsonKeyName(key string) string { + var b strings.Builder + for _, word := range strings.Split(key, "_") { + if word != "" { + b.WriteString(strings.ToUpper(word[:1]) + word[1:]) + } + } + name := b.String() + if !isIdent(name) || name[0] < 'A' || name[0] > 'Z' { + return "" + } + return name +} + +// jsonSteps are a tag-spelled path's keys, each with the wildcards that +// follow it: $.extra.contacts[*] is extra and contacts[*]. Wildcards +// before the first key, as in $[*].name, belong to no key and are dropped. +func jsonSteps(path string) []string { + rest := strings.TrimPrefix(path, "$") + for strings.HasPrefix(rest, "[") { + end := strings.IndexByte(rest, ']') + if end < 0 { + break + } + rest = rest[end+1:] + } + var steps []string + for _, step := range strings.Split(strings.TrimPrefix(rest, "."), ".") { + if step != "" { + steps = append(steps, step) + } + } + return steps +} + +// arrayLeaves are an array's members as Paths names them: each the array's +// keys, then the member's. Or why the array's members have no names there: +// a key that gives no Go name. +func arrayLeaves(a record.ElementArray) ([]jsonLeaf, string) { + leaves := make([]jsonLeaf, 0, len(a.Members)) + for _, m := range a.Members { + leaf := jsonLeaf{path: a.Path + "." + m, array: a.Path, member: m} + leaf.keys = append(jsonSteps(a.Path), strings.Split(m, ".")...) + for _, step := range leaf.keys { + key, _, _ := strings.Cut(step, "[") + name := jsonKeyName(key) + if name == "" { + return nil, fmt.Sprintf("the key %q of %s gives no Go name under Paths that starts with a letter", key, leaf.path) + } + leaf.names = append(leaf.names, name) + } + leaves = append(leaves, leaf) + } + return leaves, "" +} + +// pathsWhy finds the names under prefix (Paths.Profile) that would stand +// for two paths, and gives each array with such a path its reason in why. +// Two paths meet where their names agree and their keys do not (a_b and +// aB are both AB), or where one path's names run out along the other's (a +// member $.a[*].b, and $.a[*].b.c below it): Paths could name one of them +// only. An array that already has a reason is left out. +func pathsWhy(prefix string, leaves [][]jsonLeaf, why []string) { + type at struct{ array, leaf int } + var all []at + for i := range leaves { + if why[i] != "" { + continue + } + for j := range leaves[i] { + all = append(all, at{i, j}) + } + } + reasons := make([]string, len(leaves)) + for x := range all { + for y := x + 1; y < len(all); y++ { + p, q := leaves[all[x].array][all[x].leaf], leaves[all[y].array][all[y].leaf] + n := min(len(p.names), len(q.names)) + meet := -1 + for k := range n { + if p.names[k] != q.names[k] { + break + } + if p.keys[k] != q.keys[k] || k == n-1 { + meet = k + break + } + } + if meet < 0 { + continue + } + reason := fmt.Sprintf("%s.%s would name both %s and %s, and can name one of them only", prefix, strings.Join(p.names[:meet+1], "."), p.path, q.path) + for _, i := range []int{all[x].array, all[y].array} { + if reasons[i] == "" { + reasons[i] = reason + } + } + } + } + for i, r := range reasons { + if r != "" { + why[i] = r + } + } +} + // jsonPath is one ordered path's accessor on a document field's entry. type jsonPath struct { name string // "Age" path string // "$.age" } +// jsonArray is one array's accessor under a document field's Elements. +type jsonArray struct { + name string // "ExtraContacts" + path string // "$.extra.contacts[*]" + members []string // "relationship", "calls" + marker string // the array's type, which its predicates carry: "ProfileFieldElementsExtraContacts" + leaves []jsonLeaf // its members as Paths names them +} + +// jsonLeaf is one member of an array's elements, as Paths names it. +type jsonLeaf struct { + path string // "$.extra.contacts[*].relationship" + array string // "$.extra.contacts[*]" + member string // "relationship" + keys []string // "extra", "contacts[*]", "relationship" + names []string // "Extra", "Contacts", "Relationship" + typeExpr string // the member's Go type: "string", or "any" +} + // jsonEntry writes a document field's Fields entry: its query methods, a // method for each read, and an accessor under At for each ordered path, // whose methods are the orderings there. -func (w *writer) jsonEntry(g genField) { +func (w *writer) jsonEntry(g genField, pathsVar string) { answered := map[string]bool{} for _, op := range g.operators { answered[op.Name] = true @@ -1115,8 +1319,11 @@ func (w *writer) jsonEntry(g genField) { // public.eql_v4_json_entry), not for the stored type. leaf := strings.TrimSuffix(needle, "Query") + "LeafQuery" entry := "eql." + EQLGoName("JsonEntry") - // planJSON refuses ordered paths the engine answers no ordering on. + elementsQuery := strings.TrimSuffix(needle, "Query") + "ElementsQuery" + // planJSON refuses ordered paths the engine answers no ordering on, + // and arrays it answers no element_match on. ordered := len(g.jsonPaths) > 0 + arrays := len(g.jsonArrays) > 0 field := "gensupport.Field[" + g.valueExpr + "]" w.b.WriteString(wrapComment(fmt.Sprintf("%s provides document queries and reads for the %s field.", g.fieldType, g.GoName), 80)) @@ -1127,17 +1334,33 @@ func (w *writer) jsonEntry(g genField) { w.p("\t// are the orderings there.") w.p("\tAt %sAt", g.fieldType) } + if arrays { + w.p("\t// Elements has one accessor per array whose elements have ordered") + w.p("\t// members, whose Match tests the members of one element together.") + w.p("\tElements %sElements", g.fieldType) + } w.p("}") w.nl() w.p("func new%s(f %s) %s {", g.fieldType, field, g.fieldType) - if ordered { - w.p("\treturn %s{field: f, At: %sAt{", g.fieldType, g.fieldType) + if !ordered { + // planJSON writes an array's accessor only beneath an ordered path. + w.p("\treturn %s{field: f}", g.fieldType) + } else { + w.p("\treturn %s{", g.fieldType) + w.p("\t\tfield: f,") + w.p("\t\tAt: %sAt{", g.fieldType) for _, p := range g.jsonPaths { - w.p("\t\t%s: %sPath{field: f, path: %q},", p.name, g.fieldType, p.path) + w.p("\t\t\t%s: %sPath{field: f, path: %q},", p.name, g.fieldType, p.path) } - w.p("\t}}") - } else { - w.p("\treturn %s{field: f}", g.fieldType) + w.p("\t\t},") + if arrays { + w.p("\t\tElements: %sElements{", g.fieldType) + for _, a := range g.jsonArrays { + w.p("\t\t\t%s: %sArray[%s]{field: f, path: %q},", a.name, g.fieldType, a.marker, a.path) + } + w.p("\t\t},") + } + w.p("\t}") } w.p("}") w.nl() @@ -1202,6 +1425,9 @@ func (w *writer) jsonEntry(g genField) { w.p("\treturn gensupport.JSONNodeToken(ctx, f.field, c, \"child_count\", path)") w.p("}") } + if arrays { + w.jsonElements(g, field, elementsQuery, pathsVar) + } if !ordered { return } @@ -1243,3 +1469,121 @@ func (w *writer) jsonEntry(g genField) { w.p("}") } } + +// jsonElements writes a document field's Elements: one accessor per array +// whose elements have ordered members, each with Match, which derives the +// operand eql_v4.json_tree_element_match takes. +func (w *writer) jsonElements(g genField, field, query, pathsVar string) { + w.nl() + w.b.WriteString(wrapComment(fmt.Sprintf("%sElements names the arrays of %s whose elements have ordered members. Each accessor's Match tests the members of one element together.", g.fieldType, g.GoName), 80)) + w.p("type %sElements struct {", g.fieldType) + for _, a := range g.jsonArrays { + w.p("\t%s %sArray[%s] // %s: %s", a.name, g.fieldType, a.marker, a.path, strings.Join(a.members, ", ")) + } + w.p("}") + for _, a := range g.jsonArrays { + under := append([]string{pathsVar, g.GoName}, a.leaves[0].names[:len(jsonSteps(a.path))]...) + w.nl() + w.b.WriteString(wrapComment(fmt.Sprintf("%s is the array %s of %s, as a type: Elements.%s.Match takes an encrypt.ElementPredicate[%s], which encrypt.AtPath makes from a member under %s.", a.marker, a.path, g.GoName, a.name, a.marker, strings.Join(under, ".")), 80)) + w.p("type %s struct{}", a.marker) + } + w.nl() + w.b.WriteString(wrapComment(fmt.Sprintf("%sArray is one array of %s whose elements have ordered members. E is the array's own type, which the predicates its Match takes carry.", g.fieldType, g.GoName), 80)) + w.p("type %sArray[E any] struct {", g.fieldType) + w.p("\tfield %s", field) + w.p("\tpath string") + w.p("}") + w.nl() + w.b.WriteString(wrapComment(fmt.Sprintf("Match derives the operand that finds a row where one element of this array passes every predicate: eql_v4.json_tree_element_match(col, $1::eql_v4.query_json_elements). An EqAt and an ordering at the array's paths can each be answered by a different element; Match is answered by one. Make each predicate with encrypt.AtPath and a member of this array under %s, so a member of another array, or a value of another type, does not compile; or with Member, for a member named at run time. No predicates, and a zero ElementPredicate, are refused before the engine is asked.", pathsVar), 80)) + w.p("//") + w.b.WriteString(wrapComment("Each comparison is of the member's ordering term, so Eq is an equality of folded values: text compares case- and accent-folded (\"sibling\" equals \"Sibling\"), and integers past 2^53 can tie. EqAt at a member's full path is exact, of any element.", 80)) + w.p("//") + w.b.WriteString(wrapComment("The operand reads each array to opts.WildcardBound elements, 64 when it is 0. A row whose array is longer fails the query with SQLSTATE 22023 (invalid_parameter_value) rather than answer over part of it: raise the bound past the longest array. The function filters the rows a scan gives it, so narrow them with an indexed predicate first, such as @> on the same values.", 80)) + w.p("func (a %sArray[E]) Match(ctx context.Context, c *encrypt.Cipher, opts encrypt.JSONReadOptions, predicates ...encrypt.ElementPredicate[E]) (%s, error) {", g.fieldType, query) + w.p("\treturn gensupport.JSONElementMatch[%s](ctx, a.field, c, a.path, opts, predicates)", query) + w.p("}") + w.nl() + w.b.WriteString(wrapComment(fmt.Sprintf("Member is the comparisons of the member of this array's elements that member names, for a filter whose members are known only at run time: relationship, or address.city, written as the member is written below the array in the field's ordered paths. Its values are not checked by the compiler, and Match refuses a member the field does not order beneath this array, naming it, the array and the members the field orders there, before the engine is asked. Where the member is known when the program is written, use encrypt.AtPath and %s.", pathsVar), 80)) + w.p("func (a %sArray[E]) Member(member string) encrypt.MemberTest[E, any] {", g.fieldType) + w.p("\treturn encrypt.AtPath(encrypt.NewPath[E, any](a.path, member))") + w.p("}") +} + +// paths writes the file's Paths: for each document field with an Elements +// accessor, the ordered members of its arrays, nested as the JSON nests +// them. Paths has fields and no methods, so no JSON key is reserved: the +// operations are on the types it holds and on Fields. +func (w *writer) paths(f *genFile) { + var fields []genField + for _, g := range f.fields { + if g.isJSON() && len(g.jsonArrays) > 0 { + fields = append(fields, g) + } + } + if len(fields) == 0 { + return + } + // One tree per field; a node's children keep the order the tag gives. + type node struct { + name string + children []*node + leaf *jsonLeaf + marker string + } + child := func(n *node, name string) *node { + for _, c := range n.children { + if c.name == name { + return c + } + } + c := &node{name: name} + n.children = append(n.children, c) + return c + } + // The function's result is named apart from every import, which a + // member's type may name in the body. + local := "paths" + for f.imports.paths[local] != "" { + local += "_" + } + root := &node{} + var assigns []string + for _, g := range fields { + top := child(root, g.GoName) + for _, a := range g.jsonArrays { + for i := range a.leaves { + leaf := &a.leaves[i] + n := top + for _, name := range leaf.names { + n = child(n, name) + } + n.leaf, n.marker = leaf, a.marker + assigns = append(assigns, fmt.Sprintf("\t%s.%s.%s = encrypt.NewPath[%s, %s](%q, %q)", local, g.GoName, strings.Join(leaf.names, "."), a.marker, leaf.typeExpr, leaf.array, leaf.member)) + } + } + } + var tree func(n *node, indent string) + tree = func(n *node, indent string) { + for _, c := range n.children { + if c.leaf != nil { + w.p("%s%s encrypt.Path[%s, %s] // %s", indent, c.name, c.marker, c.leaf.typeExpr, c.leaf.path) + continue + } + w.p("%s%s struct {", indent, c.name) + tree(c, indent+"\t") + w.p("%s}", indent) + } + } + first := fields[0] + example := first.jsonArrays[0].leaves[0] + w.nl() + w.b.WriteString(wrapComment(fmt.Sprintf("%s names the ordered members of the arrays of %s's documents, nested as the JSON nests them, for the element match: encrypt.AtPath(%s.%s.%s) makes the predicates %s.%s.Elements.%s.Match takes. Each path's type names its array and the member's Go type, so a predicate on another array's member, or with a value of another type, does not compile.", f.pathsVar, f.typeName, f.pathsVar, first.GoName, strings.Join(example.names, "."), f.fieldsVar, first.GoName, first.jsonArrays[0].name), 80)) + w.p("var %s = func() (%s struct {", f.pathsVar, local) + tree(root, "\t") + w.p("}) {") + for _, a := range assigns { + w.p("%s", a) + } + w.p("\treturn %s", local) + w.p("}()") +} diff --git a/languages/golang/stashgen/read.go b/languages/golang/stashgen/read.go index a67898e34..d3a8f4bd5 100644 --- a/languages/golang/stashgen/read.go +++ b/languages/golang/stashgen/read.go @@ -46,6 +46,7 @@ type genFile struct { declVar string // "declaration" codecVar string // "codec" fieldsVar string // "Fields" + pathsVar string // "Paths" fieldTypePrefix string // "" or the -name // The kind each EncryptInto field of another integer width seals as: // the EQL type's own kind, which gensupport converts the value to. @@ -57,7 +58,7 @@ type genFile struct { notices []string printsPlaintext bool unexported []string - unqueried []string // stderr notices: indexes no generated method queries + unqueried []string // stderr notices about one field: an index no generated method queries, an array with no Elements accessor redact bool redactRecv string @@ -108,6 +109,13 @@ type genField struct { fieldType string // "EmailField" pathType string // the type with its package path, for model checks jsonPaths []jsonPath // a document field's ordered paths, each with its accessor + jsonArrays []jsonArray // a document field's arrays with ordered members, each with its accessor + // orderedType is the Go type, as the generated file writes it, of the + // value at an ordered path of a document field whose Go type the + // generator reads: "int64" at $.visits[*].at of a struct with an int64 + // At there. "any" where the type names none, as a json.RawMessage + // does, and for a field with no reader (nil). + orderedType func(path string) string // holdsStruct is set for a document field (KindJSONValue) whose type // holds a struct: its Contains refuses a needle of that type. holdsStruct bool @@ -273,12 +281,12 @@ func (r *reader) build(collected *collected, typeName string, valueNamed *types. f.shapeName = lowerFirst(baseName) + "Shape" f.shapeSource = f.typeExpr + "{}" if req.Name != "" { - f.encryptFn, f.decryptFn, f.fieldsVar = "Encrypt"+req.Name, "Decrypt"+req.Name, req.Name+"Fields" + f.encryptFn, f.decryptFn, f.fieldsVar, f.pathsVar = "Encrypt"+req.Name, "Decrypt"+req.Name, req.Name+"Fields", req.Name+"Paths" f.declVar, f.codecVar = lowerFirst(req.Name)+"Declaration", lowerFirst(req.Name)+"Codec" f.fieldTypePrefix = req.Name f.paramName = "values" } else { - f.encryptFn, f.decryptFn, f.fieldsVar = "Encrypt", "Decrypt", "Fields" + f.encryptFn, f.decryptFn, f.fieldsVar, f.pathsVar = "Encrypt", "Decrypt", "Fields", "Paths" f.declVar, f.codecVar = "declaration", "codec" f.paramName = f.pkgName } @@ -728,6 +736,7 @@ func (r *reader) buildFields(c *collected) error { // What encoding/json cannot carry both ways is refused here, when // the code is generated. holdsStruct, structElems := false, false + var docGoType types.Type if eqlType := r.eqlType(field.EQLType); field.Verb == VerbEncryptInto && eqlType != nil && eqlType.wire == record.JSON { doc, docType := &field.GoType, cf.typ if doc.Kind == KindPointer && doc.Elem != nil { @@ -741,6 +750,7 @@ func (r *reader) buildFields(c *collected) error { _, isStruct := docType.Underlying().(*types.Struct) structElems = holdsStruct && !isStruct *doc = GoType{Name: doc.Name, Kind: KindJSONValue} + docGoType = docType } } // A json.RawMessage is a document only where it names an EQL type @@ -778,6 +788,9 @@ func (r *reader) buildFields(c *collected) error { continue } g := genField{Field: field, typeExpr: r.typeExpr(cf.typ), via: cf.via, tags: cf.tags, pathType: pathType(cf.typ), holdsStruct: holdsStruct, structElems: structElems} + if docGoType != nil { + g.orderedType = func(path string) string { return r.memberTypeExpr(docGoType, path) } + } g.valueExpr = g.typeExpr if pointer { g.valueExpr = field.GoType.Name @@ -1507,3 +1520,154 @@ func valueBaseName(f *genFile) string { } return name } + +// memberTypeExpr is the Go type, as the generated file writes it, of the +// value at path in a document of Go type doc: the type encoding/json reads +// the member into, past any pointers. "any" where the generator cannot +// tell or cannot write one: a path the type does not hold, a step into a +// type that writes itself (MarshalJSON), an interface or an anonymous +// struct, an unexported type of another package, or a type whose package +// name another import of the file already has. +func (r *reader) memberTypeExpr(doc types.Type, path string) string { + t, ok := memberGoType(doc, path) + if !ok || !r.writable(t) { + return "any" + } + return r.typeExpr(types.Unalias(t)) +} + +// memberGoType walks a tag-spelled path ($, then .key, [*] and [n] steps) +// through t as encoding/json reads a document into it: a key selects the +// struct field that writes that member, or a map's element, and an index +// or wildcard a slice's or array's element. +func memberGoType(t types.Type, path string) (types.Type, bool) { + rest, ok := strings.CutPrefix(path, "$") + if !ok || strings.ContainsAny(rest, `"'`) { + return nil, false + } + for rest != "" { + t = derefAll(t) + if own, _ := ownCodec(t); own { + return nil, false + } + switch rest[0] { + case '[': + end := strings.IndexByte(rest, ']') + if end < 0 { + return nil, false + } + rest = rest[end+1:] + switch u := t.Underlying().(type) { + case *types.Slice: + t = u.Elem() + case *types.Array: + t = u.Elem() + default: + return nil, false + } + case '.': + rest = rest[1:] + end := strings.IndexAny(rest, ".[") + if end < 0 { + end = len(rest) + } + key := rest[:end] + rest = rest[end:] + switch u := t.Underlying().(type) { + case *types.Struct: + if t, ok = jsonMember(u, key, map[types.Type]bool{}); !ok { + return nil, false + } + case *types.Map: + t = u.Elem() + default: + return nil, false + } + default: + return nil, false + } + } + return derefAll(t), true +} + +// jsonMember is the type of the field of u that encoding/json writes as +// the member key: one tagged with that name, or untagged and named so, or +// one an untagged embedded struct promotes. +func jsonMember(u *types.Struct, key string, embedding map[types.Type]bool) (types.Type, bool) { + for i := range u.NumFields() { + fld := u.Field(i) + tag := reflect.StructTag(u.Tag(i)).Get("json") + if tag == "-" { + continue + } + name, _, _ := strings.Cut(tag, ",") + if fld.Embedded() && name == "" { + et := derefAll(fld.Type()) + if st, ok := et.Underlying().(*types.Struct); ok && !embedding[et] { + if own, _ := ownCodec(et); !own { + embedding[et] = true + if t, ok := jsonMember(st, key, embedding); ok { + return t, true + } + continue + } + } + } + if !fld.Exported() { + continue + } + if name == "" { + name = fld.Name() + } + if name == key { + return fld.Type(), true + } + } + return nil, false +} + +// derefAll is t past every pointer. +func derefAll(t types.Type) types.Type { + for { + p, ok := t.Underlying().(*types.Pointer) + if !ok { + return t + } + t = p.Elem() + } +} + +// writable reports whether the generated file can write t: a type built +// of basic types and of named types that are exported or in the file's +// own package, whose package names clash with no other import of the file. +func (r *reader) writable(t types.Type) bool { + switch u := types.Unalias(t).(type) { + case *types.Basic: + return u.Kind() != types.UnsafePointer && u.Info()&types.IsUntyped == 0 + case *types.Named: + obj := u.Obj() + if pkg := obj.Pkg(); pkg != nil && pkg.Path() != r.outPath() { + if !obj.Exported() { + return false + } + if other, taken := r.imports.paths[pkg.Name()]; taken && other != pkg.Path() { + return false + } + } + for i := range u.TypeArgs().Len() { + if !r.writable(u.TypeArgs().At(i)) { + return false + } + } + return true + case *types.Pointer: + return r.writable(u.Elem()) + case *types.Slice: + return r.writable(u.Elem()) + case *types.Array: + return r.writable(u.Elem()) + case *types.Map: + return r.writable(u.Key()) && r.writable(u.Elem()) + } + return false +} diff --git a/languages/golang/stashgen/testdata/stubsdk/encrypt/encrypt.go b/languages/golang/stashgen/testdata/stubsdk/encrypt/encrypt.go index 39e59465b..26821e83c 100644 --- a/languages/golang/stashgen/testdata/stubsdk/encrypt/encrypt.go +++ b/languages/golang/stashgen/testdata/stubsdk/encrypt/encrypt.go @@ -145,6 +145,29 @@ type JSONReadOptions struct{ WildcardBound uint32 } // JSONRange is both bounds of a between at a path. type JSONRange[Q ~[]byte] struct{ Low, High Q } +// Path is one ordered member of an array's elements: E is the array, T +// the member's Go type. +type Path[E, T any] struct{ _ *E } + +// NewPath is the Path of member below array. +func NewPath[E, T any](array, member string) Path[E, T] { return Path[E, T]{} } + +// AtPath is the comparisons of the member at p. +func AtPath[E, T any](p Path[E, T]) MemberTest[E, T] { return MemberTest[E, T]{} } + +// MemberTest makes the predicates on one member of an array's elements. +type MemberTest[E, T any] struct{ _ *T } + +func (m MemberTest[E, T]) Eq(v T) ElementPredicate[E] { return ElementPredicate[E]{} } +func (m MemberTest[E, T]) Lt(v T) ElementPredicate[E] { return ElementPredicate[E]{} } +func (m MemberTest[E, T]) Lte(v T) ElementPredicate[E] { return ElementPredicate[E]{} } +func (m MemberTest[E, T]) Gt(v T) ElementPredicate[E] { return ElementPredicate[E]{} } +func (m MemberTest[E, T]) Gte(v T) ElementPredicate[E] { return ElementPredicate[E]{} } +func (m MemberTest[E, T]) Between(lo, hi T) ElementPredicate[E] { return ElementPredicate[E]{} } + +// ElementPredicate is one test on a member of the elements of the array E. +type ElementPredicate[E any] struct{ _ *E } + // JSONSelection is a prepared `->` on a JSONSearch field. type JSONSelection[E ~[]byte] struct{ _ E } diff --git a/languages/golang/stashgen/testdata/stubsdk/encrypt/eql/eql.go b/languages/golang/stashgen/testdata/stubsdk/encrypt/eql/eql.go index 3bc6a8f87..eab634afd 100644 --- a/languages/golang/stashgen/testdata/stubsdk/encrypt/eql/eql.go +++ b/languages/golang/stashgen/testdata/stubsdk/encrypt/eql/eql.go @@ -19,3 +19,18 @@ type TextSearch []byte // TextSearchQuery is the query value for TextSearch. type TextSearchQuery []byte + +// JSONSearch is an encrypted JSON document. +type JSONSearch []byte + +// JSONSearchQuery is the query value for JSONSearch. +type JSONSearchQuery []byte + +// JSONSearchLeafQuery is an ordering's operand at a path of a JSONSearch. +type JSONSearchLeafQuery []byte + +// JSONSearchElementsQuery is the element match's operand on a JSONSearch. +type JSONSearchElementsQuery []byte + +// JSONEntry is one encrypted entry of a JSONSearch document. +type JSONEntry []byte diff --git a/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go b/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go index d382e3334..2c939e257 100644 --- a/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go +++ b/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go @@ -236,6 +236,11 @@ func JSONBetweenAt[Q ~[]byte, T any](context.Context, Field[T], *encrypt.Cipher, return encrypt.JSONRange[Q]{}, nil } +// JSONElementMatch derives element_match on a JSON field. +func JSONElementMatch[Q ~[]byte, T, E any](context.Context, Field[T], *encrypt.Cipher, string, encrypt.JSONReadOptions, []encrypt.ElementPredicate[E]) (Q, error) { + return nil, nil +} + // JSONSelect prepares `->` at a path on a JSON field. func JSONSelect[E ~[]byte, T any](context.Context, Field[T], *encrypt.Cipher, string) (encrypt.JSONSelection[E], error) { return encrypt.JSONSelection[E]{}, nil diff --git a/mise.toml b/mise.toml index a8ca38e6b..b90273322 100644 --- a/mise.toml +++ b/mise.toml @@ -384,7 +384,7 @@ cd languages/golang # The tests skip without DATABASE_URL and without their guest builds; -v and # the grep turn either skip into a failure here, where both are provided. DATABASE_URL="$EQL_TEST_DATABASE_URL" CGO_ENABLED=0 go test -count=1 -v \ - -run '^(TestEQLTypesRoundTripAndQueryInPostgres|TestEQLOrderMatchesPlaintextInPostgres|TestQueriesRunAgainstPostgres|TestJSONDocumentsQueryAndReadInPostgres|TestJSONTypedDocumentsInPostgres)$' ./encrypt \ + -run '^(TestEQLTypesRoundTripAndQueryInPostgres|TestEQLOrderMatchesPlaintextInPostgres|TestQueriesRunAgainstPostgres|TestJSONDocumentsQueryAndReadInPostgres|TestJSONTypedDocumentsInPostgres|TestJSONElementMatchInPostgres)$' ./encrypt \ | tee "${TMPDIR:-/tmp}/go-test-postgres.log" if grep -q -- '--- SKIP' "${TMPDIR:-/tmp}/go-test-postgres.log"; then echo "error: a PostgreSQL test skipped; build the deterministic guests first" >&2 @@ -392,7 +392,7 @@ if grep -q -- '--- SKIP' "${TMPDIR:-/tmp}/go-test-postgres.log"; then fi # A -run pattern that matches no test (one renamed, say) prints "no tests # to run" and PASS, so each test has to be seen to pass by name. -for test in TestEQLTypesRoundTripAndQueryInPostgres TestEQLOrderMatchesPlaintextInPostgres TestQueriesRunAgainstPostgres TestJSONDocumentsQueryAndReadInPostgres TestJSONTypedDocumentsInPostgres; do +for test in TestEQLTypesRoundTripAndQueryInPostgres TestEQLOrderMatchesPlaintextInPostgres TestQueriesRunAgainstPostgres TestJSONDocumentsQueryAndReadInPostgres TestJSONTypedDocumentsInPostgres TestJSONElementMatchInPostgres; do if ! grep -q -- "^--- PASS: $test " "${TMPDIR:-/tmp}/go-test-postgres.log"; then echo "error: $test did not run and pass; was it renamed?" >&2 exit 1 From ae71ed253782fc12ed81a8b18aacc43c1063ca49 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 30 Oct 2026 13:11:01 +1100 Subject: [PATCH 2/2] docs(go): show the element match in the references and the JSON example The stashgen README's "Encrypted JSON documents" table gets the Match row and its SQL, with when Elements and Paths are generated, how encrypt.AtPath makes a predicate from a path, what a path's type checks, the run-time Member form, what is refused before the engine, which arrays get no accessor, the folded equality and the 22023 bound. Its list of what stashgen writes, and -name, name Paths. The SDK README and package doc show the call, and the CHANGELOG records it under Unreleased. The typed-document example orders each visit's at and place, and finds the customer with one visit at home after 15, with the paths from Paths: Ada has a visit at home and a visit after 15, but no one visit that is both. Its generated file is regenerated from the new tag. Refs #1442 Claude-Session: https://claude.ai/code/session_01MrxG73fnX1p7oiizh3GRG5 --- languages/golang/CHANGELOG.md | 33 +++++++ languages/golang/cmd/stashgen/README.md | 48 ++++++++- languages/golang/encrypt/README.md | 31 +++++- languages/golang/encrypt/doc.go | 23 +++++ .../golang/encrypt/example/json/README.md | 29 +++++- .../encrypt/example/json/customer_stash.go | 97 ++++++++++++++++++- languages/golang/encrypt/example/json/main.go | 26 ++++- .../golang/encrypt/example/json/model.go | 8 +- 8 files changed, 279 insertions(+), 16 deletions(-) diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index 48239517d..b06df306f 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -12,6 +12,39 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. ### Added +- **Test several members of one array element of a `JsonSearch` field + (#1442).** A field whose `ordered=` paths name members beneath an array's + wildcard (`$.contacts[*].relationship;$.contacts[*].calls`) gets + `Fields..Elements..Match`, one accessor per array, named as + `At` names a path (`Elements.Contacts`). `Match(ctx, cipher, opts, + predicates...)` returns an `eql.JSONSearchElementsQuery` for + `eql_v4.json_tree_element_match(col, $1::eql_v4.query_json_elements)`, + which finds a row where one element passes every predicate. An `EqAt` and + an ordering at the array's paths can each be answered by a different + element. The generated `Paths` names each array's ordered members, nested + as the JSON nests them (`Paths.Profile.Contacts.Relationship`), and + `encrypt.AtPath(path)` makes the predicates: `Eq`, `Lt`, `Lte`, `Gt`, + `Gte` and `Between`. A path's type, `encrypt.Path[E, T]`, carries its + array and the member's Go type from the document's struct (`any` for a + `json.RawMessage`), so a value of another type, or a predicate on another + array's member, does not compile. `Paths` holds only fields, so no JSON + key is reserved. For a member named at run time, the array accessor's + `Member("relationship")` makes the same predicates with `any` values. + Each comparison is of the member's ordering term, so `Eq` compares text + case- and accent-folded and integers past 2^53 can tie; `EqAt` stays + exact. A member named at run time that the field does not order is + refused before the engine is asked, with `encrypt.ErrTerm`, naming the + member and the array; no predicates, a zero `ElementPredicate` and a + member with a wildcard are refused with `encrypt.ErrEncoding`. An array + whose members `Paths` cannot name (a key with no Go name, or one name for + two paths) gets no accessor and a notice, as two arrays with one + accessor name do. A row whose array is + longer than `JSONReadOptions.WildcardBound` (64 by default) fails the + query with SQLSTATE 22023. An `ElementPredicate` prints and logs its + member and comparison, never its values. The embedded EQL guest now names + the `eql_v4.query_json_elements` domain, so `Checker.Operators` lists + `element_match` (input `path_predicates`) on a field that orders a path. + - **A two-letter word is searchable in `TextMatch`, `TextSearch` and `TextSearchOre` (#1309).** The EQL v4 match types index each two-letter word whole, beside the 3-grams of the text, so a two-letter surname such diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index b08acdcd4..02e227448 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -178,6 +178,7 @@ The field's `Fields` entry has a method per operation: | `Selector(ctx, c, path)` | `encrypt.JSONSelection`: `Selector()` and `Open` | `col -> $1::text` | | `Subtree(ctx, c, path, opts)` | `encrypt.JSONRead`: `Tokens()`, `Open` and `OpenOne` | `eql_v4.json_tree_subtree(col, $1::text[], 64)` | | `Children(ctx, c, path)`, `ChildCount(ctx, c, path)` | `encrypt.JSONToken` | `eql_v4.json_tree_children(col, $1::text)`, `eql_v4.json_tree_child_count(col, $1::text)` | +| `Elements.Visits.Match(ctx, c, opts, predicates...)` | `eql.JSONSearchElementsQuery` | `eql_v4.json_tree_element_match(col, $1::eql_v4.query_json_elements)` | `At` has one accessor per ordered path, named for its keys (`$.visits[*].at` is `At.VisitsAt`, `$.a.1` is `At.A1`); two paths with one name, or one path given twice, stop `stashgen`. An ordering compares the value `->` extracts at its path, and EQL's `<` compares ordering terms alone, so take the selector from the same accessor (`At.Age.Selector`) rather than spelling the path again. @@ -193,6 +194,50 @@ SELECT id FROM customers ORDER BY ( ``` with `$1` from `At.VisitsAt.Selector(ctx, c)` and `$2` from `At.VisitsAt.Gt(ctx, c, v)`; for the latest visit, order by `ORDER BY 1 DESC NULLS LAST` inside and `DESC` outside. + +Two queries at a wildcard path, an `EqAt` and an ordering say, can each be answered by a different element. +To test several members of one element together, use `Match` under `Elements`. +`Elements` has one accessor per array whose elements have an ordered member, named for the array's keys as `At` names a path: with `ordered=$.visits[*].at;$.visits[*].place`, `$.visits[*]` is `Elements.Visits`, with the members `at` and `place`. +A root array (`ordered=$[*].name`) is `Elements.Root`. +An ordered path that ends at its wildcard (`$.tags[*]`), or has none, gives no accessor, and a field without one has no `Elements`. +Two arrays whose keys give one name (`$.a_b[*]` and `$.aB[*]` are both `AB`), and an array whose keys give no name, get no accessor: `stashgen` still writes the field and its `At`, and prints a notice naming the ordered paths, which the generated file repeats in a comment. + +The generated `Paths` names each accessor's members, nested as the JSON nests them: `$.visits[*].place` is `Paths.Profile.Visits.Place`. +`encrypt.AtPath` makes a predicate from one, with `Eq`, `Lt`, `Lte`, `Gt`, `Gte` or `Between` (inclusive). + +```go +// rows with one visit that is at home and after 15 +q, err := customers.Fields.Profile.Elements.Visits.Match(ctx, cipher, encrypt.JSONReadOptions{}, + encrypt.AtPath(customers.Paths.Profile.Visits.Place).Eq("home"), + encrypt.AtPath(customers.Paths.Profile.Visits.At).Gt(15), +) +``` + +```sql +SELECT id FROM customers WHERE eql_v4.json_tree_element_match(profile, $1::eql_v4.query_json_elements) +``` + +A path is an `encrypt.Path[E, T]`. +`E` is a type `stashgen` writes for the array (`ProfileFieldElementsVisits`), and `Match` takes only `encrypt.ElementPredicate[E]` of its own array, so a predicate on another array's member does not compile. +`T` is the member's Go type in the document's struct, read as `encoding/json` reads the document (through pointers, promoted fields, maps and slices), so a value of another type does not compile either. +It is `any` where the type names none: a `json.RawMessage` document, a path the struct does not hold, or a member inside a type with its own `MarshalJSON`. +`Paths` holds only fields, and its paths' methods are the SDK's, so any JSON key can name a member. +With `-name Customer`, it is `CustomerPaths`, as `Fields` is `CustomerFields`. + +For a filter built at run time, from text such as `visits:(place = "home" AND at > 15)`, name the member with the array's `Member`, written as it is below the array in the tag (`place`, or `address.city`): + +```go +visits := customers.Fields.Profile.Elements.Visits +q, err := visits.Match(ctx, cipher, encrypt.JSONReadOptions{}, visits.Member("place").Eq("home"), visits.Member("at").Gt(15)) +``` + +Its values are `any`, as the orderings take them. +A member the field does not order is refused before the engine is asked, with `encrypt.ErrTerm`, naming the member, the array and the members it orders; no predicates, a zero `ElementPredicate`, and a member with a wildcard, are refused with `encrypt.ErrEncoding`. +`Paths` names each key on its own, so an array whose members it cannot name gets no accessor and the same notice: a key that gives no Go name there (`$.a.1[*].x`, whose `1` `At` joins to `A1X`), or one name for two paths (`$.a[*].b` beside `$.a[*].b.c`, or `$.x.a_b[*].m` beside `$.x.aB.n[*].o`). +Each comparison is of the member's ordering term, so `Eq` is an equality of folded values: text compares case- and accent-folded (`"home"` equals `"Home"`), and integers past 2^53 can tie. +`EqAt` at the member's full path is exact, of any element. +`Match` reads each array to `opts.WildcardBound` elements, 64 when it is 0, and a row whose array is longer fails the query with SQLSTATE 22023 rather than answer over part of it: set the bound past the longest array. +The function runs on each row a scan gives it, so narrow the rows first with an indexed predicate, such as `@>` on the same values. `OpenOne` reads one value and refuses a read whose path has a wildcard before anything is opened; open a wildcard read with `Open`. A needle and a value are Go values: a `json.RawMessage` of JSON text crosses as it is, and anything else is marshalled with `encoding/json`. `EqAt` and the orderings take a scalar: a string, a number, a bool, or `nil` for JSON `null`. @@ -253,6 +298,7 @@ For `-type User`, the file `user_stash.go` holds: `encrypt_into=TextMatch` answers `Matches`, which returns an `encrypt.EQLMatchQuery`: the `eql.TextMatchQuery` operand, with `Verify` and `TargetLimit`. The query of `Matches` carries `Verify` and `TargetLimit`, because its answer can hold rows that do not match; an exact operator's query carries neither. See the encrypt package's README for each method's result and the query with `database/sql`. +- `Paths`, when a `JsonSearch` field orders members of an array's elements: those members, nested as the JSON nests them, for `encrypt.AtPath` and the array's `Match` (see [Encrypted JSON documents](#encrypted-json-documents)). - `String` and `LogValue` on `EncryptedUser`, which print the passthrough fields and hide the sealed ones. - A copy of the fields of `User`, so a change to them stops the build. @@ -263,7 +309,7 @@ Generated code uses no reflection, and no function in it panics. | Flag | Meaning | |---|---| | `-type T` | The struct that carries the `stash` tags. Required. | -| `-name N` | Write `EncryptN`, `DecryptN` and `NFields`. A package holds one `Encrypt`; the second struct in a package needs a name. | +| `-name N` | Write `EncryptN`, `DecryptN` and `NFields` (and `NPaths`). A package holds one `Encrypt`; the second struct in a package needs a name. | | `-for P.F` | `T` declares the tags for `F`, a type in another package. Each field of `T` names a field of `F` with the same name and type, and every exported field of `F` is named. | | `-model Name=R` | A model `R` for separate columns: one field for each column, each tagged with the output it holds, `stash:"email"` or `stash:"email,equality"`. Writes `EncryptName` and `DecryptName`. Any number. | | `-model Name=R:D` | The same, for an `R` that cannot carry tags. The struct `D` in your package declares them. | diff --git a/languages/golang/encrypt/README.md b/languages/golang/encrypt/README.md index 8b9ebc979..c6fa259a3 100644 --- a/languages/golang/encrypt/README.md +++ b/languages/golang/encrypt/README.md @@ -520,7 +520,7 @@ The EQL types the engine produces, and the Go field types each takes: | `Numeric` | `encrypt.Decimal` | none | | `NumericEq` | `encrypt.Decimal` | `=`, `<>` (`1.5 = 1.50`) | | `NumericOrd`, `NumericOrdOpe`, `NumericOrdOre` | `encrypt.Decimal` | `<`, `<=`, `>`, `>=`, `ORDER BY` | -| `JsonSearch` (`eql.JSONSearch`) | struct, map, slice, or `json.RawMessage` | containment, equality at a path, ordering at declared paths, selection, subtree reads; see [Encrypted JSON documents](#encrypted-json-documents) | +| `JsonSearch` (`eql.JSONSearch`) | struct, map, slice, or `json.RawMessage` | containment, equality at a path, ordering at declared paths, several members of one array element, selection, subtree reads; see [Encrypted JSON documents](#encrypted-json-documents) | Any built-in Go integer field takes an integer EQL type (ADR-0002): the field is sealed at the type's width, and opens back into the field's type. A type @@ -658,6 +658,35 @@ element of each row, so an ordering over every element is written over `eql_v4.json_tree_path_query(col, $sel)`, its matches (the stashgen README shows the SQL). +Two such queries can each be answered by a different element. To test +several members of one array element together, use `Match` on the array's +accessor under `Elements`. A field has one for each array whose elements +have an ordered member (`ordered=$.visits[*].at;$.visits[*].place`). The +generated `Paths` names those members, nested as the JSON nests them, and +`encrypt.AtPath` makes a predicate from one: + +```go +q, err := users.Fields.Profile.Elements.Visits.Match(ctx, cipher, encrypt.JSONReadOptions{}, + encrypt.AtPath(users.Paths.Profile.Visits.Place).Eq("home"), + encrypt.AtPath(users.Paths.Profile.Visits.At).Gt(15), +) +rows, err := db.QueryContext(ctx, `SELECT id FROM users WHERE eql_v4.json_tree_element_match(profile, $1::eql_v4.query_json_elements)`, q) +``` + +A path carries its array and the member's Go type from the document's +struct, so a value of another type, or a predicate on another array's +member, does not compile. For a filter built at run time, name the member +with the array's `Member`: +`users.Fields.Profile.Elements.Visits.Member("place").Eq("home")`. A member +named so that the field does not order is refused before the engine is +asked, naming it. + +Each comparison is of the member's ordering term, so `Eq` compares text +case- and accent-folded, and integers past 2^53 can tie; `EqAt` is exact. +A row whose array is longer than +`JSONReadOptions.WildcardBound` (64 by default) fails the query with +SQLSTATE 22023. + ## Building The package embeds `wasm/stack_encrypt_guest.wasm`, a build artefact of the diff --git a/languages/golang/encrypt/doc.go b/languages/golang/encrypt/doc.go index 903b455b9..658720b79 100644 --- a/languages/golang/encrypt/doc.go +++ b/languages/golang/encrypt/doc.go @@ -242,6 +242,29 @@ // A selection or a subtree read opens values as JSON text ([JSONValue], // [JSONRead.Open]). [DecodeJSON] reads one into a Go type. // +// Two queries at an array's paths can each be answered by a different +// element. To test several members of one element together, use Match on +// the array's accessor under Elements, which a field has for each array +// whose elements have an ordered member. The generated Paths names those +// members, nested as the JSON nests them, and [AtPath] makes a predicate +// from one: +// +// q, err := customers.Fields.Profile.Elements.Visits.Match(ctx, cipher, encrypt.JSONReadOptions{}, +// encrypt.AtPath(customers.Paths.Profile.Visits.Place).Eq("home"), +// encrypt.AtPath(customers.Paths.Profile.Visits.At).Gt(15), +// ) +// // WHERE eql_v4.json_tree_element_match(profile, $1::eql_v4.query_json_elements) +// +// A [Path] carries its array and the member's Go type, so a value of +// another type, or a predicate on another array's member, does not +// compile. For a filter built at run time, the array accessor's Member +// names the member as text: Visits.Member("place").Eq("home"). +// +// Each comparison is of the member's ordering term, so [MemberTest.Eq] +// compares text case- and accent-folded, and integers past 2^53 can tie. A row whose +// array is longer than [JSONReadOptions.WildcardBound] (64 by default) +// fails the query with SQLSTATE 22023. +// // # Cancellation // // Every call takes a [context.Context], and a call whose context is diff --git a/languages/golang/encrypt/example/json/README.md b/languages/golang/encrypt/example/json/README.md index 2fe4a9800..f4651d12e 100644 --- a/languages/golang/encrypt/example/json/README.md +++ b/languages/golang/encrypt/example/json/README.md @@ -1,8 +1,9 @@ # Typed encrypted JSON document example This program stores a Go `Profile` struct as one encrypted EQL v4 JSON column. -It finds a row by containment, compares an ordered path, decrypts a whole -profile, and opens only the visit times from another query. +It finds a row by containment, compares an ordered path, finds a row where one +visit passes two tests, decrypts a whole profile, and opens only the visit +times from another query. ## Run @@ -35,8 +36,28 @@ its `stash` tags. After changing the struct or tags, run this from go generate ./encrypt/example/json ``` -`Profile` uses `encoding/json` for the document shape. The `ordered=$.age` -tag adds an ordering term for that path. A containment needle made from a +`Profile` uses `encoding/json` for the document shape. The +`ordered=$.age;$.visits[*].at;$.visits[*].place` tag adds an ordering term +for each of those paths. Because `at` and `place` are ordered members of +each visit, the generated `Fields.Profile.Elements.Visits.Match` tests both +on one visit. The generated `Paths` names them as the JSON nests them: + +```go +q, err := Fields.Profile.Elements.Visits.Match(ctx, cipher, encrypt.JSONReadOptions{}, + encrypt.AtPath(Paths.Profile.Visits.Place).Eq("home"), + encrypt.AtPath(Paths.Profile.Visits.At).Gt(15), +) +// SELECT id FROM customers WHERE eql_v4.json_tree_element_match(profile, $1::eql_v4.query_json_elements) +``` + +`Paths.Profile.Visits.At` has the Go type of `Visit.At`, `int64`, so +`Gt("15")` does not compile, and neither does a predicate on another +array's member. Ada has a visit at home and a visit after 15, but no one +visit that is both, so only Bea's row matches. Each comparison is of the +ordering term, so text +compares case- and accent-folded. A row with more visits than +`JSONReadOptions.WildcardBound` (64 by default) fails the query with SQLSTATE +22023. A containment needle made from a struct includes every field without `omitempty`, including zero values; `main.go` uses a map to name only the member being queried. The prepared subtree read returns JSON text, which `encrypt.DecodeJSON[int64]` decodes. diff --git a/languages/golang/encrypt/example/json/customer_stash.go b/languages/golang/encrypt/example/json/customer_stash.go index 092f14762..3f68604e4 100644 --- a/languages/golang/encrypt/example/json/customer_stash.go +++ b/languages/golang/encrypt/example/json/customer_stash.go @@ -53,7 +53,7 @@ type customerShape struct { var declaration = gensupport.Declare("customers"). Passthrough("id"). EncryptInto("profile", gensupport.JSON, "JsonSearch"). - Options("profile", gensupport.TargetOptions{Ordered: []string{"$.age"}}) + Options("profile", gensupport.TargetOptions{Ordered: []string{"$.age", "$.visits[*].at", "$.visits[*].place"}}) var codec = gensupport.New(gensupport.Generated[Customer, EncryptedCustomer]{ TypeName: "Customer", @@ -119,12 +119,23 @@ type ProfileField struct { // At has one accessor per path the field orders, whose methods // are the orderings there. At ProfileFieldAt + // Elements has one accessor per array whose elements have ordered + // members, whose Match tests the members of one element together. + Elements ProfileFieldElements } func newProfileField(f gensupport.Field[Profile]) ProfileField { - return ProfileField{field: f, At: ProfileFieldAt{ - Age: ProfileFieldPath{field: f, path: "$.age"}, - }} + return ProfileField{ + field: f, + At: ProfileFieldAt{ + Age: ProfileFieldPath{field: f, path: "$.age"}, + VisitsAt: ProfileFieldPath{field: f, path: "$.visits[*].at"}, + VisitsPlace: ProfileFieldPath{field: f, path: "$.visits[*].place"}, + }, + Elements: ProfileFieldElements{ + Visits: ProfileFieldArray[ProfileFieldElementsVisits]{field: f, path: "$.visits[*]"}, + }, + } } // Encrypt seals one Profile document for an update of its stored column. @@ -196,10 +207,67 @@ func (f ProfileField) ChildCount(ctx context.Context, c *encrypt.Cipher, path st return gensupport.JSONNodeToken(ctx, f.field, c, "child_count", path) } +// ProfileFieldElements names the arrays of Profile whose elements have ordered +// members. Each accessor's Match tests the members of one element together. +type ProfileFieldElements struct { + Visits ProfileFieldArray[ProfileFieldElementsVisits] // $.visits[*]: at, place +} + +// ProfileFieldElementsVisits is the array $.visits[*] of Profile, as a type: +// Elements.Visits.Match takes an +// encrypt.ElementPredicate[ProfileFieldElementsVisits], which encrypt.AtPath +// makes from a member under Paths.Profile.Visits. +type ProfileFieldElementsVisits struct{} + +// ProfileFieldArray is one array of Profile whose elements have ordered +// members. E is the array's own type, which the predicates its Match takes +// carry. +type ProfileFieldArray[E any] struct { + field gensupport.Field[Profile] + path string +} + +// Match derives the operand that finds a row where one element of this array +// passes every predicate: eql_v4.json_tree_element_match(col, +// $1::eql_v4.query_json_elements). An EqAt and an ordering at the array's paths +// can each be answered by a different element; Match is answered by one. Make +// each predicate with encrypt.AtPath and a member of this array under Paths, so +// a member of another array, or a value of another type, does not compile; or +// with Member, for a member named at run time. No predicates, and a zero +// ElementPredicate, are refused before the engine is asked. +// +// Each comparison is of the member's ordering term, so Eq is an equality of +// folded values: text compares case- and accent-folded ("sibling" equals +// "Sibling"), and integers past 2^53 can tie. EqAt at a member's full path is +// exact, of any element. +// +// The operand reads each array to opts.WildcardBound elements, 64 when it is 0. +// A row whose array is longer fails the query with SQLSTATE 22023 +// (invalid_parameter_value) rather than answer over part of it: raise the bound +// past the longest array. The function filters the rows a scan gives it, so +// narrow them with an indexed predicate first, such as @> on the same values. +func (a ProfileFieldArray[E]) Match(ctx context.Context, c *encrypt.Cipher, opts encrypt.JSONReadOptions, predicates ...encrypt.ElementPredicate[E]) (eql.JSONSearchElementsQuery, error) { + return gensupport.JSONElementMatch[eql.JSONSearchElementsQuery](ctx, a.field, c, a.path, opts, predicates) +} + +// Member is the comparisons of the member of this array's elements that member +// names, for a filter whose members are known only at run time: relationship, +// or address.city, written as the member is written below the array in the +// field's ordered paths. Its values are not checked by the compiler, and Match +// refuses a member the field does not order beneath this array, naming it, the +// array and the members the field orders there, before the engine is asked. +// Where the member is known when the program is written, use encrypt.AtPath and +// Paths. +func (a ProfileFieldArray[E]) Member(member string) encrypt.MemberTest[E, any] { + return encrypt.AtPath(encrypt.NewPath[E, any](a.path, member)) +} + // ProfileFieldAt names the declared ordered paths of Profile. Use an accessor's // Selector with its ordering query. type ProfileFieldAt struct { - Age ProfileFieldPath // $.age + Age ProfileFieldPath // $.age + VisitsAt ProfileFieldPath // $.visits[*].at + VisitsPlace ProfileFieldPath // $.visits[*].place } // ProfileFieldPath is one ordered path of Profile. Its orderings derive the @@ -261,3 +329,22 @@ func (p ProfileFieldPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi func (p ProfileFieldPath) Selector(ctx context.Context, c *encrypt.Cipher) (encrypt.JSONSelection[eql.JSONEntry], error) { return gensupport.JSONSelect[eql.JSONEntry](ctx, p.field, c, p.path) } + +// Paths names the ordered members of the arrays of Customer's documents, nested +// as the JSON nests them, for the element match: +// encrypt.AtPath(Paths.Profile.Visits.At) makes the predicates +// Fields.Profile.Elements.Visits.Match takes. Each path's type names its array +// and the member's Go type, so a predicate on another array's member, or with a +// value of another type, does not compile. +var Paths = func() (paths struct { + Profile struct { + Visits struct { + At encrypt.Path[ProfileFieldElementsVisits, int64] // $.visits[*].at + Place encrypt.Path[ProfileFieldElementsVisits, string] // $.visits[*].place + } + } +}) { + paths.Profile.Visits.At = encrypt.NewPath[ProfileFieldElementsVisits, int64]("$.visits[*]", "at") + paths.Profile.Visits.Place = encrypt.NewPath[ProfileFieldElementsVisits, string]("$.visits[*]", "place") + return paths +}() diff --git a/languages/golang/encrypt/example/json/main.go b/languages/golang/encrypt/example/json/main.go index 58c60a328..e50fd9cc2 100644 --- a/languages/golang/encrypt/example/json/main.go +++ b/languages/golang/encrypt/example/json/main.go @@ -50,8 +50,8 @@ func run(ctx context.Context) error { defer client.Close() cipher := client.DefaultKeyset() customers := []Customer{ - {ID: 1, Profile: Profile{Name: "Ada", Age: 35, Visits: []Visit{{At: 10}, {At: 20}}}}, - {ID: 2, Profile: Profile{Name: "Bea", Age: 28, Visits: []Visit{{At: 30}}}}, + {ID: 1, Profile: Profile{Name: "Ada", Age: 35, Visits: []Visit{{At: 10, Place: "home"}, {At: 20, Place: "work"}}}}, + {ID: 2, Profile: Profile{Name: "Bea", Age: 28, Visits: []Visit{{At: 30, Place: "home"}}}}, } sealed, err := Encrypt(ctx, cipher, customers) if err != nil { @@ -101,6 +101,28 @@ func run(ctx context.Context) error { } fmt.Printf("customer older than 30: %d\n", olderID) + // Test two members of one visit together: a visit at home after 15. + // Ada has a visit at home and a visit after 15, but not one visit that + // is both, so only Bea's row matches. Match compares ordering terms, so + // "Home" would equal "home": text compares case- and accent-folded. + // Paths names each visit's members with their Go types, so a value of + // another type, or a member of another array, does not compile. + homeLate, err := Fields.Profile.Elements.Visits.Match(ctx, cipher, encrypt.JSONReadOptions{}, + encrypt.AtPath(Paths.Profile.Visits.Place).Eq("home"), + encrypt.AtPath(Paths.Profile.Visits.At).Gt(15), + ) + if err != nil { + return fmt.Errorf("prepare visit match: %w", err) + } + var homeLateID int64 + if err := conn.QueryRowContext(ctx, + `SELECT id FROM customers WHERE eql_v4.json_tree_element_match(profile, $1::eql_v4.query_json_elements)`, + homeLate, + ).Scan(&homeLateID); err != nil { + return fmt.Errorf("find a visit at home after 15: %w", err) + } + fmt.Printf("customer with a visit at home after 15: %d\n", homeLateID) + // Read only the visit times of one row. The database returns encrypted // entries; Open checks them against the prepared read and returns JSON text. read, err := Fields.Profile.Subtree(ctx, cipher, "$.visits[*].at", encrypt.JSONReadOptions{}) diff --git a/languages/golang/encrypt/example/json/model.go b/languages/golang/encrypt/example/json/model.go index eef660abc..dc60afadc 100644 --- a/languages/golang/encrypt/example/json/model.go +++ b/languages/golang/encrypt/example/json/model.go @@ -6,10 +6,12 @@ package main // Visit is one visit stored inside a customer's encrypted profile. type Visit struct { - At int64 `json:"at"` + At int64 `json:"at"` + Place string `json:"place"` } -// Profile is a typed JSON document. Its Age is searchable by order. +// Profile is a typed JSON document. Its Age, and each visit's At and Place, +// are searchable by order. type Profile struct { Name string `json:"name"` Age int64 `json:"age"` @@ -20,5 +22,5 @@ type Profile struct { type Customer struct { _ struct{} `stash:"context=customers"` ID int64 `stash:"id,passthrough"` - Profile Profile `stash:"profile,encrypt_into=JsonSearch(ordered=$.age)"` + Profile Profile `stash:"profile,encrypt_into=JsonSearch(ordered=$.age;$.visits[*].at;$.visits[*].place)"` }