From f9e0163cdc8208f9faf08d05c2ae8bbce55caeb4 Mon Sep 17 00:00:00 2001 From: Rik Allen Date: Mon, 9 Nov 2026 17:35:00 +0000 Subject: [PATCH] feat(search): add a tags_all filter that requires every tag search's tags filter matches memories with at least one of the tags given, so "status:open AND owner:me" could not be asked in one query: an agent had to fetch with one tag and filter the rest client-side, and with results capped at 100 that silently missed rows once the store grew. tags_all keeps only memories that carry every tag given. It can be combined with tags (any-of AND all-of), and an empty list means no filter, as for tags. A new migration adds a 10-argument search_memories with p_tags_all, applied inside each arm before its LIMIT like the other filters. The 9-argument function becomes a SQL wrapper, so the deployed Edge Function keeps working once the migration is applied. The new Edge Function calls the 10-argument overload, so the migration must be applied first. Tests: a pgTAP file (every mode, before-LIMIT in each arm, combination with tags, empty list, RLS, grants, the 9-argument wrapper), an integration test through PgMemoryStore, and unit tests for the handler and the registered tool schema. Co-Authored-By: Claude Opus 5.5 --- README.md | 3 + ...2026-11-07-supabase-agent-memory-design.md | 3 +- supabase/functions/_shared/pg-store.ts | 1 + supabase/functions/_shared/tools.ts | 3 + supabase/functions/_shared/types.ts | 2 + .../20261109120000_search_tags_all.sql | 154 ++++++++++++++++++ .../database/11_search_tags_all.test.sql | 91 +++++++++++ tests/integration/pg-store-read.test.ts | 14 ++ tests/unit/tools.test.ts | 24 +++ 9 files changed, 294 insertions(+), 1 deletion(-) create mode 100644 supabase/migrations/20261109120000_search_tags_all.sql create mode 100644 supabase/tests/database/11_search_tags_all.test.sql diff --git a/README.md b/README.md index 201ec7c..ead79e4 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,9 @@ The server exposes 22 tools, plus 13 `admin_` tools that only admins see (see [A thread tools (see [Threads and sessions](#threads-and-sessions)). The memory tools are `list_scopes`, `store`, `store_batch`, `search`, `recall`, `find_related`, `list_recent`, `update`, `delete`, `stats` and `prune`. They match the original server, plus a `scope` argument on the write tools. Search is hybrid (BM25 full-text plus `gte-small` vector similarity, fused with RRF) with temporal decay. +Its `tags` filter keeps memories with at least one of the tags given; `tags_all` keeps only those with every one, so a +query such as "open work in this repo" (`tags_all: ["repo:", "status:open"]`) needs no client-side filtering. +The two can be combined. Delete and prune are soft; rows are purged after 30 days. ## Threads and sessions diff --git a/docs/superpowers/specs/2026-11-07-supabase-agent-memory-design.md b/docs/superpowers/specs/2026-11-07-supabase-agent-memory-design.md index ee027ea..ad81113 100644 --- a/docs/superpowers/specs/2026-11-07-supabase-agent-memory-design.md +++ b/docs/superpowers/specs/2026-11-07-supabase-agent-memory-design.md @@ -169,7 +169,8 @@ Port the original behaviour onto Postgres: - **Hybrid** = Postgres full-text rank + pgvector cosine, fused with RRF (k = 60), with over-fetch then filter. - **Keyword** and **semantic** modes. -- Filters: category, tags (any-of), before/after, limit, scopes. +- Filters: category, tags (any-of), tags_all (every one; added later as a 10-argument `search_memories`), before/after, + limit, scopes. - **Temporal decay**: half-life 30 days default (configurable), the original importance multiplier (access frequency and recency stretch the half-life, range 1–3×), and the `evergreen` / `never-forget` exemption. Re-sort after decay. diff --git a/supabase/functions/_shared/pg-store.ts b/supabase/functions/_shared/pg-store.ts index 2624303..60174e4 100644 --- a/supabase/functions/_shared/pg-store.ts +++ b/supabase/functions/_shared/pg-store.ts @@ -357,6 +357,7 @@ export class PgMemoryStore implements MemoryStore { select * from agent_memory.search_memories( ${query}::text, ${embArg}::extensions.vector, ${effectiveMode}::text, ${scopeIds}::uuid[], ${f.category ?? null}::text, ${f.tags?.length ? tx.array(f.tags) : null}::text[], + ${f.tagsAll?.length ? tx.array(f.tagsAll) : null}::text[], ${f.after ?? null}::timestamptz, ${f.before ?? null}::timestamptz, ${limit * OVER_FETCH_FACTOR}::int)`; const results = applyDecay(this.toDecayInputs(rows), limit, this.halfLifeDays); diff --git a/supabase/functions/_shared/tools.ts b/supabase/functions/_shared/tools.ts index 746509a..4689fb7 100644 --- a/supabase/functions/_shared/tools.ts +++ b/supabase/functions/_shared/tools.ts @@ -25,6 +25,7 @@ const scopesSchema = z.array(z.string()).optional().describe('Scopes to search; const filtersShape = { category: categorySchema.optional().describe('Filter by category'), tags: z.array(z.string()).optional().describe('Filter: memory must have at least one of these tags'), + tags_all: z.array(z.string()).optional().describe('Filter: memory must have every one of these tags'), after: z.string().optional().describe('Filter: created after this ISO 8601 date'), before: z.string().optional().describe('Filter: created before this ISO 8601 date'), limit: z.number().optional().describe('Max results to return (default 10)'), @@ -122,6 +123,7 @@ export function handleSearch(store: MemoryStore) { mode?: SearchMode; category?: MemoryCategory; tags?: string[]; + tags_all?: string[]; after?: string; before?: string; limit?: number; @@ -131,6 +133,7 @@ export function handleSearch(store: MemoryStore) { const results = await store.search(args.query, args.mode ?? 'hybrid', { category: args.category, tags: args.tags, + tagsAll: args.tags_all, after: args.after, before: args.before, limit: args.limit, diff --git a/supabase/functions/_shared/types.ts b/supabase/functions/_shared/types.ts index 32e7904..68053ac 100644 --- a/supabase/functions/_shared/types.ts +++ b/supabase/functions/_shared/types.ts @@ -92,6 +92,8 @@ export interface UpdateRequest { export interface SearchFilters { category?: MemoryCategory; tags?: string[]; + /** Every one of these tags must be present (tags matches any). */ + tagsAll?: string[]; after?: string; before?: string; limit?: number; diff --git a/supabase/migrations/20261109120000_search_tags_all.sql b/supabase/migrations/20261109120000_search_tags_all.sql new file mode 100644 index 0000000..32c0123 --- /dev/null +++ b/supabase/migrations/20261109120000_search_tags_all.sql @@ -0,0 +1,154 @@ +-- search_memories gains p_tags_all: a memory must carry EVERY tag in it (p_tags keeps its "at least one" meaning, and +-- both may be given). The filter applies inside each arm, before its LIMIT, like the other filters. +-- +-- The new function is a 10-argument overload. The 9-argument one stays as a thin wrapper, so the Edge Function that is +-- already deployed keeps working once this migration is applied. The new Edge Function calls the 10-argument overload, +-- so apply this migration BEFORE deploying it. Neither overload has defaults, so a 9-argument call is never ambiguous. + +-- Contract: returns at most p_limit rows in every mode (each hybrid arm fetches p_limit, fused result is +-- trimmed to p_limit); callers over-fetch for decay re-ranking. +-- Exact similarity on purpose: an ANN index post-filtered by RLS returns short pages and leaks result counts; +-- exact scan is milliseconds at this size. +create function agent_memory.search_memories( + p_query text, + p_embedding extensions.vector(384), + p_mode text, + p_scope_ids uuid[], + p_category text, + p_tags text[], + p_tags_all text[], + p_after timestamptz, + p_before timestamptz, + p_limit int +) returns table ( + id uuid, scope_id uuid, scope text, content text, category text, tags text[], + created_at timestamptz, updated_at timestamptz, created_by text, updated_by text, + access_count int, last_accessed_at timestamptz, raw_score float8 +) +language plpgsql stable security invoker +set search_path = '' +set client_min_messages = warning -- websearch_to_tsquery NOTICEs on stop-word-only queries +as $$ +#variable_conflict use_column +declare + v_tsq tsquery; +begin + if p_mode is null or p_mode not in ('keyword', 'semantic', 'hybrid') then + raise exception 'invalid search mode: %', p_mode using errcode = '22023'; + end if; + if p_limit is null or p_limit < 1 then + raise exception 'invalid limit: %', p_limit using errcode = '22023'; + end if; + if p_mode <> 'keyword' and p_embedding is null then + raise exception 'embedding required for % search', p_mode using errcode = '22023'; + end if; + v_tsq := pg_catalog.websearch_to_tsquery('english', coalesce(p_query, '')); + + if p_mode = 'keyword' then + if pg_catalog.numnode(v_tsq) = 0 then + return; + end if; + return query + select v.id, v.scope_id, v.scope, v.content, v.category, v.tags, v.created_at, v.updated_at, + v.created_by, v.updated_by, v.access_count, v.last_accessed_at, k.score + from ( + select m.id, pg_catalog.ts_rank_cd(m.fts, v_tsq)::float8 as score + from agent_memory.memories m + where m.fts operator(pg_catalog.@@) v_tsq + and (p_scope_ids is null or m.scope_id = any (p_scope_ids)) + and (p_category is null or m.category = p_category) + and (p_tags is null or m.tags && p_tags) + and (p_tags_all is null or m.tags @> p_tags_all) + and (p_after is null or m.created_at >= p_after) + and (p_before is null or m.created_at <= p_before) + order by score desc, m.id + limit p_limit + ) k + join agent_memory.memory_view v on v.id = k.id + order by k.score desc, v.id; + + elsif p_mode = 'semantic' then + return query + select v.id, v.scope_id, v.scope, v.content, v.category, v.tags, v.created_at, v.updated_at, + v.created_by, v.updated_by, v.access_count, v.last_accessed_at, (1 / (1 + s.dist))::float8 + from ( + select m.id, (m.embedding operator(extensions.<=>) p_embedding) as dist + from agent_memory.memories m + where (p_scope_ids is null or m.scope_id = any (p_scope_ids)) + and (p_category is null or m.category = p_category) + and (p_tags is null or m.tags && p_tags) + and (p_tags_all is null or m.tags @> p_tags_all) + and (p_after is null or m.created_at >= p_after) + and (p_before is null or m.created_at <= p_before) + order by m.embedding operator(extensions.<=>) p_embedding, m.id + limit p_limit + ) s + join agent_memory.memory_view v on v.id = s.id + order by s.dist, v.id; + + else + return query + with kw as ( + select m.id, pg_catalog.row_number() over (order by pg_catalog.ts_rank_cd(m.fts, v_tsq) desc, m.id) as rnk + from agent_memory.memories m + where pg_catalog.numnode(v_tsq) > 0 and m.fts operator(pg_catalog.@@) v_tsq + and (p_scope_ids is null or m.scope_id = any (p_scope_ids)) + and (p_category is null or m.category = p_category) + and (p_tags is null or m.tags && p_tags) + and (p_tags_all is null or m.tags @> p_tags_all) + and (p_after is null or m.created_at >= p_after) + and (p_before is null or m.created_at <= p_before) + order by pg_catalog.ts_rank_cd(m.fts, v_tsq) desc, m.id + limit p_limit + ), + sem as ( + select m.id, pg_catalog.row_number() over (order by m.embedding operator(extensions.<=>) p_embedding, m.id) as rnk + from agent_memory.memories m + where (p_scope_ids is null or m.scope_id = any (p_scope_ids)) + and (p_category is null or m.category = p_category) + and (p_tags is null or m.tags && p_tags) + and (p_tags_all is null or m.tags @> p_tags_all) + and (p_after is null or m.created_at >= p_after) + and (p_before is null or m.created_at <= p_before) + order by m.embedding operator(extensions.<=>) p_embedding, m.id + limit p_limit + ), + fused as ( + select coalesce(kw.id, sem.id) as id, + (coalesce(1.0 / (60 + kw.rnk), 0) + coalesce(1.0 / (60 + sem.rnk), 0))::float8 as score + from kw full outer join sem on kw.id = sem.id + ) + select v.id, v.scope_id, v.scope, v.content, v.category, v.tags, v.created_at, v.updated_at, + v.created_by, v.updated_by, v.access_count, v.last_accessed_at, f.score + from fused f + join agent_memory.memory_view v on v.id = f.id + order by f.score desc, v.id + limit p_limit; + end if; +end +$$; + +create or replace function agent_memory.search_memories( + p_query text, + p_embedding extensions.vector(384), + p_mode text, + p_scope_ids uuid[], + p_category text, + p_tags text[], + p_after timestamptz, + p_before timestamptz, + p_limit int +) returns table ( + id uuid, scope_id uuid, scope text, content text, category text, tags text[], + created_at timestamptz, updated_at timestamptz, created_by text, updated_by text, + access_count int, last_accessed_at timestamptz, raw_score float8 +) +language sql stable security invoker +set search_path = '' +as $$ + select * from agent_memory.search_memories(p_query, p_embedding, p_mode, p_scope_ids, p_category, p_tags, null::text[], + p_after, p_before, p_limit) +$$; + +revoke execute on function agent_memory.search_memories(text, extensions.vector, text, uuid[], text, text[], text[], timestamptz, timestamptz, int) from public; +grant execute on function agent_memory.search_memories(text, extensions.vector, text, uuid[], text, text[], text[], timestamptz, timestamptz, int) to agent_memory_member; diff --git a/supabase/tests/database/11_search_tags_all.test.sql b/supabase/tests/database/11_search_tags_all.test.sql new file mode 100644 index 0000000..54c825e --- /dev/null +++ b/supabase/tests/database/11_search_tags_all.test.sql @@ -0,0 +1,91 @@ +begin; +create extension if not exists pgtap with schema extensions; +select plan(15); + +-- one-hot vector at dim i, optionally with weight w at dim j +create function pg_temp.v(i int, j int default 1, w real default 0) returns extensions.vector +language sql immutable as $$ + select (array_agg(case when d = i then 1::real when d = j and w > 0 then w else 0::real end order by d))::extensions.vector + from generate_series(1, 384) d +$$; + +-- ids of a search, sorted, so results compare as sets +create function pg_temp.ids(p_query text, p_emb extensions.vector, p_mode text, p_tags text[], p_tags_all text[], p_limit int) +returns uuid[] language sql as $$ + select coalesce(array_agg(id order by id), '{}') + from agent_memory.search_memories(p_query, p_emb, p_mode, null, null, p_tags, p_tags_all, null, null, p_limit) +$$; + +insert into agent_memory.scopes (id, name) values + ('a0000000-0000-0000-0000-00000000000a', 'alpha'), + ('b0000000-0000-0000-0000-00000000000b', 'beta'); +insert into agent_memory.members (id, display_name, kind) values + ('11111111-1111-1111-1111-111111111111', 'rw', 'human'), + ('44444444-4444-4444-4444-444444444444', 'bw', 'bot'); +insert into agent_memory.scope_grants (scope_id, member_id, access) values + ('a0000000-0000-0000-0000-00000000000a', '11111111-1111-1111-1111-111111111111', 'write'), + ('b0000000-0000-0000-0000-00000000000b', '44444444-4444-4444-4444-444444444444', 'write'); + +insert into agent_memory.memories (id, scope_id, content, category, tags, created_by, updated_by, embedding) values + ('aaaaaaaa-0000-0000-0000-000000000001', 'a0000000-0000-0000-0000-00000000000a', 'heat pump note one', 'other', + '{repo:x,status:open,owner:me}', '11111111-1111-1111-1111-111111111111', '11111111-1111-1111-1111-111111111111', pg_temp.v(5)), + ('aaaaaaaa-0000-0000-0000-000000000002', 'a0000000-0000-0000-0000-00000000000a', 'heat pump note two', 'other', + '{repo:x,status:done,owner:me}', '11111111-1111-1111-1111-111111111111', '11111111-1111-1111-1111-111111111111', pg_temp.v(6)), + ('aaaaaaaa-0000-0000-0000-000000000003', 'a0000000-0000-0000-0000-00000000000a', 'heat pump note three', 'other', + '{repo:x,status:open}', '11111111-1111-1111-1111-111111111111', '11111111-1111-1111-1111-111111111111', pg_temp.v(7)), + ('aaaaaaaa-0000-0000-0000-000000000004', 'a0000000-0000-0000-0000-00000000000a', 'heat pump note four', 'other', + '{}', '11111111-1111-1111-1111-111111111111', '11111111-1111-1111-1111-111111111111', pg_temp.v(8)), + ('bbbbbbbb-0000-0000-0000-000000000005', 'b0000000-0000-0000-0000-00000000000b', 'heat pump beta secret', 'other', + '{status:open,owner:me}', '44444444-4444-4444-4444-444444444444', '44444444-4444-4444-4444-444444444444', pg_temp.v(1)); + +-- 20 readable rows that match status:open only, all nearer to the query vector v(1), and ranking higher for the keyword +-- query 'heat pump', than any row above +insert into agent_memory.memories (scope_id, content, category, tags, created_by, updated_by, embedding) +select 'a0000000-0000-0000-0000-00000000000a', 'heat pump heat pump heat pump filler ' || i, 'other', '{status:open}', + '11111111-1111-1111-1111-111111111111', '11111111-1111-1111-1111-111111111111', pg_temp.v(1, 100 + i, 0.1) +from generate_series(1, 20) i; +analyze agent_memory.memories; + +select ok(has_function_privilege('agent_memory_member', + 'agent_memory.search_memories(text, extensions.vector, text, uuid[], text, text[], text[], timestamptz, timestamptz, int)', 'execute'), + 'member can execute the 10-argument search_memories'); +select ok(not has_function_privilege('anon', + 'agent_memory.search_memories(text, extensions.vector, text, uuid[], text, text[], text[], timestamptz, timestamptz, int)', 'execute'), + 'anon cannot execute the 10-argument search_memories'); +select ok(not has_function_privilege('authenticated', + 'agent_memory.search_memories(text, extensions.vector, text, uuid[], text, text[], text[], timestamptz, timestamptz, int)', 'execute'), + 'authenticated cannot execute the 10-argument search_memories'); + +select set_config('agent_memory.member_id', '11111111-1111-1111-1111-111111111111', true); +set local role agent_memory_member; + +select is(pg_temp.ids('x', pg_temp.v(1), 'semantic', null, '{status:open,owner:me}', 50), + array['aaaaaaaa-0000-0000-0000-000000000001']::uuid[], 'semantic: tags_all requires every tag'); +select is(pg_temp.ids('heat pump', null, 'keyword', null, '{status:open,owner:me}', 50), + array['aaaaaaaa-0000-0000-0000-000000000001']::uuid[], 'keyword: tags_all requires every tag'); +select is(pg_temp.ids('heat pump', pg_temp.v(1), 'hybrid', null, '{status:open,owner:me}', 50), + array['aaaaaaaa-0000-0000-0000-000000000001']::uuid[], 'hybrid: tags_all requires every tag'); +select is(pg_temp.ids('zzzz', pg_temp.v(1), 'hybrid', null, '{status:open,owner:me}', 1), + array['aaaaaaaa-0000-0000-0000-000000000001']::uuid[], 'hybrid tags_all filter applies before limit'); +select is(pg_temp.ids('heat pump', null, 'keyword', null, '{status:open,owner:me}', 1), + array['aaaaaaaa-0000-0000-0000-000000000001']::uuid[], 'keyword tags_all filter applies before limit'); +select is(pg_temp.ids('heat pump', pg_temp.v(1), 'hybrid', null, '{status:open,owner:me}', 1), + array['aaaaaaaa-0000-0000-0000-000000000001']::uuid[], 'hybrid (both arms matching) tags_all filter applies before limit'); +select is(pg_temp.ids('x', pg_temp.v(1), 'semantic', null, '{status:open,owner:me}', 1), + array['aaaaaaaa-0000-0000-0000-000000000001']::uuid[], 'semantic tags_all filter applies before limit'); +select is(pg_temp.ids('x', pg_temp.v(1), 'semantic', '{status:open,status:done}', '{owner:me}', 50), + array['aaaaaaaa-0000-0000-0000-000000000001', 'aaaaaaaa-0000-0000-0000-000000000002']::uuid[], + 'tags (any) and tags_all (every) combine'); +select is(pg_temp.ids('x', pg_temp.v(1), 'semantic', null, '{owner:nobody}', 50), '{}'::uuid[], + 'tags_all with a tag no memory has returns nothing'); +select ok(not ('bbbbbbbb-0000-0000-0000-000000000005' = any (pg_temp.ids('x', pg_temp.v(1), 'semantic', null, '{owner:me}', 50))), + 'tags_all does not reach an unreadable scope'); +select is(pg_temp.ids('x', pg_temp.v(1), 'semantic', null, '{}', 50), pg_temp.ids('x', pg_temp.v(1), 'semantic', null, null, 50), + 'an empty tags_all is no filter'); +select is((select coalesce(array_agg(id order by id), '{}') from agent_memory.search_memories('heat pump', pg_temp.v(1), 'hybrid', null, null, '{owner:me}', null, null, 50)), + pg_temp.ids('heat pump', pg_temp.v(1), 'hybrid', '{owner:me}', null, 50), + 'the 9-argument search_memories still works and matches tags_all null'); + +reset role; +select * from finish(); +rollback; diff --git a/tests/integration/pg-store-read.test.ts b/tests/integration/pg-store-read.test.ts index 8604f2b..4bbc76e 100644 --- a/tests/integration/pg-store-read.test.ts +++ b/tests/integration/pg-store-read.test.ts @@ -228,6 +228,20 @@ Deno.test('soft-deleted memories are invisible to every read', () => await assertRejects(() => rw.findRelated(gone.id, 5), Error, `Memory ${gone.id} not found`); })); +Deno.test('tagsAll requires every tag, and combines with tags', () => + withWorld(async (w, _a, mk) => { + const [s] = mk(w.rw.id); + const a = await s.store(w.alpha.name, req('okapi status', ['status:open', 'owner:me'])); + const b = await s.store(w.alpha.name, req('okapi status two', ['status:done', 'owner:me'])); + await s.store(w.alpha.name, req('okapi status three', ['status:open'])); + const ids = async (f: Parameters[2]) => + (await s.search('okapi status', 'keyword', f)).map((r) => r.memory.id).sort(); + assertEquals(await ids({ tagsAll: ['status:open', 'owner:me'] }), [a.id]); + assertEquals(await ids({ tags: ['status:open', 'status:done'], tagsAll: ['owner:me'] }), [a.id, b.id].sort()); + assertEquals(await ids({ tagsAll: ['owner:nobody'] }), []); + assertEquals((await ids({ tagsAll: [] })).length, 3); + })); + Deno.test('empty tags filter means no filter', () => withWorld(async (w, _a, mk) => { const [s] = mk(w.rw.id); diff --git a/tests/unit/tools.test.ts b/tests/unit/tools.test.ts index bfd124b..a08f9ea 100644 --- a/tests/unit/tools.test.ts +++ b/tests/unit/tools.test.ts @@ -169,6 +169,30 @@ Deno.test('non-empty scopes are passed through', async () => { assertEquals((calls.search[0][2] as SearchFilters).scopes, ['a']); }); +Deno.test('handleSearch passes tags and tags_all through as separate filters', async () => { + const { store, calls } = mockStore(); + await handleSearch(store)({ query: 'q', tags: ['a', 'b'], tags_all: ['c', 'd'] }); + const f = calls.search[0][2] as SearchFilters; + assertEquals(f.tags, ['a', 'b']); + assertEquals(f.tagsAll, ['c', 'd']); +}); + +Deno.test('the registered search tool accepts tags_all', () => { + const { store } = mockStore(); + const { embedder } = mockEmbedder(); + const server = new McpServer({ name: 'test', version: '0.0.0' }); + registerTools(server, store, embedder, noThreads as never); + const registered = + (server as unknown as { _registeredTools: Record unknown } }> }) + ._registeredTools; + const parsed = registered.search.inputSchema.parse({ query: 'q', tags: ['a'], tags_all: ['b', 'c'] }) as Record< + string, + unknown + >; + assertEquals(parsed.tags_all, ['b', 'c']); + assertEquals(parsed.tags, ['a']); +}); + Deno.test('handleUpdate with only tags passes only tags', async () => { const { store, calls } = mockStore(); await handleUpdate(store)({ id: 'i', tags: ['x'] });