From 1f464c544a9d1db8919c762201d558a65728d44d Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Mon, 12 Oct 2026 12:54:54 -0700 Subject: [PATCH 01/22] feat(analyst): `sql_check predicate_effect` reports what each filter removes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Analysts are told that a filter which removes nothing usually means the wrong column, but there was no cheap way to verify it. `sql_check` gains an opt-in `predicate_effect` check: given an explicit read-only population (`base_sql`, a SELECT or a table reference) and one to eight boolean predicates, it runs a single aggregate probe SELECT COUNT(*), COUNT(*) FILTER (WHERE p1), …, COUNT(*) FILTER (WHERE p1 AND …) and reports rows kept and removed per predicate plus the combined count, flagging `this predicate removes no rows at this population`. NULL counts as not kept and the output says so; an empty population is reported as such. - No rewriting of the caller's WHERE clause; predicates are validated as bare expressions through the existing read-only validator and the connection's dialect, and all supplied SQL goes through the `sql_execute` permission request before routing. - Bounded: SQLite, DuckDB and Postgres only; a catalog row estimate above `max_rows_scanned` (default 5,000,000) for a bare table refuses with a suggestion to narrow the population; query timeouts return `skipped: timeout` instead of failing the caller. - Analyst prompt: the existing "every clause must bite" sentence points at the check and says a vacuous filter is a reason to profile an alternative column, not to invent an exclusion. Co-Authored-By: Claude Fable 5.1 --- .../opencode/src/altimate/prompts/analyst.txt | 2 +- .../opencode/src/altimate/tools/sql-check.ts | 158 +++++- .../test/agent/analyst-data-explorer.test.ts | 9 + .../sql-check-predicate-effect.test.ts | 473 ++++++++++++++++++ 4 files changed, 638 insertions(+), 4 deletions(-) create mode 100644 packages/opencode/test/altimate/sql-check-predicate-effect.test.ts diff --git a/packages/opencode/src/altimate/prompts/analyst.txt b/packages/opencode/src/altimate/prompts/analyst.txt index 71c9006dd..ba90b13aa 100644 --- a/packages/opencode/src/altimate/prompts/analyst.txt +++ b/packages/opencode/src/altimate/prompts/analyst.txt @@ -1,6 +1,6 @@ You are altimate-code: a read-only analyst. Do not modify source data or files. When available, `sql_materialize` may write session-local `scratch` tables. -Inspect connections, schemas and actual rows before joining or parsing; key formats may differ across databases. Establish population, grain, denominator and assumptions. Every clause of the question must bite: a filter that removes nothing usually means the wrong column. When a phrase admits two readings (which population, denominator, transform or owner column), compute both, then choose by the question's own definitions and the data's shape; keep that reasoning out of the final message. Use `sql_check` for keys, fan-out, grain, nulls or extraction coverage when they affect the answer; trivial queries need no ritual. `sql_analyze` is optional. +Inspect connections, schemas and actual rows before joining or parsing; key formats may differ across databases. Establish population, grain, denominator and assumptions. Every clause of the question must bite: a filter that removes nothing usually means the wrong column. (`sql_check` `predicate_effect` reports what each filter removes; a filter that removes nothing at the right population is a reason to profile an alternative column, not to invent an exclusion) When a phrase admits two readings (which population, denominator, transform or owner column), compute both, then choose by the question's own definitions and the data's shape; keep that reasoning out of the final message. Use `sql_check` for keys, fan-out, grain, nulls or extraction coverage when they affect the answer; trivial queries need no ritual. `sql_analyze` is optional. Derive large intermediate sets in-engine. For a cross-engine join, move the smaller side: a few hundred keys go into an IN list or VALUES against the big table; materialize into `scratch` only the side that is small after filtering, never a multi-million-row table when a filter or aggregate could shrink it first. Preserve requested weighting: group means weighted by contributing counts can be exact; unweighted averages change the weighting. For many similar tables, prefer catalog-driven SQL (information_schema, DuckDB query_table/UNION ALL BY NAME) after confirming dialect support. diff --git a/packages/opencode/src/altimate/tools/sql-check.ts b/packages/opencode/src/altimate/tools/sql-check.ts index 4f554ac18..a8e60653c 100644 --- a/packages/opencode/src/altimate/tools/sql-check.ts +++ b/packages/opencode/src/altimate/tools/sql-check.ts @@ -1,4 +1,5 @@ import z from "zod" +import { isQueryTimeoutError } from "@altimateai/drivers/types" import { Tool } from "../../tool/tool" import * as Precedence from "../workspace/precedence" import * as Registry from "../native/connections/registry" @@ -12,7 +13,23 @@ const column = z const parameters = z .object({ warehouse: z.string().min(1), - check: z.enum(["fanout", "keys", "grain", "nulls", "coverage"]), + check: z.enum(["fanout", "keys", "grain", "nulls", "coverage", "predicate_effect"]), + base_sql: z + .string() + .optional() + .describe("Population before filtering: read-only SELECT or bare table reference for predicate_effect"), + predicates: z + .array(z.string().min(1)) + .min(1) + .max(8) + .optional() + .describe("1–8 boolean expressions over base_sql columns"), + max_rows_scanned: z + .number() + .int() + .positive() + .optional() + .describe("Bare-table catalog row estimate cap for predicate_effect (default 5000000); not a SELECT row limit"), sql: z.string().optional().describe("Read-only SELECT for grain, nulls or coverage"), columns: z.array(column).min(1).max(20).optional().describe("Output column names for grain or nulls (up to 20)"), column: column.optional().describe("Output column name for coverage"), @@ -43,7 +60,18 @@ export type SqlCheckInput = z.input const publicParameters = z .object({ warehouse: z.string().min(1), - check: z.enum(["fanout", "keys", "grain", "nulls", "coverage"]), + check: z.enum(["fanout", "keys", "grain", "nulls", "coverage", "predicate_effect"]), + base_sql: z + .string() + .optional() + .describe("Population before filtering: read-only SELECT or bare table reference for predicate_effect"), + predicates: z.array(z.string()).max(8).optional().describe("1–8 boolean expressions over base_sql columns"), + max_rows_scanned: z + .number() + .int() + .positive() + .optional() + .describe("Bare-table catalog row estimate cap for predicate_effect (default 5000000); not a SELECT row limit"), sql: z.string().optional().describe("Read-only SELECT for grain, nulls or coverage"), columns: z.array(z.string()).max(20).optional().describe("Output column names for grain or nulls (up to 20)"), column: z.string().optional().describe("Output column name for coverage"), @@ -145,6 +173,80 @@ function result(check: string, metadata: Row, lines: string[]): { title: string; return { title: `SQL check: ${check}`, metadata: { check, ...metadata }, output: lines.join("\n") } } +/** Only identifiers count as bare tables; SELECT populations are never rewritten. */ +function readBase(sql: string, dialect?: string) { + const bare = + /^(?:"(?:[^"]|"")+"|[A-Za-z_][A-Za-z0-9_$]*)(?:\s*\.\s*(?:"(?:[^"]|"")+"|[A-Za-z_][A-Za-z0-9_$]*)){0,2}$/.test( + sql.trim(), + ) + const table = bare ? sql.trim() : undefined + return { sql: readSql(table ? `SELECT * FROM ${table}` : sql, dialect), table } +} + +function readPredicate(predicate: string, dialect?: string): string { + readSql(`SELECT 1 WHERE (\n${predicate}\n)`, dialect) + // A read-only wrapper alone can still accept an expression that closes it and + // introduces another SELECT. Require the expression's own delimiters to balance. + let depth = 0 + for (const char of maskSql(tokenizeSql(predicate, dialect).tokens)) { + if (char === "(") depth++ + if (char === ")" && --depth < 0) throw new Error("sql_check predicates must be bare boolean expressions") + } + if (depth !== 0) throw new Error("sql_check predicates must be bare boolean expressions") + return `(\n${predicate}\n)` +} + +/** Catalog reads only; never scan the population to obtain a guard estimate. */ +async function tableEstimate(table: string, dialect: Dialect, run: (sql: string) => Promise) { + try { + if (dialect === "postgres") { + const [row] = await run(`SELECT reltuples AS estimate FROM pg_catalog.pg_class + WHERE oid = pg_catalog.to_regclass(E${literal(table.replace(/\\/g, "\\\\"))}) AND reltuples >= 0`) + return row ? number(row.estimate) : undefined + } + const parts = table + .match(/"(?:[^"]|"")+"|[A-Za-z_][A-Za-z0-9_$]*/g)! + .map((part) => (part.startsWith('"') ? part.slice(1, -1).replaceAll('""', '"') : part)) + const name = literal(parts.at(-1)!) + if (dialect === "duckdb") { + const matches = (column: string, value: string) => `LOWER(${column}) = LOWER(${literal(value)})` + const qualifier = + parts.length === 3 + ? `${matches("database_name", parts[0])} AND ${matches("schema_name", parts[1])}` + : parts.length === 2 + ? `((${matches("schema_name", parts[0])} AND database_name IN (current_database(), 'temp')) + OR (${matches("database_name", parts[0])} AND schema_name = 'main'))` + : "LOWER(schema_name) IN (SELECT LOWER(unnest(current_schemas(true))))" + const rows = await run(`SELECT estimate FROM ( + SELECT database_name, schema_name, table_name, estimated_size AS estimate FROM duckdb_tables() + UNION ALL SELECT database_name, schema_name, view_name AS table_name, NULL AS estimate FROM duckdb_views() + ) relations WHERE LOWER(table_name) = LOWER(${name}) AND (${qualifier})`) + // Search paths and temporary tables can shadow another catalog match. + // An ambiguous estimate is unknown, rather than a reason to refuse a probe. + return rows.length === 1 && rows[0].estimate != null ? number(rows[0].estimate) : undefined + } + if (parts.length > 2) return undefined + const schema = + parts.length === 2 + ? parts[0] + : ( + await run(`SELECT t.schema FROM pragma_table_list(${name}) t + JOIN pragma_database_list d ON d.name = t.schema + ORDER BY CASE WHEN t.schema = 'temp' THEN -1 ELSE d.seq END LIMIT 1`) + )[0]?.schema + if (!schema) return undefined + const rows = await run(`SELECT stat FROM ${ident(schema)}.sqlite_stat1 WHERE tbl = ${name} COLLATE NOCASE`) + const estimates = rows + .map((row) => Number(String(row.stat).split(/\s+/)[0])) + .filter((n) => Number.isFinite(n) && n >= 0) + return estimates.length ? Math.max(...estimates) : undefined + } catch (error) { + if (isQueryTimeoutError(error)) throw error + // Missing ANALYZE statistics or unavailable catalog access means unknown. + return undefined + } +} + /** Counts run in the warehouse; only aggregate rows and at most five samples per side leave it. */ export async function checkSql(input: SqlCheckPublicInput) { const args = parameters.parse(dropEmpty(input)) @@ -156,6 +258,8 @@ export async function checkSql(input: SqlCheckPublicInput) { const sql = args.sql === undefined ? undefined : readSql(args.sql, sourceDialect) const left = args.left_sql === undefined ? undefined : readSql(args.left_sql, sourceDialect) const right = args.right_sql === undefined ? undefined : readSql(args.right_sql, sourceDialect) + const base = args.base_sql === undefined ? undefined : readBase(args.base_sql, sourceDialect) + const predicates = args.predicates?.map((predicate) => readPredicate(predicate, sourceDialect)) // Keep internal CTEs from shadowing any relation in the supplied queries. let scope = "__sql_check_0_" for (let i = 1; [sql, left, right].some((s) => s?.toLowerCase().includes(scope)); i++) scope = `__sql_check_${i}_` @@ -179,6 +283,49 @@ export async function checkSql(input: SqlCheckPublicInput) { const two = () => `WITH ${scope}lhs AS (\n${required(left, "left_sql")}\n), ${scope}rhs AS (\n${required(right, "right_sql")}\n)` + if (args.check === "predicate_effect") { + const population = required(base, "base_sql") + const expressions = required(predicates, "predicates") + try { + const estimate = population.table === undefined ? undefined : await tableEstimate(population.table, dialect, run) + const cap = args.max_rows_scanned ?? 5_000_000 + if (estimate !== undefined && estimate > cap) { + throw new Error( + `sql_check population estimate ${estimate} exceeds max_rows_scanned ${cap}; use a narrower base_sql.`, + ) + } + const count = (predicate: string) => `COUNT(*) FILTER (WHERE ${predicate})` + const [row] = await run(`SELECT COUNT(*) AS total, + ${expressions.map((predicate, i) => `${count(predicate)} AS keep_${i + 1}`).join(", ")}, + ${count(expressions.join(" AND "))} AS keep_all FROM (\n${population.sql}\n) t`) + const total = number(row.total) + const effects = args.predicates!.map((predicate, i) => { + const kept = number(row[`keep_${i + 1}`]) + return { + predicate, + kept_rows: kept, + removed_rows: total - kept, + ...(total > 0 && total === kept ? { warning: "this predicate removes no rows at this population" } : {}), + } + }) + const combined = number(row.keep_all) + return result(args.check, { total_rows: total, predicates: effects, combined_kept_rows: combined }, [ + `predicate_effect: ${total ? `${total} rows in population` : "empty population"}`, + ...effects.map( + (effect, i) => + `predicate ${i + 1} ${display(effect.predicate)}: kept ${effect.kept_rows}; removed ${effect.removed_rows}${effect.warning ? `; ${effect.warning}` : ""}`, + ), + `combined (AND): kept ${combined}; removed ${total - combined}`, + "A predicate that returns NULL counts as not kept.", + ]) + } catch (error) { + if (!isQueryTimeoutError(error)) throw error + return result(args.check, { skipped: "timeout" }, [ + "predicate_effect: skipped: timeout; use a narrower base_sql.", + ]) + } + } + if (args.check === "fanout") { const on = required(args.on, "on") const names = on.map((_pair, i) => `k${i}`) @@ -389,10 +536,12 @@ async function checkKeys( export const SqlCheckTool = Tool.define("sql_check", { description: - "Check SQL result invariants: fanout (join defaults to inner), keys, grain, nulls, or extraction coverage. " + + "Check SQL result invariants: fanout (join defaults to inner), keys, grain, nulls, extraction coverage, or predicate_effect. " + "Read-only SELECTs run in one SQLite, DuckDB or Postgres warehouse, with their original LIMITs preserved; samples cap at five. " + "Use sql_materialize to put cross-warehouse or other-engine sources into scratch first. " + "Key normalization is diagnostic and may collapse unrelated values; no automatic fixes. Coverage uses regex except SQLite's explicit LIKE fallback. " + + "predicate_effect takes base_sql before filtering and 1–8 explicit predicates; reports individual and combined removals, with NULL not kept. " + + "Bare-table estimates above max_rows_scanned (default 5000000) are refused; unknown estimates use the query timeout and expiry is reported as skipped. " + "Leave fields a check does not use empty.", parameters: publicParameters, async execute(args, ctx) { @@ -401,6 +550,9 @@ export const SqlCheckTool = Tool.define("sql_check", { const input = dropEmpty(args) const dialect = Registry.canonicalType(Registry.getConfig(input.warehouse)?.type) ?? undefined const queries = [input.sql, input.left_sql, input.right_sql].filter((sql): sql is string => sql !== undefined) + if (input.base_sql !== undefined) queries.push(readBase(input.base_sql, dialect).sql) + for (const predicate of input.predicates ?? []) + queries.push(`SELECT 1 WHERE ${readPredicate(predicate, dialect)}`) for (const sql of queries) readSql(sql, dialect) await ctx.ask({ permission: "sql_execute", diff --git a/packages/opencode/test/agent/analyst-data-explorer.test.ts b/packages/opencode/test/agent/analyst-data-explorer.test.ts index 89c81d91c..9ceac1b39 100644 --- a/packages/opencode/test/agent/analyst-data-explorer.test.ts +++ b/packages/opencode/test/agent/analyst-data-explorer.test.ts @@ -23,6 +23,15 @@ it.instance("analyst can load skills", () => }), ) +it.instance("analyst explains how to use predicate_effect without inventing exclusions", () => + Effect.gen(function* () { + const analyst = yield* Agent.use.get("analyst") + expect(analyst?.prompt).toContain( + "Every clause of the question must bite: a filter that removes nothing usually means the wrong column. (`sql_check` `predicate_effect` reports what each filter removes; a filter that removes nothing at the right population is a reason to profile an alternative column, not to invent an exclusion)", + ) + }), +) + it.instance("analyst may delegate only to data-explorer", () => Effect.gen(function* () { const analyst = yield* Agent.use.get("analyst") diff --git a/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts b/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts new file mode 100644 index 000000000..df60dc2b9 --- /dev/null +++ b/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts @@ -0,0 +1,473 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, spyOn, test } from "bun:test" +import { QueryTimeoutError } from "@altimateai/drivers/types" +import * as Registry from "../../src/altimate/native/connections/registry" +import { closeScratch } from "../../src/altimate/native/connections/scratch" +import * as Precedence from "../../src/altimate/workspace/precedence" +import { checkSql, SqlCheckTool } from "../../src/altimate/tools/sql-check" +import { initTool } from "./tool-fixture" + +const warehouses = ["sqlite_predicates", "scratch"] as const +const baseSql = "SELECT * FROM orders" +const noRemoval = "this predicate removes no rows at this population" +let previousTelemetry: string | undefined + +beforeAll(() => { + previousTelemetry = process.env.ALTIMATE_TELEMETRY_DISABLED + process.env.ALTIMATE_TELEMETRY_DISABLED = "true" +}) + +beforeEach(async () => { + Registry.setConfigs({ sqlite_predicates: { type: "sqlite", path: ":memory:" } }) + await closeScratch() + for (const warehouse of warehouses) { + const connector = await Registry.get(warehouse) + await connector.execute("CREATE TABLE orders (id INTEGER, kind TEXT, amount INTEGER, note TEXT)") + await connector.execute( + "INSERT INTO orders VALUES (1, 'a', 5, 'first'), (2, 'b', 7, NULL), (3, 'a', 1, NULL), (4, 'z', 3, 'tail'), (5, 'a', NULL, NULL)", + ) + } +}) + +afterEach(async () => { + await Registry.closeAll() + await closeScratch() +}) + +afterAll(() => { + if (previousTelemetry === undefined) delete process.env.ALTIMATE_TELEMETRY_DISABLED + else process.env.ALTIMATE_TELEMETRY_DISABLED = previousTelemetry +}) + +for (const warehouse of warehouses) { + describe(`predicate_effect with ${warehouse === "scratch" ? "DuckDB" : "SQLite"}`, () => { + test("reports independent removals and flags only predicates that remove no rows", async () => { + const connector = await Registry.get(warehouse) + const execute = spyOn(connector, "execute") + try { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: baseSql, + predicates: ["id > 0", "kind = 'a'"], + }) + expect(result.metadata).toMatchObject({ + check: "predicate_effect", + total_rows: 5, + predicates: [ + { predicate: "id > 0", kept_rows: 5, removed_rows: 0, warning: noRemoval }, + { predicate: "kind = 'a'", kept_rows: 3, removed_rows: 2 }, + ], + combined_kept_rows: 3, + }) + expect(result.metadata.predicates[1].warning).toBeUndefined() + expect(result.output).toContain(noRemoval) + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][3]).toMatchObject({ noLimit: true }) + } finally { + execute.mockRestore() + } + }) + + test("counts NULL-valued predicates as not kept and explains that rule", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: baseSql, + predicates: ["amount > 0", "note = 'first'"], + }) + expect(result.metadata).toMatchObject({ + total_rows: 5, + predicates: [ + { kept_rows: 4, removed_rows: 1 }, + { kept_rows: 1, removed_rows: 4 }, + ], + combined_kept_rows: 1, + }) + expect(result.output).toMatch(/NULL.*not kept/i) + }) + + test("reports an empty population without flagging predicates", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: `${baseSql} WHERE 1 = 0`, + predicates: ["id > 0", "amount > 10"], + }) + expect(result.metadata).toMatchObject({ + total_rows: 0, + predicates: [ + { kept_rows: 0, removed_rows: 0 }, + { kept_rows: 0, removed_rows: 0 }, + ], + combined_kept_rows: 0, + }) + expect(result.metadata.predicates.every((predicate: { warning?: string }) => !predicate.warning)).toBe(true) + expect(result.output).toContain("empty population") + expect(result.output).not.toContain(noRemoval) + }) + + test("combined count matches direct AND filtering while preserving OR precedence", async () => { + const predicates = ["kind = 'a' OR kind = 'b'", "amount > 4"] + const result = await checkSql({ warehouse, check: "predicate_effect", base_sql: baseSql, predicates }) + const connector = await Registry.get(warehouse) + const direct = await connector.execute( + `SELECT COUNT(*) AS kept FROM (${baseSql}) t WHERE (${predicates[0]}) AND (${predicates[1]})`, + ) + expect(result.metadata.combined_kept_rows).toBe(Number(direct.rows[0][0])) + expect(result.metadata).toMatchObject({ + predicates: [ + { kept_rows: 4, removed_rows: 1 }, + { kept_rows: 2, removed_rows: 3 }, + ], + combined_kept_rows: 2, + }) + }) + + test("accepts a bare table reference as the population", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 2"], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + }) + + test("refuses a bare table whose catalog estimate exceeds the custom cap", async () => { + const connector = await Registry.get(warehouse) + if (warehouse === "sqlite_predicates") await connector.execute("ANALYZE orders") + const execute = spyOn(connector, "execute") + try { + await expect( + checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 0"], + max_rows_scanned: 4, + }), + ).rejects.toThrow(/narrower.*base_sql/i) + expect(execute.mock.calls.some(([query]) => /COUNT\(\*\)\s+FILTER/i.test(query))).toBe(false) + } finally { + execute.mockRestore() + } + }) + + test("applies the estimate guard only to bare table populations", async () => { + const connector = await Registry.get(warehouse) + const execute = spyOn(connector, "execute") + try { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: baseSql, + predicates: ["id > 0"], + max_rows_scanned: 1, + }) + expect(result.metadata.total_rows).toBe(5) + expect(execute).toHaveBeenCalledTimes(1) + } finally { + execute.mockRestore() + } + }) + + test("accepts eight predicates and preserves an explicit population LIMIT", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: `${baseSql} ORDER BY id LIMIT 2`, + predicates: Array.from({ length: 8 }, (_, i) => `id > ${i}`), + }) + expect(result.metadata.total_rows).toBe(2) + expect(result.metadata.predicates).toHaveLength(8) + expect(result.metadata.combined_kept_rows).toBe(0) + }) + + test("rejects nine predicates before query execution", async () => { + const connector = await Registry.get(warehouse) + const execute = spyOn(connector, "execute") + try { + await expect( + checkSql({ warehouse, check: "predicate_effect", base_sql: baseSql, predicates: Array(9).fill("id > 0") }), + ).rejects.toThrow(/8/) + expect(execute).not.toHaveBeenCalled() + } finally { + execute.mockRestore() + } + }) + + test("requires at least one predicate before query execution", async () => { + const connector = await Registry.get(warehouse) + const execute = spyOn(connector, "execute") + try { + await expect( + checkSql({ warehouse, check: "predicate_effect", base_sql: baseSql, predicates: [] }), + ).rejects.toThrow(/predicate/i) + expect(execute).not.toHaveBeenCalled() + } finally { + execute.mockRestore() + } + }) + + test("rejects writes and statement escapes inside predicates before query execution", async () => { + const connector = await Registry.get(warehouse) + const execute = spyOn(connector, "execute") + try { + for (const predicate of [ + "1=1); DELETE FROM orders; --", + "1=1) UNION SELECT 1 --", + "1=1) UNION SELECT (1", + "1=1) OR (1=1", + ]) { + await expect( + checkSql({ warehouse, check: "predicate_effect", base_sql: baseSql, predicates: [predicate] }), + ).rejects.toThrow(/read|SELECT|expression|predicate/i) + } + expect(execute).not.toHaveBeenCalled() + } finally { + execute.mockRestore() + } + }) + + test("rejects a write population before query execution", async () => { + const connector = await Registry.get(warehouse) + const execute = spyOn(connector, "execute") + try { + for (const base_sql of ["DELETE FROM orders", "SELECT * FROM orders; DELETE FROM orders"]) { + await expect( + checkSql({ warehouse, check: "predicate_effect", base_sql, predicates: ["id > 0"] }), + ).rejects.toThrow(/read|SELECT/i) + } + expect(execute).not.toHaveBeenCalled() + } finally { + execute.mockRestore() + } + }) + + test("accepts quoted semicolons and a read-only scalar expression", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "SELECT 'a;b' AS value", + predicates: ["REPLACE(value, ';', '') = 'ab'"], + }) + expect(result.metadata).toMatchObject({ total_rows: 1, combined_kept_rows: 1 }) + }) + + test("preserves trailing comments in the population and predicate", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: `${baseSql} -- population comment`, + predicates: ["id > 2 -- predicate comment"], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + }) + + test("propagates an unexpected query error instead of reporting a timeout", async () => { + const connector = await Registry.get(warehouse) + const execute = spyOn(connector, "execute").mockRejectedValue(new Error("unexpected query failure")) + try { + await expect( + checkSql({ warehouse, check: "predicate_effect", base_sql: baseSql, predicates: ["id > 0"] }), + ).rejects.toThrow("unexpected query failure") + expect(execute).toHaveBeenCalledTimes(1) + } finally { + execute.mockRestore() + } + }) + + test("skips without a probe when a catalog lookup times out", async () => { + const connector = await Registry.get(warehouse) + const execute = spyOn(connector, "execute").mockRejectedValue( + new QueryTimeoutError({ engine: warehouse === "scratch" ? "DuckDB" : "SQLite", timeoutMs: 4321 }), + ) + try { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 0"], + query_timeout_ms: 4321, + }) + expect(result.metadata).toMatchObject({ skipped: "timeout" }) + expect(result.metadata.total_rows).toBeUndefined() + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][0]).not.toMatch(/COUNT\(\*\)\s+FILTER/i) + expect(execute.mock.calls[0][3]).toMatchObject({ noLimit: true, queryTimeoutMs: 4321 }) + } finally { + execute.mockRestore() + } + }) + + test("returns skipped timeout without a verdict when the query expires", async () => { + const connector = await Registry.get(warehouse) + const execute = spyOn(connector, "execute").mockRejectedValue( + new QueryTimeoutError({ engine: warehouse === "scratch" ? "DuckDB" : "SQLite", timeoutMs: 4321 }), + ) + try { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: baseSql, + predicates: ["id > 0"], + query_timeout_ms: 4321, + }) + expect(result.metadata).toMatchObject({ check: "predicate_effect", skipped: "timeout" }) + expect(result.metadata.error).toBeUndefined() + expect(result.metadata.total_rows).toBeUndefined() + expect(result.output).toContain("skipped: timeout") + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][3]).toMatchObject({ noLimit: true, queryTimeoutMs: 4321 }) + } finally { + execute.mockRestore() + } + }) + }) +} + +test("DuckDB predicate_effect respects a view shadowing a larger table on the schema search path", async () => { + const connector = await Registry.get("scratch") + await connector.execute("CREATE SCHEMA smaller") + await connector.execute("CREATE VIEW smaller.orders AS SELECT * FROM main.orders LIMIT 1") + await connector.execute("SET schema = 'smaller'") + const result = await checkSql({ + warehouse: "scratch", + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 0"], + max_rows_scanned: 2, + }) + expect(result.metadata).toMatchObject({ total_rows: 1, combined_kept_rows: 1 }) +}) + +test("DuckDB predicate_effect guards a two-part catalog and table reference", async () => { + const connector = await Registry.get("scratch") + const database = await connector.execute("SELECT current_database()") + const catalog = String(database.rows[0][0]).replaceAll('"', '""') + const execute = spyOn(connector, "execute") + try { + await expect( + checkSql({ + warehouse: "scratch", + check: "predicate_effect", + base_sql: `"${catalog}".orders`, + predicates: ["id > 0"], + max_rows_scanned: 2, + }), + ).rejects.toThrow(/narrower.*base_sql/i) + expect(execute.mock.calls.some(([query]) => /COUNT\(\*\)\s+FILTER/i.test(query))).toBe(false) + } finally { + execute.mockRestore() + } +}) + +test("DuckDB predicate_effect resolves mixed-case schema names on the search path", async () => { + const connector = await Registry.get("scratch") + await connector.execute('CREATE SCHEMA "Foo"') + await connector.execute('CREATE TABLE "Foo"."Orders" AS SELECT * FROM main.orders') + await connector.execute("DROP TABLE main.orders") + await connector.execute("SET schema = 'foo'") + const execute = spyOn(connector, "execute") + try { + await expect( + checkSql({ + warehouse: "scratch", + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 0"], + max_rows_scanned: 2, + }), + ).rejects.toThrow(/narrower.*base_sql/i) + expect(execute.mock.calls.some(([query]) => /COUNT\(\*\)\s+FILTER/i.test(query))).toBe(false) + } finally { + execute.mockRestore() + } +}) + +test("SQLite predicate_effect respects a temporary table shadowing larger main-table statistics", async () => { + const connector = await Registry.get("sqlite_predicates") + await connector.execute("ANALYZE main.orders") + await connector.execute("CREATE TEMP TABLE orders AS SELECT * FROM main.orders LIMIT 1") + const result = await checkSql({ + warehouse: "sqlite_predicates", + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 0"], + max_rows_scanned: 2, + }) + expect(result.metadata).toMatchObject({ total_rows: 1, combined_kept_rows: 1 }) +}) + +test("SQLite predicate_effect defaults the bare-table estimate cap to five million", async () => { + const connector = await Registry.get("sqlite_predicates") + await connector.execute("ANALYZE orders") + await connector.execute("UPDATE sqlite_stat1 SET stat = '5000001' WHERE tbl = 'orders'") + const execute = spyOn(connector, "execute") + try { + await expect( + checkSql({ + warehouse: "sqlite_predicates", + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 0"], + }), + ).rejects.toThrow(/narrower.*base_sql/i) + expect(execute.mock.calls.some(([query]) => /COUNT\(\*\)\s+FILTER/i.test(query))).toBe(false) + } finally { + execute.mockRestore() + } +}) + +test("predicate_effect returns the existing unsupported-dialect message before connecting", async () => { + Registry.setConfigs({ unsupported: { type: "mysql", host: "unused" } }) + const get = spyOn(Registry, "get") + try { + await expect( + checkSql({ + warehouse: "unsupported", + check: "predicate_effect", + base_sql: "SELECT 1 AS id", + predicates: ["id > 0"], + }), + ).rejects.toThrow("sql_check supports SQLite, DuckDB and Postgres; use sql_materialize") + expect(get).not.toHaveBeenCalled() + } finally { + get.mockRestore() + } +}) + +test("sql_execute denied prevents predicate_effect routing and query execution", async () => { + const connector = await Registry.get("sqlite_predicates") + const execute = spyOn(connector, "execute") + const precedence = spyOn(Precedence, "check") + const asked: { permission: string; patterns: string[] }[] = [] + try { + const tool = await initTool(SqlCheckTool) + const result = await tool.execute( + { warehouse: "sqlite_predicates", check: "predicate_effect", base_sql: baseSql, predicates: ["id > 0"] }, + { + sessionID: "test-session", + messageID: "test-message", + callID: "test-call", + agent: "test", + abort: new AbortController().signal, + messages: [], + metadata: () => {}, + ask: async (request: { permission: string; patterns: string[] }) => { + asked.push(request) + throw new Error("sql_execute denied") + }, + }, + ) + expect(result.output).toContain("sql_execute denied") + expect(asked).toHaveLength(1) + expect(asked[0]).toMatchObject({ permission: "sql_execute" }) + expect(asked[0].patterns.join("\n")).toContain(baseSql) + expect(execute).not.toHaveBeenCalled() + expect(precedence).not.toHaveBeenCalled() + } finally { + execute.mockRestore() + precedence.mockRestore() + } +}) From 52f0e5ed744341b48c7adff09eaf887d09c9d83a Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Mon, 12 Oct 2026 13:24:57 -0700 Subject: [PATCH 02/22] fix(analyst): `predicate_effect` reports unknown estimates, blank inputs and all-row removal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups to the predicate-effect probe: - When the catalog row estimate is unknown (views, SQLite without ANALYZE, unanalysed or partitioned Postgres tables, ambiguous relations) the output says `row_estimate: unknown — probe runs under the query timeout` instead of proceeding silently. - Strict-schema fillers (`max_rows_scanned: 0`, blank predicates, empty `base_sql`) are treated as absent and produce the tool's own message, never a raw schema validation dump. - A predicate that removes every row is flagged; counts over a `LIMIT`ed SELECT population say so. - Postgres branch covered by tests (probe shape, `to_regclass`/`reltuples` guard, unknown-estimate path). - The analyst prompt sentence ends with a period. Co-Authored-By: Claude Fable 5.1 --- .../opencode/src/altimate/prompts/analyst.txt | 2 +- .../opencode/src/altimate/tools/sql-check.ts | 29 ++- .../test/agent/analyst-data-explorer.test.ts | 7 +- .../test/altimate/sql-check-postgres.test.ts | 134 ++++++++++++++ .../sql-check-predicate-effect.test.ts | 165 +++++++++++++++++- 5 files changed, 327 insertions(+), 10 deletions(-) create mode 100644 packages/opencode/test/altimate/sql-check-postgres.test.ts diff --git a/packages/opencode/src/altimate/prompts/analyst.txt b/packages/opencode/src/altimate/prompts/analyst.txt index ba90b13aa..ab915c58b 100644 --- a/packages/opencode/src/altimate/prompts/analyst.txt +++ b/packages/opencode/src/altimate/prompts/analyst.txt @@ -1,6 +1,6 @@ You are altimate-code: a read-only analyst. Do not modify source data or files. When available, `sql_materialize` may write session-local `scratch` tables. -Inspect connections, schemas and actual rows before joining or parsing; key formats may differ across databases. Establish population, grain, denominator and assumptions. Every clause of the question must bite: a filter that removes nothing usually means the wrong column. (`sql_check` `predicate_effect` reports what each filter removes; a filter that removes nothing at the right population is a reason to profile an alternative column, not to invent an exclusion) When a phrase admits two readings (which population, denominator, transform or owner column), compute both, then choose by the question's own definitions and the data's shape; keep that reasoning out of the final message. Use `sql_check` for keys, fan-out, grain, nulls or extraction coverage when they affect the answer; trivial queries need no ritual. `sql_analyze` is optional. +Inspect connections, schemas and actual rows before joining or parsing; key formats may differ across databases. Establish population, grain, denominator and assumptions. Every clause of the question must bite: a filter that removes nothing usually means the wrong column. (`sql_check` `predicate_effect` reports what each filter removes; a filter that removes nothing at the right population is a reason to profile an alternative column, not to invent an exclusion). When a phrase admits two readings (which population, denominator, transform or owner column), compute both, then choose by the question's own definitions and the data's shape; keep that reasoning out of the final message. Use `sql_check` for keys, fan-out, grain, nulls or extraction coverage when they affect the answer; trivial queries need no ritual. `sql_analyze` is optional. Derive large intermediate sets in-engine. For a cross-engine join, move the smaller side: a few hundred keys go into an IN list or VALUES against the big table; materialize into `scratch` only the side that is small after filtering, never a multi-million-row table when a filter or aggregate could shrink it first. Preserve requested weighting: group means weighted by contributing counts can be exact; unweighted averages change the weighting. For many similar tables, prefer catalog-driven SQL (information_schema, DuckDB query_table/UNION ALL BY NAME) after confirming dialect support. diff --git a/packages/opencode/src/altimate/tools/sql-check.ts b/packages/opencode/src/altimate/tools/sql-check.ts index a8e60653c..c2ab31de5 100644 --- a/packages/opencode/src/altimate/tools/sql-check.ts +++ b/packages/opencode/src/altimate/tools/sql-check.ts @@ -69,7 +69,7 @@ const publicParameters = z max_rows_scanned: z .number() .int() - .positive() + .nonnegative() .optional() .describe("Bare-table catalog row estimate cap for predicate_effect (default 5000000); not a SELECT row limit"), sql: z.string().optional().describe("Read-only SELECT for grain, nulls or coverage"), @@ -96,16 +96,22 @@ export type SqlCheckPublicInput = z.input /** * Treat "", [] and null as absent so strict-schema callers can leave unused fields empty. + * Blank predicates are dropped; a zero max_rows_scanned uses the default cap. * A coverage check's `pattern` is the one field where "" is a value (match empty strings). */ export function dropEmpty(input: SqlCheckPublicInput): SqlCheckInput { const out: Record = {} - for (const [key, value] of Object.entries(input)) { + for (const [key, raw] of Object.entries(input)) { + const value = + key === "predicates" && Array.isArray(raw) + ? raw.filter((predicate) => typeof predicate !== "string" || predicate.trim() !== "") + : raw if (key === "pattern" && input.check === "coverage" && value === "") { out[key] = value continue } if (value === "" || value === null || (Array.isArray(value) && value.length === 0)) continue + if (key === "max_rows_scanned" && value === 0) continue out[key] = value } return out as SqlCheckInput @@ -201,7 +207,9 @@ async function tableEstimate(table: string, dialect: Dialect, run: (sql: string) try { if (dialect === "postgres") { const [row] = await run(`SELECT reltuples AS estimate FROM pg_catalog.pg_class - WHERE oid = pg_catalog.to_regclass(E${literal(table.replace(/\\/g, "\\\\"))}) AND reltuples >= 0`) + WHERE oid = pg_catalog.to_regclass(E${literal(table.replace(/\\/g, "\\\\"))}) + AND relkind IN ('r', 'm') AND reltuples > 0`) + // Views/partitioned parents have no usable population estimate; zero can mean no statistics. return row ? number(row.estimate) : undefined } const parts = table @@ -249,7 +257,13 @@ async function tableEstimate(table: string, dialect: Dialect, run: (sql: string) /** Counts run in the warehouse; only aggregate rows and at most five samples per side leave it. */ export async function checkSql(input: SqlCheckPublicInput) { - const args = parameters.parse(dropEmpty(input)) + const parsed = parameters.safeParse(dropEmpty(input)) + if (!parsed.success) { + throw new Error( + `sql_check invalid input: ${parsed.error.issues.map((issue) => `${issue.path.join(".")}: ${issue.message}`).join("; ")}`, + ) + } + const args = parsed.data if ([args.left_warehouse, args.right_warehouse].some((w) => w !== undefined && w !== args.warehouse)) { throw new Error("sql_check runs in one warehouse; use sql_materialize to put both sides in scratch first.") } @@ -306,16 +320,21 @@ export async function checkSql(input: SqlCheckPublicInput) { kept_rows: kept, removed_rows: total - kept, ...(total > 0 && total === kept ? { warning: "this predicate removes no rows at this population" } : {}), + ...(total > 0 && kept === 0 ? { warning: "this predicate removes all rows" } : {}), } }) const combined = number(row.keep_all) return result(args.check, { total_rows: total, predicates: effects, combined_kept_rows: combined }, [ `predicate_effect: ${total ? `${total} rows in population` : "empty population"}`, + ...(estimate === undefined ? ["row_estimate: unknown — probe runs under the query timeout"] : []), + ...(/\bLIMIT\b/i.test(maskSql(tokenizeSql(population.sql, dialect).tokens)) + ? ["counts are over the limited population (base_sql includes LIMIT)"] + : []), ...effects.map( (effect, i) => `predicate ${i + 1} ${display(effect.predicate)}: kept ${effect.kept_rows}; removed ${effect.removed_rows}${effect.warning ? `; ${effect.warning}` : ""}`, ), - `combined (AND): kept ${combined}; removed ${total - combined}`, + `combined (AND): kept ${combined}; removed ${total - combined}${total > 0 && combined === 0 ? "; this combination removes all rows" : ""}`, "A predicate that returns NULL counts as not kept.", ]) } catch (error) { diff --git a/packages/opencode/test/agent/analyst-data-explorer.test.ts b/packages/opencode/test/agent/analyst-data-explorer.test.ts index 9ceac1b39..a86d80d75 100644 --- a/packages/opencode/test/agent/analyst-data-explorer.test.ts +++ b/packages/opencode/test/agent/analyst-data-explorer.test.ts @@ -26,9 +26,10 @@ it.instance("analyst can load skills", () => it.instance("analyst explains how to use predicate_effect without inventing exclusions", () => Effect.gen(function* () { const analyst = yield* Agent.use.get("analyst") - expect(analyst?.prompt).toContain( - "Every clause of the question must bite: a filter that removes nothing usually means the wrong column. (`sql_check` `predicate_effect` reports what each filter removes; a filter that removes nothing at the right population is a reason to profile an alternative column, not to invent an exclusion)", - ) + expect(analyst?.prompt).toContain("a filter that removes nothing usually means the wrong column") + expect(analyst?.prompt).toContain("`sql_check` `predicate_effect` reports what each filter removes") + expect(analyst?.prompt).toContain("a reason to profile an alternative column, not to invent an exclusion") + expect(analyst?.prompt).toContain("not to invent an exclusion).") }), ) diff --git a/packages/opencode/test/altimate/sql-check-postgres.test.ts b/packages/opencode/test/altimate/sql-check-postgres.test.ts new file mode 100644 index 000000000..734038ed2 --- /dev/null +++ b/packages/opencode/test/altimate/sql-check-postgres.test.ts @@ -0,0 +1,134 @@ +import { afterEach, beforeEach, describe, expect, mock, spyOn, test } from "bun:test" +import { Database } from "bun:sqlite" +import type { Connector } from "@altimateai/drivers" +import * as Registry from "../../src/altimate/native/connections/registry" +import { checkSql } from "../../src/altimate/tools/sql-check" + +const warehouse = "postgres_predicates" +const unknownEstimate = "row_estimate: unknown — probe runs under the query timeout" +let database: Database +let connectorSpy: { mockRestore(): void } + +beforeEach(() => { + Registry.setConfigs({ [warehouse]: { type: "postgres" } }) + database = new Database(":memory:") + database.run("CREATE TABLE pg_class (oid INTEGER, relkind TEXT, reltuples REAL)") + database.run("CREATE TABLE orders (id INTEGER, kind TEXT, amount INTEGER)") + database.run("INSERT INTO orders VALUES (1, 'a', 5), (2, 'b', 7), (3, 'a', 1), (4, 'z', 3), (5, 'a', NULL)") +}) + +afterEach(() => { + connectorSpy?.mockRestore() + database.close() + Registry.reset() +}) + +function postgres(relkind = "r", reltuples = 5) { + database.run("INSERT INTO pg_class VALUES (1, ?, ?)", [relkind, reltuples]) + const execute = mock(async (...[sql]: Parameters) => { + // Keep the generated WHERE and FILTER clauses executable: removing either + // guard condition must change results, rather than leave a canned mock row. + const query = sql + .replace("pg_catalog.pg_class", "pg_class") + .replace(/pg_catalog\.to_regclass\(E'(?:[^']|'')*'\)/, "1") + const statement = database.query(query) + const rows = statement.values() + return { columns: statement.columnNames, rows, row_count: rows.length, truncated: false } + }) + connectorSpy = spyOn(Registry, "get").mockResolvedValue({ execute } as unknown as Connector) + return execute +} + +describe("predicate_effect with mocked Postgres", () => { + test("uses independent FILTER counts and a parenthesized AND probe with newline-safe comments", async () => { + const execute = postgres() + const base_sql = "SELECT * FROM orders -- population comment" + const predicates = ["kind = 'a' OR kind = 'b' -- first predicate", "amount > 4 -- second predicate"] + const result = await checkSql({ warehouse, check: "predicate_effect", base_sql, predicates, query_timeout_ms: 4321 }) + expect(result.metadata).toMatchObject({ + total_rows: 5, + predicates: [ + { kept_rows: 4, removed_rows: 1 }, + { kept_rows: 2, removed_rows: 3 }, + ], + combined_kept_rows: 2, + }) + expect(execute).toHaveBeenCalledTimes(1) + const sql = execute.mock.calls[0][0] + expect(sql).toContain("COUNT(*) AS total") + expect(sql).toContain(`COUNT(*) FILTER (WHERE (\n${predicates[0]}\n)) AS keep_1`) + expect(sql).toContain(`COUNT(*) FILTER (WHERE (\n${predicates[1]}\n)) AS keep_2`) + expect(sql).toContain(`COUNT(*) FILTER (WHERE (\n${predicates[0]}\n) AND (\n${predicates[1]}\n)) AS keep_all`) + expect(sql).toContain(`FROM (\n${base_sql}\n) t`) + expect(execute.mock.calls[0][3]).toMatchObject({ noLimit: true, queryTimeoutMs: 4321 }) + }) + + test.each([ + ["orders", "E'orders'"], + ['"Foo"."Bar"', 'E\'"Foo"."Bar"\''], + [String.raw`"Foo"."O'rd\\ers"`, String.raw`E'"Foo"."O''rd\\\\ers"'`], + ])("resolves %s through an escaped to_regclass literal before enforcing reltuples", async (base_sql, literal) => { + const execute = postgres("r", 6) + await expect( + checkSql({ warehouse, check: "predicate_effect", base_sql, predicates: ["id > 0"], max_rows_scanned: 5 }), + ).rejects.toThrow("population estimate 6 exceeds max_rows_scanned 5") + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][0]).toContain("SELECT reltuples AS estimate FROM pg_catalog.pg_class") + expect(execute.mock.calls[0][0]).toContain(`oid = pg_catalog.to_regclass(${literal})`) + expect(execute.mock.calls[0][0]).not.toContain("COUNT(*)") + }) + + test("allows a known estimate equal to the cap and does not label it unknown", async () => { + const execute = postgres() + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 2"], + max_rows_scanned: 5, + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + expect(result.output).not.toContain(unknownEstimate) + expect(execute).toHaveBeenCalledTimes(2) + }) + + test("guards materialized views using their positive reltuples estimate", async () => { + const execute = postgres("m", 6) + await expect( + checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 0"], + max_rows_scanned: 5, + }), + ).rejects.toThrow("population estimate 6 exceeds max_rows_scanned 5") + expect(execute).toHaveBeenCalledTimes(1) + }) + + test.each([ + ["unanalyzed table with -1 reltuples", "r", -1], + ["unanalyzed table with 0 reltuples", "r", 0], + ["view with 0 reltuples", "v", 0], + ["view with positive reltuples", "v", 6], + ["partitioned parent with 0 reltuples", "p", 0], + ["partitioned parent with positive reltuples", "p", 6], + ])("reports an unknown estimate for %s and probes under the timeout", async (_name, relkind, reltuples) => { + const execute = postgres(String(relkind), Number(reltuples)) + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 2"], + max_rows_scanned: 2, + query_timeout_ms: 4321, + }) + expect(result.output).toContain(unknownEstimate) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + expect(execute).toHaveBeenCalledTimes(2) + expect(execute.mock.calls[0][0]).toContain("pg_catalog.to_regclass(E'orders')") + for (const call of execute.mock.calls) { + expect(call[3]).toMatchObject({ noLimit: true, queryTimeoutMs: 4321 }) + } + }) +}) diff --git a/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts b/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts index df60dc2b9..5c5e076e4 100644 --- a/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts +++ b/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts @@ -9,6 +9,8 @@ import { initTool } from "./tool-fixture" const warehouses = ["sqlite_predicates", "scratch"] as const const baseSql = "SELECT * FROM orders" const noRemoval = "this predicate removes no rows at this population" +const allRemoval = "this predicate removes all rows" +const unknownEstimate = "row_estimate: unknown — probe runs under the query timeout" let previousTelemetry: string | undefined beforeAll(() => { @@ -104,6 +106,30 @@ for (const warehouse of warehouses) { expect(result.metadata.predicates.every((predicate: { warning?: string }) => !predicate.warning)).toBe(true) expect(result.output).toContain("empty population") expect(result.output).not.toContain(noRemoval) + expect(result.output).not.toContain("removes all rows") + }) + + test("warns when a predicate removes all rows", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: baseSql, + predicates: ["id > 100", "id > 2"], + }) + expect(result.metadata.predicates[0]).toMatchObject({ kept_rows: 0, removed_rows: 5, warning: allRemoval }) + expect(result.metadata.predicates[1].warning).toBeUndefined() + expect(result.output).toContain(`predicate 1 "id > 100": kept 0; removed 5; ${allRemoval}`) + }) + + test("warns when the combination removes all rows but individual predicates do not", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: baseSql, + predicates: ["id < 3", "id > 3"], + }) + expect(result.metadata.predicates.every((predicate: { warning?: string }) => !predicate.warning)).toBe(true) + expect(result.output).toContain("combined (AND): kept 0; removed 5; this combination removes all rows") }) test("combined count matches direct AND filtering while preserving OR precedence", async () => { @@ -153,6 +179,35 @@ for (const warehouse of warehouses) { } }) + test("reports unknown estimates for views and still probes under the timeout", async () => { + const connector = await Registry.get(warehouse) + await connector.execute("CREATE VIEW order_view AS SELECT * FROM orders") + const execute = spyOn(connector, "execute") + try { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "order_view", + predicates: ["id > 2"], + max_rows_scanned: 1, + query_timeout_ms: 4321, + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + expect(result.output).toContain(unknownEstimate) + expect(execute.mock.calls.at(-1)?.[3]).toMatchObject({ noLimit: true, queryTimeoutMs: 4321 }) + } finally { + execute.mockRestore() + } + }) + + test("does not report a known catalog estimate as unknown", async () => { + const connector = await Registry.get(warehouse) + if (warehouse === "sqlite_predicates") await connector.execute("ANALYZE orders") + const result = await checkSql({ warehouse, check: "predicate_effect", base_sql: "orders", predicates: ["id > 2"] }) + expect(result.metadata.total_rows).toBe(5) + expect(result.output).not.toContain(unknownEstimate) + }) + test("applies the estimate guard only to bare table populations", async () => { const connector = await Registry.get(warehouse) const execute = spyOn(connector, "execute") @@ -165,6 +220,7 @@ for (const warehouse of warehouses) { max_rows_scanned: 1, }) expect(result.metadata.total_rows).toBe(5) + expect(result.output).toContain(unknownEstimate) expect(execute).toHaveBeenCalledTimes(1) } finally { execute.mockRestore() @@ -181,6 +237,79 @@ for (const warehouse of warehouses) { expect(result.metadata.total_rows).toBe(2) expect(result.metadata.predicates).toHaveLength(8) expect(result.metadata.combined_kept_rows).toBe(0) + expect(result.output).toContain("counts are over the limited population (base_sql includes LIMIT)") + }) + + test("does not mistake quoted or commented LIMIT text for a population limit", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: `SELECT id, 'LIMIT 1' AS "LIMIT" FROM orders -- LIMIT 2`, + predicates: ["id > 2"], + }) + expect(result.metadata.total_rows).toBe(5) + expect(result.output).not.toContain("limited population") + }) + + test("strict-schema zero cap and blank predicates are treated as absent by the registered tool", async () => { + const precedence = spyOn(Precedence, "check").mockResolvedValue({}) + try { + const tool = await initTool(SqlCheckTool) + const result = await tool.execute( + { + warehouse, + check: "predicate_effect", + base_sql: "SELECT 1 AS id UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5", + predicates: ["", "id > 2", " "], + max_rows_scanned: 0, + }, + { + sessionID: "test-session", + messageID: "test-message", + agent: "test", + abort: new AbortController().signal, + messages: [], + }, + ) + expect(result.metadata.error).toBeUndefined() + expect(result.metadata).toMatchObject({ + total_rows: 5, + predicates: [{ predicate: "id > 2", kept_rows: 3 }], + combined_kept_rows: 3, + }) + expect(result.metadata.predicates).toHaveLength(1) + } finally { + precedence.mockRestore() + } + }) + + test.each([ + { base_sql: baseSql, predicates: [""], missing: "predicates" }, + { base_sql: baseSql, predicates: ["", " \t "], missing: "predicates" }, + { base_sql: "", predicates: ["id > 0"], missing: "base_sql" }, + ])("strict-schema empty inputs return the tool's requires $missing message", async ({ base_sql, predicates, missing }) => { + const connector = await Registry.get(warehouse) + const execute = spyOn(connector, "execute") + const precedence = spyOn(Precedence, "check").mockResolvedValue({}) + try { + const tool = await initTool(SqlCheckTool) + const result = await tool.execute( + { warehouse, check: "predicate_effect", base_sql, predicates }, + { + sessionID: "test-session", + messageID: "test-message", + agent: "test", + abort: new AbortController().signal, + messages: [], + }, + ) + expect(result.output).toBe(`predicate_effect: no verdict — sql_check requires ${missing} for this check`) + expect(result.metadata.error).toBe(`sql_check requires ${missing} for this check`) + expect(execute).not.toHaveBeenCalled() + } finally { + execute.mockRestore() + precedence.mockRestore() + } }) test("rejects nine predicates before query execution", async () => { @@ -189,7 +318,7 @@ for (const warehouse of warehouses) { try { await expect( checkSql({ warehouse, check: "predicate_effect", base_sql: baseSql, predicates: Array(9).fill("id > 0") }), - ).rejects.toThrow(/8/) + ).rejects.toThrow(/sql_check.*predicates.*8/) expect(execute).not.toHaveBeenCalled() } finally { execute.mockRestore() @@ -339,6 +468,7 @@ test("DuckDB predicate_effect respects a view shadowing a larger table on the sc max_rows_scanned: 2, }) expect(result.metadata).toMatchObject({ total_rows: 1, combined_kept_rows: 1 }) + expect(result.output).toContain(unknownEstimate) }) test("DuckDB predicate_effect guards a two-part catalog and table reference", async () => { @@ -419,6 +549,39 @@ test("SQLite predicate_effect defaults the bare-table estimate cap to five milli } }) +test("SQLite predicate_effect reports an unknown estimate without ANALYZE", async () => { + const result = await checkSql({ + warehouse: "sqlite_predicates", + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 2"], + max_rows_scanned: 1, + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + expect(result.output).toContain(unknownEstimate) +}) + +test("SQLite predicate_effect uses the default estimate cap when max_rows_scanned is zero", async () => { + const connector = await Registry.get("sqlite_predicates") + await connector.execute("ANALYZE orders") + await connector.execute("UPDATE sqlite_stat1 SET stat = '5000001' WHERE tbl = 'orders'") + const execute = spyOn(connector, "execute") + try { + await expect( + checkSql({ + warehouse: "sqlite_predicates", + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 0"], + max_rows_scanned: 0, + }), + ).rejects.toThrow("population estimate 5000001 exceeds max_rows_scanned 5000000") + expect(execute.mock.calls.some(([query]) => /COUNT\(\*\)\s+FILTER/i.test(query))).toBe(false) + } finally { + execute.mockRestore() + } +}) + test("predicate_effect returns the existing unsupported-dialect message before connecting", async () => { Registry.setConfigs({ unsupported: { type: "mysql", host: "unused" } }) const get = spyOn(Registry, "get") From 25baadb1c2058ea5178c85fa100a4fe82c02d9eb Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Mon, 12 Oct 2026 13:41:56 -0700 Subject: [PATCH 03/22] fix(analyst): `predicate_effect` accepts table-qualified predicates and keeps caller positions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Independent-review follow-ups: - A bare-table population is wrapped under the table's own name, so a predicate copied from the caller's WHERE clause (`orders.kind = 'a'`) works; SELECT populations keep the `t` alias and the description says so. - The limited-population note fires only for a top-level `LIMIT`, `FETCH FIRST … ROWS ONLY` or `TOP` on the population statement, not for a LIMIT inside a subquery. - Blank predicate entries are reported as skipped at their original position, so predicate numbers match the caller's array. Co-Authored-By: Claude Fable 5.1 --- .../opencode/src/altimate/tools/sql-check.ts | 49 ++++++++--- .../test/altimate/sql-check-postgres.test.ts | 59 ++++++++++++- .../sql-check-predicate-effect.test.ts | 83 +++++++++++++++++++ 3 files changed, 178 insertions(+), 13 deletions(-) diff --git a/packages/opencode/src/altimate/tools/sql-check.ts b/packages/opencode/src/altimate/tools/sql-check.ts index c2ab31de5..54aa041e1 100644 --- a/packages/opencode/src/altimate/tools/sql-check.ts +++ b/packages/opencode/src/altimate/tools/sql-check.ts @@ -23,7 +23,9 @@ const parameters = z .min(1) .max(8) .optional() - .describe("1–8 boolean expressions over base_sql columns"), + .describe( + "1–8 boolean expressions; predicates reference the population's columns; for a bare table you may qualify with the table name; for a SELECT population use bare column names or the alias `t`", + ), max_rows_scanned: z .number() .int() @@ -65,7 +67,13 @@ const publicParameters = z .string() .optional() .describe("Population before filtering: read-only SELECT or bare table reference for predicate_effect"), - predicates: z.array(z.string()).max(8).optional().describe("1–8 boolean expressions over base_sql columns"), + predicates: z + .array(z.string()) + .max(8) + .optional() + .describe( + "1–8 boolean expressions; predicates reference the population's columns; for a bare table you may qualify with the table name; for a SELECT population use bare column names or the alias `t`", + ), max_rows_scanned: z .number() .int() @@ -186,7 +194,23 @@ function readBase(sql: string, dialect?: string) { sql.trim(), ) const table = bare ? sql.trim() : undefined - return { sql: readSql(table ? `SELECT * FROM ${table}` : sql, dialect), table } + // Preserve quoting and case folding, including dots inside a quoted table name. + const alias = table?.match(/"(?:[^"]|"")+"|[A-Za-z_][A-Za-z0-9_$]*/g)?.at(-1) ?? "t" + return { sql: readSql(table ? `SELECT * FROM ${table}` : sql, dialect), table, alias } +} + +function populationLimit(sql: string, dialect: Dialect): string | undefined { + // Hide subqueries and CTE bodies, retaining the parentheses for TOP (n). + let depth = 0 + let topLevel = "" + for (const char of maskSql(tokenizeSql(sql, dialect).tokens)) { + if (char === ")") depth-- + topLevel += depth === 0 ? char : " " + if (char === "(") depth++ + } + if (/(?:^|[^\w$])LIMIT(?![\w$])/i.test(topLevel)) return "LIMIT" + if (/(?:^|[^\w$])FETCH\s+FIRST(?![\w$])[\s\S]*?\bROWS?\s+ONLY(?![\w$])/i.test(topLevel)) return "FETCH FIRST" + if (/(?:^|[^\w$])SELECT\s+(?:(?:ALL|DISTINCT)\s+)?TOP\b\s*(?:\d|\()/i.test(topLevel)) return "TOP" } function readPredicate(predicate: string, dialect?: string): string { @@ -311,7 +335,7 @@ export async function checkSql(input: SqlCheckPublicInput) { const count = (predicate: string) => `COUNT(*) FILTER (WHERE ${predicate})` const [row] = await run(`SELECT COUNT(*) AS total, ${expressions.map((predicate, i) => `${count(predicate)} AS keep_${i + 1}`).join(", ")}, - ${count(expressions.join(" AND "))} AS keep_all FROM (\n${population.sql}\n) t`) + ${count(expressions.join(" AND "))} AS keep_all FROM (\n${population.sql}\n) ${population.alias}`) const total = number(row.total) const effects = args.predicates!.map((predicate, i) => { const kept = number(row[`keep_${i + 1}`]) @@ -324,16 +348,18 @@ export async function checkSql(input: SqlCheckPublicInput) { } }) const combined = number(row.keep_all) + const limit = populationLimit(population.sql, dialect) + let effectIndex = 0 return result(args.check, { total_rows: total, predicates: effects, combined_kept_rows: combined }, [ `predicate_effect: ${total ? `${total} rows in population` : "empty population"}`, ...(estimate === undefined ? ["row_estimate: unknown — probe runs under the query timeout"] : []), - ...(/\bLIMIT\b/i.test(maskSql(tokenizeSql(population.sql, dialect).tokens)) - ? ["counts are over the limited population (base_sql includes LIMIT)"] - : []), - ...effects.map( - (effect, i) => - `predicate ${i + 1} ${display(effect.predicate)}: kept ${effect.kept_rows}; removed ${effect.removed_rows}${effect.warning ? `; ${effect.warning}` : ""}`, - ), + ...(limit ? [`counts are over the limited population (base_sql includes ${limit})`] : []), + ...input.predicates!.map((predicate, i) => { + const label = `predicate ${i + 1} ${display(predicate)}` + if (!predicate.trim()) return `${label}: skipped (blank)` + const effect = effects[effectIndex++] + return `${label}: kept ${effect.kept_rows}; removed ${effect.removed_rows}${effect.warning ? `; ${effect.warning}` : ""}` + }), `combined (AND): kept ${combined}; removed ${total - combined}${total > 0 && combined === 0 ? "; this combination removes all rows" : ""}`, "A predicate that returns NULL counts as not kept.", ]) @@ -560,6 +586,7 @@ export const SqlCheckTool = Tool.define("sql_check", { "Use sql_materialize to put cross-warehouse or other-engine sources into scratch first. " + "Key normalization is diagnostic and may collapse unrelated values; no automatic fixes. Coverage uses regex except SQLite's explicit LIKE fallback. " + "predicate_effect takes base_sql before filtering and 1–8 explicit predicates; reports individual and combined removals, with NULL not kept. " + + "predicates reference the population's columns; for a bare table you may qualify with the table name; for a SELECT population use bare column names or the alias `t`. " + "Bare-table estimates above max_rows_scanned (default 5000000) are refused; unknown estimates use the query timeout and expiry is reported as skipped. " + "Leave fields a check does not use empty.", parameters: publicParameters, diff --git a/packages/opencode/test/altimate/sql-check-postgres.test.ts b/packages/opencode/test/altimate/sql-check-postgres.test.ts index 734038ed2..a91a8238c 100644 --- a/packages/opencode/test/altimate/sql-check-postgres.test.ts +++ b/packages/opencode/test/altimate/sql-check-postgres.test.ts @@ -23,12 +23,12 @@ afterEach(() => { Registry.reset() }) -function postgres(relkind = "r", reltuples = 5) { +function postgres(relkind = "r", reltuples = 5, translate = (sql: string) => sql) { database.run("INSERT INTO pg_class VALUES (1, ?, ?)", [relkind, reltuples]) const execute = mock(async (...[sql]: Parameters) => { // Keep the generated WHERE and FILTER clauses executable: removing either // guard condition must change results, rather than leave a canned mock row. - const query = sql + const query = translate(sql) .replace("pg_catalog.pg_class", "pg_class") .replace(/pg_catalog\.to_regclass\(E'(?:[^']|'')*'\)/, "1") const statement = database.query(query) @@ -40,6 +40,61 @@ function postgres(relkind = "r", reltuples = 5) { } describe("predicate_effect with mocked Postgres", () => { + test("qualifies a bare table using its quoted unqualified name with a dot and escaped quote", async () => { + database.run('CREATE TABLE "Odd.orders""quoted" AS SELECT * FROM orders') + const execute = postgres() + const base_sql = 'main."Odd.orders""quoted"' + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql, + predicates: ['"Odd.orders""quoted".kind = \'a\''], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + expect(execute.mock.calls.at(-1)?.[0]).toContain(`FROM (\nSELECT * FROM ${base_sql}\n) "Odd.orders""quoted"`) + }) + + test("retains the t qualifier for SELECT populations", async () => { + const execute = postgres() + const base_sql = "SELECT * FROM orders" + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql, + predicates: ["t.kind = 'a'"], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + expect(execute.mock.calls[0][0]).toContain(`FROM (\n${base_sql}\n) t`) + }) + + test.each([ + { name: "FETCH FIRST", population: "SELECT * FROM orders ORDER BY id FETCH FIRST 2 ROWS ONLY" }, + { name: "TOP", population: "SELECT TOP (2) * FROM orders ORDER BY id" }, + ])("notes a top-level $name while preserving the population SQL", async ({ population }) => { + const execute = postgres("r", 5, (sql) => sql.replace(population, "SELECT * FROM orders ORDER BY id LIMIT 2")) + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: population, + predicates: ["id > 1"], + }) + expect(result.metadata).toMatchObject({ total_rows: 2, combined_kept_rows: 1 }) + expect(execute.mock.calls[0][0]).toContain(`FROM (\n${population}\n) t`) + expect(result.output).toContain("counts are over the limited population") + }) + + test.each([ + { name: "FETCH FIRST", inner: "SELECT * FROM orders ORDER BY id FETCH FIRST 2 ROWS ONLY" }, + { name: "TOP", inner: "SELECT TOP (2) * FROM orders ORDER BY id" }, + ])("does not note $name confined to a subquery", async ({ inner }) => { + const execute = postgres("r", 5, (sql) => sql.replace(inner, "SELECT * FROM orders ORDER BY id LIMIT 2")) + const base_sql = `SELECT * FROM (${inner}) limited_orders` + const result = await checkSql({ warehouse, check: "predicate_effect", base_sql, predicates: ["id > 1"] }) + expect(result.metadata).toMatchObject({ total_rows: 2, combined_kept_rows: 1 }) + expect(execute.mock.calls[0][0]).toContain(`FROM (\n${base_sql}\n) t`) + expect(result.output).not.toContain("limited population") + }) + test("uses independent FILTER counts and a parenthesized AND probe with newline-safe comments", async () => { const execute = postgres() const base_sql = "SELECT * FROM orders -- population comment" diff --git a/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts b/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts index 5c5e076e4..9e5f21308 100644 --- a/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts +++ b/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts @@ -159,6 +159,38 @@ for (const warehouse of warehouses) { expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) }) + test.each(["orders", "main.orders"])("accepts table-qualified predicates for bare population %s", async (base_sql) => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql, + predicates: ["orders.kind = 'a'"], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + }) + + test("preserves quoted unqualified table names containing dots and escaped quotes", async () => { + const connector = await Registry.get(warehouse) + await connector.execute('CREATE TABLE "Order.Items""Archive" AS SELECT * FROM orders') + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: 'main."Order.Items""Archive"', + predicates: ['"Order.Items""Archive".kind = \'a\''], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + }) + + test("keeps alias t for SELECT populations", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: baseSql, + predicates: ["t.kind = 'a'"], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + }) + test("refuses a bare table whose catalog estimate exceeds the custom cap", async () => { const connector = await Registry.get(warehouse) if (warehouse === "sqlite_predicates") await connector.execute("ANALYZE orders") @@ -251,6 +283,54 @@ for (const warehouse of warehouses) { expect(result.output).not.toContain("limited population") }) + test("does not mistake a LIMIT suffix in an unquoted identifier for a population limit", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: 'SELECT row$LIMIT FROM (SELECT id AS "row$LIMIT" FROM orders) subset', + predicates: ['"row$LIMIT" > 2'], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + expect(result.output).not.toContain("limited population") + }) + + test("does not mistake column top2 for a population TOP clause", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "SELECT top2 FROM (SELECT id AS top2 FROM orders) subset", + predicates: ["top2 > 2"], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + expect(result.output).not.toContain("limited population") + }) + + test.each([ + "SELECT * FROM (SELECT * FROM orders ORDER BY id LIMIT 2) subset", + "WITH subset AS (SELECT * FROM orders ORDER BY id LIMIT 2) SELECT * FROM subset", + ])("does not label an inner LIMIT as a population limit: %s", async (base_sql) => { + const result = await checkSql({ warehouse, check: "predicate_effect", base_sql, predicates: ["id > 0"] }) + expect(result.metadata).toMatchObject({ total_rows: 2, combined_kept_rows: 2 }) + expect(result.output).not.toContain("limited population") + }) + + test("preserves caller positions and lists leading, middle and trailing blank predicates", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: baseSql, + predicates: ["", "id > 2", " \t ", "kind = 'a'", " "], + }) + expect(result.output.match(/^predicate \d+.*$/gm)).toEqual([ + 'predicate 1 "": skipped (blank)', + 'predicate 2 "id > 2": kept 3; removed 2', + 'predicate 3 " \\t ": skipped (blank)', + `predicate 4 "kind = 'a'": kept 3; removed 2`, + 'predicate 5 " ": skipped (blank)', + ]) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 2 }) + }) + test("strict-schema zero cap and blank predicates are treated as absent by the registered tool", async () => { const precedence = spyOn(Precedence, "check").mockResolvedValue({}) try { @@ -278,6 +358,9 @@ for (const warehouse of warehouses) { combined_kept_rows: 3, }) expect(result.metadata.predicates).toHaveLength(1) + expect(result.output).toContain('predicate 1 "": skipped (blank)') + expect(result.output).toContain('predicate 2 "id > 2": kept 3; removed 2') + expect(result.output).toContain('predicate 3 " ": skipped (blank)') } finally { precedence.mockRestore() } From 66236b50e7e781bb30f0b4e728312f3409eb7be5 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Mon, 12 Oct 2026 14:10:47 -0700 Subject: [PATCH 04/22] fix(analyst): `predicate_effect` refuses backend-signalling functions in predicates and populations A review finding of the highest severity tier: on PostgreSQL connections allowed to signal other backends, `pg_terminate_backend(pid)` or `pg_cancel_backend(pid)` passes the read-only validator as a boolean expression. The shared validator now blocks backend-signalling calls (including escaped function names and CR-comment variants) before any query runs; harmless literals, comments, column names and scalar expressions remain accepted. Tests: `rejects backend-signaling predicate before query execution` and the population variant (14 cases). Co-Authored-By: Claude Fable 5.1 --- .../opencode/src/altimate/tools/sql-check.ts | 22 +++++- .../src/altimate/tools/sql-tokenize.ts | 7 +- .../test/altimate/sql-check-postgres.test.ts | 70 ++++++++++++++++++- 3 files changed, 93 insertions(+), 6 deletions(-) diff --git a/packages/opencode/src/altimate/tools/sql-check.ts b/packages/opencode/src/altimate/tools/sql-check.ts index 54aa041e1..d1d2c3bc2 100644 --- a/packages/opencode/src/altimate/tools/sql-check.ts +++ b/packages/opencode/src/altimate/tools/sql-check.ts @@ -45,7 +45,9 @@ const parameters = z .optional() .describe("Required for fanout: pairs of [left output column, right output column] for equality joins"), join: z.enum(["inner", "left"]).optional().default("inner"), - left_col: column.optional().describe("Left output column for keys; compared as text, with type differences reported"), + left_col: column + .optional() + .describe("Left output column for keys; compared as text, with type differences reported"), right_col: column.optional().describe("Right output column for keys"), left_warehouse: z.string().optional().describe("Optional assertion: must equal warehouse"), right_warehouse: z.string().optional().describe("Optional assertion: must equal warehouse"), @@ -92,7 +94,10 @@ const publicParameters = z .optional() .describe("Required for fanout: pairs of [left output column, right output column] for equality joins"), join: z.string().optional().describe('"inner" (default) or "left"'), - left_col: z.string().optional().describe("Left output column for keys; compared as text, with type differences reported"), + left_col: z + .string() + .optional() + .describe("Left output column for keys; compared as text, with type differences reported"), right_col: z.string().optional().describe("Right output column for keys"), left_warehouse: z.string().optional().describe("Optional assertion: must equal warehouse"), right_warehouse: z.string().optional().describe("Optional assertion: must equal warehouse"), @@ -151,6 +156,16 @@ export function readSql(sql: string, dialect?: string): string { // SELECT alone does not make sequence setters, extension loaders or file writers read-only. // Preserve quoted function identifiers while hiding string literals and comments. const functions = maskSql(tokens, true) + // The tokenizer does not decode PostgreSQL U&"..." function names. Refuse + // those calls so Unicode escapes cannot hide a side-effect function. + const escapedFunction = tokens.some( + (token, i) => + token.kind === "identifier" && + token.text.startsWith('"') && + tokens[i - 1]?.kind === "code" && + /(?:^|[^\w$])U&$/i.test(tokens[i - 1].text) && + /^\s*(?:UESCAPE\s+)?\(/i.test(maskSql(tokens.slice(i + 1))), + ) const body = masked.trim().replace(/;$/, "").trim() let depth = 0 let balanced = true @@ -163,6 +178,7 @@ export function readSql(sql: string, dialect?: string): string { queryType !== "read" || !isSingleQuery(sql, dialect) || !complete || + escapedFunction || !balanced || depth !== 0 || !/^(SELECT|WITH)\b/i.test(body) || @@ -171,7 +187,7 @@ export function readSql(sql: string, dialect?: string): string { body, ) || /\bREPLACE\b(?!\s*\()/i.test(body) || - /\b(nextval|setval|set_config|load_extension|writefile|dblink_exec|lo_import|lo_export|lo_create|lo_unlink|lo_put|pg_write_file|pg_append_file|pg_file_write|pg_file_rename|pg_file_unlink|pg_advisory_lock|pg_advisory_xact_lock)\s*\(/i.test( + /\b(nextval|setval|set_config|load_extension|writefile|dblink_exec|lo_import|lo_export|lo_create|lo_unlink|lo_put|pg_write_file|pg_append_file|pg_file_write|pg_file_rename|pg_file_unlink|pg_advisory_lock|pg_advisory_xact_lock|pg_terminate_backend|pg_cancel_backend)\s*\(/i.test( functions, ) ) { diff --git a/packages/opencode/src/altimate/tools/sql-tokenize.ts b/packages/opencode/src/altimate/tools/sql-tokenize.ts index d1cd50a5d..0c949f3f1 100644 --- a/packages/opencode/src/altimate/tools/sql-tokenize.ts +++ b/packages/opencode/src/altimate/tools/sql-tokenize.ts @@ -16,8 +16,11 @@ export function tokenizeSql(sql: string, dialect?: string): { tokens: SqlToken[] // to the write-clause checks even when another dialect treats it as a comment. if (sql.startsWith("--", i) && (!mysqlComments || i + 2 === sql.length || /[\s\x00-\x1f]/.test(sql[i + 2]))) { kind = "comment" - i = sql.indexOf("\n", i + 2) - if (i < 0) i = sql.length + // PostgreSQL also ends line comments at CR. Keep following code visible + // when the dialect is unknown; other dialects retain their LF handling. + const newline = !dialect || /^postgres$/i.test(dialect) ? /[\r\n]/ : /\n/ + const end = sql.slice(i + 2).search(newline) + i = end < 0 ? sql.length : i + 2 + end } else if (sql.startsWith("/*", i)) { // MySQL/MariaDB execute these comments. Generic SQL parsers discard them, // so they cannot establish that a query is read-only or scratch-confined. diff --git a/packages/opencode/test/altimate/sql-check-postgres.test.ts b/packages/opencode/test/altimate/sql-check-postgres.test.ts index a91a8238c..42adac2e8 100644 --- a/packages/opencode/test/altimate/sql-check-postgres.test.ts +++ b/packages/opencode/test/altimate/sql-check-postgres.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, expect, mock, spyOn, test } from "bun:test" import { Database } from "bun:sqlite" +import { rejects } from "node:assert/strict" import type { Connector } from "@altimateai/drivers" import * as Registry from "../../src/altimate/native/connections/registry" import { checkSql } from "../../src/altimate/tools/sql-check" @@ -40,6 +41,67 @@ function postgres(relkind = "r", reltuples = 5, translate = (sql: string) => sql } describe("predicate_effect with mocked Postgres", () => { + test.each([ + "pg_terminate_backend(id)", + "PG_TERMINATE_BACKEND(id)", + 'pg_catalog."pg_terminate_backend" /* signal */ (id)', + "pg_cancel_backend(id)", + 'pg_catalog."pg_cancel_backend" -- signal\n(id)', + "EXISTS (SELECT 1 WHERE pg_catalog.pg_terminate_backend(id))", + String.raw`U&"pg\005fterminate_backend"(id)`, + String.raw`pg_catalog.U&"pg_terminate_\+000062ackend"(id)`, + `U&"pg!005fcancel_backend" /* escape */ UESCAPE '!' (id)`, + "true -- comment\rAND pg_terminate_backend(id)", + "true -- comment\rAND pg_cancel_backend(id)", + ])("rejects backend-signaling predicate %s before query execution", async (predicate) => { + // Substitute only at the connector so an unguarded probe succeeds instead + // of failing because SQLite does not provide PostgreSQL's signaling functions. + const execute = postgres("r", 5, (sql) => sql.replaceAll(predicate, "id > 0")) + await rejects( + checkSql({ warehouse, check: "predicate_effect", base_sql: "orders", predicates: [predicate] }), + /sql_check only accepts a single read-only SELECT/, + ) + expect(execute).not.toHaveBeenCalled() + }) + + test.each(["pg_terminate_backend", "pg_cancel_backend"])( + "rejects backend-signaling population %s before query execution", + async (name) => { + const execute = postgres("r", 5, (sql) => sql.replaceAll(`${name}(id)`, "id > 0")) + await rejects( + checkSql({ + warehouse, + check: "predicate_effect", + base_sql: `SELECT ${name}(id) AS signaled FROM orders`, + predicates: ["signaled"], + }), + /sql_check only accepts a single read-only SELECT/, + ) + expect(execute).not.toHaveBeenCalled() + }, + ) + + test("accepts backend-signaling names in literals, comments and column identifiers", async () => { + const escapedColumn = String.raw`U&"pg\005fterminate_backend"` + const execute = postgres("r", 5, (sql) => sql.replaceAll(escapedColumn, '"pg_terminate_backend"')) + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: 'SELECT id AS "pg_terminate_backend" FROM orders', + predicates: [ + '"pg_terminate_backend" > 2 -- pg_terminate_backend(id)', + "length('pg_cancel_backend(id)') > 0 /* pg_terminate_backend(id) */", + `${escapedColumn} > 2`, + ], + }) + expect(result.metadata).toMatchObject({ + total_rows: 5, + predicates: [{ kept_rows: 3 }, { kept_rows: 5 }, { kept_rows: 3 }], + combined_kept_rows: 3, + }) + expect(execute).toHaveBeenCalledTimes(1) + }) + test("qualifies a bare table using its quoted unqualified name with a dot and escaped quote", async () => { database.run('CREATE TABLE "Odd.orders""quoted" AS SELECT * FROM orders') const execute = postgres() @@ -99,7 +161,13 @@ describe("predicate_effect with mocked Postgres", () => { const execute = postgres() const base_sql = "SELECT * FROM orders -- population comment" const predicates = ["kind = 'a' OR kind = 'b' -- first predicate", "amount > 4 -- second predicate"] - const result = await checkSql({ warehouse, check: "predicate_effect", base_sql, predicates, query_timeout_ms: 4321 }) + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql, + predicates, + query_timeout_ms: 4321, + }) expect(result.metadata).toMatchObject({ total_rows: 5, predicates: [ From 3fa390c8782ca4c0a86e7fb482a1173bb533ee1e Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Mon, 12 Oct 2026 14:26:38 -0700 Subject: [PATCH 05/22] fix(analyst): `predicate_effect` runs read-only and denies server-control and dblink functions Three review findings of the highest severity tier, each with red-first tests: - The probe executes with the connector's read-only mode, so a user-defined function that writes cannot be reached through a predicate. - `pg_promote` and `pg_reload_conf` are denied in predicates and populations. - `dblink_send_query`, `dblink_connect` and `dblink_connect_u` are denied in predicates and populations. Co-Authored-By: Claude Fable 5.1 --- .../opencode/src/altimate/tools/sql-check.ts | 3 +- .../test/altimate/sql-check-postgres.test.ts | 100 +++++++++++++++++- 2 files changed, 99 insertions(+), 4 deletions(-) diff --git a/packages/opencode/src/altimate/tools/sql-check.ts b/packages/opencode/src/altimate/tools/sql-check.ts index d1d2c3bc2..e4ad509a2 100644 --- a/packages/opencode/src/altimate/tools/sql-check.ts +++ b/packages/opencode/src/altimate/tools/sql-check.ts @@ -187,7 +187,7 @@ export function readSql(sql: string, dialect?: string): string { body, ) || /\bREPLACE\b(?!\s*\()/i.test(body) || - /\b(nextval|setval|set_config|load_extension|writefile|dblink_exec|lo_import|lo_export|lo_create|lo_unlink|lo_put|pg_write_file|pg_append_file|pg_file_write|pg_file_rename|pg_file_unlink|pg_advisory_lock|pg_advisory_xact_lock|pg_terminate_backend|pg_cancel_backend)\s*\(/i.test( + /\b(nextval|setval|set_config|load_extension|writefile|dblink_exec|dblink_send_query|dblink_connect|dblink_connect_u|lo_import|lo_export|lo_create|lo_unlink|lo_put|pg_write_file|pg_append_file|pg_file_write|pg_file_rename|pg_file_unlink|pg_advisory_lock|pg_advisory_xact_lock|pg_terminate_backend|pg_cancel_backend|pg_promote|pg_reload_conf)\s*\(/i.test( functions, ) ) { @@ -328,6 +328,7 @@ export async function checkSql(input: SqlCheckPublicInput) { async function run(query: string): Promise { const response = await connector.execute(query, undefined, undefined, { noLimit: true, + readOnly: true, queryTimeoutMs: args.query_timeout_ms, }) if (response.truncated) throw new Error("sql_check received truncated aggregates; no verdict is available") diff --git a/packages/opencode/test/altimate/sql-check-postgres.test.ts b/packages/opencode/test/altimate/sql-check-postgres.test.ts index 42adac2e8..2c421c365 100644 --- a/packages/opencode/test/altimate/sql-check-postgres.test.ts +++ b/packages/opencode/test/altimate/sql-check-postgres.test.ts @@ -24,12 +24,16 @@ afterEach(() => { Registry.reset() }) -function postgres(relkind = "r", reltuples = 5, translate = (sql: string) => sql) { +function postgres( + relkind = "r", + reltuples = 5, + translate = (sql: string, _options?: Parameters[3]) => sql, +) { database.run("INSERT INTO pg_class VALUES (1, ?, ?)", [relkind, reltuples]) - const execute = mock(async (...[sql]: Parameters) => { + const execute = mock(async (...[sql, , , options]: Parameters) => { // Keep the generated WHERE and FILTER clauses executable: removing either // guard condition must change results, rather than leave a canned mock row. - const query = translate(sql) + const query = translate(sql, options) .replace("pg_catalog.pg_class", "pg_class") .replace(/pg_catalog\.to_regclass\(E'(?:[^']|'')*'\)/, "1") const statement = database.query(query) @@ -41,6 +45,96 @@ function postgres(relkind = "r", reltuples = 5, translate = (sql: string) => sql } describe("predicate_effect with mocked Postgres", () => { + test.each([ + { name: "predicates", base_sql: "orders", predicates: ["custom_write_udf() > 0"] as string[], calls: 2 }, + { + name: "populations", + base_sql: "SELECT * FROM orders WHERE custom_write_udf() > 0", + predicates: ["id > 0"] as string[], + calls: 1, + }, + ])("uses read-only execution to reject writing UDFs in $name", async ({ base_sql, predicates, calls }) => { + let writes = 0 + const execute = postgres("r", 5, (sql, options) => { + if (!sql.includes("custom_write_udf()")) return sql + // Model the connector's transaction boundary for a function unknown to the denylist. + if (options?.readOnly) throw new Error("cannot execute INSERT in a read-only transaction") + writes++ + return sql.replaceAll("custom_write_udf()", "1") + }) + await rejects( + checkSql({ warehouse, check: "predicate_effect", base_sql, predicates, query_timeout_ms: 4321 }), + /read-only transaction/, + ) + expect(writes).toBe(0) + expect(execute).toHaveBeenCalledTimes(calls) + for (const call of execute.mock.calls) { + expect(call[3]).toMatchObject({ noLimit: true, queryTimeoutMs: 4321, readOnly: true }) + } + }) + + for (const [kind, calls] of [ + [ + "server-control", + [ + "pg_promote()", + 'pg_catalog."pg_promote" /* promote */ ()', + "PG_RELOAD_CONF()", + 'pg_catalog."pg_reload_conf" -- reload\n()', + ], + ], + [ + "dblink-write", + [ + "dblink_send_query('remote', 'DELETE FROM important_table')", + `public."dblink_send_query" /* send */ ('remote', 'DELETE FROM important_table')`, + "DBLINK_CONNECT('remote', 'dbname=remote')", + `public."dblink_connect" -- connect\n('remote', 'dbname=remote')`, + "dblink_connect_u('remote', 'dbname=remote')", + `public."dblink_connect_u" /* connect */ ('remote', 'dbname=remote')`, + ], + ], + ] satisfies [string, string[]][]) { + test.each(calls)(`rejects ${kind} predicate %s before query execution`, async (call) => { + const predicate = `${call} IS NOT NULL` + const execute = postgres("r", 5, (sql) => sql.replaceAll(call, "1")) + await rejects( + checkSql({ warehouse, check: "predicate_effect", base_sql: "orders", predicates: [predicate] }), + /sql_check only accepts a single read-only SELECT/, + ) + expect(execute).not.toHaveBeenCalled() + }) + + test.each(calls)(`rejects ${kind} population %s before query execution`, async (call) => { + const execute = postgres("r", 5, (sql) => sql.replaceAll(call, "1")) + await rejects( + checkSql({ + warehouse, + check: "predicate_effect", + base_sql: `SELECT ${call} AS value FROM orders`, + predicates: ["value IS NOT NULL"], + }), + /sql_check only accepts a single read-only SELECT/, + ) + expect(execute).not.toHaveBeenCalled() + }) + } + + test.each(["pg_promote", "pg_reload_conf", "dblink_send_query", "dblink_connect", "dblink_connect_u"])( + "accepts %s in literals, comments and column identifiers", + async (name) => { + const execute = postgres() + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: `SELECT id AS "${name}" FROM orders`, + predicates: [`"${name}" > 2 -- ${name}()`, `length('${name}()') > 0 /* ${name}() */`], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + expect(execute).toHaveBeenCalledTimes(1) + }, + ) + test.each([ "pg_terminate_backend(id)", "PG_TERMINATE_BACKEND(id)", From c566efc289499fa7ba154d9769eb88c3cab43e37 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Mon, 12 Oct 2026 16:18:19 -0700 Subject: [PATCH 06/22] fix(analyst): close the remaining write paths in `sql_check predicate_effect` - Read-only classification: deny `pg_logical_emit_message` (its non-transactional form is written to WAL immediately and survives the rollback), bare `dblink` (the synchronous form runs its string argument on a remote connection outside the local read-only transaction) and every `pg_advisory_lock` / `pg_try_advisory_lock` / `pg_advisory_xact_lock` variant (session-level locks survive ROLLBACK and the session returns to the pool). Applies to the population and to every predicate; case-insensitive and schema-qualified spellings covered. - Permission: `PermissionNext.ask` evaluated patterns in order and returned a prompt on the first `ask`, so a later pattern with an explicit `deny` would execute once the earlier one was approved. It now evaluates every pattern before publishing the approval request (`upstream_fix`). - Local engines honour `readOnly`: SQLite runs the probe under a scoped `query_only` pragma restored afterwards; DuckDB runs it in a READ ONLY transaction that is rolled back, and refuses to start one while the caller has its own transaction open rather than aborting it. Previously the flag was silently ignored on both engines. - Regression tests for each denied function family, for the deny-before-prompt ordering, and for SQLite/DuckDB read-only enforcement (hidden sequence writes in a view and in a predicate subquery, pragma restoration after failure, and the active-transaction refusal). Co-Authored-By: Claude Fable 5.1 --- packages/drivers/src/duckdb.ts | 4 +- .../drivers/src/local-engine-child-source.ts | 35 +++++- packages/drivers/src/sqlite.ts | 12 +- packages/drivers/src/types.ts | 2 +- .../opencode/src/altimate/tools/sql-check.ts | 2 +- packages/opencode/src/permission/next.ts | 38 +++--- .../test/altimate/sql-check-postgres.test.ts | 37 +++++- .../sql-check-predicate-effect.test.ts | 111 +++++++++++++++++- 8 files changed, 215 insertions(+), 26 deletions(-) diff --git a/packages/drivers/src/duckdb.ts b/packages/drivers/src/duckdb.ts index c44ec60d8..1bd20100b 100644 --- a/packages/drivers/src/duckdb.ts +++ b/packages/drivers/src/duckdb.ts @@ -166,10 +166,11 @@ export async function connect( params: any[] | undefined, timeoutMs: number | undefined, maxResultBytes?: number, + readOnly?: boolean, ): Promise { if (!shared) throw new Error("DuckDB connection not open") try { - return await shared.engine.call("all", { sql, params, connectionId, maxResultBytes }, timeoutMs) + return await shared.engine.call("all", { sql, params, connectionId, maxResultBytes, readOnly }, timeoutMs) } catch (err) { throw err instanceof Error ? wrapDuckDBError(err) : err } @@ -319,6 +320,7 @@ export async function connect( binds?.length ? binds : undefined, resolveQueryTimeout(config, options).ms, options?.maxResultBytes, + options?.readOnly, ) // altimate_change end const columns = diff --git a/packages/drivers/src/local-engine-child-source.ts b/packages/drivers/src/local-engine-child-source.ts index bdb515948..b6246a0ed 100644 --- a/packages/drivers/src/local-engine-child-source.ts +++ b/packages/drivers/src/local-engine-child-source.ts @@ -222,10 +222,41 @@ async function close() { return null; } +// Guard setup, result consumption and cleanup share one serialized request. +// Separate requests would let another caller run between setup and cleanup. +async function withReadOnly(p, execute) { + if (engine === "sqlite") { + const previous = sqliteDb.query("PRAGMA query_only").get().query_only; + sqliteDb.exec("PRAGMA query_only = ON"); + try { + return await execute(); + } finally { + sqliteDb.exec("PRAGMA query_only = " + (previous ? "ON" : "OFF")); + } + } + const control = (sql) => all({ sql, connectionId: p.connectionId }); + // The legacy binding has no autocommit getter. Implicit transactions receive + // different IDs; an explicit transaction keeps the same ID across statements. + // Refuse before BEGIN: a nested BEGIN would abort the caller's transaction. + const first = await control("SELECT current_transaction_id() AS id"); + const second = await control("SELECT current_transaction_id() AS id"); + if (first[0].id === second[0].id) { + throw new Error("DuckDB read-only execution cannot run inside an active transaction; finish it before running the probe."); + } + await control("BEGIN TRANSACTION READ ONLY"); + try { + return await execute(); + } finally { + await control("ROLLBACK"); + } +} + async function handle(req) { if (req.op === "open") return open(req.payload); - if (req.op === "all") return all(req.payload); - if (req.op === "run") return run(req.payload); + if (req.op === "all" || req.op === "run") { + const execute = () => req.op === "all" ? all(req.payload) : run(req.payload); + return req.payload.readOnly ? withReadOnly(req.payload, execute) : execute(); + } if (req.op === "disconnect") return disconnect(req.payload); if (req.op === "close") return close(); throw new Error("unknown op " + req.op); diff --git a/packages/drivers/src/sqlite.ts b/packages/drivers/src/sqlite.ts index 00b031ec5..9b5a2b9f4 100644 --- a/packages/drivers/src/sqlite.ts +++ b/packages/drivers/src/sqlite.ts @@ -73,7 +73,11 @@ export async function connect(config: ConnectionConfig): Promise { if (!isSelect) { // Non-SELECT statements (INSERT, UPDATE, DELETE, CREATE, etc.) - const info = await engine.call<{ changes: number; lastInsertRowid: number | bigint }>("run", { sql }, timeoutMs) + const info = await engine.call<{ changes: number; lastInsertRowid: number | bigint }>( + "run", + { sql, readOnly: options?.readOnly }, + timeoutMs, + ) return { columns: ["changes", "lastInsertRowid"], rows: [[info.changes, info.lastInsertRowid]], @@ -83,7 +87,11 @@ export async function connect(config: ConnectionConfig): Promise { } // altimate_change start — forward the budget before the helper collects rows - const rows = await engine.call("all", { sql: query, maxResultBytes: options?.maxResultBytes }, timeoutMs) + const rows = await engine.call( + "all", + { sql: query, maxResultBytes: options?.maxResultBytes, readOnly: options?.readOnly }, + timeoutMs, + ) // altimate_change end const columns = rows.length > 0 ? Object.keys(rows[0]) : [] const truncated = effectiveLimit > 0 && rows.length > effectiveLimit diff --git a/packages/drivers/src/types.ts b/packages/drivers/src/types.ts index df45541ea..e188edfc7 100644 --- a/packages/drivers/src/types.ts +++ b/packages/drivers/src/types.ts @@ -26,7 +26,7 @@ export interface ExecuteOptions { maxResultBytes?: number // altimate_change end // altimate_change start — PostgreSQL source extraction uses a pinned, rolled-back transaction - /** PostgreSQL only: execute in a READ ONLY transaction and roll it back before releasing the client. */ + /** PostgreSQL/DuckDB: rolled-back READ ONLY transaction; SQLite: scoped query_only pragma. */ readOnly?: boolean // altimate_change end /** Skip the default LIMIT injection and post-truncation. Use when the caller diff --git a/packages/opencode/src/altimate/tools/sql-check.ts b/packages/opencode/src/altimate/tools/sql-check.ts index e4ad509a2..be94b125b 100644 --- a/packages/opencode/src/altimate/tools/sql-check.ts +++ b/packages/opencode/src/altimate/tools/sql-check.ts @@ -187,7 +187,7 @@ export function readSql(sql: string, dialect?: string): string { body, ) || /\bREPLACE\b(?!\s*\()/i.test(body) || - /\b(nextval|setval|set_config|load_extension|writefile|dblink_exec|dblink_send_query|dblink_connect|dblink_connect_u|lo_import|lo_export|lo_create|lo_unlink|lo_put|pg_write_file|pg_append_file|pg_file_write|pg_file_rename|pg_file_unlink|pg_advisory_lock|pg_advisory_xact_lock|pg_terminate_backend|pg_cancel_backend|pg_promote|pg_reload_conf)\s*\(/i.test( + /\b(nextval|setval|set_config|load_extension|writefile|dblink|dblink_exec|dblink_send_query|dblink_connect|dblink_connect_u|lo_import|lo_export|lo_create|lo_unlink|lo_put|pg_write_file|pg_append_file|pg_file_write|pg_file_rename|pg_file_unlink|pg_logical_emit_message|pg_(?:try_)?advisory_(?:xact_)?lock\w*|pg_terminate_backend|pg_cancel_backend|pg_promote|pg_reload_conf)\s*\(/i.test( functions, ) ) { diff --git a/packages/opencode/src/permission/next.ts b/packages/opencode/src/permission/next.ts index edc2ca1d0..c67acb4c5 100644 --- a/packages/opencode/src/permission/next.ts +++ b/packages/opencode/src/permission/next.ts @@ -135,6 +135,10 @@ export namespace PermissionNext { async (input) => { const s = await state() const { ruleset, ...request } = input + // altimate_change start — upstream_fix: evaluate every pattern before prompting so a + // later explicit deny cannot be bypassed by approving an earlier pattern + let needsAsk = false + // altimate_change end for (const pattern of request.patterns ?? []) { const rule = evaluate(request.permission, pattern, ruleset, s.approved) log.info("evaluated", { permission: request.permission, pattern, action: rule }) @@ -149,23 +153,25 @@ export namespace PermissionNext { }) throw new DeniedError(ruleset.filter((r) => Wildcard.match(request.permission, r.permission))) } - if (rule.action === "ask") { - const id = input.id ?? PermissionID.ascending() - return new Promise((resolve, reject) => { - const info: Request = { - id, - ...request, - } - s.pending.set(id, { - info, - resolve, - reject, - }) - Bus.publish(Event.Asked, info) - }) - } - if (rule.action === "allow") continue + // altimate_change start — upstream_fix: defer the prompt until all patterns are evaluated + if (rule.action === "ask") needsAsk = true } + // A later pattern can deny the whole request; evaluate all of them before prompting. + if (!needsAsk) return + // altimate_change end + const id = input.id ?? PermissionID.ascending() + return new Promise((resolve, reject) => { + const info: Request = { + id, + ...request, + } + s.pending.set(id, { + info, + resolve, + reject, + }) + Bus.publish(Event.Asked, info) + }) }, ) diff --git a/packages/opencode/test/altimate/sql-check-postgres.test.ts b/packages/opencode/test/altimate/sql-check-postgres.test.ts index 2c421c365..ac307d5d8 100644 --- a/packages/opencode/test/altimate/sql-check-postgres.test.ts +++ b/packages/opencode/test/altimate/sql-check-postgres.test.ts @@ -74,6 +74,35 @@ describe("predicate_effect with mocked Postgres", () => { }) for (const [kind, calls] of [ + [ + "WAL-message", + [ + "pg_logical_emit_message(false, 'probe', 'message')", + "PG_CATALOG.PG_LOGICAL_EMIT_MESSAGE(false, 'probe', 'message')", + `pg_catalog."pg_logical_emit_message" /* WAL */ (false, 'probe', 'message')`, + ], + ], + [ + "synchronous-dblink", + [ + "dblink('remote', 'DELETE FROM important_table RETURNING id')", + "PG_CATALOG.DBLINK('remote', 'DELETE FROM important_table RETURNING id')", + `public."dblink" /* remote */ ('remote', 'DELETE FROM important_table RETURNING id')`, + ], + ], + ...[ + "pg_advisory_lock", + "pg_advisory_lock_shared", + "pg_try_advisory_lock", + "pg_try_advisory_lock_shared", + "pg_advisory_xact_lock", + "pg_advisory_xact_lock_shared", + "pg_try_advisory_xact_lock", + "pg_try_advisory_xact_lock_shared", + ].map((name): [string, string[]] => [ + "advisory-lock", + [`${name}(1)`, `PG_CATALOG.${name.toUpperCase()}(1, 2)`, `pg_catalog."${name}" /* lock */ (1)`], + ]), [ "server-control", [ @@ -99,7 +128,7 @@ describe("predicate_effect with mocked Postgres", () => { const predicate = `${call} IS NOT NULL` const execute = postgres("r", 5, (sql) => sql.replaceAll(call, "1")) await rejects( - checkSql({ warehouse, check: "predicate_effect", base_sql: "orders", predicates: [predicate] }), + checkSql({ warehouse, check: "predicate_effect", base_sql: "orders", predicates: ["id > 0", predicate] }), /sql_check only accepts a single read-only SELECT/, ) expect(execute).not.toHaveBeenCalled() @@ -120,7 +149,11 @@ describe("predicate_effect with mocked Postgres", () => { }) } - test.each(["pg_promote", "pg_reload_conf", "dblink_send_query", "dblink_connect", "dblink_connect_u"])( + test.each([ + "pg_promote", "pg_reload_conf", "dblink_send_query", "dblink_connect", "dblink_connect_u", + "pg_logical_emit_message", "dblink", "pg_advisory_lock_shared", "pg_try_advisory_lock_shared", + "pg_advisory_xact_lock_shared", "pg_try_advisory_xact_lock_shared", + ])( "accepts %s in literals, comments and column identifiers", async (name) => { const execute = postgres() diff --git a/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts b/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts index 9e5f21308..797605aa4 100644 --- a/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts +++ b/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts @@ -5,6 +5,11 @@ import { closeScratch } from "../../src/altimate/native/connections/scratch" import * as Precedence from "../../src/altimate/workspace/precedence" import { checkSql, SqlCheckTool } from "../../src/altimate/tools/sql-check" import { initTool } from "./tool-fixture" +import { Bus } from "../../src/bus" +import { Instance } from "../../src/project/instance" +import { PermissionNext } from "../../src/permission/next" +import { SessionID } from "../../src/session/schema" +import { tmpdir } from "../fixture/fixture" const warehouses = ["sqlite_predicates", "scratch"] as const const baseSql = "SELECT * FROM orders" @@ -64,7 +69,7 @@ for (const warehouse of warehouses) { expect(result.metadata.predicates[1].warning).toBeUndefined() expect(result.output).toContain(noRemoval) expect(execute).toHaveBeenCalledTimes(1) - expect(execute.mock.calls[0][3]).toMatchObject({ noLimit: true }) + expect(execute.mock.calls[0][3]).toMatchObject({ noLimit: true, readOnly: true }) } finally { execute.mockRestore() } @@ -717,3 +722,107 @@ test("sql_execute denied prevents predicate_effect routing and query execution", precedence.mockRestore() } }) + +test("predicate_effect refuses a later denied predicate before requesting population approval", async () => { + await using tmp = await tmpdir({ git: true }) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const connector = await Registry.get("sqlite_predicates") + const execute = spyOn(connector, "execute") + const precedence = spyOn(Precedence, "check").mockResolvedValue({}) + const asked: PermissionNext.Request[] = [] + const publish = spyOn(Bus, "publish").mockImplementation(async (event, properties) => { + if (event.type !== PermissionNext.Event.Asked.type) return + const request = properties as PermissionNext.Request + asked.push(request) + // Reproduce the bypass: approving the first pattern must never authorize a later deny. + await PermissionNext.reply({ requestID: request.id, reply: "once" }) + }) + try { + const tool = await initTool(SqlCheckTool) + const sessionID = SessionID.make("session_predicate_permissions") + const result = await tool.execute( + { warehouse: "sqlite_predicates", check: "predicate_effect", base_sql: baseSql, predicates: ["id > 0", "amount > 4"] }, + { + sessionID, + messageID: "test-message", + agent: "test", + abort: new AbortController().signal, + messages: [], + ask: (request: Omit) => PermissionNext.ask({ + ...request, + sessionID, + ruleset: [ + { permission: "sql_execute", pattern: "*", action: "ask" }, + { permission: "sql_execute", pattern: "*amount*", action: "deny" }, + ], + }), + }, + ) + expect(result.metadata.error).toContain("rule which prevents") + expect(asked).toEqual([]) + expect(await PermissionNext.list()).toEqual([]) + expect(precedence).not.toHaveBeenCalled() + expect(execute).not.toHaveBeenCalled() + } finally { + publish.mockRestore() + execute.mockRestore() + precedence.mockRestore() + await Instance.dispose() + } + }, + }) +}) + +test("SQLite predicate_effect enables query_only and restores the previous setting after success and failure", async () => { + const connector = await Registry.get("sqlite_predicates") + const result = await checkSql({ + warehouse: "sqlite_predicates", + check: "predicate_effect", + base_sql: "SELECT query_only AS guarded FROM pragma_query_only()", + predicates: ["guarded = 1"], + }) + expect(result.metadata.combined_kept_rows).toBe(1) + expect((await connector.execute("PRAGMA query_only")).rows).toEqual([[0]]) + await expect(connector.execute("INSERT INTO orders (id) VALUES (6)", undefined, undefined, { readOnly: true })) + .rejects.toThrow(/readonly/i) + expect((await connector.execute("SELECT COUNT(*) FROM orders")).rows).toEqual([[5]]) + expect((await connector.execute("PRAGMA query_only")).rows).toEqual([[0]]) + await connector.execute("INSERT INTO orders (id) VALUES (6)") + await connector.execute("PRAGMA query_only = ON") + await checkSql({ warehouse: "sqlite_predicates", check: "predicate_effect", base_sql: baseSql, predicates: ["id > 0"] }) + await expect(checkSql({ warehouse: "sqlite_predicates", check: "predicate_effect", base_sql: "SELECT * FROM missing", predicates: ["id > 0"] })) + .rejects.toThrow(/missing/) + expect((await connector.execute("PRAGMA query_only")).rows).toEqual([[1]]) +}) + +test.each([ + { location: "population view", base_sql: "sequence_writer", predicates: ["id > 0"] }, + { location: "predicate subquery", base_sql: baseSql, predicates: ["(SELECT id FROM sequence_writer) > 0"] }, +])("DuckDB predicate_effect blocks hidden sequence writes in $location and restores the session", async ({ base_sql, predicates }) => { + const connector = await Registry.get("scratch") + await connector.execute("CREATE SEQUENCE probe_sequence") + await connector.execute("CREATE VIEW sequence_writer AS SELECT nextval('probe_sequence') AS id") + await expect(checkSql({ warehouse: "scratch", check: "predicate_effect", base_sql, predicates: [...predicates] })) + .rejects.toThrow(/read.only/i) + expect((await connector.execute("SELECT last_value FROM duckdb_sequences() WHERE sequence_name = 'probe_sequence'")).rows).toEqual([[null]]) + await connector.execute("INSERT INTO orders (id) VALUES (6)") + const result = await checkSql({ warehouse: "scratch", check: "predicate_effect", base_sql: baseSql, predicates: ["id > 0"] }) + expect(result.metadata.total_rows).toBe(6) +}) + +test("DuckDB predicate_effect refuses an active transaction without aborting caller changes", async () => { + const connector = await Registry.get("scratch") + await connector.execute("BEGIN TRANSACTION") + try { + await connector.execute("INSERT INTO orders (id) VALUES (6)") + await expect(checkSql({ warehouse: "scratch", check: "predicate_effect", base_sql: baseSql, predicates: ["id > 0"] })) + .rejects.toThrow(/DuckDB.*active transaction/i) + await connector.execute("COMMIT") + expect(Number((await connector.execute("SELECT COUNT(*) FROM orders")).rows[0][0])).toBe(6) + } catch (error) { + await connector.execute("ROLLBACK").catch(() => {}) + throw error + } +}) From 8e567511e3327a3eb148e92b673f218e0f6efbcf Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Mon, 12 Oct 2026 16:28:17 -0700 Subject: [PATCH 07/22] fix(analyst): deny the advisory-unlock functions in sql_check predicate_effect pg_advisory_unlock, pg_advisory_unlock_shared and pg_advisory_unlock_all release session-level locks held by the pooled connection, and a read-only rollback does not undo that. The denylist now covers every pg_advisory_* / pg_try_advisory_* function; regression cases added for the three unlock variants. Co-Authored-By: Claude Fable 5.1 --- packages/opencode/src/altimate/tools/sql-check.ts | 2 +- packages/opencode/test/altimate/sql-check-postgres.test.ts | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/altimate/tools/sql-check.ts b/packages/opencode/src/altimate/tools/sql-check.ts index be94b125b..68773770a 100644 --- a/packages/opencode/src/altimate/tools/sql-check.ts +++ b/packages/opencode/src/altimate/tools/sql-check.ts @@ -187,7 +187,7 @@ export function readSql(sql: string, dialect?: string): string { body, ) || /\bREPLACE\b(?!\s*\()/i.test(body) || - /\b(nextval|setval|set_config|load_extension|writefile|dblink|dblink_exec|dblink_send_query|dblink_connect|dblink_connect_u|lo_import|lo_export|lo_create|lo_unlink|lo_put|pg_write_file|pg_append_file|pg_file_write|pg_file_rename|pg_file_unlink|pg_logical_emit_message|pg_(?:try_)?advisory_(?:xact_)?lock\w*|pg_terminate_backend|pg_cancel_backend|pg_promote|pg_reload_conf)\s*\(/i.test( + /\b(nextval|setval|set_config|load_extension|writefile|dblink|dblink_exec|dblink_send_query|dblink_connect|dblink_connect_u|lo_import|lo_export|lo_create|lo_unlink|lo_put|pg_write_file|pg_append_file|pg_file_write|pg_file_rename|pg_file_unlink|pg_logical_emit_message|pg_(?:try_)?advisory_\w*|pg_terminate_backend|pg_cancel_backend|pg_promote|pg_reload_conf)\s*\(/i.test( functions, ) ) { diff --git a/packages/opencode/test/altimate/sql-check-postgres.test.ts b/packages/opencode/test/altimate/sql-check-postgres.test.ts index ac307d5d8..7d81e96a8 100644 --- a/packages/opencode/test/altimate/sql-check-postgres.test.ts +++ b/packages/opencode/test/altimate/sql-check-postgres.test.ts @@ -99,6 +99,9 @@ describe("predicate_effect with mocked Postgres", () => { "pg_advisory_xact_lock_shared", "pg_try_advisory_xact_lock", "pg_try_advisory_xact_lock_shared", + "pg_advisory_unlock", + "pg_advisory_unlock_shared", + "pg_advisory_unlock_all", ].map((name): [string, string[]] => [ "advisory-lock", [`${name}(1)`, `PG_CATALOG.${name.toUpperCase()}(1, 2)`, `pg_catalog."${name}" /* lock */ (1)`], From 4e0f5dbdc7dbe5a180f78aab2e7a0073e67fc594 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Tue, 13 Oct 2026 10:14:57 -0700 Subject: [PATCH 08/22] fix(analyst): refuse every dblink function and the WAL replay controls in read-only `sql_check` sources The denylist named individual dblink functions, so `dblink_open`/`dblink_fetch`/`dblink_close` (a remote data-modifying cursor whose close commits) passed, and it omitted `pg_wal_replay_pause`/`pg_wal_replay_resume`, which change recovery state on a standby and are not undone by rollback. The list now matches `dblink\w*` and `pg_wal_replay_\w*`. Tests: dblink cursor open/fetch/close (incl. a data-modifying CTE), and both WAL replay controls, each rejected before query execution in plain, schema-qualified and commented spellings. Co-Authored-By: Claude Fable 5.1 --- packages/opencode/src/altimate/tools/sql-check.ts | 2 +- .../test/altimate/sql-check-postgres.test.ts | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/altimate/tools/sql-check.ts b/packages/opencode/src/altimate/tools/sql-check.ts index efeec02d8..ccc9a26ab 100644 --- a/packages/opencode/src/altimate/tools/sql-check.ts +++ b/packages/opencode/src/altimate/tools/sql-check.ts @@ -187,7 +187,7 @@ export function readSql(sql: string, dialect?: string): string { body, ) || /\bREPLACE\b(?!\s*\()/i.test(body) || - /\b(nextval|setval|set_config|load_extension|writefile|dblink|dblink_exec|dblink_send_query|dblink_connect|dblink_connect_u|lo_import|lo_export|lo_create|lo_unlink|lo_put|pg_write_file|pg_append_file|pg_file_write|pg_file_rename|pg_file_unlink|pg_logical_emit_message|pg_(?:try_)?advisory_\w*|pg_terminate_backend|pg_cancel_backend|pg_promote|pg_reload_conf)\s*\(/i.test( + /\b(nextval|setval|set_config|load_extension|writefile|dblink\w*|pg_wal_replay_\w*|lo_import|lo_export|lo_create|lo_unlink|lo_put|pg_write_file|pg_append_file|pg_file_write|pg_file_rename|pg_file_unlink|pg_logical_emit_message|pg_(?:try_)?advisory_\w*|pg_terminate_backend|pg_cancel_backend|pg_promote|pg_reload_conf)\s*\(/i.test( functions, ) ) { diff --git a/packages/opencode/test/altimate/sql-check-postgres.test.ts b/packages/opencode/test/altimate/sql-check-postgres.test.ts index 7d81e96a8..efa01a0d8 100644 --- a/packages/opencode/test/altimate/sql-check-postgres.test.ts +++ b/packages/opencode/test/altimate/sql-check-postgres.test.ts @@ -124,6 +124,19 @@ describe("predicate_effect with mocked Postgres", () => { `public."dblink_connect" -- connect\n('remote', 'dbname=remote')`, "dblink_connect_u('remote', 'dbname=remote')", `public."dblink_connect_u" /* connect */ ('remote', 'dbname=remote')`, + "dblink_open('remote', 'c', 'WITH w AS (DELETE FROM important_table RETURNING 1) SELECT 1')", + `public."dblink_open" /* open */ ('remote', 'c', 'SELECT 1')`, + "dblink_fetch('remote', 'c', 10)", + "DBLINK_CLOSE('remote', 'c')", + `public."dblink_close" -- close\n('remote', 'c')`, + ], + ], + [ + "recovery-control", + [ + "pg_wal_replay_pause()", + 'pg_catalog."pg_wal_replay_pause" /* pause */ ()', + "PG_WAL_REPLAY_RESUME()", ], ], ] satisfies [string, string[]][]) { From be91b03b9c40adc929bbe6c319593a2e31e13ef6 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Tue, 13 Oct 2026 10:24:08 -0700 Subject: [PATCH 09/22] fix(analyst): refuse the PostgreSQL WAL, backup, replication and server-control function class in read-only `sql_check` sources Naming individual control functions left `pg_create_restore_point`, `pg_switch_wal`, the replication-slot functions and `pg_logical_slot_get_*` reachable from a read-only probe; a read-only transaction and rollback do not undo WAL records or slot consumption. The denylist now rejects the class: `pg_wal_replay_*`, `pg_logical_*`, `pg_switch_wal/xlog`, `pg_create_restore_point`, backup start/stop, create/drop/copy replication slots, `pg_replication_*`, backend termination/cancel, promote, reload, log rotation and memory-context dumps. Tests: restore point with a per-row name, switch WAL, drop and create replication slots, log rotation, logical slot consumption, each rejected before execution. Co-Authored-By: Claude Fable 5.1 --- packages/opencode/src/altimate/tools/sql-check.ts | 2 +- packages/opencode/test/altimate/sql-check-postgres.test.ts | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/altimate/tools/sql-check.ts b/packages/opencode/src/altimate/tools/sql-check.ts index ccc9a26ab..de5643611 100644 --- a/packages/opencode/src/altimate/tools/sql-check.ts +++ b/packages/opencode/src/altimate/tools/sql-check.ts @@ -187,7 +187,7 @@ export function readSql(sql: string, dialect?: string): string { body, ) || /\bREPLACE\b(?!\s*\()/i.test(body) || - /\b(nextval|setval|set_config|load_extension|writefile|dblink\w*|pg_wal_replay_\w*|lo_import|lo_export|lo_create|lo_unlink|lo_put|pg_write_file|pg_append_file|pg_file_write|pg_file_rename|pg_file_unlink|pg_logical_emit_message|pg_(?:try_)?advisory_\w*|pg_terminate_backend|pg_cancel_backend|pg_promote|pg_reload_conf)\s*\(/i.test( + /\b(nextval|setval|set_config|load_extension|writefile|dblink\w*|lo_import|lo_export|lo_create|lo_unlink|lo_put|pg_write_file|pg_append_file|pg_file_write|pg_file_rename|pg_file_unlink|pg_(?:try_)?advisory_\w*|pg_(?:wal_replay_\w*|logical_\w*|switch_wal|switch_xlog|create_restore_point|(?:start|stop)_backup|backup_(?:start|stop)|(?:create|drop|copy)_\w*replication_slot|replication_\w*|terminate_backend|cancel_backend|promote|reload_conf|rotate_logfile|log_backend_memory_contexts))\s*\(/i.test( functions, ) ) { diff --git a/packages/opencode/test/altimate/sql-check-postgres.test.ts b/packages/opencode/test/altimate/sql-check-postgres.test.ts index efa01a0d8..a0e23b337 100644 --- a/packages/opencode/test/altimate/sql-check-postgres.test.ts +++ b/packages/opencode/test/altimate/sql-check-postgres.test.ts @@ -137,6 +137,13 @@ describe("predicate_effect with mocked Postgres", () => { "pg_wal_replay_pause()", 'pg_catalog."pg_wal_replay_pause" /* pause */ ()', "PG_WAL_REPLAY_RESUME()", + "pg_create_restore_point('probe-' || id::text)", + 'pg_catalog."pg_create_restore_point" /* mark */ (\'p\')', + "PG_SWITCH_WAL()", + "pg_drop_replication_slot('slot')", + "pg_create_logical_replication_slot('slot', 'pgoutput')", + "pg_rotate_logfile()", + "pg_logical_slot_get_changes('slot', NULL, NULL)", ], ], ] satisfies [string, string[]][]) { From 4a7c61e110aaa3ba5dfe7ce62504092cde37fb4a Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Tue, 13 Oct 2026 10:30:05 -0700 Subject: [PATCH 10/22] fix(analyst): refuse every PostgreSQL replication function in read-only `sql_check` sources The replication class named create/drop/copy slot functions and `pg_replication_*`, which left `pg_sync_replication_slots` (and `pg_replication_origin_*` by prefix) reachable. The class is now `pg_*replication*`. Tests: `pg_sync_replication_slots`, `pg_replication_origin_create`, `pg_replication_slot_advance` rejected before execution. Co-Authored-By: Claude Fable 5.1 --- packages/opencode/src/altimate/tools/sql-check.ts | 2 +- packages/opencode/test/altimate/sql-check-postgres.test.ts | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/altimate/tools/sql-check.ts b/packages/opencode/src/altimate/tools/sql-check.ts index de5643611..950937b8e 100644 --- a/packages/opencode/src/altimate/tools/sql-check.ts +++ b/packages/opencode/src/altimate/tools/sql-check.ts @@ -187,7 +187,7 @@ export function readSql(sql: string, dialect?: string): string { body, ) || /\bREPLACE\b(?!\s*\()/i.test(body) || - /\b(nextval|setval|set_config|load_extension|writefile|dblink\w*|lo_import|lo_export|lo_create|lo_unlink|lo_put|pg_write_file|pg_append_file|pg_file_write|pg_file_rename|pg_file_unlink|pg_(?:try_)?advisory_\w*|pg_(?:wal_replay_\w*|logical_\w*|switch_wal|switch_xlog|create_restore_point|(?:start|stop)_backup|backup_(?:start|stop)|(?:create|drop|copy)_\w*replication_slot|replication_\w*|terminate_backend|cancel_backend|promote|reload_conf|rotate_logfile|log_backend_memory_contexts))\s*\(/i.test( + /\b(nextval|setval|set_config|load_extension|writefile|dblink\w*|lo_import|lo_export|lo_create|lo_unlink|lo_put|pg_write_file|pg_append_file|pg_file_write|pg_file_rename|pg_file_unlink|pg_(?:try_)?advisory_\w*|pg_(?:wal_replay_\w*|logical_\w*|switch_wal|switch_xlog|create_restore_point|(?:start|stop)_backup|backup_(?:start|stop)|\w*replication\w*|terminate_backend|cancel_backend|promote|reload_conf|rotate_logfile|log_backend_memory_contexts))\s*\(/i.test( functions, ) ) { diff --git a/packages/opencode/test/altimate/sql-check-postgres.test.ts b/packages/opencode/test/altimate/sql-check-postgres.test.ts index a0e23b337..2ee0e1ea6 100644 --- a/packages/opencode/test/altimate/sql-check-postgres.test.ts +++ b/packages/opencode/test/altimate/sql-check-postgres.test.ts @@ -144,6 +144,9 @@ describe("predicate_effect with mocked Postgres", () => { "pg_create_logical_replication_slot('slot', 'pgoutput')", "pg_rotate_logfile()", "pg_logical_slot_get_changes('slot', NULL, NULL)", + "pg_sync_replication_slots()", + "pg_replication_origin_create('origin')", + "pg_replication_slot_advance('slot', '0/1')", ], ], ] satisfies [string, string[]][]) { From 2f824cf9d094c12f05b2791558658270f7c74636 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Tue, 13 Oct 2026 10:39:53 -0700 Subject: [PATCH 11/22] fix(analyst): refuse the PostgreSQL statistics-reset functions in read-only `sql_check` sources `pg_stat_reset`, `pg_stat_reset_shared` and the other `pg_stat_reset*` functions clear cumulative counters that a read-only transaction and rollback cannot restore; they are now part of the denied control-function class. Tests: the three forms rejected before execution. Co-Authored-By: Claude Fable 5.1 --- packages/opencode/src/altimate/tools/sql-check.ts | 2 +- packages/opencode/test/altimate/sql-check-postgres.test.ts | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/altimate/tools/sql-check.ts b/packages/opencode/src/altimate/tools/sql-check.ts index 950937b8e..7d8fa66df 100644 --- a/packages/opencode/src/altimate/tools/sql-check.ts +++ b/packages/opencode/src/altimate/tools/sql-check.ts @@ -187,7 +187,7 @@ export function readSql(sql: string, dialect?: string): string { body, ) || /\bREPLACE\b(?!\s*\()/i.test(body) || - /\b(nextval|setval|set_config|load_extension|writefile|dblink\w*|lo_import|lo_export|lo_create|lo_unlink|lo_put|pg_write_file|pg_append_file|pg_file_write|pg_file_rename|pg_file_unlink|pg_(?:try_)?advisory_\w*|pg_(?:wal_replay_\w*|logical_\w*|switch_wal|switch_xlog|create_restore_point|(?:start|stop)_backup|backup_(?:start|stop)|\w*replication\w*|terminate_backend|cancel_backend|promote|reload_conf|rotate_logfile|log_backend_memory_contexts))\s*\(/i.test( + /\b(nextval|setval|set_config|load_extension|writefile|dblink\w*|lo_import|lo_export|lo_create|lo_unlink|lo_put|pg_write_file|pg_append_file|pg_file_write|pg_file_rename|pg_file_unlink|pg_(?:try_)?advisory_\w*|pg_(?:stat_reset\w*|wal_replay_\w*|logical_\w*|switch_wal|switch_xlog|create_restore_point|(?:start|stop)_backup|backup_(?:start|stop)|\w*replication\w*|terminate_backend|cancel_backend|promote|reload_conf|rotate_logfile|log_backend_memory_contexts))\s*\(/i.test( functions, ) ) { diff --git a/packages/opencode/test/altimate/sql-check-postgres.test.ts b/packages/opencode/test/altimate/sql-check-postgres.test.ts index 2ee0e1ea6..db08d4f99 100644 --- a/packages/opencode/test/altimate/sql-check-postgres.test.ts +++ b/packages/opencode/test/altimate/sql-check-postgres.test.ts @@ -147,6 +147,9 @@ describe("predicate_effect with mocked Postgres", () => { "pg_sync_replication_slots()", "pg_replication_origin_create('origin')", "pg_replication_slot_advance('slot', '0/1')", + "pg_stat_reset()", + "PG_STAT_RESET_SHARED('bgwriter')", + 'pg_catalog."pg_stat_reset_single_table_counters" /* reset */ (1)', ], ], ] satisfies [string, string[]][]) { From c9cfe1750af716ce0e3788a679e14c8799dbebed Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Tue, 13 Oct 2026 11:25:22 -0700 Subject: [PATCH 12/22] fix(analyst): refuse every PostgreSQL large-object write path in read-only `sql_check` sources; drop a redundant read-only spread `lo_from_bytea`, `lo_truncate`/`lo_truncate64`, `lo_open` and `lowrite` join the denied large-object group (the first creates or overwrites a large object from bytea; the others write through a descriptor), and the PostgreSQL-only `readOnly` spread that duplicated the unconditional `readOnly: true` is removed. Tests: the five large-object forms rejected before execution. Co-Authored-By: Claude Fable 5.1 --- packages/opencode/src/altimate/tools/sql-check.ts | 3 +-- .../opencode/test/altimate/sql-check-postgres.test.ts | 10 ++++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/opencode/src/altimate/tools/sql-check.ts b/packages/opencode/src/altimate/tools/sql-check.ts index 7d8fa66df..888e97ff2 100644 --- a/packages/opencode/src/altimate/tools/sql-check.ts +++ b/packages/opencode/src/altimate/tools/sql-check.ts @@ -187,7 +187,7 @@ export function readSql(sql: string, dialect?: string): string { body, ) || /\bREPLACE\b(?!\s*\()/i.test(body) || - /\b(nextval|setval|set_config|load_extension|writefile|dblink\w*|lo_import|lo_export|lo_create|lo_unlink|lo_put|pg_write_file|pg_append_file|pg_file_write|pg_file_rename|pg_file_unlink|pg_(?:try_)?advisory_\w*|pg_(?:stat_reset\w*|wal_replay_\w*|logical_\w*|switch_wal|switch_xlog|create_restore_point|(?:start|stop)_backup|backup_(?:start|stop)|\w*replication\w*|terminate_backend|cancel_backend|promote|reload_conf|rotate_logfile|log_backend_memory_contexts))\s*\(/i.test( + /\b(nextval|setval|set_config|load_extension|writefile|dblink\w*|lo_import|lo_export|lo_create|lo_unlink|lo_put|lo_from_bytea|lo_truncate\w*|lo_open|lowrite|pg_write_file|pg_append_file|pg_file_write|pg_file_rename|pg_file_unlink|pg_(?:try_)?advisory_\w*|pg_(?:stat_reset\w*|wal_replay_\w*|logical_\w*|switch_wal|switch_xlog|create_restore_point|(?:start|stop)_backup|backup_(?:start|stop)|\w*replication\w*|terminate_backend|cancel_backend|promote|reload_conf|rotate_logfile|log_backend_memory_contexts))\s*\(/i.test( functions, ) ) { @@ -330,7 +330,6 @@ export async function checkSql(input: SqlCheckPublicInput) { noLimit: true, readOnly: true, queryTimeoutMs: args.query_timeout_ms, - ...(dialect === "postgres" ? { readOnly: true } : {}), }) if (response.truncated) throw new Error("sql_check received truncated aggregates; no verdict is available") return response.rows.map((row) => Object.fromEntries(response.columns.map((col, i) => [col, row[i]]))) diff --git a/packages/opencode/test/altimate/sql-check-postgres.test.ts b/packages/opencode/test/altimate/sql-check-postgres.test.ts index db08d4f99..6c18bf185 100644 --- a/packages/opencode/test/altimate/sql-check-postgres.test.ts +++ b/packages/opencode/test/altimate/sql-check-postgres.test.ts @@ -131,6 +131,16 @@ describe("predicate_effect with mocked Postgres", () => { `public."dblink_close" -- close\n('remote', 'c')`, ], ], + [ + "large-object-write", + [ + "lo_from_bytea(0, '\\x00'::bytea)", + 'pg_catalog."lo_from_bytea" /* write */ (0, \'\\x00\'::bytea)', + "LO_TRUNCATE(1, 0)", + "lo_truncate64(1, 0)", + "lowrite(lo_open(1, 131072), '\\x00'::bytea)", + ], + ], [ "recovery-control", [ From 5a69583806059750c86669a415e015960b3eb8e1 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Tue, 13 Oct 2026 11:41:06 -0700 Subject: [PATCH 13/22] fix(analyst): deny PostgreSQL control, large-object, dblink and checkpoint function classes structurally in read-only `sql_check` sources The function denylist grew one name per review round (`pg_log_standby_snapshot`, `lo_import_with_oid`, `lo_creat`, DuckDB `checkpoint`/`force_checkpoint` were the latest escapes). It is replaced by a structural rule: every `pg_*` call is refused unless it is a known pure read (type/size/catalog inspection, `pg_get_*`, stat getters other than `pg_stat_reset*`, LSN and snapshot reads, privilege checks); every `lo_*`, `lowrite`/`loread`, `dblink*`, `checkpoint`/`force_checkpoint`, sequence, config and file-write call is refused. A future server-control function is therefore denied by default. tests: class-denied cases (lo_import_with_oid, lo_creat, pg_log_standby_snapshot, pg_read_file, pg_sleep, checkpoint, force_checkpoint, an invented future pg_ function) rejected before execution in predicates and populations; eight pure reads (pg_typeof, pg_size_pretty, pg_column_size, pg_get_expr, pg_stat_get_numscans, pg_relation_size, pg_current_wal_lsn, pg_has_role) still execute; all earlier denylist and literal/comment/identifier cases unchanged. Co-Authored-By: Claude Fable 5.1 --- .../opencode/src/altimate/tools/sql-check.ts | 23 +++++++++++- .../test/altimate/sql-check-postgres.test.ts | 35 +++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/altimate/tools/sql-check.ts b/packages/opencode/src/altimate/tools/sql-check.ts index 888e97ff2..753b1c68a 100644 --- a/packages/opencode/src/altimate/tools/sql-check.ts +++ b/packages/opencode/src/altimate/tools/sql-check.ts @@ -6,6 +6,27 @@ import * as Registry from "../native/connections/registry" import { classifyAndCheck, isSingleQuery } from "./sql-classify" import { maskSql, tokenizeSql } from "./sql-tokenize" +// Function calls that can change state outside the read-only transaction. PostgreSQL administrative, +// WAL, replication, statistics-reset, large-object, file and dblink functions are denied as whole classes: +// every `pg_*` call is refused unless it is a known pure read (type/size/catalog inspection, stat getters, +// LSN and snapshot reads), so a new server-control function cannot slip through one name at a time. +const PURE_PG_FUNCTIONS = + "typeof|size_pretty|column_size|\\w*_size|backend_pid|is_in_recovery|get_\\w*|describe_object|identify_object\\w*|" + + "input_is_valid|trigger_depth|has_role|\\w*_is_visible|current_\\w*_lsn|last_\\w*_lsn|wal_lsn_diff|walfile_name\\w*|" + + "split_walfile_name|postmaster_start_time|conf_load_time|my_temp_schema|is_other_temp_schema|jit_available|" + + "collation_for|tablespace_\\w*|relation_filepath|relation_filenode|filenode_relation|char_to_encoding|" + + "encoding_to_char|client_encoding|column_compression|xact_commit_timestamp\\w*|current_xact_id\\w*|current_snapshot|" + + "snapshot_\\w*|visible_in_snapshot|notification_queue_usage|listening_channels|blocking_pids|" + + "safe_snapshot_blocking_pids|isolation_test_session_is_blocked|stat_(?!reset)\\w*|statio_\\w*|mcv_list_\\w*|" + + "index\\w*_has_property|options_to_table|settings_get_flags|size_bytes|lsn\\w*|partition_\\w*|" + + "indexes_size|table_is_visible|has_\\w*_privilege|get_userbyid|database_collation_actual_version|" + + "collation_actual_version|available_extension\\w*|control_\\w*|ls_\\w*dir|cursor\\w*|event_trigger_\\w*" +const SIDE_EFFECT_FUNCTION_CALL = new RegExp( + "\\b(nextval|setval|set_config|load_extension|writefile|dblink\\w*|lo_\\w*|lowrite|loread|lo_creat|checkpoint|force_checkpoint|" + + `pg_(?!(?:${PURE_PG_FUNCTIONS})\\s*\\()\\w*)\\s*\\(`, + "i", +) + const column = z .string() .min(1) @@ -187,7 +208,7 @@ export function readSql(sql: string, dialect?: string): string { body, ) || /\bREPLACE\b(?!\s*\()/i.test(body) || - /\b(nextval|setval|set_config|load_extension|writefile|dblink\w*|lo_import|lo_export|lo_create|lo_unlink|lo_put|lo_from_bytea|lo_truncate\w*|lo_open|lowrite|pg_write_file|pg_append_file|pg_file_write|pg_file_rename|pg_file_unlink|pg_(?:try_)?advisory_\w*|pg_(?:stat_reset\w*|wal_replay_\w*|logical_\w*|switch_wal|switch_xlog|create_restore_point|(?:start|stop)_backup|backup_(?:start|stop)|\w*replication\w*|terminate_backend|cancel_backend|promote|reload_conf|rotate_logfile|log_backend_memory_contexts))\s*\(/i.test( + SIDE_EFFECT_FUNCTION_CALL.test( functions, ) ) { diff --git a/packages/opencode/test/altimate/sql-check-postgres.test.ts b/packages/opencode/test/altimate/sql-check-postgres.test.ts index 6c18bf185..31b08080e 100644 --- a/packages/opencode/test/altimate/sql-check-postgres.test.ts +++ b/packages/opencode/test/altimate/sql-check-postgres.test.ts @@ -141,6 +141,19 @@ describe("predicate_effect with mocked Postgres", () => { "lowrite(lo_open(1, 131072), '\\x00'::bytea)", ], ], + [ + "class-denied", + [ + "lo_import_with_oid('/tmp/x', 1)", + "lo_creat(131072)", + "pg_log_standby_snapshot()", + "PG_READ_FILE('/etc/passwd')", + "pg_sleep(30)", + "checkpoint()", + "force_checkpoint(current_database())", + "pg_some_future_control_function(1)", + ], + ], [ "recovery-control", [ @@ -188,6 +201,28 @@ describe("predicate_effect with mocked Postgres", () => { }) } + test.each([ + "pg_typeof(id)", + "PG_SIZE_PRETTY(id::bigint)", + "pg_column_size(id)", + "pg_get_expr('1', 1)", + "pg_stat_get_numscans(1)", + "pg_relation_size('orders')", + "pg_current_wal_lsn()", + "pg_has_role('x', 'USAGE')", + ])("accepts the pure read function %s in predicates and populations", async (call) => { + const execute = postgres("r", 5, (sql) => sql.replaceAll(call, "1")) + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 0", `${call} IS NOT NULL`], + }) + expect(result.metadata).toMatchObject({ total_rows: 5 }) + // The pure call reached execution (a bare table base also issues the row-estimate query). + expect(execute).toHaveBeenCalled() + }) + test.each([ "pg_promote", "pg_reload_conf", "dblink_send_query", "dblink_connect", "dblink_connect_u", "pg_logical_emit_message", "dblink", "pg_advisory_lock_shared", "pg_try_advisory_lock_shared", From 9ae7d7707eeecde5415ab81addf6ee25fd0f966b Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Tue, 13 Oct 2026 11:55:14 -0700 Subject: [PATCH 14/22] fix(analyst): DuckDB read-only guard uses txid_current, dynamic query() is refused, estimate-lookup infrastructure errors propagate - The DuckDB read-only guard probes the transaction id with `txid_current()`, which every supported DuckDB release exposes, instead of `current_transaction_id()` - DuckDB `query()` executes the SQL in its string argument, so a masked literal could smuggle an already-denied call into the probe; it is now part of the denied function class. `pg_ls_*dir` is no longer allowlisted (server directory listings are file access); redundant allowlist/denylist entries removed - The bare-table estimate lookup re-throws a lost helper session (`stateLost`) and any other infrastructure error instead of treating it as an unknown estimate; only recognised missing-statistics or catalog-access failures still mean unknown, so a probe never runs under silently reset session state - tests: query() smuggling and pg_ls_dir/pg_ls_waldir rejected; lost-session estimate error propagates; missing-catalog error still yields an unknown estimate Co-Authored-By: Claude Fable 5.1 --- .../drivers/src/local-engine-child-source.ts | 4 +-- .../opencode/src/altimate/tools/sql-check.ts | 17 +++++++++---- .../test/altimate/sql-check-postgres.test.ts | 25 +++++++++++++++++++ 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/packages/drivers/src/local-engine-child-source.ts b/packages/drivers/src/local-engine-child-source.ts index b6246a0ed..20f174f0b 100644 --- a/packages/drivers/src/local-engine-child-source.ts +++ b/packages/drivers/src/local-engine-child-source.ts @@ -238,8 +238,8 @@ async function withReadOnly(p, execute) { // The legacy binding has no autocommit getter. Implicit transactions receive // different IDs; an explicit transaction keeps the same ID across statements. // Refuse before BEGIN: a nested BEGIN would abort the caller's transaction. - const first = await control("SELECT current_transaction_id() AS id"); - const second = await control("SELECT current_transaction_id() AS id"); + const first = await control("SELECT txid_current() AS id"); + const second = await control("SELECT txid_current() AS id"); if (first[0].id === second[0].id) { throw new Error("DuckDB read-only execution cannot run inside an active transaction; finish it before running the probe."); } diff --git a/packages/opencode/src/altimate/tools/sql-check.ts b/packages/opencode/src/altimate/tools/sql-check.ts index 753b1c68a..502e33197 100644 --- a/packages/opencode/src/altimate/tools/sql-check.ts +++ b/packages/opencode/src/altimate/tools/sql-check.ts @@ -19,10 +19,10 @@ const PURE_PG_FUNCTIONS = "snapshot_\\w*|visible_in_snapshot|notification_queue_usage|listening_channels|blocking_pids|" + "safe_snapshot_blocking_pids|isolation_test_session_is_blocked|stat_(?!reset)\\w*|statio_\\w*|mcv_list_\\w*|" + "index\\w*_has_property|options_to_table|settings_get_flags|size_bytes|lsn\\w*|partition_\\w*|" + - "indexes_size|table_is_visible|has_\\w*_privilege|get_userbyid|database_collation_actual_version|" + - "collation_actual_version|available_extension\\w*|control_\\w*|ls_\\w*dir|cursor\\w*|event_trigger_\\w*" + "table_is_visible|has_\\w*_privilege|database_collation_actual_version|" + + "collation_actual_version|available_extension\\w*|control_\\w*|cursor\\w*|event_trigger_\\w*" const SIDE_EFFECT_FUNCTION_CALL = new RegExp( - "\\b(nextval|setval|set_config|load_extension|writefile|dblink\\w*|lo_\\w*|lowrite|loread|lo_creat|checkpoint|force_checkpoint|" + + "\\b(nextval|setval|set_config|load_extension|writefile|dblink\\w*|lo_\\w*|lowrite|loread|checkpoint|force_checkpoint|query|" + `pg_(?!(?:${PURE_PG_FUNCTIONS})\\s*\\()\\w*)\\s*\\(`, "i", ) @@ -311,8 +311,15 @@ async function tableEstimate(table: string, dialect: Dialect, run: (sql: string) return estimates.length ? Math.max(...estimates) : undefined } catch (error) { if (isQueryTimeoutError(error)) throw error - // Missing ANALYZE statistics or unavailable catalog access means unknown. - return undefined + // A lost helper session or any other infrastructure failure must reach the caller: the probe + // would otherwise run under reset session state. Only recognised missing-statistics or + // catalog-access failures mean "unknown estimate". + if (typeof error === "object" && error !== null && "stateLost" in error) throw error + const message = error instanceof Error ? error.message : String(error) + if (/no such table|does not exist|sqlite_stat1|pragma_table_list|permission denied|insufficient privilege|catalog/i.test(message)) { + return undefined + } + throw error } } diff --git a/packages/opencode/test/altimate/sql-check-postgres.test.ts b/packages/opencode/test/altimate/sql-check-postgres.test.ts index 31b08080e..f0e6676be 100644 --- a/packages/opencode/test/altimate/sql-check-postgres.test.ts +++ b/packages/opencode/test/altimate/sql-check-postgres.test.ts @@ -152,6 +152,9 @@ describe("predicate_effect with mocked Postgres", () => { "checkpoint()", "force_checkpoint(current_database())", "pg_some_future_control_function(1)", + "query('SELECT force_checkpoint(current_database())')", + "pg_ls_dir('.')", + "pg_ls_waldir()", ], ], [ @@ -402,6 +405,28 @@ describe("predicate_effect with mocked Postgres", () => { expect(execute.mock.calls[0][0]).not.toContain("COUNT(*)") }) + test("a lost helper session during the estimate lookup reaches the caller instead of becoming an unknown estimate", async () => { + const execute = postgres("r", 5, (sql) => { + if (/pg_class/.test(sql)) throw Object.assign(new Error("session state was lost"), { code: "SESSION_STATE_LOST", stateLost: true }) + return sql + }) + await expect( + checkSql({ warehouse, check: "predicate_effect", base_sql: "orders", predicates: ["id > 2"] }), + ).rejects.toMatchObject({ stateLost: true }) + expect(execute).toHaveBeenCalledTimes(1) + }) + + test("a missing-catalog error during the estimate lookup still means an unknown estimate", async () => { + const execute = postgres("r", 5, (sql) => { + if (/pg_class/.test(sql)) throw new Error('relation "pg_catalog.pg_class" does not exist') + return sql + }) + const result = await checkSql({ warehouse, check: "predicate_effect", base_sql: "orders", predicates: ["id > 2"] }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + expect(result.output).toContain(unknownEstimate) + expect(execute).toHaveBeenCalledTimes(2) + }) + test("allows a known estimate equal to the cap and does not label it unknown", async () => { const execute = postgres() const result = await checkSql({ From 214e47ec19d8efae0e10529b449593e5213cc5ab Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Tue, 13 Oct 2026 15:47:14 -0700 Subject: [PATCH 15/22] fix(analyst): `sql_check` rethrows probe timeouts that lost local session state - The `predicate_effect` probe converted every `QueryTimeoutError` into `skipped: "timeout"`, including SQLite/DuckDB timeouts whose helper restart discarded temporary tables, settings, open transactions and `:memory:` databases. A timeout carrying `stateLost: true` is now rethrown so the caller receives the state-loss disclosure; timeouts without it still skip. The catalog-estimate path reaches the same catch, so both probes are covered. - tests: catalog-estimate and predicate-query timeouts with and without `stateLost`, on SQLite and DuckDB. Co-Authored-By: Claude Fable 5.1 --- .../opencode/src/altimate/tools/sql-check.ts | 2 +- .../test/altimate/tools/sql-check.test.ts | 41 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/altimate/tools/sql-check.ts b/packages/opencode/src/altimate/tools/sql-check.ts index 502e33197..2789ec995 100644 --- a/packages/opencode/src/altimate/tools/sql-check.ts +++ b/packages/opencode/src/altimate/tools/sql-check.ts @@ -409,7 +409,7 @@ export async function checkSql(input: SqlCheckPublicInput) { "A predicate that returns NULL counts as not kept.", ]) } catch (error) { - if (!isQueryTimeoutError(error)) throw error + if (!isQueryTimeoutError(error) || (error as { stateLost?: boolean }).stateLost === true) throw error return result(args.check, { skipped: "timeout" }, [ "predicate_effect: skipped: timeout; use a narrower base_sql.", ]) diff --git a/packages/opencode/test/altimate/tools/sql-check.test.ts b/packages/opencode/test/altimate/tools/sql-check.test.ts index ee02e7499..dde6d48f4 100644 --- a/packages/opencode/test/altimate/tools/sql-check.test.ts +++ b/packages/opencode/test/altimate/tools/sql-check.test.ts @@ -592,6 +592,47 @@ for (const warehouse of ["sqlite_fixture", "scratch"] as const) { expect((await checkSql({ warehouse, check: "grain", sql: leftSql, columns: ["id"] })).metadata.total_rows).toBe(5) }) + for (const [probe, baseSql, query] of [ + ["catalog estimate", "orders", /pragma_table_list|duckdb_tables/], + ["predicate query", leftSql, /SELECT COUNT\(\*\) AS total/], + ] as const) { + for (const stateLost of [true, undefined]) { + test(`predicate_effect ${stateLost ? "rethrows" : "skips"} ${probe} timeouts ${stateLost ? "with" : "without"} state loss`, async () => { + const connector = await Registry.get(warehouse) + const error = Object.assign(new Error("query deadline exceeded"), { + name: "QueryTimeoutError", + code: "QUERY_TIMEOUT", + engine: warehouse === "scratch" ? "duckdb" : "sqlite", + timeoutMs: 4321, + cancelled: true, + ...(stateLost ? { stateLost: true } : {}), + }) + const execute = spyOn(connector, "execute").mockRejectedValue(error) + try { + const pending = checkSql({ + warehouse, + check: "predicate_effect", + base_sql: baseSql, + predicates: ["amount > 0"], + query_timeout_ms: 4321, + }) + if (stateLost) { + await expect(pending).rejects.toBe(error) + await expect(pending).rejects.toMatchObject({ stateLost: true }) + } else { + const result = await pending + expect(result.metadata.skipped).toBe("timeout") + expect(result.output).toContain("skipped: timeout") + } + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][0]).toMatch(query) + } finally { + execute.mockRestore() + } + }) + } + } + test("forwards the query timeout and suppresses connector LIMIT injection", async () => { const connector = await Registry.get(warehouse) const execute = spyOn(connector, "execute") From d152ddd85b2cf9e2483c9eec015e1d1f75b5be24 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Tue, 13 Oct 2026 20:38:25 -0700 Subject: [PATCH 16/22] fix(analyst): `predicate_effect` admits only built-in pure functions and guards the PostgreSQL estimate atomically - Predicates may call only functions in a fixed allowlist of built-in pure functions (the earlier `PURE_PG_FUNCTIONS` names plus ordinary scalar built-ins); every other call, including schema-qualified and user-defined wrappers such as `emit_wrapper(id)` or DuckDB `postgres_execute(...)`, is refused before any SQL runs with the function named. The side-effect denylist stays as a second check. Applies to PostgreSQL, SQLite and DuckDB alike. Identifier handling preserves quoted names and folds only ASCII case. - Bare-table PostgreSQL probes run the `pg_class`/`to_regclass` estimate and the aggregate in one statement, with the estimate condition gating population rows, so two pool clients can no longer see different relations (temporary tables, `search_path`) between the guard and the scan; an exceeded estimate returns `skipped: "estimate"` without a population scan. - tests: wrapper, qualified, quoted and pg-prefixed calls refused on every dialect; scalar built-ins, casts and boolean grouping accepted; single-statement estimate guard, exceeded estimate gated, unknown estimate vs empty population, CTE name collisions. Co-Authored-By: Claude Fable 5.1 --- .../opencode/src/altimate/tools/sql-check.ts | 502 +++++++++++++++++- .../test/altimate/tools/sql-check.test.ts | 269 ++++++++++ 2 files changed, 758 insertions(+), 13 deletions(-) diff --git a/packages/opencode/src/altimate/tools/sql-check.ts b/packages/opencode/src/altimate/tools/sql-check.ts index 2789ec995..0b184f747 100644 --- a/packages/opencode/src/altimate/tools/sql-check.ts +++ b/packages/opencode/src/altimate/tools/sql-check.ts @@ -27,6 +27,424 @@ const SIDE_EFFECT_FUNCTION_CALL = new RegExp( "i", ) +// Predicates accept only these built-in pure functions; keep this list flat and sorted. +const PURE_PREDICATE_FUNCTIONS = new Set([ + "abs", + "acos", + "age", + "array_agg", + "array_append", + "array_cat", + "array_dims", + "array_length", + "array_lower", + "array_ndims", + "array_position", + "array_positions", + "array_prepend", + "array_remove", + "array_replace", + "array_to_string", + "array_upper", + "ascii", + "asin", + "atan", + "atan2", + "avg", + "bit_length", + "bool_and", + "bool_or", + "btrim", + "cardinality", + "cast", + "cbrt", + "ceil", + "ceiling", + "char", + "char_length", + "character", + "character_length", + "chr", + "coalesce", + "concat", + "concat_ws", + "cos", + "count", + "current_date", + "current_time", + "current_timestamp", + "date", + "date_part", + "date_trunc", + "decimal", + "degrees", + "div", + "exp", + "extract", + "floor", + "format", + "greatest", + "initcap", + "json_array_length", + "json_extract_path", + "json_extract_path_text", + "json_typeof", + "jsonb_array_length", + "jsonb_extract_path", + "jsonb_extract_path_text", + "jsonb_typeof", + "least", + "left", + "length", + "ln", + "log", + "lower", + "lpad", + "ltrim", + "make_date", + "make_interval", + "make_time", + "make_timestamp", + "max", + "md5", + "min", + "mod", + "now", + "nullif", + "numeric", + "octet_length", + "overlay", + "pg_available_extension_versions", + "pg_available_extensions", + "pg_backend_pid", + "pg_blocking_pids", + "pg_char_to_encoding", + "pg_client_encoding", + "pg_collation_actual_version", + "pg_collation_for", + "pg_collation_is_visible", + "pg_column_compression", + "pg_column_size", + "pg_conf_load_time", + "pg_control_checkpoint", + "pg_control_init", + "pg_control_recovery", + "pg_control_system", + "pg_conversion_is_visible", + "pg_current_snapshot", + "pg_current_wal_flush_lsn", + "pg_current_wal_insert_lsn", + "pg_current_wal_lsn", + "pg_current_xact_id", + "pg_current_xact_id_if_assigned", + "pg_cursor", + "pg_database_collation_actual_version", + "pg_database_size", + "pg_describe_object", + "pg_encoding_to_char", + "pg_event_trigger_ddl_commands", + "pg_event_trigger_dropped_objects", + "pg_event_trigger_table_rewrite_oid", + "pg_event_trigger_table_rewrite_reason", + "pg_filenode_relation", + "pg_function_is_visible", + "pg_get_acl", + "pg_get_aios", + "pg_get_backend_memory_contexts", + "pg_get_catalog_foreign_keys", + "pg_get_constraintdef", + "pg_get_expr", + "pg_get_function_arg_default", + "pg_get_function_arguments", + "pg_get_function_identity_arguments", + "pg_get_function_result", + "pg_get_function_sqlbody", + "pg_get_functiondef", + "pg_get_indexdef", + "pg_get_keywords", + "pg_get_loaded_modules", + "pg_get_multixact_members", + "pg_get_object_address", + "pg_get_partition_constraintdef", + "pg_get_partkeydef", + "pg_get_publication_tables", + "pg_get_replica_identity_index", + "pg_get_replication_slots", + "pg_get_ruledef", + "pg_get_sequence_data", + "pg_get_serial_sequence", + "pg_get_shmem_allocations", + "pg_get_shmem_allocations_numa", + "pg_get_statisticsobjdef", + "pg_get_statisticsobjdef_columns", + "pg_get_statisticsobjdef_expressions", + "pg_get_triggerdef", + "pg_get_userbyid", + "pg_get_viewdef", + "pg_get_wait_events", + "pg_get_wal_replay_pause_state", + "pg_get_wal_resource_managers", + "pg_get_wal_summarizer_state", + "pg_has_role", + "pg_identify_object", + "pg_identify_object_as_address", + "pg_index_column_has_property", + "pg_index_has_property", + "pg_indexam_has_property", + "pg_indexes_size", + "pg_input_is_valid", + "pg_is_in_recovery", + "pg_is_other_temp_schema", + "pg_isolation_test_session_is_blocked", + "pg_jit_available", + "pg_last_wal_receive_lsn", + "pg_last_wal_replay_lsn", + "pg_listening_channels", + "pg_lsn", + "pg_lsn_cmp", + "pg_lsn_eq", + "pg_lsn_ge", + "pg_lsn_gt", + "pg_lsn_hash", + "pg_lsn_hash_extended", + "pg_lsn_in", + "pg_lsn_larger", + "pg_lsn_le", + "pg_lsn_lt", + "pg_lsn_mi", + "pg_lsn_mii", + "pg_lsn_ne", + "pg_lsn_out", + "pg_lsn_pli", + "pg_lsn_recv", + "pg_lsn_send", + "pg_lsn_smaller", + "pg_mcv_list_in", + "pg_mcv_list_items", + "pg_mcv_list_out", + "pg_mcv_list_recv", + "pg_mcv_list_send", + "pg_my_temp_schema", + "pg_notification_queue_usage", + "pg_opclass_is_visible", + "pg_operator_is_visible", + "pg_opfamily_is_visible", + "pg_options_to_table", + "pg_partition_ancestors", + "pg_partition_root", + "pg_partition_tree", + "pg_postmaster_start_time", + "pg_relation_filenode", + "pg_relation_filepath", + "pg_relation_size", + "pg_safe_snapshot_blocking_pids", + "pg_settings_get_flags", + "pg_size_bytes", + "pg_size_pretty", + "pg_snapshot_in", + "pg_snapshot_out", + "pg_snapshot_recv", + "pg_snapshot_send", + "pg_snapshot_xip", + "pg_snapshot_xmax", + "pg_snapshot_xmin", + "pg_split_walfile_name", + "pg_stat_file", + "pg_stat_get_activity", + "pg_stat_get_analyze_count", + "pg_stat_get_archiver", + "pg_stat_get_autoanalyze_count", + "pg_stat_get_autovacuum_count", + "pg_stat_get_backend_activity", + "pg_stat_get_backend_activity_start", + "pg_stat_get_backend_client_addr", + "pg_stat_get_backend_client_port", + "pg_stat_get_backend_dbid", + "pg_stat_get_backend_idset", + "pg_stat_get_backend_io", + "pg_stat_get_backend_pid", + "pg_stat_get_backend_start", + "pg_stat_get_backend_subxact", + "pg_stat_get_backend_userid", + "pg_stat_get_backend_wait_event", + "pg_stat_get_backend_wait_event_type", + "pg_stat_get_backend_wal", + "pg_stat_get_backend_xact_start", + "pg_stat_get_bgwriter_buf_written_checkpoints", + "pg_stat_get_bgwriter_buf_written_clean", + "pg_stat_get_bgwriter_maxwritten_clean", + "pg_stat_get_bgwriter_requested_checkpoints", + "pg_stat_get_bgwriter_stat_reset_time", + "pg_stat_get_bgwriter_timed_checkpoints", + "pg_stat_get_blocks_fetched", + "pg_stat_get_blocks_hit", + "pg_stat_get_buf_alloc", + "pg_stat_get_buf_fsync_backend", + "pg_stat_get_buf_written_backend", + "pg_stat_get_checkpoint_sync_time", + "pg_stat_get_checkpoint_write_time", + "pg_stat_get_checkpointer_buffers_written", + "pg_stat_get_checkpointer_num_performed", + "pg_stat_get_checkpointer_num_requested", + "pg_stat_get_checkpointer_num_timed", + "pg_stat_get_checkpointer_restartpoints_performed", + "pg_stat_get_checkpointer_restartpoints_requested", + "pg_stat_get_checkpointer_restartpoints_timed", + "pg_stat_get_checkpointer_slru_written", + "pg_stat_get_checkpointer_stat_reset_time", + "pg_stat_get_checkpointer_sync_time", + "pg_stat_get_checkpointer_write_time", + "pg_stat_get_db_active_time", + "pg_stat_get_db_blk_read_time", + "pg_stat_get_db_blk_write_time", + "pg_stat_get_db_blocks_fetched", + "pg_stat_get_db_blocks_hit", + "pg_stat_get_db_checksum_failures", + "pg_stat_get_db_checksum_last_failure", + "pg_stat_get_db_conflict_all", + "pg_stat_get_db_conflict_bufferpin", + "pg_stat_get_db_conflict_lock", + "pg_stat_get_db_conflict_logicalslot", + "pg_stat_get_db_conflict_snapshot", + "pg_stat_get_db_conflict_startup_deadlock", + "pg_stat_get_db_conflict_tablespace", + "pg_stat_get_db_deadlocks", + "pg_stat_get_db_idle_in_transaction_time", + "pg_stat_get_db_numbackends", + "pg_stat_get_db_parallel_workers_launched", + "pg_stat_get_db_parallel_workers_to_launch", + "pg_stat_get_db_session_time", + "pg_stat_get_db_sessions", + "pg_stat_get_db_sessions_abandoned", + "pg_stat_get_db_sessions_fatal", + "pg_stat_get_db_sessions_killed", + "pg_stat_get_db_stat_reset_time", + "pg_stat_get_db_temp_bytes", + "pg_stat_get_db_temp_files", + "pg_stat_get_db_tuples_deleted", + "pg_stat_get_db_tuples_fetched", + "pg_stat_get_db_tuples_inserted", + "pg_stat_get_db_tuples_returned", + "pg_stat_get_db_tuples_updated", + "pg_stat_get_db_xact_commit", + "pg_stat_get_db_xact_rollback", + "pg_stat_get_dead_tuples", + "pg_stat_get_function_calls", + "pg_stat_get_function_self_time", + "pg_stat_get_function_total_time", + "pg_stat_get_ins_since_vacuum", + "pg_stat_get_io", + "pg_stat_get_last_analyze_time", + "pg_stat_get_last_autoanalyze_time", + "pg_stat_get_last_autovacuum_time", + "pg_stat_get_last_vacuum_time", + "pg_stat_get_lastscan", + "pg_stat_get_live_tuples", + "pg_stat_get_mod_since_analyze", + "pg_stat_get_numscans", + "pg_stat_get_progress_info", + "pg_stat_get_recovery_prefetch", + "pg_stat_get_replication_slot", + "pg_stat_get_slru", + "pg_stat_get_snapshot_timestamp", + "pg_stat_get_subscription", + "pg_stat_get_subscription_stats", + "pg_stat_get_total_analyze_time", + "pg_stat_get_total_autoanalyze_time", + "pg_stat_get_total_autovacuum_time", + "pg_stat_get_total_vacuum_time", + "pg_stat_get_tuples_deleted", + "pg_stat_get_tuples_fetched", + "pg_stat_get_tuples_hot_updated", + "pg_stat_get_tuples_inserted", + "pg_stat_get_tuples_newpage_updated", + "pg_stat_get_tuples_returned", + "pg_stat_get_tuples_updated", + "pg_stat_get_vacuum_count", + "pg_stat_get_wal", + "pg_stat_get_wal_receiver", + "pg_stat_get_wal_senders", + "pg_stat_get_xact_blocks_fetched", + "pg_stat_get_xact_blocks_hit", + "pg_stat_get_xact_function_calls", + "pg_stat_get_xact_function_self_time", + "pg_stat_get_xact_function_total_time", + "pg_stat_get_xact_numscans", + "pg_stat_get_xact_tuples_deleted", + "pg_stat_get_xact_tuples_fetched", + "pg_stat_get_xact_tuples_hot_updated", + "pg_stat_get_xact_tuples_inserted", + "pg_stat_get_xact_tuples_newpage_updated", + "pg_stat_get_xact_tuples_returned", + "pg_stat_get_xact_tuples_updated", + "pg_stat_have_stats", + "pg_statistics_obj_is_visible", + "pg_table_is_visible", + "pg_table_size", + "pg_tablespace_databases", + "pg_tablespace_location", + "pg_tablespace_size", + "pg_total_relation_size", + "pg_trigger_depth", + "pg_ts_config_is_visible", + "pg_ts_dict_is_visible", + "pg_ts_parser_is_visible", + "pg_ts_template_is_visible", + "pg_type_is_visible", + "pg_typeof", + "pg_visible_in_snapshot", + "pg_wal_lsn_diff", + "pg_walfile_name", + "pg_walfile_name_offset", + "pg_xact_commit_timestamp", + "pg_xact_commit_timestamp_origin", + "pi", + "position", + "power", + "quote_ident", + "quote_literal", + "quote_nullable", + "radians", + "regexp_count", + "regexp_instr", + "regexp_like", + "regexp_match", + "regexp_matches", + "regexp_replace", + "regexp_split_to_array", + "regexp_substr", + "repeat", + "replace", + "reverse", + "right", + "round", + "rpad", + "rtrim", + "sign", + "sin", + "split_part", + "sqrt", + "starts_with", + "string_agg", + "strpos", + "substr", + "substring", + "sum", + "tan", + "time", + "timestamp", + "to_char", + "to_date", + "to_number", + "to_timestamp", + "translate", + "trim", + "trunc", + "upper", + "varchar", + "width_bucket", +]) + const column = z .string() .min(1) @@ -251,11 +669,36 @@ function populationLimit(sql: string, dialect: Dialect): string | undefined { } function readPredicate(predicate: string, dialect?: string): string { + const { tokens } = tokenizeSql(predicate, dialect) + // Retain identifier boundaries and case; hide strings and comments, which cannot call functions. + const functions = tokens + .map(({ kind, text }) => (kind === "code" || kind === "identifier" ? text : text.replace(/[^\r\n]/g, " "))) + .join("") + const identifier = + '(?:"(?:[^"\\x00]|"")+"|`(?:[^`]|``)+`|\\[(?:[^\\]]|\\]\\])+\\]|[A-Za-z_\\u0080-\\uFFFF][\\w$\\u0080-\\uFFFF]*)' + const calls = new RegExp(`${identifier}(?:\\s*\\.\\s*${identifier})*`, "g") + for (const match of functions.matchAll(calls)) { + if (!/^\s*\(/.test(functions.slice(match.index + match[0].length))) continue + const call = match[0].trim() + const parts = call.match(new RegExp(identifier, "g"))! + const quoted = /^["`\[]/.test(parts[0]) + const name = quoted ? parts[0].slice(1, -1) : parts[0].replace(/[A-Z]/g, (char) => char.toLowerCase()) + // These unquoted SQL keywords introduce grouped expressions, not function calls. + if ( + parts.length === 1 && + !quoted && + /^(all|and|any|as|between|else|exists|from|having|in|not|on|or|select|some|then|when|where)$/.test(name) + ) + continue + if (parts.length !== 1 || !PURE_PREDICATE_FUNCTIONS.has(name)) { + throw new Error(`sql_check refuses function ${call}: predicates are limited to built-in pure functions.`) + } + } readSql(`SELECT 1 WHERE (\n${predicate}\n)`, dialect) // A read-only wrapper alone can still accept an expression that closes it and // introduces another SELECT. Require the expression's own delimiters to balance. let depth = 0 - for (const char of maskSql(tokenizeSql(predicate, dialect).tokens)) { + for (const char of maskSql(tokens)) { if (char === "(") depth++ if (char === ")" && --depth < 0) throw new Error("sql_check predicates must be bare boolean expressions") } @@ -264,15 +707,12 @@ function readPredicate(predicate: string, dialect?: string): string { } /** Catalog reads only; never scan the population to obtain a guard estimate. */ -async function tableEstimate(table: string, dialect: Dialect, run: (sql: string) => Promise) { +async function tableEstimate( + table: string, + dialect: Exclude, + run: (sql: string) => Promise, +) { try { - if (dialect === "postgres") { - const [row] = await run(`SELECT reltuples AS estimate FROM pg_catalog.pg_class - WHERE oid = pg_catalog.to_regclass(E${literal(table.replace(/\\/g, "\\\\"))}) - AND relkind IN ('r', 'm') AND reltuples > 0`) - // Views/partitioned parents have no usable population estimate; zero can mean no statistics. - return row ? number(row.estimate) : undefined - } const parts = table .match(/"(?:[^"]|"")+"|[A-Za-z_][A-Za-z0-9_$]*/g)! .map((part) => (part.startsWith('"') ? part.slice(1, -1).replaceAll('""', '"') : part)) @@ -344,7 +784,13 @@ export async function checkSql(input: SqlCheckPublicInput) { const predicates = args.predicates?.map((predicate) => readPredicate(predicate, sourceDialect)) // Keep internal CTEs from shadowing any relation in the supplied queries. let scope = "__sql_check_0_" - for (let i = 1; [sql, left, right].some((s) => s?.toLowerCase().includes(scope)); i++) scope = `__sql_check_${i}_` + for ( + let i = 1; + [sql, left, right, base?.sql, ...(predicates ?? [])].some((s) => s?.toLowerCase().includes(scope)); + i++ + ) { + scope = `__sql_check_${i}_` + } const rawType = Registry.getConfig(args.warehouse)?.type.toLowerCase() const dialect = rawType === "postgresql" ? "postgres" : rawType if (dialect !== "sqlite" && dialect !== "duckdb" && dialect !== "postgres") { @@ -370,7 +816,10 @@ export async function checkSql(input: SqlCheckPublicInput) { const population = required(base, "base_sql") const expressions = required(predicates, "predicates") try { - const estimate = population.table === undefined ? undefined : await tableEstimate(population.table, dialect, run) + let estimate = + population.table === undefined || dialect === "postgres" + ? undefined + : await tableEstimate(population.table, dialect, run) const cap = args.max_rows_scanned ?? 5_000_000 if (estimate !== undefined && estimate > cap) { throw new Error( @@ -378,9 +827,36 @@ export async function checkSql(input: SqlCheckPublicInput) { ) } const count = (predicate: string) => `COUNT(*) FILTER (WHERE ${predicate})` - const [row] = await run(`SELECT COUNT(*) AS total, + const probe = `SELECT COUNT(*) AS total, ${expressions.map((predicate, i) => `${count(predicate)} AS keep_${i + 1}`).join(", ")}, - ${count(expressions.join(" AND "))} AS keep_all FROM (\n${population.sql}\n) ${population.alias}`) + ${count(expressions.join(" AND "))} AS keep_all FROM (\n${population.sql}\n) ${population.alias}` + const atomicTable = dialect === "postgres" ? population.table : undefined + // One statement resolves the catalog estimate and population in the same session; + // the uncorrelated WHERE guard rejects an oversized population before scanning it. + const [row] = await run( + atomicTable === undefined + ? probe + : `WITH ${scope}est AS ( + SELECT reltuples AS estimate FROM pg_catalog.pg_class + WHERE oid = pg_catalog.to_regclass(E${literal(atomicTable.replace(/\\/g, "\\\\"))}) + AND relkind IN ('r', 'm') AND reltuples > 0 + ), ${scope}probe AS ( + ${probe} + WHERE COALESCE((SELECT estimate FROM ${scope}est) <= ${cap}, TRUE) + ) + SELECT ${scope}probe.*, (SELECT estimate FROM ${scope}est) AS estimate, + COALESCE((SELECT estimate FROM ${scope}est) > ${cap}, FALSE) AS estimate_exceeded + FROM ${scope}probe`, + ) + if (atomicTable !== undefined) { + // Views, partitioned parents and uncollected statistics retain an unknown estimate. + estimate = row.estimate == null ? undefined : number(row.estimate) + if (row.estimate_exceeded) { + return result(args.check, { skipped: "estimate" }, [ + `predicate_effect: skipped: estimate; population estimate ${estimate} exceeds max_rows_scanned ${cap}; use a narrower base_sql.`, + ]) + } + } const total = number(row.total) const effects = args.predicates!.map((predicate, i) => { const kept = number(row[`keep_${i + 1}`]) diff --git a/packages/opencode/test/altimate/tools/sql-check.test.ts b/packages/opencode/test/altimate/tools/sql-check.test.ts index dde6d48f4..2eab3836a 100644 --- a/packages/opencode/test/altimate/tools/sql-check.test.ts +++ b/packages/opencode/test/altimate/tools/sql-check.test.ts @@ -727,6 +727,140 @@ test("refuses cross-warehouse checks and explains the materialization prerequisi ).rejects.toThrow(/sql_materialize/) }) +test("PostgreSQL predicate_effect checks the estimate and population in one statement", async () => { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute").mockImplementation(async (sql) => + sql.includes("COUNT(*)") + ? { + columns: ["total", "keep_1", "keep_all", "estimate", "estimate_exceeded"], + rows: [[5, 3, 3, 5, false]], + row_count: 1, + truncated: false, + } + : { columns: ["estimate"], rows: [[5]], row_count: 1, truncated: false }, + ) + const get = spyOn(Registry, "get").mockResolvedValue(connector) + try { + for (const type of ["postgres", "postgresql"]) { + Registry.setConfigs({ pg_fixture: { type } }) + execute.mockClear() + const result = await checkSql({ + warehouse: "pg_fixture", + check: "predicate_effect", + base_sql: '"orders"', + predicates: ["amount > 0"], + max_rows_scanned: 10, + query_timeout_ms: 4321, + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + expect(execute).toHaveBeenCalledTimes(1) + const [sql, , , options] = execute.mock.calls[0] + expect(sql).toContain("pg_catalog.pg_class") + expect(sql).toContain(`pg_catalog.to_regclass(E'"orders"')`) + expect(sql).toContain("COUNT(*) AS total") + expect(sql).toMatch(/WHERE COALESCE\(\(SELECT estimate FROM \w+\) <= 10, TRUE\)/) + expect(sql).toContain("AS estimate_exceeded") + expect(options).toMatchObject({ readOnly: true, noLimit: true, queryTimeoutMs: 4321 }) + } + } finally { + get.mockRestore() + execute.mockRestore() + } +}) + +test("PostgreSQL predicate_effect skips an exceeded estimate with the population scan gated", async () => { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute").mockImplementation(async (sql) => + sql.includes("COUNT(*)") + ? { + columns: ["total", "keep_1", "keep_all", "estimate", "estimate_exceeded"], + rows: [[0, 0, 0, 1000, true]], + row_count: 1, + truncated: false, + } + : { columns: ["estimate"], rows: [[1000]], row_count: 1, truncated: false }, + ) + const get = spyOn(Registry, "get").mockResolvedValue(connector) + Registry.setConfigs({ pg_fixture: { type: "postgres" } }) + try { + const result = await checkSql({ + warehouse: "pg_fixture", + check: "predicate_effect", + base_sql: "orders", + predicates: ["amount > 0"], + max_rows_scanned: 10, + }) + expect(result.metadata).toMatchObject({ skipped: "estimate" }) + expect(result.metadata.total_rows).toBeUndefined() + expect(result.output).toContain("1000 exceeds max_rows_scanned 10") + expect(execute).toHaveBeenCalledTimes(1) + const sql = execute.mock.calls[0][0] + // The uncorrelated catalog guard belongs below the aggregate, so it can + // reject the population before any row or predicate is evaluated. + expect(sql).toMatch( + /FROM \(\s*SELECT \* FROM orders\s*\) orders\s+WHERE COALESCE\(\(SELECT estimate FROM \w+\) <= 10, TRUE\)/, + ) + expect(sql).toContain("pg_catalog.pg_class") + } finally { + get.mockRestore() + execute.mockRestore() + } +}) + +test("PostgreSQL predicate_effect keeps an unknown atomic estimate and empty population distinct", async () => { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute").mockResolvedValue({ + columns: ["total", "keep_1", "keep_all", "estimate", "estimate_exceeded"], + rows: [[0, 0, 0, null, false]], + row_count: 1, + truncated: false, + }) + const get = spyOn(Registry, "get").mockResolvedValue(connector) + Registry.setConfigs({ pg_fixture: { type: "postgres" } }) + try { + const result = await checkSql({ + warehouse: "pg_fixture", + check: "predicate_effect", + base_sql: "orders", + predicates: ["amount > 0"], + }) + expect(result.metadata).toMatchObject({ total_rows: 0, combined_kept_rows: 0 }) + expect(result.metadata.skipped).toBeUndefined() + expect(result.output).toContain("row_estimate: unknown") + expect(result.output).toContain("empty population") + expect(execute).toHaveBeenCalledTimes(1) + } finally { + get.mockRestore() + execute.mockRestore() + } +}) + +test("PostgreSQL predicate_effect avoids catalog CTE collisions with its population table", async () => { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute").mockResolvedValue({ + columns: ["total", "keep_1", "keep_all", "estimate", "estimate_exceeded"], + rows: [[1, 1, 1, 1, false]], + row_count: 1, + truncated: false, + }) + const get = spyOn(Registry, "get").mockResolvedValue(connector) + Registry.setConfigs({ pg_fixture: { type: "postgres" } }) + try { + await checkSql({ + warehouse: "pg_fixture", + check: "predicate_effect", + base_sql: "__sql_check_0_est", + predicates: ["amount > 0"], + }) + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][0]).toMatch(/^WITH __sql_check_1_est AS/) + expect(execute.mock.calls[0][0]).toContain("SELECT * FROM __sql_check_0_est") + } finally { + get.mockRestore() + execute.mockRestore() + } +}) + test("PostgreSQL checks execute with readOnly for postgres and postgresql connectors", async () => { const connector = await Registry.get("sqlite_fixture") const execute = spyOn(connector, "execute").mockResolvedValue({ @@ -796,6 +930,141 @@ test("PostgreSQL checks report a write-performing SELECT as an error under readO } }) +test("predicate_effect refuses emit_wrapper before SQL execution on every filtered dialect", async () => { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute").mockResolvedValue({ + columns: ["total", "keep_1", "keep_all"], + rows: [[1, 1, 1]], + row_count: 1, + truncated: false, + }) + const get = spyOn(Registry, "get").mockResolvedValue(connector) + try { + for (const type of ["postgres", "postgresql", "duckdb", "sqlite"]) { + Registry.setConfigs({ filtered_fixture: { type } }) + await expect( + checkSql({ + warehouse: "filtered_fixture", + check: "predicate_effect", + base_sql: "SELECT 1 AS id", + predicates: ["emit_wrapper(id) IS NOT NULL"], + }), + ).rejects.toThrow(/emit_wrapper.*predicates are limited to built-in pure functions/i) + } + expect(get).not.toHaveBeenCalled() + expect(execute).not.toHaveBeenCalled() + } finally { + get.mockRestore() + execute.mockRestore() + } +}) + +test("predicate_effect refuses qualified, quoted, and pg-prefixed wrappers before SQL execution", async () => { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute").mockResolvedValue({ + columns: ["total", "keep_1", "keep_all"], + rows: [[1, 1, 1]], + row_count: 1, + truncated: false, + }) + const get = spyOn(Registry, "get").mockResolvedValue(connector) + Registry.setConfigs({ pg_fixture: { type: "postgres" } }) + try { + for (const fn of [ + "public.my_fn", + "public.lower", + '"public" /* gap */ . "my_fn"', + '"emit_wrapper"', + '"LOWER"', + '"custom.lower"', + "pg_get_wrapper", + "émit_wrapper", + "width_bucKet", + "wrapper$fn", + ]) { + const pending = checkSql({ + warehouse: "pg_fixture", + check: "predicate_effect", + base_sql: "SELECT 2 AS x", + predicates: [`${fn}(x) IS NOT NULL`], + }) + await expect(pending).rejects.toThrow(/predicates are limited to built-in pure functions/i) + if (fn === "public.my_fn") await expect(pending).rejects.toThrow("public.my_fn") + } + expect(get).not.toHaveBeenCalled() + expect(execute).not.toHaveBeenCalled() + } finally { + get.mockRestore() + execute.mockRestore() + } +}) + +test("predicate_effect accepts coalesce and lower on every filtered dialect", async () => { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute").mockResolvedValue({ + columns: ["total", "keep_1", "keep_2", "keep_all"], + rows: [[1, 1, 1, 1]], + row_count: 1, + truncated: false, + }) + const get = spyOn(Registry, "get").mockResolvedValue(connector) + try { + for (const type of ["postgres", "postgresql", "duckdb", "sqlite"]) { + Registry.setConfigs({ filtered_fixture: { type } }) + const result = await checkSql({ + warehouse: "filtered_fixture", + check: "predicate_effect", + base_sql: "SELECT 2 AS x, 'A' AS name", + predicates: ["coalesce(x, 0) > 1", "lower(name) = 'a'"], + }) + expect(result.metadata).toMatchObject({ total_rows: 1, combined_kept_rows: 1 }) + } + expect(execute).toHaveBeenCalledTimes(4) + } finally { + get.mockRestore() + execute.mockRestore() + } +}) + +test("predicate_effect accepts scalar built-ins, casts, and boolean grouping", async () => { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute").mockResolvedValue({ + columns: ["total", "keep_1", "keep_all"], + rows: [[1, 1, 1]], + row_count: 1, + truncated: false, + }) + const get = spyOn(Registry, "get").mockResolvedValue(connector) + Registry.setConfigs({ pg_fixture: { type: "postgres" } }) + const predicates = [ + "CAST(x AS numeric(10, 2)) > 0 AND (x IN (1, 2) OR NOT (x = 0))", + "x::numeric(10, 2) > 0 AND CAST(name AS varchar(10)) IS NOT NULL", + "extract(year FROM current_date) > 2000 AND position('a' IN lower(name)) > 0", + "substring(name FROM 1 FOR 1) = 'a' AND trim(name) <> ''", + "pg_typeof(x) IS NOT NULL AND pg_column_size(x) > 0", + "\"lower\"(name) = 'a' AND x = ANY (ARRAY[1, 2])", + "name = 'emit_wrapper(x)' /* public.my_fn(x) */ AND lower(name) <> ''", + "CASE WHEN (x > 0) THEN (abs(x) > 0) ELSE (false) END", + '"emit_wrapper(id)" IS NOT NULL', + "EXISTS (SELECT 1 WHERE (x > 0))", + ] + try { + for (const predicate of predicates) { + const result = await checkSql({ + warehouse: "pg_fixture", + check: "predicate_effect", + base_sql: "SELECT 2 AS x, 'a' AS name", + predicates: [predicate], + }) + expect(result.metadata.total_rows).toBe(1) + } + expect(execute).toHaveBeenCalledTimes(predicates.length) + } finally { + get.mockRestore() + execute.mockRestore() + } +}) + test("the registered tool reports SQL refusal as an error result", async () => { const tool = await initTool(SqlCheckTool) const result = await tool.execute( From bc5c34726a21ef4b40bd5d9b750a1e5564739717 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Tue, 13 Oct 2026 21:12:02 -0700 Subject: [PATCH 17/22] fix(analyst): `predicate_effect` resolves functions and operators against the catalog and bounds local populations in one statement - The built-in pure-function allowlist now applies to the population SQL as well as the predicates, on every dialect, so a DuckDB `postgres_execute(...)` or a PostgreSQL wrapper population is refused before any SQL runs. - PostgreSQL: before the probe, one read-only `pg_operator` query and one `pg_proc` query (bound `ANY($1)` arrays) cover every operator and function name referenced by the population and predicates, including implicit operators behind `LIKE`/`IN`/`BETWEEN`/`IS DISTINCT FROM` and the `!=` alias; any definition outside `pg_catalog` refuses the probe with the name. DuckDB checks `duckdb_functions()` for non-internal macros the same way; SQLite has no SQL-level function catalog (commented). - SQLite and DuckDB run the estimate and the aggregate on one population bounded to `cap + 1` rows in a single statement, so a concurrent peer cannot swap the relation between the guard and the scan; an exceeded bound returns `skipped: "estimate"`. PostgreSQL keeps its atomic catalog-estimate gate. - Allowlist gains the SQLite/DuckDB date built-ins (`strftime`, `julianday`, `unixepoch`, `datetime`, `timediff`) and the bare `array` constructor. - Drivers: the PostgreSQL connector forwards bind parameters on both the regular and byte-limited read paths; it previously dropped them. - tests: wrapper and remote-execute populations refused per dialect; user-defined operator/overload/macro refused and built-in-only accepted after read-only catalog queries; implicit operators and trailing-sign splits resolved; SQLite/DuckDB single-statement bound with exceeded skip; date built-ins accepted; PostgreSQL bind forwarding on both read paths. Co-Authored-By: Claude Fable 5.1 --- packages/drivers/src/postgres.ts | 8 +- .../drivers/test/postgres-readonly.test.ts | 59 ++ .../opencode/src/altimate/tools/sql-check.ts | 247 +++++--- .../src/altimate/tools/sql-tokenize.ts | 5 +- .../sql-check-predicate-effect.test.ts | 139 ++-- .../test/altimate/tools/sql-check.test.ts | 599 ++++++++++++++++-- 6 files changed, 842 insertions(+), 215 deletions(-) diff --git a/packages/drivers/src/postgres.ts b/packages/drivers/src/postgres.ts index 9f99a65d8..50ee8b878 100644 --- a/packages/drivers/src/postgres.ts +++ b/packages/drivers/src/postgres.ts @@ -94,7 +94,7 @@ export async function connect(config: ConnectionConfig): Promise { pool = new Pool(poolConfig) }, - async execute(sql: string, limit?: number, _binds?: any[], options?: ExecuteOptions): Promise { + async execute(sql: string, limit?: number, binds?: any[], options?: ExecuteOptions): Promise { const client = await acquireClient(pool, options?.signal) // Tighter of the explicit statement_timeout and the common execution // deadline: a configured limit stricter than the default is preserved. @@ -155,7 +155,7 @@ export async function connect(config: ConnectionConfig): Promise { const consume = createResultByteBudget(options.maxResultBytes) const Query = pg.default?.Query ?? pg.Query return await new Promise((resolve, reject) => { - const stream = new Query({ text: query, rowMode: "array" }) + const stream = new Query({ text: query, values: binds, rowMode: "array" }) const rows: unknown[][] = [] let columns: string[] = [] let truncated = false @@ -192,7 +192,9 @@ export async function connect(config: ConnectionConfig): Promise { }) } // altimate_change end - const result = await client.query(query) + // altimate_change start — preserve bound catalog names and other query parameters + const result = await client.query(query, binds) + // altimate_change end const columns = result.fields?.map((f: any) => f.name) ?? [] const truncated = effectiveLimit > 0 && result.rows.length > effectiveLimit const rows = truncated diff --git a/packages/drivers/test/postgres-readonly.test.ts b/packages/drivers/test/postgres-readonly.test.ts index 162db957d..de5b91f79 100644 --- a/packages/drivers/test/postgres-readonly.test.ts +++ b/packages/drivers/test/postgres-readonly.test.ts @@ -1,4 +1,5 @@ import { expect, mock, test } from "bun:test" +import { EventEmitter } from "node:events" import { connect } from "../src/postgres" import { QueryTimeoutError } from "../src/types" @@ -182,3 +183,61 @@ test("Postgres never starts user SQL when cancellation arrives during session se await connector.close() } }) + +for (const maxResultBytes of [undefined, 1024]) { + test(`Postgres forwards bound catalog arrays in ${maxResultBytes === undefined ? "regular" : "byte-limited"} read-only queries`, async () => { + const calls: { sql: string; values?: unknown[] }[] = [] + const releases: unknown[] = [] + class Query extends EventEmitter { + constructor(readonly config: { text: string; values?: unknown[]; rowMode: string }) { + super() + } + } + mock.module("pg", () => ({ + default: { + Query, + Pool: class { + async connect() { + return { + query(query: string | Query, values?: unknown[]) { + const sql = typeof query === "string" ? query : query.config.text + calls.push({ sql, values: typeof query === "string" ? values : query.config.values }) + const fields = [{ name: "proname" }] + if (query instanceof Query) { + queueMicrotask(() => { + query.emit("row", ["lower"], { fields }) + query.emit("end", { fields }) + }) + return query + } + return Promise.resolve({ fields, rows: [{ proname: "lower" }] }) + }, + release(discard?: boolean) { releases.push(discard) }, + } + } + async end() {} + }, + }, + })) + const connector = await connect({ type: "postgres", query_timeout_ms: 0 }) + await connector.connect() + const sql = "SELECT proname FROM pg_catalog.pg_proc WHERE proname = ANY($1)" + const values = [["lower", "upper"]] + try { + const result = await connector.execute(sql, undefined, values, { + noLimit: true, + readOnly: true, + maxResultBytes, + }) + expect(result.rows).toEqual([["lower"]]) + expect(calls).toEqual([ + { sql: "BEGIN READ ONLY", values: undefined }, + { sql, values }, + { sql: "ROLLBACK", values: undefined }, + ]) + expect(releases).toEqual([undefined]) + } finally { + await connector.close() + } + }) +} diff --git a/packages/opencode/src/altimate/tools/sql-check.ts b/packages/opencode/src/altimate/tools/sql-check.ts index 0b184f747..c1bfc5468 100644 --- a/packages/opencode/src/altimate/tools/sql-check.ts +++ b/packages/opencode/src/altimate/tools/sql-check.ts @@ -27,11 +27,12 @@ const SIDE_EFFECT_FUNCTION_CALL = new RegExp( "i", ) -// Predicates accept only these built-in pure functions; keep this list flat and sorted. +// Predicate probes and their populations accept only these pure functions; keep this list flat and sorted. const PURE_PREDICATE_FUNCTIONS = new Set([ "abs", "acos", "age", + "array", "array_agg", "array_append", "array_cat", @@ -76,6 +77,7 @@ const PURE_PREDICATE_FUNCTIONS = new Set([ "date", "date_part", "date_trunc", + "datetime", "decimal", "degrees", "div", @@ -93,6 +95,7 @@ const PURE_PREDICATE_FUNCTIONS = new Set([ "jsonb_extract_path", "jsonb_extract_path_text", "jsonb_typeof", + "julianday", "least", "left", "length", @@ -425,6 +428,7 @@ const PURE_PREDICATE_FUNCTIONS = new Set([ "split_part", "sqrt", "starts_with", + "strftime", "string_agg", "strpos", "substr", @@ -432,6 +436,7 @@ const PURE_PREDICATE_FUNCTIONS = new Set([ "sum", "tan", "time", + "timediff", "timestamp", "to_char", "to_date", @@ -440,6 +445,7 @@ const PURE_PREDICATE_FUNCTIONS = new Set([ "translate", "trim", "trunc", + "unixepoch", "upper", "varchar", "width_bucket", @@ -470,7 +476,7 @@ const parameters = z .int() .positive() .optional() - .describe("Bare-table catalog row estimate cap for predicate_effect (default 5000000); not a SELECT row limit"), + .describe("Bare-table row estimate cap for predicate_effect (default 5000000); not a SELECT row limit"), sql: z.string().optional().describe("Read-only SELECT for grain, nulls or coverage"), columns: z.array(column).min(1).max(20).optional().describe("Output column names for grain or nulls (up to 20)"), column: column.optional().describe("Output column name for coverage"), @@ -520,7 +526,7 @@ const publicParameters = z .int() .nonnegative() .optional() - .describe("Bare-table catalog row estimate cap for predicate_effect (default 5000000); not a SELECT row limit"), + .describe("Bare-table row estimate cap for predicate_effect (default 5000000); not a SELECT row limit"), sql: z.string().optional().describe("Read-only SELECT for grain, nulls or coverage"), columns: z.array(z.string()).max(20).optional().describe("Output column names for grain or nulls (up to 20)"), column: z.string().optional().describe("Output column name for coverage"), @@ -585,12 +591,54 @@ function required(value: T | undefined, name: string): T { return value } +const OPERATOR_TOKEN = /[+\-*/<>=~!@#%^&|`?]+/g + +function postgresOperators(sql: string): string[] { + const code = maskSql(tokenizeSql(sql, "postgres").tokens) + const names = new Set(code.match(OPERATOR_TOKEN) ?? []) + for (const operator of [...names]) { + // PostgreSQL splits trailing unary signs from ordinary operator runs: =- is = and -. + if (!/[~!@#%^&|`?]/.test(operator)) { + const signs = /[+-]+$/.exec(operator)?.[0] + if (signs) { + const head = operator.slice(0, -signs.length) + if (head) names.add(head) + for (const sign of signs) names.add(sign) + } + } + } + // The parser rewrites these spellings to catalog operator names. + if (names.has("!=")) names.add("<>") + for (const [keyword, operators] of [ + [/\bLIKE\b/i, ["~~", "!~~"]], + [/\bILIKE\b/i, ["~~*", "!~~*"]], + [/\bSIMILAR\s+TO\b/i, ["~", "!~"]], + [/\bBETWEEN\b/i, ["<", ">", "<=", ">="]], + [/\bIN\b/i, ["=", "<>"]], + [/\bIS\s+(?:NOT\s+)?DISTINCT\s+FROM\b/i, ["="]], + ] as const) { + if (keyword.test(code)) for (const operator of operators) names.add(operator) + } + return [...names] +} + /** Ignore quoted content while checking structure; preserve the original SELECT and its LIMIT. */ -export function readSql(sql: string, dialect?: string): string { - const { queryType, blocked } = classifyAndCheck(sql, dialect) +export function readSql(sql: string, dialect?: string, catalogOperators = false): string { // Independently reject write CTEs, SELECT INTO and statement/CTE escapes before // embedding user SQL, even when the parser reports a query category. const { tokens, complete } = tokenizeSql(sql, dialect) + // The structural parser does not know arbitrary PostgreSQL operator names. Only + // catalog-checked probes may substitute their spelling for structural parsing; + // all lexical checks and the executed SQL still use the original text. + const structuralSql = + dialect === "postgres" && catalogOperators + ? tokens + .map(({ kind, text }) => + kind === "code" ? text.replace(OPERATOR_TOKEN, (operator) => (operator === "*" ? "*" : "+")) : text, + ) + .join("") + : sql + const { queryType, blocked } = classifyAndCheck(structuralSql, dialect) const masked = maskSql(tokens) // SELECT alone does not make sequence setters, extension loaders or file writers read-only. // Preserve quoted function identifiers while hiding string literals and comments. @@ -615,7 +663,7 @@ export function readSql(sql: string, dialect?: string): string { if ( blocked || queryType !== "read" || - !isSingleQuery(sql, dialect) || + !isSingleQuery(structuralSql, dialect) || !complete || escapedFunction || !balanced || @@ -626,9 +674,7 @@ export function readSql(sql: string, dialect?: string): string { body, ) || /\bREPLACE\b(?!\s*\()/i.test(body) || - SIDE_EFFECT_FUNCTION_CALL.test( - functions, - ) + SIDE_EFFECT_FUNCTION_CALL.test(functions) ) { throw new Error( "sql_check only accepts a single read-only SELECT (or WITH ... SELECT); it never writes or fixes data.", @@ -651,7 +697,9 @@ function readBase(sql: string, dialect?: string) { const table = bare ? sql.trim() : undefined // Preserve quoting and case folding, including dots inside a quoted table name. const alias = table?.match(/"(?:[^"]|"")+"|[A-Za-z_][A-Za-z0-9_$]*/g)?.at(-1) ?? "t" - return { sql: readSql(table ? `SELECT * FROM ${table}` : sql, dialect), table, alias } + const population = readSql(table ? `SELECT * FROM ${table}` : sql, dialect, true) + pureFunctions(population, dialect) + return { sql: population, table, alias } } function populationLimit(sql: string, dialect: Dialect): string | undefined { @@ -668,15 +716,17 @@ function populationLimit(sql: string, dialect: Dialect): string | undefined { if (/(?:^|[^\w$])SELECT\s+(?:(?:ALL|DISTINCT)\s+)?TOP\b\s*(?:\d|\()/i.test(topLevel)) return "TOP" } -function readPredicate(predicate: string, dialect?: string): string { - const { tokens } = tokenizeSql(predicate, dialect) +function pureFunctions(sql: string, dialect?: string): string[] { + const { tokens } = tokenizeSql(sql, dialect) // Retain identifier boundaries and case; hide strings and comments, which cannot call functions. const functions = tokens .map(({ kind, text }) => (kind === "code" || kind === "identifier" ? text : text.replace(/[^\r\n]/g, " "))) .join("") - const identifier = - '(?:"(?:[^"\\x00]|"")+"|`(?:[^`]|``)+`|\\[(?:[^\\]]|\\]\\])+\\]|[A-Za-z_\\u0080-\\uFFFF][\\w$\\u0080-\\uFFFF]*)' + const identifier = /^(postgres|duckdb)$/i.test(dialect ?? "") + ? '(?:"(?:[^"\\x00]|"")+"|[A-Za-z_\\u0080-\\uFFFF][\\w$\\u0080-\\uFFFF]*)' + : '(?:"(?:[^"\\x00]|"")+"|`(?:[^`]|``)+`|\\[(?:[^\\]]|\\]\\])+\\]|[A-Za-z_\\u0080-\\uFFFF][\\w$\\u0080-\\uFFFF]*)' const calls = new RegExp(`${identifier}(?:\\s*\\.\\s*${identifier})*`, "g") + const names = new Set() for (const match of functions.matchAll(calls)) { if (!/^\s*\(/.test(functions.slice(match.index + match[0].length))) continue const call = match[0].trim() @@ -693,8 +743,15 @@ function readPredicate(predicate: string, dialect?: string): string { if (parts.length !== 1 || !PURE_PREDICATE_FUNCTIONS.has(name)) { throw new Error(`sql_check refuses function ${call}: predicates are limited to built-in pure functions.`) } + names.add(name) } - readSql(`SELECT 1 WHERE (\n${predicate}\n)`, dialect) + return [...names] +} + +function readPredicate(predicate: string, dialect?: string): string { + pureFunctions(predicate, dialect) + const { tokens } = tokenizeSql(predicate, dialect) + readSql(`SELECT 1 WHERE (\n${predicate}\n)`, dialect, true) // A read-only wrapper alone can still accept an expression that closes it and // introduces another SELECT. Require the expression's own delimiters to balance. let depth = 0 @@ -706,60 +763,62 @@ function readPredicate(predicate: string, dialect?: string): string { return `(\n${predicate}\n)` } -/** Catalog reads only; never scan the population to obtain a guard estimate. */ -async function tableEstimate( - table: string, - dialect: Exclude, - run: (sql: string) => Promise, +/** Reject server-defined overloads even when a call's spelling is on the pure allowlist. */ +async function checkPredicateCatalog( + population: string, + predicates: string[], + dialect: Dialect, + run: (sql: string, binds?: unknown[]) => Promise, ) { - try { - const parts = table - .match(/"(?:[^"]|"")+"|[A-Za-z_][A-Za-z0-9_$]*/g)! - .map((part) => (part.startsWith('"') ? part.slice(1, -1).replaceAll('""', '"') : part)) - const name = literal(parts.at(-1)!) - if (dialect === "duckdb") { - const matches = (column: string, value: string) => `LOWER(${column}) = LOWER(${literal(value)})` - const qualifier = - parts.length === 3 - ? `${matches("database_name", parts[0])} AND ${matches("schema_name", parts[1])}` - : parts.length === 2 - ? `((${matches("schema_name", parts[0])} AND database_name IN (current_database(), 'temp')) - OR (${matches("database_name", parts[0])} AND schema_name = 'main'))` - : "LOWER(schema_name) IN (SELECT LOWER(unnest(current_schemas(true))))" - const rows = await run(`SELECT estimate FROM ( - SELECT database_name, schema_name, table_name, estimated_size AS estimate FROM duckdb_tables() - UNION ALL SELECT database_name, schema_name, view_name AS table_name, NULL AS estimate FROM duckdb_views() - ) relations WHERE LOWER(table_name) = LOWER(${name}) AND (${qualifier})`) - // Search paths and temporary tables can shadow another catalog match. - // An ambiguous estimate is unknown, rather than a reason to refuse a probe. - return rows.length === 1 && rows[0].estimate != null ? number(rows[0].estimate) : undefined + // SQLite SQL cannot persist functions; custom functions must be registered by the application. + if (dialect === "sqlite") return + const queries = [population, ...predicates] + const functionNames = queries.map((sql) => pureFunctions(sql, dialect)) + const functions = [...new Set(functionNames.flat())] + const origin = (name: string, names: string[][]) => + names.slice(1).some((query) => query.includes(name)) ? "predicate" : "population" + if (dialect === "postgres") { + const operatorNames = queries.map(postgresOperators) + const operators = [...new Set(operatorNames.flat())] + if (operators.length) { + const [operator] = await run( + `SELECT o.oprname, n.nspname + FROM pg_catalog.pg_operator o JOIN pg_catalog.pg_namespace n ON n.oid = o.oprnamespace + WHERE o.oprname = ANY($1) AND n.nspname <> 'pg_catalog'`, + [operators], + ) + if (operator) { + throw new Error( + `sql_check ${origin(operator.oprname, operatorNames)} uses operator \`${operator.oprname}\`, which is not a built-in operator`, + ) + } } - if (parts.length > 2) return undefined - const schema = - parts.length === 2 - ? parts[0] - : ( - await run(`SELECT t.schema FROM pragma_table_list(${name}) t - JOIN pragma_database_list d ON d.name = t.schema - ORDER BY CASE WHEN t.schema = 'temp' THEN -1 ELSE d.seq END LIMIT 1`) - )[0]?.schema - if (!schema) return undefined - const rows = await run(`SELECT stat FROM ${ident(schema)}.sqlite_stat1 WHERE tbl = ${name} COLLATE NOCASE`) - const estimates = rows - .map((row) => Number(String(row.stat).split(/\s+/)[0])) - .filter((n) => Number.isFinite(n) && n >= 0) - return estimates.length ? Math.max(...estimates) : undefined - } catch (error) { - if (isQueryTimeoutError(error)) throw error - // A lost helper session or any other infrastructure failure must reach the caller: the probe - // would otherwise run under reset session state. Only recognised missing-statistics or - // catalog-access failures mean "unknown estimate". - if (typeof error === "object" && error !== null && "stateLost" in error) throw error - const message = error instanceof Error ? error.message : String(error) - if (/no such table|does not exist|sqlite_stat1|pragma_table_list|permission denied|insufficient privilege|catalog/i.test(message)) { - return undefined + if (functions.length) { + const [fn] = await run( + `SELECT p.proname, n.nspname + FROM pg_catalog.pg_proc p JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE p.proname = ANY($1) AND p.pronamespace <> 'pg_catalog'::regnamespace`, + [functions], + ) + if (fn) { + throw new Error( + `sql_check ${origin(fn.proname, functionNames)} calls \`${fn.proname}\`, which has a user-defined overload in schema \`${fn.nspname}\``, + ) + } + } + return + } + if (functions.length) { + const [fn] = await run( + `SELECT function_name, schema_name FROM duckdb_functions() + WHERE internal = false AND function_name COLLATE NOCASE IN (${functions.map(() => "?").join(", ")})`, + functions, + ) + if (fn) { + throw new Error( + `sql_check ${origin(fn.function_name.toLowerCase(), functionNames)} calls \`${fn.function_name}\`, which has a user-defined overload in schema \`${fn.schema_name}\``, + ) } - throw error } } @@ -799,8 +858,8 @@ export async function checkSql(input: SqlCheckPublicInput) { ) } const connector = await Registry.get(args.warehouse) - async function run(query: string): Promise { - const response = await connector.execute(query, undefined, undefined, { + async function run(query: string, binds?: unknown[]): Promise { + const response = await connector.execute(query, undefined, binds, { noLimit: true, readOnly: true, queryTimeoutMs: args.query_timeout_ms, @@ -816,30 +875,33 @@ export async function checkSql(input: SqlCheckPublicInput) { const population = required(base, "base_sql") const expressions = required(predicates, "predicates") try { - let estimate = - population.table === undefined || dialect === "postgres" - ? undefined - : await tableEstimate(population.table, dialect, run) + await checkPredicateCatalog(population.sql, expressions, dialect, run) const cap = args.max_rows_scanned ?? 5_000_000 - if (estimate !== undefined && estimate > cap) { - throw new Error( - `sql_check population estimate ${estimate} exceeds max_rows_scanned ${cap}; use a narrower base_sql.`, - ) - } + const localTable = population.table !== undefined && dialect !== "postgres" const count = (predicate: string) => `COUNT(*) FILTER (WHERE ${predicate})` const probe = `SELECT COUNT(*) AS total, ${expressions.map((predicate, i) => `${count(predicate)} AS keep_${i + 1}`).join(", ")}, - ${count(expressions.join(" AND "))} AS keep_all FROM (\n${population.sql}\n) ${population.alias}` - const atomicTable = dialect === "postgres" ? population.table : undefined - // One statement resolves the catalog estimate and population in the same session; - // the uncorrelated WHERE guard rejects an oversized population before scanning it. + ${count(expressions.join(" AND "))} AS keep_all FROM (\n${localTable ? `SELECT * FROM ${scope}source` : population.sql}\n) ${population.alias}` + // Local engines reuse one bounded, materialized population for both the + // estimate and aggregate. A peer cannot swap the relation between them, + // and an exceeded cap never sends an unbounded relation to the aggregate. const [row] = await run( - atomicTable === undefined + population.table === undefined ? probe - : `WITH ${scope}est AS ( - SELECT reltuples AS estimate FROM pg_catalog.pg_class - WHERE oid = pg_catalog.to_regclass(E${literal(atomicTable.replace(/\\/g, "\\\\"))}) - AND relkind IN ('r', 'm') AND reltuples > 0 + : `WITH ${ + localTable + ? `${scope}source AS MATERIALIZED ( + ${population.sql} LIMIT ${BigInt(cap) + 1n} + ), ` + : "" + }${scope}est AS ( + ${ + localTable + ? `SELECT COUNT(*) AS estimate FROM ${scope}source` + : `SELECT reltuples AS estimate FROM pg_catalog.pg_class + WHERE oid = pg_catalog.to_regclass(E${literal(population.table.replace(/\\/g, "\\\\"))}) + AND relkind IN ('r', 'm') AND reltuples > 0` + } ), ${scope}probe AS ( ${probe} WHERE COALESCE((SELECT estimate FROM ${scope}est) <= ${cap}, TRUE) @@ -848,14 +910,11 @@ export async function checkSql(input: SqlCheckPublicInput) { COALESCE((SELECT estimate FROM ${scope}est) > ${cap}, FALSE) AS estimate_exceeded FROM ${scope}probe`, ) - if (atomicTable !== undefined) { - // Views, partitioned parents and uncollected statistics retain an unknown estimate. - estimate = row.estimate == null ? undefined : number(row.estimate) - if (row.estimate_exceeded) { - return result(args.check, { skipped: "estimate" }, [ - `predicate_effect: skipped: estimate; population estimate ${estimate} exceeds max_rows_scanned ${cap}; use a narrower base_sql.`, - ]) - } + const estimate = row.estimate == null ? undefined : number(row.estimate) + if (row.estimate_exceeded) { + return result(args.check, { skipped: "estimate" }, [ + `predicate_effect: skipped: estimate; population estimate ${estimate} exceeds max_rows_scanned ${cap}; use a narrower base_sql.`, + ]) } const total = number(row.total) const effects = args.predicates!.map((predicate, i) => { diff --git a/packages/opencode/src/altimate/tools/sql-tokenize.ts b/packages/opencode/src/altimate/tools/sql-tokenize.ts index 0c949f3f1..15dc9053f 100644 --- a/packages/opencode/src/altimate/tools/sql-tokenize.ts +++ b/packages/opencode/src/altimate/tools/sql-tokenize.ts @@ -6,6 +6,9 @@ export function tokenizeSql(sql: string, dialect?: string): { tokens: SqlToken[] const tokens: SqlToken[] = [] const mysqlComments = !dialect || /^(mysql|mariadb)$/i.test(dialect) const dollarQuotes = /^(postgres|redshift|duckdb|snowflake)$/i.test(dialect ?? "") + // PostgreSQL backticks are operators; PostgreSQL/DuckDB brackets contain array expressions. + const backtickIdentifiers = !/^(postgres|duckdb)$/i.test(dialect ?? "") + const bracketIdentifiers = !/^(postgres|duckdb)$/i.test(dialect ?? "") let complete = true let i = 0 let code = 0 @@ -41,7 +44,7 @@ export function tokenizeSql(sql: string, dialect?: string): { tokens: SqlToken[] } else { const escaped = /[eE]/.test(sql[i]) && sql[i + 1] === "'" && !/[\w$]/.test(sql[i - 1] ?? "") const quote = escaped ? "'" : sql[i] - if (quote === "'" || quote === '"' || quote === "`" || quote === "[") { + if (quote === "'" || quote === '"' || (quote === "`" && backtickIdentifiers) || (quote === "[" && bracketIdentifiers)) { kind = quote === "'" ? "literal" : "identifier" const end = quote === "[" ? "]" : quote i += escaped ? 2 : 1 diff --git a/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts b/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts index 797605aa4..c82398a13 100644 --- a/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts +++ b/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts @@ -196,27 +196,28 @@ for (const warehouse of warehouses) { expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) }) - test("refuses a bare table whose catalog estimate exceeds the custom cap", async () => { + test("skips a bare table whose bounded estimate exceeds the custom cap", async () => { const connector = await Registry.get(warehouse) if (warehouse === "sqlite_predicates") await connector.execute("ANALYZE orders") const execute = spyOn(connector, "execute") try { - await expect( - checkSql({ - warehouse, - check: "predicate_effect", - base_sql: "orders", - predicates: ["id > 0"], - max_rows_scanned: 4, - }), - ).rejects.toThrow(/narrower.*base_sql/i) - expect(execute.mock.calls.some(([query]) => /COUNT\(\*\)\s+FILTER/i.test(query))).toBe(false) + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 0"], + max_rows_scanned: 4, + }) + expect(result.metadata).toMatchObject({ skipped: "estimate" }) + expect(result.output).toMatch(/narrower.*base_sql/i) + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][0]).toContain("LIMIT 5") } finally { execute.mockRestore() } }) - test("reports unknown estimates for views and still probes under the timeout", async () => { + test("bounds view populations and skips an exceeded cap under the timeout", async () => { const connector = await Registry.get(warehouse) await connector.execute("CREATE VIEW order_view AS SELECT * FROM orders") const execute = spyOn(connector, "execute") @@ -229,18 +230,24 @@ for (const warehouse of warehouses) { max_rows_scanned: 1, query_timeout_ms: 4321, }) - expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) - expect(result.output).toContain(unknownEstimate) + expect(result.metadata).toMatchObject({ skipped: "estimate" }) + expect(result.output).toContain("2 exceeds max_rows_scanned 1") + expect(execute).toHaveBeenCalledTimes(1) expect(execute.mock.calls.at(-1)?.[3]).toMatchObject({ noLimit: true, queryTimeoutMs: 4321 }) } finally { execute.mockRestore() } }) - test("does not report a known catalog estimate as unknown", async () => { + test("does not report a bounded bare-table estimate as unknown", async () => { const connector = await Registry.get(warehouse) if (warehouse === "sqlite_predicates") await connector.execute("ANALYZE orders") - const result = await checkSql({ warehouse, check: "predicate_effect", base_sql: "orders", predicates: ["id > 2"] }) + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 2"], + }) expect(result.metadata.total_rows).toBe(5) expect(result.output).not.toContain(unknownEstimate) }) @@ -494,7 +501,7 @@ for (const warehouse of warehouses) { } }) - test("skips without a probe when a catalog lookup times out", async () => { + test("skips without a verdict when the bounded estimate statement times out", async () => { const connector = await Registry.get(warehouse) const execute = spyOn(connector, "execute").mockRejectedValue( new QueryTimeoutError({ engine: warehouse === "scratch" ? "DuckDB" : "SQLite", timeoutMs: 4321 }), @@ -510,7 +517,7 @@ for (const warehouse of warehouses) { expect(result.metadata).toMatchObject({ skipped: "timeout" }) expect(result.metadata.total_rows).toBeUndefined() expect(execute).toHaveBeenCalledTimes(1) - expect(execute.mock.calls[0][0]).not.toMatch(/COUNT\(\*\)\s+FILTER/i) + expect(execute.mock.calls[0][0]).toMatch(/COUNT\(\*\)\s+FILTER/i) expect(execute.mock.calls[0][3]).toMatchObject({ noLimit: true, queryTimeoutMs: 4321 }) } finally { execute.mockRestore() @@ -556,7 +563,7 @@ test("DuckDB predicate_effect respects a view shadowing a larger table on the sc max_rows_scanned: 2, }) expect(result.metadata).toMatchObject({ total_rows: 1, combined_kept_rows: 1 }) - expect(result.output).toContain(unknownEstimate) + expect(result.output).not.toContain(unknownEstimate) }) test("DuckDB predicate_effect guards a two-part catalog and table reference", async () => { @@ -565,16 +572,16 @@ test("DuckDB predicate_effect guards a two-part catalog and table reference", as const catalog = String(database.rows[0][0]).replaceAll('"', '""') const execute = spyOn(connector, "execute") try { - await expect( - checkSql({ - warehouse: "scratch", - check: "predicate_effect", - base_sql: `"${catalog}".orders`, - predicates: ["id > 0"], - max_rows_scanned: 2, - }), - ).rejects.toThrow(/narrower.*base_sql/i) - expect(execute.mock.calls.some(([query]) => /COUNT\(\*\)\s+FILTER/i.test(query))).toBe(false) + const result = await checkSql({ + warehouse: "scratch", + check: "predicate_effect", + base_sql: `"${catalog}".orders`, + predicates: ["id > 0"], + max_rows_scanned: 2, + }) + expect(result.metadata).toMatchObject({ skipped: "estimate" }) + expect(result.output).toMatch(/narrower.*base_sql/i) + expect(execute).toHaveBeenCalledTimes(1) } finally { execute.mockRestore() } @@ -588,16 +595,16 @@ test("DuckDB predicate_effect resolves mixed-case schema names on the search pat await connector.execute("SET schema = 'foo'") const execute = spyOn(connector, "execute") try { - await expect( - checkSql({ - warehouse: "scratch", - check: "predicate_effect", - base_sql: "orders", - predicates: ["id > 0"], - max_rows_scanned: 2, - }), - ).rejects.toThrow(/narrower.*base_sql/i) - expect(execute.mock.calls.some(([query]) => /COUNT\(\*\)\s+FILTER/i.test(query))).toBe(false) + const result = await checkSql({ + warehouse: "scratch", + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 0"], + max_rows_scanned: 2, + }) + expect(result.metadata).toMatchObject({ skipped: "estimate" }) + expect(result.output).toMatch(/narrower.*base_sql/i) + expect(execute).toHaveBeenCalledTimes(1) } finally { execute.mockRestore() } @@ -623,21 +630,21 @@ test("SQLite predicate_effect defaults the bare-table estimate cap to five milli await connector.execute("UPDATE sqlite_stat1 SET stat = '5000001' WHERE tbl = 'orders'") const execute = spyOn(connector, "execute") try { - await expect( - checkSql({ - warehouse: "sqlite_predicates", - check: "predicate_effect", - base_sql: "orders", - predicates: ["id > 0"], - }), - ).rejects.toThrow(/narrower.*base_sql/i) - expect(execute.mock.calls.some(([query]) => /COUNT\(\*\)\s+FILTER/i.test(query))).toBe(false) + const result = await checkSql({ + warehouse: "sqlite_predicates", + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 0"], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 5 }) + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][0]).toContain("LIMIT 5000001") } finally { execute.mockRestore() } }) -test("SQLite predicate_effect reports an unknown estimate without ANALYZE", async () => { +test("SQLite predicate_effect enforces the bounded estimate without ANALYZE", async () => { const result = await checkSql({ warehouse: "sqlite_predicates", check: "predicate_effect", @@ -645,8 +652,8 @@ test("SQLite predicate_effect reports an unknown estimate without ANALYZE", asyn predicates: ["id > 2"], max_rows_scanned: 1, }) - expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) - expect(result.output).toContain(unknownEstimate) + expect(result.metadata).toMatchObject({ skipped: "estimate" }) + expect(result.output).toContain("2 exceeds max_rows_scanned 1") }) test("SQLite predicate_effect uses the default estimate cap when max_rows_scanned is zero", async () => { @@ -655,16 +662,16 @@ test("SQLite predicate_effect uses the default estimate cap when max_rows_scanne await connector.execute("UPDATE sqlite_stat1 SET stat = '5000001' WHERE tbl = 'orders'") const execute = spyOn(connector, "execute") try { - await expect( - checkSql({ - warehouse: "sqlite_predicates", - check: "predicate_effect", - base_sql: "orders", - predicates: ["id > 0"], - max_rows_scanned: 0, - }), - ).rejects.toThrow("population estimate 5000001 exceeds max_rows_scanned 5000000") - expect(execute.mock.calls.some(([query]) => /COUNT\(\*\)\s+FILTER/i.test(query))).toBe(false) + const result = await checkSql({ + warehouse: "sqlite_predicates", + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 0"], + max_rows_scanned: 0, + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 5 }) + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][0]).toContain("LIMIT 5000001") } finally { execute.mockRestore() } @@ -777,13 +784,17 @@ test("predicate_effect refuses a later denied predicate before requesting popula test("SQLite predicate_effect enables query_only and restores the previous setting after success and failure", async () => { const connector = await Registry.get("sqlite_predicates") + const guarded = await connector.execute("SELECT query_only FROM pragma_query_only()", undefined, undefined, { + readOnly: true, + }) + expect(guarded.rows).toEqual([[1]]) const result = await checkSql({ warehouse: "sqlite_predicates", check: "predicate_effect", - base_sql: "SELECT query_only AS guarded FROM pragma_query_only()", - predicates: ["guarded = 1"], + base_sql: "orders", + predicates: ["id > 0"], }) - expect(result.metadata.combined_kept_rows).toBe(1) + expect(result.metadata.combined_kept_rows).toBe(5) expect((await connector.execute("PRAGMA query_only")).rows).toEqual([[0]]) await expect(connector.execute("INSERT INTO orders (id) VALUES (6)", undefined, undefined, { readOnly: true })) .rejects.toThrow(/readonly/i) diff --git a/packages/opencode/test/altimate/tools/sql-check.test.ts b/packages/opencode/test/altimate/tools/sql-check.test.ts index 2eab3836a..c0a3300fe 100644 --- a/packages/opencode/test/altimate/tools/sql-check.test.ts +++ b/packages/opencode/test/altimate/tools/sql-check.test.ts @@ -593,7 +593,7 @@ for (const warehouse of ["sqlite_fixture", "scratch"] as const) { }) for (const [probe, baseSql, query] of [ - ["catalog estimate", "orders", /pragma_table_list|duckdb_tables/], + ["bounded estimate", "orders", /LIMIT 5000001[\s\S]*SELECT COUNT\(\*\) AS estimate/], ["predicate query", leftSql, /SELECT COUNT\(\*\) AS total/], ] as const) { for (const stateLost of [true, undefined]) { @@ -727,17 +727,57 @@ test("refuses cross-warehouse checks and explains the materialization prerequisi ).rejects.toThrow(/sql_materialize/) }) +function emptyPredicateCatalog(sql: string) { + const columns = sql.includes("pg_catalog.pg_operator") + ? ["oprname", "nspname"] + : sql.includes("pg_catalog.pg_proc") + ? ["proname", "nspname"] + : sql.includes("duckdb_functions()") + ? ["function_name", "schema_name"] + : undefined + return columns ? { columns, rows: [], row_count: 0, truncated: false } : undefined +} + +async function predicateCatalogFixture(type: string, catalog: { operators?: string[][]; functions?: string[][] } = {}) { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute").mockImplementation(async (sql) => { + const response = emptyPredicateCatalog(sql) + if (response) { + const entries = sql.includes("pg_catalog.pg_operator") ? catalog.operators : catalog.functions + const rows = (entries ?? []).filter(([, schema]) => schema !== "pg_catalog") + return { ...response, rows, row_count: rows.length } + } + return { + columns: ["total", "keep_1", "keep_all"], + rows: [[1, 1, 1]], + row_count: 1, + truncated: false, + } + }) + const get = spyOn(Registry, "get").mockResolvedValue(connector) + Registry.setConfigs({ catalog_fixture: { type } }) + return { + execute, + restore() { + get.mockRestore() + execute.mockRestore() + }, + } +} + test("PostgreSQL predicate_effect checks the estimate and population in one statement", async () => { const connector = await Registry.get("sqlite_fixture") - const execute = spyOn(connector, "execute").mockImplementation(async (sql) => - sql.includes("COUNT(*)") - ? { - columns: ["total", "keep_1", "keep_all", "estimate", "estimate_exceeded"], - rows: [[5, 3, 3, 5, false]], - row_count: 1, - truncated: false, - } - : { columns: ["estimate"], rows: [[5]], row_count: 1, truncated: false }, + const execute = spyOn(connector, "execute").mockImplementation( + async (sql) => + emptyPredicateCatalog(sql) ?? + (sql.includes("COUNT(*)") + ? { + columns: ["total", "keep_1", "keep_all", "estimate", "estimate_exceeded"], + rows: [[5, 3, 3, 5, false]], + row_count: 1, + truncated: false, + } + : { columns: ["estimate"], rows: [[5]], row_count: 1, truncated: false }), ) const get = spyOn(Registry, "get").mockResolvedValue(connector) try { @@ -753,8 +793,8 @@ test("PostgreSQL predicate_effect checks the estimate and population in one stat query_timeout_ms: 4321, }) expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) - expect(execute).toHaveBeenCalledTimes(1) - const [sql, , , options] = execute.mock.calls[0] + expect(execute).toHaveBeenCalledTimes(2) + const [sql, , , options] = execute.mock.calls[1] expect(sql).toContain("pg_catalog.pg_class") expect(sql).toContain(`pg_catalog.to_regclass(E'"orders"')`) expect(sql).toContain("COUNT(*) AS total") @@ -770,15 +810,17 @@ test("PostgreSQL predicate_effect checks the estimate and population in one stat test("PostgreSQL predicate_effect skips an exceeded estimate with the population scan gated", async () => { const connector = await Registry.get("sqlite_fixture") - const execute = spyOn(connector, "execute").mockImplementation(async (sql) => - sql.includes("COUNT(*)") - ? { - columns: ["total", "keep_1", "keep_all", "estimate", "estimate_exceeded"], - rows: [[0, 0, 0, 1000, true]], - row_count: 1, - truncated: false, - } - : { columns: ["estimate"], rows: [[1000]], row_count: 1, truncated: false }, + const execute = spyOn(connector, "execute").mockImplementation( + async (sql) => + emptyPredicateCatalog(sql) ?? + (sql.includes("COUNT(*)") + ? { + columns: ["total", "keep_1", "keep_all", "estimate", "estimate_exceeded"], + rows: [[0, 0, 0, 1000, true]], + row_count: 1, + truncated: false, + } + : { columns: ["estimate"], rows: [[1000]], row_count: 1, truncated: false }), ) const get = spyOn(Registry, "get").mockResolvedValue(connector) Registry.setConfigs({ pg_fixture: { type: "postgres" } }) @@ -793,8 +835,8 @@ test("PostgreSQL predicate_effect skips an exceeded estimate with the population expect(result.metadata).toMatchObject({ skipped: "estimate" }) expect(result.metadata.total_rows).toBeUndefined() expect(result.output).toContain("1000 exceeds max_rows_scanned 10") - expect(execute).toHaveBeenCalledTimes(1) - const sql = execute.mock.calls[0][0] + expect(execute).toHaveBeenCalledTimes(2) + const sql = execute.mock.calls[1][0] // The uncorrelated catalog guard belongs below the aggregate, so it can // reject the population before any row or predicate is evaluated. expect(sql).toMatch( @@ -809,12 +851,15 @@ test("PostgreSQL predicate_effect skips an exceeded estimate with the population test("PostgreSQL predicate_effect keeps an unknown atomic estimate and empty population distinct", async () => { const connector = await Registry.get("sqlite_fixture") - const execute = spyOn(connector, "execute").mockResolvedValue({ - columns: ["total", "keep_1", "keep_all", "estimate", "estimate_exceeded"], - rows: [[0, 0, 0, null, false]], - row_count: 1, - truncated: false, - }) + const execute = spyOn(connector, "execute").mockImplementation( + async (sql) => + emptyPredicateCatalog(sql) ?? { + columns: ["total", "keep_1", "keep_all", "estimate", "estimate_exceeded"], + rows: [[0, 0, 0, null, false]], + row_count: 1, + truncated: false, + }, + ) const get = spyOn(Registry, "get").mockResolvedValue(connector) Registry.setConfigs({ pg_fixture: { type: "postgres" } }) try { @@ -828,7 +873,7 @@ test("PostgreSQL predicate_effect keeps an unknown atomic estimate and empty pop expect(result.metadata.skipped).toBeUndefined() expect(result.output).toContain("row_estimate: unknown") expect(result.output).toContain("empty population") - expect(execute).toHaveBeenCalledTimes(1) + expect(execute).toHaveBeenCalledTimes(2) } finally { get.mockRestore() execute.mockRestore() @@ -837,12 +882,15 @@ test("PostgreSQL predicate_effect keeps an unknown atomic estimate and empty pop test("PostgreSQL predicate_effect avoids catalog CTE collisions with its population table", async () => { const connector = await Registry.get("sqlite_fixture") - const execute = spyOn(connector, "execute").mockResolvedValue({ - columns: ["total", "keep_1", "keep_all", "estimate", "estimate_exceeded"], - rows: [[1, 1, 1, 1, false]], - row_count: 1, - truncated: false, - }) + const execute = spyOn(connector, "execute").mockImplementation( + async (sql) => + emptyPredicateCatalog(sql) ?? { + columns: ["total", "keep_1", "keep_all", "estimate", "estimate_exceeded"], + rows: [[1, 1, 1, 1, false]], + row_count: 1, + truncated: false, + }, + ) const get = spyOn(Registry, "get").mockResolvedValue(connector) Registry.setConfigs({ pg_fixture: { type: "postgres" } }) try { @@ -852,9 +900,9 @@ test("PostgreSQL predicate_effect avoids catalog CTE collisions with its populat base_sql: "__sql_check_0_est", predicates: ["amount > 0"], }) - expect(execute).toHaveBeenCalledTimes(1) - expect(execute.mock.calls[0][0]).toMatch(/^WITH __sql_check_1_est AS/) - expect(execute.mock.calls[0][0]).toContain("SELECT * FROM __sql_check_0_est") + expect(execute).toHaveBeenCalledTimes(2) + expect(execute.mock.calls[1][0]).toMatch(/^WITH __sql_check_1_est AS/) + expect(execute.mock.calls[1][0]).toContain("SELECT * FROM __sql_check_0_est") } finally { get.mockRestore() execute.mockRestore() @@ -1001,12 +1049,15 @@ test("predicate_effect refuses qualified, quoted, and pg-prefixed wrappers befor test("predicate_effect accepts coalesce and lower on every filtered dialect", async () => { const connector = await Registry.get("sqlite_fixture") - const execute = spyOn(connector, "execute").mockResolvedValue({ - columns: ["total", "keep_1", "keep_2", "keep_all"], - rows: [[1, 1, 1, 1]], - row_count: 1, - truncated: false, - }) + const execute = spyOn(connector, "execute").mockImplementation( + async (sql) => + emptyPredicateCatalog(sql) ?? { + columns: ["total", "keep_1", "keep_2", "keep_all"], + rows: [[1, 1, 1, 1]], + row_count: 1, + truncated: false, + }, + ) const get = spyOn(Registry, "get").mockResolvedValue(connector) try { for (const type of ["postgres", "postgresql", "duckdb", "sqlite"]) { @@ -1019,7 +1070,7 @@ test("predicate_effect accepts coalesce and lower on every filtered dialect", as }) expect(result.metadata).toMatchObject({ total_rows: 1, combined_kept_rows: 1 }) } - expect(execute).toHaveBeenCalledTimes(4) + expect(execute.mock.calls.filter(([sql]) => !emptyPredicateCatalog(sql))).toHaveLength(4) } finally { get.mockRestore() execute.mockRestore() @@ -1028,12 +1079,15 @@ test("predicate_effect accepts coalesce and lower on every filtered dialect", as test("predicate_effect accepts scalar built-ins, casts, and boolean grouping", async () => { const connector = await Registry.get("sqlite_fixture") - const execute = spyOn(connector, "execute").mockResolvedValue({ - columns: ["total", "keep_1", "keep_all"], - rows: [[1, 1, 1]], - row_count: 1, - truncated: false, - }) + const execute = spyOn(connector, "execute").mockImplementation( + async (sql) => + emptyPredicateCatalog(sql) ?? { + columns: ["total", "keep_1", "keep_all"], + rows: [[1, 1, 1]], + row_count: 1, + truncated: false, + }, + ) const get = spyOn(Registry, "get").mockResolvedValue(connector) Registry.setConfigs({ pg_fixture: { type: "postgres" } }) const predicates = [ @@ -1058,7 +1112,7 @@ test("predicate_effect accepts scalar built-ins, casts, and boolean grouping", a }) expect(result.metadata.total_rows).toBe(1) } - expect(execute).toHaveBeenCalledTimes(predicates.length) + expect(execute.mock.calls.filter(([sql]) => !emptyPredicateCatalog(sql))).toHaveLength(predicates.length) } finally { get.mockRestore() execute.mockRestore() @@ -1083,3 +1137,442 @@ test("the registered tool reports SQL refusal as an error result", async () => { expect(result.metadata.error).toBeDefined() expect(result.output).toMatch(/read|SELECT/i) }) + +for (const type of ["postgres", "postgresql", "duckdb", "sqlite"]) { + test(`predicate_effect refuses wrapper population functions before execution on ${type}`, async () => { + const fixture = await predicateCatalogFixture(type) + try { + await expect( + checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: "SELECT emit_wrapper(id) AS id FROM orders", + predicates: ["id > 0"], + }), + ).rejects.toThrow(/emit_wrapper.*built-in pure functions/i) + expect(fixture.execute).not.toHaveBeenCalled() + } finally { + fixture.restore() + } + }) +} + +test("predicate_effect refuses DuckDB postgres_execute population before execution", async () => { + const fixture = await predicateCatalogFixture("duckdb") + try { + await expect( + checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: "SELECT * FROM postgres_execute('remote', 'DELETE FROM orders', use_transaction=false)", + predicates: ["id > 0"], + }), + ).rejects.toThrow(/postgres_execute.*built-in pure functions/i) + expect(fixture.execute).not.toHaveBeenCalled() + } finally { + fixture.restore() + } +}) + +for (const field of ["predicate", "population"] as const) { + test(`PostgreSQL predicate_effect refuses a user-defined ${field} operator before the probe`, async () => { + const fixture = await predicateCatalogFixture("postgres", { operators: [["##", "public"]] }) + try { + await expect( + checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: field === "population" ? "SELECT id ## 0 AS id FROM orders" : "SELECT 1 AS id", + predicates: [field === "predicate" ? "id ## 0" : "id > 0"], + query_timeout_ms: 4321, + }), + ).rejects.toThrow(new RegExp(`${field} uses operator .##., which is not a built-in operator`)) + const calls = fixture.execute.mock.calls + const catalogs = calls.filter(([sql]) => sql.includes("pg_catalog.pg_operator")) + expect(catalogs).toHaveLength(1) + expect(calls.every(([sql]) => !!emptyPredicateCatalog(sql))).toBe(true) + const [sql, limit, binds, options] = catalogs[0] + expect(sql).toContain("pg_catalog.pg_namespace") + expect(sql).toMatch(/oprname\s*=\s*ANY\(\$1\)/) + expect(sql).toMatch(/(?:nspname\s*<>\s*'pg_catalog'|oprnamespace\s*<>\s*'pg_catalog'::regnamespace)/) + expect(limit).toBeUndefined() + expect(binds).toEqual([expect.arrayContaining(["##"])]) + expect(options).toMatchObject({ readOnly: true, noLimit: true, queryTimeoutMs: 4321 }) + } finally { + fixture.restore() + } + }) +} + +test("PostgreSQL predicate_effect accepts built-in-only operator names after one read-only catalog query", async () => { + const fixture = await predicateCatalogFixture("postgres", { operators: [["##", "pg_catalog"]] }) + try { + const result = await checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: "SELECT 1 AS id", + predicates: ["id ## 0"], + }) + expect(result.metadata.total_rows).toBe(1) + expect(fixture.execute).toHaveBeenCalledTimes(2) + const [catalog, probe] = fixture.execute.mock.calls + expect(catalog[0]).toContain("pg_catalog.pg_operator") + expect(catalog[2]).toEqual([["##"]]) + expect(catalog[3]).toMatchObject({ readOnly: true }) + expect(probe[0]).toContain("COUNT(*) AS total") + expect(probe[3]).toMatchObject({ readOnly: true }) + } finally { + fixture.restore() + } +}) + +test("PostgreSQL predicate_effect masks non-code operator text and deduplicates catalog names", async () => { + const fixture = await predicateCatalogFixture("postgres") + try { + await checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: "SELECT 1 + 1 AS id, '##' AS name, 1 AS \"@@\"", + predicates: ["id >= 1 AND id >= 0 AND name = '~~' AND \"@@\" = 1 /* !! */ -- ??\n"], + }) + const catalogs = fixture.execute.mock.calls.filter(([sql]) => sql.includes("pg_catalog.pg_operator")) + expect(catalogs).toHaveLength(1) + expect(catalogs[0][2]?.[0]).toEqual(expect.arrayContaining(["+", ">=", "="])) + expect(catalogs[0][2]?.[0]).toHaveLength(3) + expect(fixture.execute.mock.calls.at(-1)![0]).toContain("COUNT(*) AS total") + } finally { + fixture.restore() + } +}) + +for (const field of ["predicate", "population"] as const) { + test(`PostgreSQL predicate_effect refuses a user-defined ${field} overload before the probe`, async () => { + const fixture = await predicateCatalogFixture("postgres", { functions: [["lower", "public"]] }) + try { + await expect( + checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: field === "population" ? "SELECT lower(id) AS id FROM orders" : "SELECT 1 AS id", + predicates: [field === "predicate" ? "lower(id) = 'x'" : "id = 'x'"], + query_timeout_ms: 4321, + }), + ).rejects.toThrow(new RegExp(`${field} calls .lower., which has a user-defined overload in schema .public.`)) + const calls = fixture.execute.mock.calls + expect(calls.every(([sql]) => !!emptyPredicateCatalog(sql))).toBe(true) + const catalogs = calls.filter(([sql]) => sql.includes("pg_catalog.pg_proc")) + expect(catalogs).toHaveLength(1) + const [sql, limit, binds, options] = catalogs[0] + expect(sql).toContain("pg_catalog.pg_namespace") + expect(sql).toMatch(/proname\s*=\s*ANY\(\$1\)/) + expect(sql).toMatch(/pronamespace\s*<>\s*'pg_catalog'::regnamespace/) + expect(limit).toBeUndefined() + expect(binds).toEqual([["lower"]]) + expect(options).toMatchObject({ readOnly: true, noLimit: true, queryTimeoutMs: 4321 }) + } finally { + fixture.restore() + } + }) +} + +test("PostgreSQL predicate_effect accepts lower after read-only function and operator catalogs before the probe", async () => { + const fixture = await predicateCatalogFixture("postgres", { functions: [["lower", "pg_catalog"]] }) + try { + const result = await checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: "SELECT lower('X') AS id", + predicates: ["lower(id) = 'x' AND lower(id) = 'x'"], + query_timeout_ms: 4321, + }) + expect(result.metadata.total_rows).toBe(1) + expect(fixture.execute).toHaveBeenCalledTimes(3) + const calls = fixture.execute.mock.calls + expect(calls.slice(0, -1).every(([sql]) => !!emptyPredicateCatalog(sql))).toBe(true) + expect(calls.at(-1)![0]).toContain("COUNT(*) AS total") + expect(calls.find(([sql]) => sql.includes("pg_catalog.pg_proc"))![2]).toEqual([["lower"]]) + expect(calls.find(([sql]) => sql.includes("pg_catalog.pg_operator"))![2]).toEqual([["="]]) + for (const [, , , options] of calls) + expect(options).toMatchObject({ readOnly: true, noLimit: true, queryTimeoutMs: 4321 }) + } finally { + fixture.restore() + } +}) + +for (const field of ["predicate", "population"] as const) { + test(`DuckDB predicate_effect refuses a user-defined ${field} macro before the probe`, async () => { + const fixture = await predicateCatalogFixture("duckdb", { functions: [["lower", "main"]] }) + try { + await expect( + checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: field === "population" ? "SELECT lower(id) AS id FROM orders" : "SELECT 1 AS id", + predicates: [field === "predicate" ? "lower(id) = 'x'" : "id = 'x'"], + query_timeout_ms: 4321, + }), + ).rejects.toThrow(new RegExp(`${field} calls .lower., which has a user-defined overload in schema .main.`)) + expect(fixture.execute).toHaveBeenCalledTimes(1) + const [sql, limit, binds, options] = fixture.execute.mock.calls[0] + expect(sql).toContain("duckdb_functions()") + expect(sql).toMatch(/internal\s*=\s*false/i) + expect(binds).toEqual(["lower"]) + expect(limit).toBeUndefined() + expect(options).toMatchObject({ readOnly: true, noLimit: true, queryTimeoutMs: 4321 }) + } finally { + fixture.restore() + } + }) +} + +for (const warehouse of ["sqlite_fixture", "scratch"] as const) { + const dialect = warehouse === "scratch" ? "DuckDB" : "SQLite" + for (const cap of [10, 2]) { + test(`${dialect} predicate_effect bounds the estimate and aggregate in one execute call${cap === 2 ? " and skips an exceeded cap" : ""}`, async () => { + const connector = await Registry.get(warehouse) + const execute = spyOn(connector, "execute") + try { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "orders", + predicates: ["amount > 0"], + max_rows_scanned: cap, + }) + if (cap === 2) { + expect(result.metadata).toMatchObject({ skipped: "estimate" }) + expect(result.metadata.total_rows).toBeUndefined() + expect(result.output).toContain("3 exceeds max_rows_scanned 2") + } else expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 4 }) + expect(execute).toHaveBeenCalledTimes(1) + const [sql, , , options] = execute.mock.calls[0] + expect(sql).toMatch(new RegExp(`AS MATERIALIZED \\(\\s*SELECT \\* FROM orders LIMIT ${cap + 1}\\s*\\)`)) + expect(sql).toMatch(/SELECT COUNT\(\*\) AS estimate FROM \w+/) + expect(sql).toContain("COUNT(*) AS total") + expect(sql).toContain("AS estimate_exceeded") + expect(options).toMatchObject({ readOnly: true, noLimit: true }) + } finally { + execute.mockRestore() + } + }) + } +} + +test("SQLite predicate_effect accepts strftime without a function catalog lookup", async () => { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute") + try { + const result = await checkSql({ + warehouse: "sqlite_fixture", + check: "predicate_effect", + base_sql: "SELECT '2024-01-01' AS created_at UNION ALL SELECT '2023-12-31'", + predicates: ["strftime('%Y', created_at) = '2024'"], + }) + expect(result.metadata).toMatchObject({ total_rows: 2, combined_kept_rows: 1 }) + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][0]).toContain("COUNT(*) AS total") + } finally { + execute.mockRestore() + } +}) + +test("predicate_effect accepts pure date built-ins and the bare array constructor", async () => { + const fixture = await predicateCatalogFixture("sqlite") + try { + for (const predicate of [ + "strftime('%Y', created_at) = '2024'", + "julianday(created_at) > 0", + "unixepoch(created_at) > 0", + "date(created_at) IS NOT NULL", + "time(created_at) IS NOT NULL", + "datetime(created_at) IS NOT NULL", + "timediff(created_at, created_at) IS NOT NULL", + "array(SELECT 1) IS NOT NULL", + ]) { + const result = await checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: "SELECT '2024-01-01' AS created_at", + predicates: [predicate], + }) + expect(result.metadata.total_rows).toBe(1) + } + expect(fixture.execute).toHaveBeenCalledTimes(8) + } finally { + fixture.restore() + } +}) + +test("PostgreSQL predicate_effect refuses a wrapper inside an array constructor before execution", async () => { + const fixture = await predicateCatalogFixture("postgres") + try { + await expect( + checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: "SELECT 1 AS id", + predicates: ["ARRAY[emit_wrapper(id)] IS NOT NULL"], + }), + ).rejects.toThrow(/emit_wrapper.*built-in pure functions/i) + expect(fixture.execute).not.toHaveBeenCalled() + } finally { + fixture.restore() + } +}) + +for (const [description, operator, predicate] of [ + ["array", "##", "ARRAY[id ## 0] IS NOT NULL"], + ["backtick", "`", "id ` 0"], +]) { + test(`PostgreSQL predicate_effect resolves a user-defined ${description} operator before execution`, async () => { + const fixture = await predicateCatalogFixture("postgres", { operators: [[operator, "public"]] }) + try { + await expect( + checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: "SELECT 1 AS id", + predicates: [predicate], + }), + ).rejects.toThrow(/uses operator .*not a built-in operator/) + expect(fixture.execute.mock.calls.every(([sql]) => !!emptyPredicateCatalog(sql))).toBe(true) + expect(fixture.execute.mock.calls.find(([sql]) => sql.includes("pg_catalog.pg_operator"))?.[2]).toEqual([ + [operator], + ]) + } finally { + fixture.restore() + } + }) +} + +test("DuckDB predicate_effect refuses a mixed-case macro from the live function catalog", async () => { + const connector = await Registry.get("scratch") + await connector.execute("CREATE MACRO LoWeR(value) AS value") + const execute = spyOn(connector, "execute") + try { + await expect( + checkSql({ + warehouse: "scratch", + check: "predicate_effect", + base_sql: "SELECT 'X' AS id", + predicates: ["lower(id) = 'x'"], + }), + ).rejects.toThrow(/predicate calls .lower., which has a user-defined overload in schema .main./i) + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][0]).toContain("duckdb_functions()") + expect(execute.mock.calls[0][0]).toContain("COLLATE NOCASE") + expect(execute.mock.calls[0][2]).toEqual(["lower"]) + expect(execute.mock.calls[0][3]).toMatchObject({ readOnly: true }) + } finally { + execute.mockRestore() + } +}) + +for (const [run, predicate] of [ + ["=-", "id=-1"], + [">+-", "id>+-1"], +]) { + test(`PostgreSQL predicate_effect checks overloads after splitting trailing signs from ${run}`, async () => { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute").mockImplementation(async (sql, _limit, binds) => { + const catalog = emptyPredicateCatalog(sql) + if (catalog) { + const names = binds?.[0] as string[] | undefined + const name = run[0] + const rows = sql.includes("pg_catalog.pg_operator") && names?.includes(name) ? [[name, "public"]] : [] + return { ...catalog, rows, row_count: rows.length } + } + return { columns: ["total", "keep_1", "keep_all"], rows: [[1, 1, 1]], row_count: 1, truncated: false } + }) + const get = spyOn(Registry, "get").mockResolvedValue(connector) + Registry.setConfigs({ pg_fixture: { type: "postgres" } }) + try { + await expect( + checkSql({ + warehouse: "pg_fixture", + check: "predicate_effect", + base_sql: "SELECT 1 AS id", + predicates: [predicate], + }), + ).rejects.toThrow(/predicate uses operator .*not a built-in operator/) + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][0]).toContain("pg_catalog.pg_operator") + expect(execute.mock.calls[0][2]?.[0]).toEqual(expect.arrayContaining([run[0], "-"])) + expect(execute.mock.calls[0][3]).toMatchObject({ readOnly: true }) + } finally { + get.mockRestore() + execute.mockRestore() + } + }) +} + +test("PostgreSQL predicate_effect checks the normalized <> overload for !=", async () => { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute").mockImplementation(async (sql, _limit, binds) => { + const catalog = emptyPredicateCatalog(sql) + if (catalog) { + const names = binds?.[0] as string[] | undefined + const rows = sql.includes("pg_catalog.pg_operator") && names?.includes("<>") ? [["<>", "public"]] : [] + return { ...catalog, rows, row_count: rows.length } + } + return { columns: ["total", "keep_1", "keep_all"], rows: [[1, 1, 1]], row_count: 1, truncated: false } + }) + const get = spyOn(Registry, "get").mockResolvedValue(connector) + Registry.setConfigs({ pg_fixture: { type: "postgres" } }) + try { + await expect( + checkSql({ + warehouse: "pg_fixture", + check: "predicate_effect", + base_sql: "SELECT 1 AS id", + predicates: ["id != 0"], + }), + ).rejects.toThrow(/predicate uses operator .<>., which is not a built-in operator/) + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][0]).toContain("pg_catalog.pg_operator") + expect(execute.mock.calls[0][2]?.[0]).toEqual(expect.arrayContaining(["<>"])) + expect(execute.mock.calls[0][3]).toMatchObject({ readOnly: true }) + } finally { + get.mockRestore() + execute.mockRestore() + } +}) + +for (const [predicate, operator] of [ + ["id LIKE 'x'", "~~"], + ["id IN (0, 1)", "="], + ["id BETWEEN 0 AND 1", ">="], + ["id IS DISTINCT FROM 0", "="], +]) { + test(`PostgreSQL predicate_effect resolves implicit operator ${operator} in ${predicate}`, async () => { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute").mockImplementation(async (sql, _limit, binds) => { + const catalog = emptyPredicateCatalog(sql) + if (catalog) { + const names = binds?.[0] as string[] | undefined + const rows = sql.includes("pg_catalog.pg_operator") && names?.includes(operator) ? [[operator, "public"]] : [] + return { ...catalog, rows, row_count: rows.length } + } + return { columns: ["total", "keep_1", "keep_all"], rows: [[1, 1, 1]], row_count: 1, truncated: false } + }) + const get = spyOn(Registry, "get").mockResolvedValue(connector) + Registry.setConfigs({ pg_fixture: { type: "postgres" } }) + try { + await expect( + checkSql({ + warehouse: "pg_fixture", + check: "predicate_effect", + base_sql: "SELECT 1 AS id", + predicates: [predicate], + }), + ).rejects.toThrow(/predicate uses operator .*not a built-in operator/) + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][0]).toContain("pg_catalog.pg_operator") + expect(execute.mock.calls[0][2]?.[0]).toEqual(expect.arrayContaining([operator])) + expect(execute.mock.calls[0][3]).toMatchObject({ readOnly: true }) + } finally { + get.mockRestore() + execute.mockRestore() + } + }) +} From 7db53c16d00a20e02c336941f9ca98208042c722 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Tue, 13 Oct 2026 21:41:20 -0700 Subject: [PATCH 18/22] fix(analyst): `predicate_effect` checks explicit cast targets against the catalog; repairs the PostgreSQL test fixture - Explicit casts (`x::type`, `CAST(x AS type)`) in populations and predicates are resolved against `pg_catalog.pg_type` with the same read-only bound catalog query; a target type outside `pg_catalog` refuses the probe before any SQL runs. Casts selected by a user-defined SOURCE type cannot be seen textually; the code comment and PR body disclose that. - CTE declarations and their column lists are no longer mistaken for function calls, so the advertised `WITH population(id) AS (...) SELECT ...` form is accepted; projection, qualified and `count(*)` wildcards (including `DISTINCT ON`) are excluded from the operator catalog check while multiplication is still checked. - `sql-check-postgres.test.ts` had been left outside the previous round's verification and was red on the branch: its fixture now answers the `pg_operator`/`pg_proc`/`pg_type` catalog queries (seedable), preserves bind arguments, and the unsafe-function cases assert the pure-function refusal with no population or probe SQL executed. - tests: custom cast targets refused before the probe (predicate and population, `::` and `CAST`, qualified/quoted/array/typmod, Unicode-escaped), built-in casts accepted, CTE column lists accepted, wildcard populations accepted beside a seeded custom `*`, multiplication still checked; fixture controls for an ordinary population, seeded catalog rows and bind preservation. Co-Authored-By: Claude Fable 5.1 --- .../opencode/src/altimate/tools/sql-check.ts | 175 +++++++++++- .../test/altimate/sql-check-postgres.test.ts | 216 +++++++++----- .../test/altimate/tools/sql-check.test.ts | 266 +++++++++++++++++- 3 files changed, 580 insertions(+), 77 deletions(-) diff --git a/packages/opencode/src/altimate/tools/sql-check.ts b/packages/opencode/src/altimate/tools/sql-check.ts index c1bfc5468..b1dd704d5 100644 --- a/packages/opencode/src/altimate/tools/sql-check.ts +++ b/packages/opencode/src/altimate/tools/sql-check.ts @@ -592,10 +592,164 @@ function required(value: T | undefined, name: string): T { } const OPERATOR_TOKEN = /[+\-*/<>=~!@#%^&|`?]+/g +const POSTGRES_IDENTIFIER = /^(?:"(?:[^"\x00]|"")+"|[A-Za-z_\u0080-\uFFFF][\w$\u0080-\uFFFF]*)$/ + +/** Keep quoted tokens opaque and retain source positions for syntax-only exclusions. */ +function sqlSyntax(sql: string, dialect?: string) { + let offset = 0 + return tokenizeSql(sql, dialect).tokens.flatMap(({ kind, text }) => { + const start = offset + offset += text.length + if (kind === "comment") return [] + if (kind !== "code") return [{ text, index: start }] + return [ + ...text.matchAll( + /[A-Za-z_\u0080-\uFFFF][\w$\u0080-\uFFFF]*|(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?|::|[+\-*/<>=~!@#%^&|`?]+|[^\s]/g, + ), + ].map((match) => ({ text: match[0], index: start + match.index })) + }) +} + +function closingParen(tokens: ReturnType, start: number) { + let depth = 0 + for (let i = start; i < tokens.length; i++) { + if (tokens[i].text === "(") depth++ + if (tokens[i].text === ")" && --depth === 0) return i + } + return -1 +} + +function postgresCasts(sql: string) { + const tokens = sqlSyntax(sql, "postgres") + const targets: { name: string; index: number }[] = [] + function typeIdentifier(start: number) { + let end = start + let type = tokens[end]?.text ?? "" + if (/^u$/i.test(type) && tokens[end + 1]?.text === "&" && tokens[end + 2]?.text.startsWith('"')) { + end += 2 + type = tokens[end].text.slice(1, -1).replaceAll('""', '"') + let escape = "\\" + if (/^uescape$/i.test(tokens[end + 1]?.text ?? "")) { + const literal = tokens[end + 2]?.text ?? "" + if (!/^'[^']'$/.test(literal)) throw new Error("sql_check cannot inspect this Unicode cast type escape") + escape = literal[1] + end += 2 + } + let decoded = "" + for (let i = 0; i < type.length; i++) { + if (type[i] !== escape) decoded += type[i] + else if (type[i + 1] === escape) decoded += type[++i] + else { + const length = type[i + 1] === "+" ? 6 : 4 + const start = i + (length === 6 ? 2 : 1) + const hex = type.slice(start, start + length) + const point = parseInt(hex, 16) + if (hex.length !== length || !/^[\da-f]+$/i.test(hex) || point === 0 || point > 0x10ffff) + throw new Error("sql_check cannot inspect this Unicode cast type escape") + decoded += String.fromCodePoint(point) + i = start + length - 1 + } + } + return { name: decoded, end } + } + if (!POSTGRES_IDENTIFIER.test(type)) return + return { name: type.startsWith('"') ? type.slice(1, -1).replaceAll('""', '"') : type.toLowerCase(), end } + } + for (let i = 0; i < tokens.length; i++) { + let target = -1 + if (tokens[i].text === "::") target = i + 1 + if (/^cast$/i.test(tokens[i].text) && tokens[i + 1]?.text === "(") { + for (let j = i + 2; j < tokens.length && tokens[j].text !== ")"; j++) { + if (tokens[j].text === "(") { + j = closingParen(tokens, j) + if (j < 0) break + } else if (/^as$/i.test(tokens[j].text)) { + target = j + 1 + break + } + } + } + if (target < 0) continue + let type = typeIdentifier(target) + if (!type) continue + while (tokens[type.end + 1]?.text === ".") { + const qualified = typeIdentifier(type.end + 2) + if (!qualified) break + type = qualified + } + targets.push({ name: type.name, index: tokens[target].index }) + } + return targets +} + +function cteDeclarations(sql: string, dialect?: string) { + const tokens = sqlSyntax(sql, dialect) + const declarations = new Set() + for (let i = 0; i < tokens.length; i++) { + if (!/^with$/i.test(tokens[i].text)) continue + let j = i + 1 + if (/^recursive$/i.test(tokens[j]?.text ?? "")) j++ + while (POSTGRES_IDENTIFIER.test(tokens[j]?.text ?? "")) { + const name = tokens[j++] + if (tokens[j]?.text === "(") { + const end = closingParen(tokens, j) + if (end < 0) break + j = end + 1 + } + if (!/^as$/i.test(tokens[j++]?.text ?? "")) break + if (/^not$/i.test(tokens[j]?.text ?? "")) j++ + const materialized = /^materialized$/i.test(tokens[j]?.text ?? "") ? tokens[j++] : undefined + if (tokens[j]?.text !== "(") break + declarations.add(name.index) + if (materialized) declarations.add(materialized.index) + const end = closingParen(tokens, j) + if (end < 0 || tokens[end + 1]?.text !== ",") break + j = end + 2 + } + } + return declarations +} function postgresOperators(sql: string): string[] { const code = maskSql(tokenizeSql(sql, "postgres").tokens) - const names = new Set(code.match(OPERATOR_TOKEN) ?? []) + const tokens = sqlSyntax(sql, "postgres") + const selectList = [false] + const names = new Set() + for (let i = 0; i < tokens.length; i++) { + const token = tokens[i].text + if (token === "(") selectList.push(false) + else if (token === ")") selectList.pop() + else if (/^select$/i.test(token)) selectList[selectList.length - 1] = true + else if (/^(from|where|group|having|order|limit|offset|fetch|union|intersect|except)$/i.test(token)) + selectList[selectList.length - 1] = false + if (!/^[+\-*/<>=~!@#%^&|`?]+$/.test(token)) continue + const previous = tokens[i - 1]?.text ?? "" + let afterDistinctOn = false + if (token === "*" && previous === ")" && selectList.at(-1)) { + let depth = 0 + for (let j = i - 1; j >= 0; j--) { + if (tokens[j].text === ")") depth++ + if (tokens[j].text === "(" && --depth === 0) { + afterDistinctOn = /^select distinct on$/i.test( + tokens + .slice(j - 3, j) + .map((token) => token.text) + .join(" "), + ) + break + } + } + } + if ( + token === "*" && + (previous === "." || + afterDistinctOn || + (selectList.at(-1) && /^(select|all|distinct|,)$/i.test(previous)) || + (previous === "(" && /^(count|"count")$/i.test(tokens[i - 2]?.text ?? ""))) + ) + continue + names.add(token) + } for (const operator of [...names]) { // PostgreSQL splits trailing unary signs from ordinary operator runs: =- is = and -. if (!/[~!@#%^&|`?]/.test(operator)) { @@ -726,9 +880,12 @@ function pureFunctions(sql: string, dialect?: string): string[] { ? '(?:"(?:[^"\\x00]|"")+"|[A-Za-z_\\u0080-\\uFFFF][\\w$\\u0080-\\uFFFF]*)' : '(?:"(?:[^"\\x00]|"")+"|`(?:[^`]|``)+`|\\[(?:[^\\]]|\\]\\])+\\]|[A-Za-z_\\u0080-\\uFFFF][\\w$\\u0080-\\uFFFF]*)' const calls = new RegExp(`${identifier}(?:\\s*\\.\\s*${identifier})*`, "g") + const declarations = cteDeclarations(sql, dialect) + if (dialect === "postgres") for (const target of postgresCasts(sql)) declarations.add(target.index) const names = new Set() for (const match of functions.matchAll(calls)) { if (!/^\s*\(/.test(functions.slice(match.index + match[0].length))) continue + if (declarations.has(match.index)) continue const call = match[0].trim() const parts = call.match(new RegExp(identifier, "g"))! const quoted = /^["`\[]/.test(parts[0]) @@ -806,6 +963,22 @@ async function checkPredicateCatalog( ) } } + // Text cannot determine whether a cast's source column has a user-defined type. + const typeNames = queries.map((sql) => postgresCasts(sql).map((target) => target.name)) + const types = [...new Set(typeNames.flat())] + if (types.length) { + const [type] = await run( + `SELECT t.typname, n.nspname + FROM pg_catalog.pg_type t JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace + WHERE t.typname = ANY($1) AND n.nspname <> 'pg_catalog'`, + [types], + ) + if (type) { + throw new Error( + `sql_check ${origin(type.typname, typeNames)} casts to \`${type.typname}\`, which is not a built-in type`, + ) + } + } return } if (functions.length) { diff --git a/packages/opencode/test/altimate/sql-check-postgres.test.ts b/packages/opencode/test/altimate/sql-check-postgres.test.ts index f0e6676be..b26e695ce 100644 --- a/packages/opencode/test/altimate/sql-check-postgres.test.ts +++ b/packages/opencode/test/altimate/sql-check-postgres.test.ts @@ -28,49 +28,111 @@ function postgres( relkind = "r", reltuples = 5, translate = (sql: string, _options?: Parameters[3]) => sql, + catalog: Partial> = {}, ) { database.run("INSERT INTO pg_class VALUES (1, ?, ?)", [relkind, reltuples]) - const execute = mock(async (...[sql, , , options]: Parameters) => { + const execute = mock(async (...[sql, , binds, options]: Parameters) => { + for (const [table, column] of [ + ["pg_operator", "oprname"], + ["pg_proc", "proname"], + ["pg_type", "typname"], + ] as const) { + if (!sql.includes(`pg_catalog.${table} `)) continue + expect(sql).toContain("ANY($1)") + expect(binds).toHaveLength(1) + const names = binds![0] as string[] + const rows = (catalog[table] ?? []).filter(([name, schema]) => names.includes(name) && schema !== "pg_catalog") + return { columns: [column, "nspname"], rows, row_count: rows.length, truncated: false } + } // Keep the generated WHERE and FILTER clauses executable: removing either // guard condition must change results, rather than leave a canned mock row. const query = translate(sql, options) .replace("pg_catalog.pg_class", "pg_class") .replace(/pg_catalog\.to_regclass\(E'(?:[^']|'')*'\)/, "1") const statement = database.query(query) - const rows = statement.values() + const rows = statement.values(...(binds ?? [])) return { columns: statement.columnNames, rows, row_count: rows.length, truncated: false } }) connectorSpy = spyOn(Registry, "get").mockResolvedValue({ execute } as unknown as Connector) return execute } +// Keep assertions about estimates and population SQL separate from safety-catalog lookups. +function dataCalls(execute: ReturnType) { + return execute.mock.calls.filter(([sql]) => !/pg_catalog\.(?:pg_operator|pg_proc|pg_type)\b/.test(sql)) +} + describe("predicate_effect with mocked Postgres", () => { + test("an ordinary population and id > 0 pass the catalog checks and reach the probe", async () => { + const execute = postgres() + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "SELECT * FROM orders", + predicates: ["id > 0"], + query_timeout_ms: 4321, + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 5 }) + const [catalog, probe] = execute.mock.calls + expect(catalog[0]).toContain("pg_catalog.pg_operator") + expect(catalog[2]).toEqual([[">"]]) + expect(probe[0]).toContain("COUNT(*) AS total") + expect(dataCalls(execute)).toHaveLength(1) + for (const call of execute.mock.calls) { + expect(call[3]).toMatchObject({ noLimit: true, queryTimeoutMs: 4321, readOnly: true }) + } + }) + test.each([ - { name: "predicates", base_sql: "orders", predicates: ["custom_write_udf() > 0"] as string[], calls: 2 }, + { catalog: "pg_operator" as const, name: ">", predicate: "id > 0", message: "uses operator `>`" }, + { catalog: "pg_proc" as const, name: "length", predicate: "length(kind) > 0", message: "calls `length`" }, + ])( + "seeded $catalog rows refuse matching custom definitions before the probe", + async ({ catalog, name, predicate, message }) => { + const execute = postgres("r", 5, undefined, { [catalog]: [[name, "custom"]] }) + await expect( + checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "SELECT * FROM orders", + predicates: [predicate], + }), + ).rejects.toThrow(message) + expect(dataCalls(execute)).toEqual([]) + expect(execute.mock.calls.at(-1)?.[2]?.[0]).toContain(name) + }, + ) + + test("the Postgres fixture preserves bind arguments for data SQL", async () => { + const execute = postgres() + const result = await execute("SELECT ? AS value", undefined, [42]) + expect(result.rows).toEqual([[42]]) + expect(execute.mock.calls[0][2]).toEqual([42]) + }) + + test.each([ + { name: "predicates", base_sql: "orders", predicates: ["custom_write_udf() > 0"] as string[] }, { name: "populations", base_sql: "SELECT * FROM orders WHERE custom_write_udf() > 0", predicates: ["id > 0"] as string[], - calls: 1, }, - ])("uses read-only execution to reject writing UDFs in $name", async ({ base_sql, predicates, calls }) => { + ])("rejects writing UDFs in $name before query execution", async ({ base_sql, predicates }) => { let writes = 0 const execute = postgres("r", 5, (sql, options) => { if (!sql.includes("custom_write_udf()")) return sql - // Model the connector's transaction boundary for a function unknown to the denylist. + // A missing function guard would reach this simulated write-capable call. if (options?.readOnly) throw new Error("cannot execute INSERT in a read-only transaction") writes++ return sql.replaceAll("custom_write_udf()", "1") }) await rejects( checkSql({ warehouse, check: "predicate_effect", base_sql, predicates, query_timeout_ms: 4321 }), - /read-only transaction/, + /sql_check refuses function custom_write_udf: predicates are limited to built-in pure functions/, ) expect(writes).toBe(0) - expect(execute).toHaveBeenCalledTimes(calls) - for (const call of execute.mock.calls) { - expect(call[3]).toMatchObject({ noLimit: true, queryTimeoutMs: 4321, readOnly: true }) - } + expect(execute).not.toHaveBeenCalled() + expect(dataCalls(execute)).toEqual([]) }) for (const [kind, calls] of [ @@ -135,7 +197,7 @@ describe("predicate_effect with mocked Postgres", () => { "large-object-write", [ "lo_from_bytea(0, '\\x00'::bytea)", - 'pg_catalog."lo_from_bytea" /* write */ (0, \'\\x00\'::bytea)', + "pg_catalog.\"lo_from_bytea\" /* write */ (0, '\\x00'::bytea)", "LO_TRUNCATE(1, 0)", "lo_truncate64(1, 0)", "lowrite(lo_open(1, 131072), '\\x00'::bytea)", @@ -164,7 +226,7 @@ describe("predicate_effect with mocked Postgres", () => { 'pg_catalog."pg_wal_replay_pause" /* pause */ ()', "PG_WAL_REPLAY_RESUME()", "pg_create_restore_point('probe-' || id::text)", - 'pg_catalog."pg_create_restore_point" /* mark */ (\'p\')', + "pg_catalog.\"pg_create_restore_point\" /* mark */ ('p')", "PG_SWITCH_WAL()", "pg_drop_replication_slot('slot')", "pg_create_logical_replication_slot('slot', 'pgoutput')", @@ -184,7 +246,7 @@ describe("predicate_effect with mocked Postgres", () => { const execute = postgres("r", 5, (sql) => sql.replaceAll(call, "1")) await rejects( checkSql({ warehouse, check: "predicate_effect", base_sql: "orders", predicates: ["id > 0", predicate] }), - /sql_check only accepts a single read-only SELECT/, + /sql_check refuses function .*: predicates are limited to built-in pure functions/, ) expect(execute).not.toHaveBeenCalled() }) @@ -222,28 +284,33 @@ describe("predicate_effect with mocked Postgres", () => { predicates: ["id > 0", `${call} IS NOT NULL`], }) expect(result.metadata).toMatchObject({ total_rows: 5 }) - // The pure call reached execution (a bare table base also issues the row-estimate query). + // The pure call reached execution (a bare table probe also includes the row-estimate lookup). expect(execute).toHaveBeenCalled() }) test.each([ - "pg_promote", "pg_reload_conf", "dblink_send_query", "dblink_connect", "dblink_connect_u", - "pg_logical_emit_message", "dblink", "pg_advisory_lock_shared", "pg_try_advisory_lock_shared", - "pg_advisory_xact_lock_shared", "pg_try_advisory_xact_lock_shared", - ])( - "accepts %s in literals, comments and column identifiers", - async (name) => { - const execute = postgres() - const result = await checkSql({ - warehouse, - check: "predicate_effect", - base_sql: `SELECT id AS "${name}" FROM orders`, - predicates: [`"${name}" > 2 -- ${name}()`, `length('${name}()') > 0 /* ${name}() */`], - }) - expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) - expect(execute).toHaveBeenCalledTimes(1) - }, - ) + "pg_promote", + "pg_reload_conf", + "dblink_send_query", + "dblink_connect", + "dblink_connect_u", + "pg_logical_emit_message", + "dblink", + "pg_advisory_lock_shared", + "pg_try_advisory_lock_shared", + "pg_advisory_xact_lock_shared", + "pg_try_advisory_xact_lock_shared", + ])("accepts %s in literals, comments and column identifiers", async (name) => { + const execute = postgres() + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: `SELECT id AS "${name}" FROM orders`, + predicates: [`"${name}" > 2 -- ${name}()`, `length('${name}()') > 0 /* ${name}() */`], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + expect(dataCalls(execute)).toHaveLength(1) + }) test.each([ "pg_terminate_backend(id)", @@ -263,7 +330,7 @@ describe("predicate_effect with mocked Postgres", () => { const execute = postgres("r", 5, (sql) => sql.replaceAll(predicate, "id > 0")) await rejects( checkSql({ warehouse, check: "predicate_effect", base_sql: "orders", predicates: [predicate] }), - /sql_check only accepts a single read-only SELECT/, + /sql_check refuses function .*: predicates are limited to built-in pure functions/, ) expect(execute).not.toHaveBeenCalled() }) @@ -303,7 +370,7 @@ describe("predicate_effect with mocked Postgres", () => { predicates: [{ kept_rows: 3 }, { kept_rows: 5 }, { kept_rows: 3 }], combined_kept_rows: 3, }) - expect(execute).toHaveBeenCalledTimes(1) + expect(dataCalls(execute)).toHaveLength(1) }) test("qualifies a bare table using its quoted unqualified name with a dot and escaped quote", async () => { @@ -330,12 +397,12 @@ describe("predicate_effect with mocked Postgres", () => { predicates: ["t.kind = 'a'"], }) expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) - expect(execute.mock.calls[0][0]).toContain(`FROM (\n${base_sql}\n) t`) + expect(dataCalls(execute)[0][0]).toContain(`FROM (\n${base_sql}\n) t`) }) test.each([ { name: "FETCH FIRST", population: "SELECT * FROM orders ORDER BY id FETCH FIRST 2 ROWS ONLY" }, - { name: "TOP", population: "SELECT TOP (2) * FROM orders ORDER BY id" }, + { name: "TOP", population: "SELECT TOP 2 * FROM orders ORDER BY id" }, ])("notes a top-level $name while preserving the population SQL", async ({ population }) => { const execute = postgres("r", 5, (sql) => sql.replace(population, "SELECT * FROM orders ORDER BY id LIMIT 2")) const result = await checkSql({ @@ -345,19 +412,19 @@ describe("predicate_effect with mocked Postgres", () => { predicates: ["id > 1"], }) expect(result.metadata).toMatchObject({ total_rows: 2, combined_kept_rows: 1 }) - expect(execute.mock.calls[0][0]).toContain(`FROM (\n${population}\n) t`) + expect(dataCalls(execute)[0][0]).toContain(`FROM (\n${population}\n) t`) expect(result.output).toContain("counts are over the limited population") }) test.each([ { name: "FETCH FIRST", inner: "SELECT * FROM orders ORDER BY id FETCH FIRST 2 ROWS ONLY" }, - { name: "TOP", inner: "SELECT TOP (2) * FROM orders ORDER BY id" }, + { name: "TOP", inner: "SELECT TOP 2 * FROM orders ORDER BY id" }, ])("does not note $name confined to a subquery", async ({ inner }) => { const execute = postgres("r", 5, (sql) => sql.replace(inner, "SELECT * FROM orders ORDER BY id LIMIT 2")) const base_sql = `SELECT * FROM (${inner}) limited_orders` const result = await checkSql({ warehouse, check: "predicate_effect", base_sql, predicates: ["id > 1"] }) expect(result.metadata).toMatchObject({ total_rows: 2, combined_kept_rows: 1 }) - expect(execute.mock.calls[0][0]).toContain(`FROM (\n${base_sql}\n) t`) + expect(dataCalls(execute)[0][0]).toContain(`FROM (\n${base_sql}\n) t`) expect(result.output).not.toContain("limited population") }) @@ -380,14 +447,14 @@ describe("predicate_effect with mocked Postgres", () => { ], combined_kept_rows: 2, }) - expect(execute).toHaveBeenCalledTimes(1) - const sql = execute.mock.calls[0][0] + expect(dataCalls(execute)).toHaveLength(1) + const sql = dataCalls(execute)[0][0] expect(sql).toContain("COUNT(*) AS total") expect(sql).toContain(`COUNT(*) FILTER (WHERE (\n${predicates[0]}\n)) AS keep_1`) expect(sql).toContain(`COUNT(*) FILTER (WHERE (\n${predicates[1]}\n)) AS keep_2`) expect(sql).toContain(`COUNT(*) FILTER (WHERE (\n${predicates[0]}\n) AND (\n${predicates[1]}\n)) AS keep_all`) expect(sql).toContain(`FROM (\n${base_sql}\n) t`) - expect(execute.mock.calls[0][3]).toMatchObject({ noLimit: true, queryTimeoutMs: 4321 }) + expect(dataCalls(execute)[0][3]).toMatchObject({ noLimit: true, queryTimeoutMs: 4321 }) }) test.each([ @@ -395,36 +462,47 @@ describe("predicate_effect with mocked Postgres", () => { ['"Foo"."Bar"', 'E\'"Foo"."Bar"\''], [String.raw`"Foo"."O'rd\\ers"`, String.raw`E'"Foo"."O''rd\\\\ers"'`], ])("resolves %s through an escaped to_regclass literal before enforcing reltuples", async (base_sql, literal) => { + if (base_sql !== "orders") { + database.run(`ATTACH DATABASE ':memory:' AS "Foo"`) + database.run(`CREATE TABLE ${base_sql} AS SELECT * FROM orders`) + } const execute = postgres("r", 6) - await expect( - checkSql({ warehouse, check: "predicate_effect", base_sql, predicates: ["id > 0"], max_rows_scanned: 5 }), - ).rejects.toThrow("population estimate 6 exceeds max_rows_scanned 5") - expect(execute).toHaveBeenCalledTimes(1) - expect(execute.mock.calls[0][0]).toContain("SELECT reltuples AS estimate FROM pg_catalog.pg_class") - expect(execute.mock.calls[0][0]).toContain(`oid = pg_catalog.to_regclass(${literal})`) - expect(execute.mock.calls[0][0]).not.toContain("COUNT(*)") + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql, + predicates: ["id > 0"], + max_rows_scanned: 5, + }) + expect(result.metadata).toMatchObject({ skipped: "estimate" }) + expect(result.output).toContain("population estimate 6 exceeds max_rows_scanned 5") + expect(dataCalls(execute)).toHaveLength(1) + expect(dataCalls(execute)[0][0]).toContain("SELECT reltuples AS estimate FROM pg_catalog.pg_class") + expect(dataCalls(execute)[0][0]).toContain(`oid = pg_catalog.to_regclass(${literal})`) + expect(dataCalls(execute)[0][0]).toContain("WHERE COALESCE((SELECT estimate FROM __sql_check_0_est) <= 5, TRUE)") }) test("a lost helper session during the estimate lookup reaches the caller instead of becoming an unknown estimate", async () => { const execute = postgres("r", 5, (sql) => { - if (/pg_class/.test(sql)) throw Object.assign(new Error("session state was lost"), { code: "SESSION_STATE_LOST", stateLost: true }) + if (/pg_class/.test(sql)) + throw Object.assign(new Error("session state was lost"), { code: "SESSION_STATE_LOST", stateLost: true }) return sql }) await expect( checkSql({ warehouse, check: "predicate_effect", base_sql: "orders", predicates: ["id > 2"] }), ).rejects.toMatchObject({ stateLost: true }) - expect(execute).toHaveBeenCalledTimes(1) + expect(dataCalls(execute)).toHaveLength(1) }) - test("a missing-catalog error during the estimate lookup still means an unknown estimate", async () => { + test("a missing-catalog error in the combined estimate and probe query reaches the caller", async () => { const execute = postgres("r", 5, (sql) => { if (/pg_class/.test(sql)) throw new Error('relation "pg_catalog.pg_class" does not exist') return sql }) - const result = await checkSql({ warehouse, check: "predicate_effect", base_sql: "orders", predicates: ["id > 2"] }) - expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) - expect(result.output).toContain(unknownEstimate) - expect(execute).toHaveBeenCalledTimes(2) + await expect( + checkSql({ warehouse, check: "predicate_effect", base_sql: "orders", predicates: ["id > 2"] }), + ).rejects.toThrow('relation "pg_catalog.pg_class" does not exist') + expect(dataCalls(execute)).toHaveLength(1) }) test("allows a known estimate equal to the cap and does not label it unknown", async () => { @@ -438,21 +516,21 @@ describe("predicate_effect with mocked Postgres", () => { }) expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) expect(result.output).not.toContain(unknownEstimate) - expect(execute).toHaveBeenCalledTimes(2) + expect(dataCalls(execute)).toHaveLength(1) }) test("guards materialized views using their positive reltuples estimate", async () => { const execute = postgres("m", 6) - await expect( - checkSql({ - warehouse, - check: "predicate_effect", - base_sql: "orders", - predicates: ["id > 0"], - max_rows_scanned: 5, - }), - ).rejects.toThrow("population estimate 6 exceeds max_rows_scanned 5") - expect(execute).toHaveBeenCalledTimes(1) + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 0"], + max_rows_scanned: 5, + }) + expect(result.metadata).toMatchObject({ skipped: "estimate" }) + expect(result.output).toContain("population estimate 6 exceeds max_rows_scanned 5") + expect(dataCalls(execute)).toHaveLength(1) }) test.each([ @@ -474,8 +552,8 @@ describe("predicate_effect with mocked Postgres", () => { }) expect(result.output).toContain(unknownEstimate) expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) - expect(execute).toHaveBeenCalledTimes(2) - expect(execute.mock.calls[0][0]).toContain("pg_catalog.to_regclass(E'orders')") + expect(dataCalls(execute)).toHaveLength(1) + expect(dataCalls(execute)[0][0]).toContain("pg_catalog.to_regclass(E'orders')") for (const call of execute.mock.calls) { expect(call[3]).toMatchObject({ noLimit: true, queryTimeoutMs: 4321 }) } diff --git a/packages/opencode/test/altimate/tools/sql-check.test.ts b/packages/opencode/test/altimate/tools/sql-check.test.ts index c0a3300fe..4ace0bd36 100644 --- a/packages/opencode/test/altimate/tools/sql-check.test.ts +++ b/packages/opencode/test/altimate/tools/sql-check.test.ts @@ -732,19 +732,29 @@ function emptyPredicateCatalog(sql: string) { ? ["oprname", "nspname"] : sql.includes("pg_catalog.pg_proc") ? ["proname", "nspname"] - : sql.includes("duckdb_functions()") - ? ["function_name", "schema_name"] - : undefined + : sql.includes("pg_catalog.pg_type") + ? ["typname", "nspname"] + : sql.includes("duckdb_functions()") + ? ["function_name", "schema_name"] + : undefined return columns ? { columns, rows: [], row_count: 0, truncated: false } : undefined } -async function predicateCatalogFixture(type: string, catalog: { operators?: string[][]; functions?: string[][] } = {}) { +async function predicateCatalogFixture( + type: string, + catalog: { operators?: string[][]; functions?: string[][]; types?: string[][] } = {}, +) { const connector = await Registry.get("sqlite_fixture") - const execute = spyOn(connector, "execute").mockImplementation(async (sql) => { + const execute = spyOn(connector, "execute").mockImplementation(async (sql, _limit, binds) => { const response = emptyPredicateCatalog(sql) if (response) { - const entries = sql.includes("pg_catalog.pg_operator") ? catalog.operators : catalog.functions - const rows = (entries ?? []).filter(([, schema]) => schema !== "pg_catalog") + const entries = sql.includes("pg_catalog.pg_operator") + ? catalog.operators + : sql.includes("pg_catalog.pg_type") + ? catalog.types + : catalog.functions + const names = sql.includes("duckdb_functions()") ? binds : (binds?.[0] as string[] | undefined) + const rows = (entries ?? []).filter(([name, schema]) => schema !== "pg_catalog" && names?.includes(name)) return { ...response, rows, row_count: rows.length } } return { @@ -1204,6 +1214,248 @@ for (const field of ["predicate", "population"] as const) { }) } +for (const field of ["predicate", "population"] as const) { + for (const cast of ["x::custom_type", "CAST(x AS custom_type)"]) { + test(`PostgreSQL predicate_effect refuses ${field} cast ${cast} before the probe`, async () => { + const fixture = await predicateCatalogFixture("postgres", { types: [["custom_type", "public"]] }) + try { + await expect( + checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: field === "population" ? `SELECT ${cast} AS x FROM orders` : "SELECT 1 AS x", + predicates: [field === "predicate" ? `${cast} > 0` : "x > 0"], + query_timeout_ms: 4321, + }), + ).rejects.toThrow(`${field} casts to \`custom_type\`, which is not a built-in type`) + const calls = fixture.execute.mock.calls + expect(calls.every(([sql]) => !!emptyPredicateCatalog(sql))).toBe(true) + const catalogs = calls.filter(([sql]) => sql.includes("pg_catalog.pg_type")) + expect(catalogs).toHaveLength(1) + const [sql, limit, binds, options] = catalogs[0] + expect(sql).toContain("pg_catalog.pg_namespace") + expect(sql).toMatch(/typname\s*=\s*ANY\(\$1\)/) + expect(sql).toMatch(/(?:nspname\s*<>\s*'pg_catalog'|typnamespace\s*<>\s*'pg_catalog'::regnamespace)/) + expect(limit).toBeUndefined() + expect(binds).toEqual([["custom_type"]]) + expect(options).toMatchObject({ readOnly: true, noLimit: true, queryTimeoutMs: 4321 }) + } finally { + fixture.restore() + } + }) + } +} + +test("PostgreSQL predicate_effect accepts built-in cast targets from population and predicates", async () => { + const fixture = await predicateCatalogFixture("postgres", { types: [["integer", "pg_catalog"]] }) + try { + const result = await checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: "SELECT CAST(id AS integer) AS x FROM orders", + predicates: ["x::integer > 0 AND CAST(x AS integer) > 0"], + query_timeout_ms: 4321, + }) + expect(result.metadata.total_rows).toBe(1) + const calls = fixture.execute.mock.calls + const catalogs = calls.filter(([sql]) => sql.includes("pg_catalog.pg_type")) + expect(catalogs).toHaveLength(1) + expect(catalogs[0][2]).toEqual([["integer"]]) + expect(catalogs[0][3]).toMatchObject({ readOnly: true, noLimit: true, queryTimeoutMs: 4321 }) + expect(calls.slice(0, -1).every(([sql]) => !!emptyPredicateCatalog(sql))).toBe(true) + expect(calls.at(-1)![0]).toContain("COUNT(*) AS total") + } finally { + fixture.restore() + } +}) + +test("PostgreSQL predicate_effect resolves qualified, quoted, array, nested, and typmod cast targets", async () => { + const fixture = await predicateCatalogFixture("postgres", { + types: [ + ["custom_type", "public"], + ['custom"type', "public"], + ], + }) + const casts = [ + ["x::public.custom_type", "custom_type", ["custom_type"]], + ['CAST(x AS "public"."custom""type")', 'custom"type', ['custom"type']], + ["x::custom_type[]", "custom_type", ["custom_type"]], + ["x::custom_type::integer", "custom_type", ["custom_type", "integer"]], + ["CAST(CAST(x AS custom_type) AS integer)", "custom_type", ["custom_type", "integer"]], + ["CAST(x AS custom_type(12))", "custom_type", ["custom_type"]], + ] as const + try { + for (const [cast, target, names] of casts) { + fixture.execute.mockClear() + await expect( + checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: "SELECT 1 AS x", + predicates: [`${cast} IS NOT NULL`], + }), + ).rejects.toThrow(`predicate casts to \`${target}\`, which is not a built-in type`) + const calls = fixture.execute.mock.calls + expect(calls.every(([sql]) => !!emptyPredicateCatalog(sql))).toBe(true) + const catalogs = calls.filter(([sql]) => sql.includes("pg_catalog.pg_type")) + expect(catalogs).toHaveLength(1) + expect(catalogs[0][2]?.[0]).toEqual(expect.arrayContaining([...names])) + expect(catalogs[0][2]?.[0]).toHaveLength(names.length) + } + } finally { + fixture.restore() + } +}) + +test("PostgreSQL predicate_effect ignores cast syntax inside literals, comments, and column identifiers", async () => { + const fixture = await predicateCatalogFixture("postgres", { types: [["custom_type", "public"]] }) + try { + const result = await checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: "SELECT 1 AS x, 'x::custom_type' AS note, 1 AS \"x::custom_type\"", + predicates: ["x > 0 /* CAST(x AS custom_type) */ AND note <> 'CAST(x AS custom_type)' -- x::custom_type\n"], + }) + expect(result.metadata.total_rows).toBe(1) + expect(fixture.execute.mock.calls.some(([sql]) => sql.includes("pg_catalog.pg_type"))).toBe(false) + expect(fixture.execute.mock.calls.at(-1)![0]).toContain("COUNT(*) AS total") + } finally { + fixture.restore() + } +}) + +for (const [description, cast] of [ + ["default", String.raw`x::U&"custom_\0074ype"`], + ["custom", String.raw`x::U&"custom_!0074ype" UESCAPE '!'`], +]) { + test(`PostgreSQL predicate_effect refuses ${description} Unicode escaped cast targets before the probe`, async () => { + const fixture = await predicateCatalogFixture("postgres", { types: [["custom_type", "public"]] }) + try { + await expect( + checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: "SELECT 1 AS x", + predicates: [`${cast} IS NOT NULL`], + }), + ).rejects.toThrow( + description === "custom" + ? "sql_check only accepts a single read-only SELECT" + : "predicate casts to `custom_type`, which is not a built-in type", + ) + const calls = fixture.execute.mock.calls + expect(calls.every(([sql]) => !!emptyPredicateCatalog(sql))).toBe(true) + // The structural parser already refuses UESCAPE; ordinary Unicode names need the type catalog. + if (description === "custom") expect(calls).toHaveLength(0) + else expect(calls.find(([sql]) => sql.includes("pg_catalog.pg_type"))?.[2]).toEqual([["custom_type"]]) + } finally { + fixture.restore() + } + }) +} + +test("PostgreSQL predicate_effect accepts CTE column lists in populations", async () => { + const fixture = await predicateCatalogFixture("postgres") + const populations = [ + "WITH population(id) AS (SELECT 1) SELECT id FROM population", + 'WITH first(id) AS (SELECT 1), "population"(id) AS (SELECT id FROM first) SELECT id FROM "population"', + "WITH RECURSIVE population(id) AS (SELECT 1) SELECT id FROM population", + "WITH population(id) AS MATERIALIZED (SELECT 1) SELECT id FROM population", + ] + try { + for (const base_sql of populations) { + const result = await checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql, + predicates: ["id > 0"], + }) + expect(result.metadata.total_rows).toBe(1) + } + expect(fixture.execute.mock.calls.filter(([sql]) => !emptyPredicateCatalog(sql))).toHaveLength(populations.length) + expect(fixture.execute.mock.calls.some(([sql]) => sql.includes("pg_catalog.pg_proc"))).toBe(false) + } finally { + fixture.restore() + } +}) + +test("PostgreSQL predicate_effect refuses actual function calls that share a CTE declaration name", async () => { + const fixture = await predicateCatalogFixture("postgres") + try { + for (const base_sql of [ + "WITH population(id) AS (SELECT population(1)) SELECT id FROM population", + "WITH population(id) AS (SELECT 1) SELECT population(id) FROM population", + ]) { + await expect( + checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql, + predicates: ["id > 0"], + }), + ).rejects.toThrow(/refuses function population.*built-in pure functions/) + } + expect(fixture.execute).not.toHaveBeenCalled() + } finally { + fixture.restore() + } +}) + +test("PostgreSQL predicate_effect excludes projection and count wildcards from the operator catalog", async () => { + const fixture = await predicateCatalogFixture("postgres", { operators: [["*", "public"]] }) + const populations = [ + "SELECT * FROM orders", + "SELECT t.* FROM orders t", + "SELECT id, * FROM orders", + "SELECT count(*) AS id FROM orders", + "SELECT ALL * FROM orders", + "SELECT DISTINCT * FROM orders", + "SELECT DISTINCT ON (id) * FROM orders", + "SELECT count(/* rows */ *) AS id FROM orders", + ] + try { + for (const base_sql of populations) { + const result = await checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql, + predicates: ["id > 0"], + }) + expect(result.metadata.total_rows).toBe(1) + } + const calls = fixture.execute.mock.calls + expect(calls.filter(([sql]) => !emptyPredicateCatalog(sql))).toHaveLength(populations.length) + const catalogs = calls.filter(([sql]) => sql.includes("pg_catalog.pg_operator")) + expect(catalogs).toHaveLength(populations.length) + for (const [, , binds] of catalogs) expect(binds).toEqual([[">"]]) + } finally { + fixture.restore() + } +}) + +test("PostgreSQL predicate_effect still checks multiplication beside projection wildcards", async () => { + const fixture = await predicateCatalogFixture("postgres", { operators: [["*", "public"]] }) + try { + for (const base_sql of ["SELECT *, a * b AS id FROM orders", "SELECT 1.*2 AS id"]) { + await expect( + checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql, + predicates: ["id > 0"], + }), + ).rejects.toThrow("population uses operator `*`, which is not a built-in operator") + } + const calls = fixture.execute.mock.calls + expect(calls.every(([sql]) => !!emptyPredicateCatalog(sql))).toBe(true) + const catalogs = calls.filter(([sql]) => sql.includes("pg_catalog.pg_operator")) + expect(catalogs).toHaveLength(2) + for (const [, , binds] of catalogs) expect(binds).toEqual([expect.arrayContaining(["*", ">"])]) + } finally { + fixture.restore() + } +}) + test("PostgreSQL predicate_effect accepts built-in-only operator names after one read-only catalog query", async () => { const fixture = await predicateCatalogFixture("postgres", { operators: [["##", "pg_catalog"]] }) try { From defa2ba41957912046f95492dcbc82bdce47c3ce Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Tue, 13 Oct 2026 22:05:17 -0700 Subject: [PATCH 19/22] fix(analyst): `predicate_effect` authorizes the combined probe and accepts FILTER/OVER clauses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Permission rules are matched per pattern, so a `sql_execute` deny rule scoped to a column-and-table combination (`*secret_col*FROM*sensitive_table*`) matched neither the population nor the predicate on its own although the probe joins them. The tool now also asks for the combined probe in the shape it executes (`SELECT COUNT(*) FILTER (WHERE ) … FROM () AS t`), so such a rule denies before any SQL runs. - `FILTER (…)`, `OVER (…)`, `WITHIN GROUP (…)`, `USING (…)` and similar clause introducers are no longer mistaken for function calls, so valid aggregate and window populations are accepted while unknown functions are still refused. - The permission regression tests use the package `it.instance`/`TestInstance` fixture instead of a manual `Instance.provide`/`dispose` lifecycle. - tests: combined-probe deny before SQL and a non-matching control; FILTER/OVER population accepted and `filterx(id)` refused. Co-Authored-By: Claude Fable 5.1 --- .../opencode/src/altimate/tools/sql-check.ts | 18 +- .../sql-check-predicate-effect.test.ts | 159 ++++++++++++------ 2 files changed, 123 insertions(+), 54 deletions(-) diff --git a/packages/opencode/src/altimate/tools/sql-check.ts b/packages/opencode/src/altimate/tools/sql-check.ts index b1dd704d5..2751a2285 100644 --- a/packages/opencode/src/altimate/tools/sql-check.ts +++ b/packages/opencode/src/altimate/tools/sql-check.ts @@ -890,11 +890,14 @@ function pureFunctions(sql: string, dialect?: string): string[] { const parts = call.match(new RegExp(identifier, "g"))! const quoted = /^["`\[]/.test(parts[0]) const name = quoted ? parts[0].slice(1, -1) : parts[0].replace(/[A-Z]/g, (char) => char.toLowerCase()) - // These unquoted SQL keywords introduce grouped expressions, not function calls. + // These unquoted SQL keywords introduce grouped expressions or clauses (`FILTER (WHERE …)`, + // `OVER (PARTITION BY …)`, `WITHIN GROUP (…)`, `USING (…)`), not function calls. if ( parts.length === 1 && !quoted && - /^(all|and|any|as|between|else|exists|from|having|in|not|on|or|select|some|then|when|where)$/.test(name) + /^(all|and|any|as|between|distinct|else|exists|filter|from|groups|having|in|lateral|not|on|or|ordinality|over|partition|range|rows|select|some|then|using|values|when|where|within)$/.test( + name, + ) ) continue if (parts.length !== 1 || !PURE_PREDICATE_FUNCTIONS.has(name)) { @@ -1353,9 +1356,18 @@ export const SqlCheckTool = Tool.define("sql_check", { for (const predicate of input.predicates ?? []) queries.push(`SELECT 1 WHERE ${readPredicate(predicate, dialect)}`) for (const sql of queries) readSql(sql, dialect) + // Permission rules are matched per pattern, so a rule scoped to a column-and-table + // combination must also see the probe that joins them: authorize the combined + // population + predicates in the shape the probe executes, not only the parts. + const combined = + input.base_sql !== undefined && input.predicates?.length + ? [ + `SELECT ${input.predicates.map((predicate) => `COUNT(*) FILTER (WHERE ${predicate})`).join(", ")} FROM (${readBase(input.base_sql, dialect).sql}) AS t`, + ] + : [] await ctx.ask({ permission: "sql_execute", - patterns: queries, + patterns: [...queries, ...combined], always: ["*"], metadata: { warehouse: input.warehouse }, }) diff --git a/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts b/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts index c82398a13..4dce399df 100644 --- a/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts +++ b/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts @@ -6,10 +6,15 @@ import * as Precedence from "../../src/altimate/workspace/precedence" import { checkSql, SqlCheckTool } from "../../src/altimate/tools/sql-check" import { initTool } from "./tool-fixture" import { Bus } from "../../src/bus" -import { Instance } from "../../src/project/instance" import { PermissionNext } from "../../src/permission/next" import { SessionID } from "../../src/session/schema" -import { tmpdir } from "../fixture/fixture" +import { Effect, Layer } from "effect" +import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner" +import { Instance } from "../../src/project/instance" +import { TestInstance, testInstanceStoreLayer } from "../fixture/fixture" +import { testEffect } from "../lib/effect" + +const it = testEffect(Layer.mergeAll(CrossSpawnSpawner.defaultLayer, testInstanceStoreLayer)) const warehouses = ["sqlite_predicates", "scratch"] as const const baseSql = "SELECT * FROM orders" @@ -730,56 +735,108 @@ test("sql_execute denied prevents predicate_effect routing and query execution", } }) -test("predicate_effect refuses a later denied predicate before requesting population approval", async () => { - await using tmp = await tmpdir({ git: true }) - await Instance.provide({ - directory: tmp.path, - fn: async () => { - const connector = await Registry.get("sqlite_predicates") - const execute = spyOn(connector, "execute") - const precedence = spyOn(Precedence, "check").mockResolvedValue({}) - const asked: PermissionNext.Request[] = [] - const publish = spyOn(Bus, "publish").mockImplementation(async (event, properties) => { - if (event.type !== PermissionNext.Event.Asked.type) return - const request = properties as PermissionNext.Request - asked.push(request) - // Reproduce the bypass: approving the first pattern must never authorize a later deny. - await PermissionNext.reply({ requestID: request.id, reply: "once" }) - }) - try { - const tool = await initTool(SqlCheckTool) - const sessionID = SessionID.make("session_predicate_permissions") - const result = await tool.execute( - { warehouse: "sqlite_predicates", check: "predicate_effect", base_sql: baseSql, predicates: ["id > 0", "amount > 4"] }, - { - sessionID, - messageID: "test-message", - agent: "test", - abort: new AbortController().signal, - messages: [], - ask: (request: Omit) => PermissionNext.ask({ - ...request, - sessionID, - ruleset: [ - { permission: "sql_execute", pattern: "*", action: "ask" }, - { permission: "sql_execute", pattern: "*amount*", action: "deny" }, - ], - }), - }, - ) - expect(result.metadata.error).toContain("rule which prevents") - expect(asked).toEqual([]) - expect(await PermissionNext.list()).toEqual([]) - expect(precedence).not.toHaveBeenCalled() - expect(execute).not.toHaveBeenCalled() - } finally { - publish.mockRestore() - execute.mockRestore() - precedence.mockRestore() - await Instance.dispose() - } - }, +/** Run predicate_effect through the tool boundary with a permission ruleset; returns the result and the asks that reached the bus. */ +async function checkWithRules( + input: { base_sql: string; predicates: string[] }, + ruleset: PermissionNext.Rule[], + sessionName: string, +) { + const connector = await Registry.get("sqlite_predicates") + const execute = spyOn(connector, "execute") + const precedence = spyOn(Precedence, "check").mockResolvedValue({}) + const asked: PermissionNext.Request[] = [] + const publish = spyOn(Bus, "publish").mockImplementation(async (event, properties) => { + if (event.type !== PermissionNext.Event.Asked.type) return + const request = properties as PermissionNext.Request + asked.push(request) + // Approving one pattern must never authorize a later deny. + await PermissionNext.reply({ requestID: request.id, reply: "once" }) + }) + try { + const tool = await initTool(SqlCheckTool) + const sessionID = SessionID.make(sessionName) + const result = await tool.execute( + { warehouse: "sqlite_predicates", check: "predicate_effect", ...input }, + { + sessionID, + messageID: "test-message", + agent: "test", + abort: new AbortController().signal, + messages: [], + ask: (request: Omit) => + PermissionNext.ask({ ...request, sessionID, ruleset }), + }, + ) + return { result, asked, executed: execute.mock.calls.length, precedenceChecked: precedence.mock.calls.length } + } finally { + publish.mockRestore() + execute.mockRestore() + precedence.mockRestore() + } +} + +/** Run promise-based tool code inside the fixture's instance (the tool reads the instance through async context). */ +const inInstance = (fn: () => Promise) => + Effect.gen(function* () { + const { directory } = yield* TestInstance + yield* Effect.promise(() => Instance.provide({ directory, fn })) + }) + +it.instance("predicate_effect refuses a later denied predicate before requesting population approval", () => + inInstance(async () => { + const { result, asked, executed, precedenceChecked } = await checkWithRules( + { base_sql: baseSql, predicates: ["id > 0", "amount > 4"] }, + [ + { permission: "sql_execute", pattern: "*", action: "ask" }, + { permission: "sql_execute", pattern: "*amount*", action: "deny" }, + ], + "session_predicate_permissions", + ) + expect(result.metadata.error).toContain("rule which prevents") + expect(asked).toEqual([]) + expect(await PermissionNext.list()).toEqual([]) + expect(precedenceChecked).toBe(0) + expect(executed).toBe(0) + }), +) + +it.instance("predicate_effect authorizes the combined probe so a table-and-column deny rule applies", () => + inInstance(async () => { + const ruleset: PermissionNext.Rule[] = [ + { permission: "sql_execute", pattern: "*", action: "ask" }, + { permission: "sql_execute", pattern: "*amount*FROM*orders*", action: "deny" }, + ] + // Neither part matches the rule on its own; the combined probe (predicate before FROM) does. + const denied = await checkWithRules({ base_sql: baseSql, predicates: ["amount > 4"] }, ruleset, "session_combined_deny") + expect(denied.result.metadata.error).toContain("rule which prevents") + expect(denied.executed).toBe(0) + expect(denied.precedenceChecked).toBe(0) + // The same rule does not touch a probe over another table. + const allowed = await checkWithRules( + { base_sql: "SELECT id, amount FROM orders AS archive", predicates: ["id > 0"] }, + [ + { permission: "sql_execute", pattern: "*", action: "allow" }, + { permission: "sql_execute", pattern: "*amount*FROM*sensitive_table*", action: "deny" }, + ], + "session_combined_allow", + ) + expect(allowed.result.metadata.error).toBeUndefined() + expect(allowed.result.metadata).toMatchObject({ check: "predicate_effect" }) + }), +) + +test("predicate_effect accepts FILTER and OVER clauses in populations and still refuses unknown functions", async () => { + const accepted = await checkSql({ + warehouse: "sqlite_predicates", + check: "predicate_effect", + base_sql: "SELECT id, COUNT(*) FILTER (WHERE amount > 0) OVER (PARTITION BY kind) AS positives FROM orders", + predicates: ["positives >= 0"], }) + expect(accepted.metadata.error).toBeUndefined() + expect(accepted.metadata).toMatchObject({ check: "predicate_effect" }) + await expect( + checkSql({ warehouse: "sqlite_predicates", check: "predicate_effect", base_sql: baseSql, predicates: ["filterx(id) = 1"] }), + ).rejects.toThrow(/refuses function filterx/) }) test("SQLite predicate_effect enables query_only and restores the previous setting after success and failure", async () => { From f9dbf9ad03d869f9338f89d17f73835b470a354c Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Wed, 14 Oct 2026 11:15:31 -0700 Subject: [PATCH 20/22] feat: make `sql_check` `predicate_effect` opt-in and accept common SQL shapes - Add `sql_check.predicate_effect` config (default false). When unset or false the tool returns a disabled error before any permission prompt, routing or SQL. The probe executes caller-supplied SQL whose server-side definitions (views, cast functions, operators) cannot be fully validated from the text. - Allow the general-purpose window functions (`row_number`, `rank`, `dense_rank`, `percent_rank`, `cume_dist`, `ntile`, `lag`, `lead`, `first_value`, `last_value`, `nth_value`). - Accept SQLite bracket and backtick quoted bare tables, including qualified names and names with spaces. - Treat derived-table and table-function column alias lists (`AS v(id)`) as declarations at their source position, so real calls named `v` stay checked. Co-Authored-By: Claude Opus 5.5 --- packages/core/src/v1/config/config.ts | 10 + .../opencode/src/altimate/tools/sql-check.ts | 55 +++- .../sql-check-predicate-effect.test.ts | 242 +++++++++++++++++- packages/opencode/test/config/config.test.ts | 28 ++ 4 files changed, 323 insertions(+), 12 deletions(-) diff --git a/packages/core/src/v1/config/config.ts b/packages/core/src/v1/config/config.ts index b6fe79e97..c9a39e21d 100644 --- a/packages/core/src/v1/config/config.ts +++ b/packages/core/src/v1/config/config.ts @@ -169,6 +169,16 @@ export const Info = Schema.Struct({ description: "Thresholds for truncating tool output. When output exceeds either limit, the full text is written to the truncation directory and a preview is returned.", }), + // altimate_change start — require an explicit opt-in for the predicate-effect SQL probe + sql_check: Schema.optional( + Schema.Struct({ + predicate_effect: Schema.optional(Schema.Boolean).annotate({ + description: + "Enable predicate_effect for trusted databases (default: false). Executes caller-supplied SQL whose server-side definitions cannot be fully validated.", + }), + }), + ).annotate({ description: "SQL check tool options" }), + // altimate_change end compaction: Schema.optional( Schema.Struct({ auto: Schema.optional(Schema.Boolean).annotate({ diff --git a/packages/opencode/src/altimate/tools/sql-check.ts b/packages/opencode/src/altimate/tools/sql-check.ts index 2751a2285..4a21a67b8 100644 --- a/packages/opencode/src/altimate/tools/sql-check.ts +++ b/packages/opencode/src/altimate/tools/sql-check.ts @@ -1,6 +1,7 @@ import z from "zod" import { isQueryTimeoutError } from "@altimateai/drivers/types" import { Tool } from "../../tool/tool" +import { Config } from "../../config/config" import * as Precedence from "../workspace/precedence" import * as Registry from "../native/connections/registry" import { classifyAndCheck, isSingleQuery } from "./sql-classify" @@ -71,6 +72,7 @@ const PURE_PREDICATE_FUNCTIONS = new Set([ "concat_ws", "cos", "count", + "cume_dist", "current_date", "current_time", "current_timestamp", @@ -80,9 +82,11 @@ const PURE_PREDICATE_FUNCTIONS = new Set([ "datetime", "decimal", "degrees", + "dense_rank", "div", "exp", "extract", + "first_value", "floor", "format", "greatest", @@ -96,6 +100,9 @@ const PURE_PREDICATE_FUNCTIONS = new Set([ "jsonb_extract_path_text", "jsonb_typeof", "julianday", + "lag", + "last_value", + "lead", "least", "left", "length", @@ -113,10 +120,13 @@ const PURE_PREDICATE_FUNCTIONS = new Set([ "min", "mod", "now", + "nth_value", + "ntile", "nullif", "numeric", "octet_length", "overlay", + "percent_rank", "pg_available_extension_versions", "pg_available_extensions", "pg_backend_pid", @@ -408,6 +418,7 @@ const PURE_PREDICATE_FUNCTIONS = new Set([ "quote_literal", "quote_nullable", "radians", + "rank", "regexp_count", "regexp_instr", "regexp_like", @@ -421,6 +432,7 @@ const PURE_PREDICATE_FUNCTIONS = new Set([ "reverse", "right", "round", + "row_number", "rpad", "rtrim", "sign", @@ -682,7 +694,7 @@ function postgresCasts(sql: string) { return targets } -function cteDeclarations(sql: string, dialect?: string) { +function sqlDeclarations(sql: string, dialect?: string) { const tokens = sqlSyntax(sql, dialect) const declarations = new Set() for (let i = 0; i < tokens.length; i++) { @@ -707,6 +719,23 @@ function cteDeclarations(sql: string, dialect?: string) { j = end + 2 } } + // A derived table or table function may declare output columns after its alias. + // Exclude only this source position, so calls sharing the alias name remain checked. + for (let i = 0; i < tokens.length; i++) { + if (tokens[i].text !== ")") continue + let j = i + 1 + if (/^as$/i.test(tokens[j]?.text ?? "")) j++ + const alias = tokens[j++] + if (!alias || !POSTGRES_IDENTIFIER.test(alias.text) || tokens[j++]?.text !== "(") continue + while (POSTGRES_IDENTIFIER.test(tokens[j]?.text ?? "")) { + j++ + if (tokens[j]?.text === ")") { + declarations.add(alias.index) + break + } + if (tokens[j++]?.text !== ",") break + } + } return declarations } @@ -844,13 +873,14 @@ function result(check: string, metadata: Row, lines: string[]): { title: string; /** Only identifiers count as bare tables; SELECT populations are never rewritten. */ function readBase(sql: string, dialect?: string) { - const bare = - /^(?:"(?:[^"]|"")+"|[A-Za-z_][A-Za-z0-9_$]*)(?:\s*\.\s*(?:"(?:[^"]|"")+"|[A-Za-z_][A-Za-z0-9_$]*)){0,2}$/.test( - sql.trim(), - ) + const identifier = + dialect === "sqlite" + ? '(?:"(?:[^"]|"")+"|`(?:[^`]|``)+`|\\[[^\\]]+\\]|[A-Za-z_][A-Za-z0-9_$]*)' + : '(?:"(?:[^"]|"")+"|[A-Za-z_][A-Za-z0-9_$]*)' + const bare = new RegExp(`^${identifier}(?:\\s*\\.\\s*${identifier}){0,2}$`).test(sql.trim()) const table = bare ? sql.trim() : undefined // Preserve quoting and case folding, including dots inside a quoted table name. - const alias = table?.match(/"(?:[^"]|"")+"|[A-Za-z_][A-Za-z0-9_$]*/g)?.at(-1) ?? "t" + const alias = table?.match(new RegExp(identifier, "g"))?.at(-1) ?? "t" const population = readSql(table ? `SELECT * FROM ${table}` : sql, dialect, true) pureFunctions(population, dialect) return { sql: population, table, alias } @@ -880,7 +910,7 @@ function pureFunctions(sql: string, dialect?: string): string[] { ? '(?:"(?:[^"\\x00]|"")+"|[A-Za-z_\\u0080-\\uFFFF][\\w$\\u0080-\\uFFFF]*)' : '(?:"(?:[^"\\x00]|"")+"|`(?:[^`]|``)+`|\\[(?:[^\\]]|\\]\\])+\\]|[A-Za-z_\\u0080-\\uFFFF][\\w$\\u0080-\\uFFFF]*)' const calls = new RegExp(`${identifier}(?:\\s*\\.\\s*${identifier})*`, "g") - const declarations = cteDeclarations(sql, dialect) + const declarations = sqlDeclarations(sql, dialect) if (dialect === "postgres") for (const target of postgresCasts(sql)) declarations.add(target.index) const names = new Set() for (const match of functions.matchAll(calls)) { @@ -1341,7 +1371,7 @@ export const SqlCheckTool = Tool.define("sql_check", { "Read-only SELECTs run in one SQLite, DuckDB or Postgres warehouse, with their original LIMITs preserved; samples cap at five. " + "Use sql_materialize to put cross-warehouse or other-engine sources into scratch first. " + "Key normalization is diagnostic and may collapse unrelated values; no automatic fixes. Coverage uses regex except SQLite's explicit LIKE fallback. " + - "predicate_effect takes base_sql before filtering and 1–8 explicit predicates; reports individual and combined removals, with NULL not kept. " + + "predicate_effect is opt-in via sql_check.predicate_effect (default false); takes base_sql before filtering and 1–8 explicit predicates and reports individual and combined removals, with NULL not kept. " + "predicates reference the population's columns; for a bare table you may qualify with the table name; for a SELECT population use bare column names or the alias `t`. " + "Bare-table estimates above max_rows_scanned (default 5000000) are refused; unknown estimates use the query timeout and expiry is reported as skipped. " + "Leave fields a check does not use empty.", @@ -1349,6 +1379,15 @@ export const SqlCheckTool = Tool.define("sql_check", { async execute(args, ctx) { try { // altimate_change start — validate and authorize reads before workspace redirects + if (args.check === "predicate_effect" && (await Config.get()).sql_check?.predicate_effect !== true) { + const error = + "predicate_effect is disabled by default because it executes caller-supplied population and predicate SQL whose server-side definitions (views, casts, operators) cannot be fully validated; enable it in config (sql_check.predicate_effect: true) for trusted databases." + return { + title: "SQL check: predicate_effect ERROR", + metadata: { check: args.check, error, disabled: true }, + output: `predicate_effect: no verdict — ${error}`, + } + } const input = dropEmpty(args) const dialect = Registry.canonicalType(Registry.getConfig(input.warehouse)?.type) ?? undefined const queries = [input.sql, input.left_sql, input.right_sql].filter((sql): sql is string => sql !== undefined) diff --git a/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts b/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts index 4dce399df..e736fb8bc 100644 --- a/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts +++ b/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts @@ -4,14 +4,14 @@ import * as Registry from "../../src/altimate/native/connections/registry" import { closeScratch } from "../../src/altimate/native/connections/scratch" import * as Precedence from "../../src/altimate/workspace/precedence" import { checkSql, SqlCheckTool } from "../../src/altimate/tools/sql-check" -import { initTool } from "./tool-fixture" +import { initTool as initTestTool } from "./tool-fixture" import { Bus } from "../../src/bus" import { PermissionNext } from "../../src/permission/next" import { SessionID } from "../../src/session/schema" import { Effect, Layer } from "effect" import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner" import { Instance } from "../../src/project/instance" -import { TestInstance, testInstanceStoreLayer } from "../fixture/fixture" +import { TestInstance, testInstanceStoreLayer, tmpdir } from "../fixture/fixture" import { testEffect } from "../lib/effect" const it = testEffect(Layer.mergeAll(CrossSpawnSpawner.defaultLayer, testInstanceStoreLayer)) @@ -22,10 +22,15 @@ const noRemoval = "this predicate removes no rows at this population" const allRemoval = "this predicate removes all rows" const unknownEstimate = "row_estimate: unknown — probe runs under the query timeout" let previousTelemetry: string | undefined +let previousConfig: string | undefined +let configFixture: Awaited> -beforeAll(() => { +beforeAll(async () => { previousTelemetry = process.env.ALTIMATE_TELEMETRY_DISABLED process.env.ALTIMATE_TELEMETRY_DISABLED = "true" + previousConfig = process.env.OPENCODE_CONFIG_CONTENT + process.env.OPENCODE_CONFIG_CONTENT = JSON.stringify({ sql_check: { predicate_effect: true } }) + configFixture = await tmpdir() }) beforeEach(async () => { @@ -45,9 +50,113 @@ afterEach(async () => { await closeScratch() }) -afterAll(() => { +afterAll(async () => { if (previousTelemetry === undefined) delete process.env.ALTIMATE_TELEMETRY_DISABLED else process.env.ALTIMATE_TELEMETRY_DISABLED = previousTelemetry + if (previousConfig === undefined) delete process.env.OPENCODE_CONFIG_CONTENT + else process.env.OPENCODE_CONFIG_CONTENT = previousConfig + await Instance.provide({ directory: configFixture.path, fn: () => Instance.dispose() }) + await configFixture[Symbol.asyncDispose]() +}) + +// Tool config is instance-scoped; retain an existing permission fixture's instance when present. +async function initTool(tool: typeof SqlCheckTool) { + const initialized = await initTestTool(tool) + return { + ...initialized, + execute: (args: Parameters[0], ctx: Parameters[1]) => { + let directory = configFixture.path + try { + directory = Instance.directory + } catch {} + return Instance.provide({ directory, fn: () => initialized.execute(args, ctx) }) + }, + } +} + +test.each([ + { label: "unset", config: {} }, + { label: "false", config: { sql_check: { predicate_effect: false } } }, +])( + "predicate_effect disabled with flag $label asks no permission and performs no routing or SQL", + async ({ config }) => { + const enabledConfig = process.env.OPENCODE_CONFIG_CONTENT + process.env.OPENCODE_CONFIG_CONTENT = JSON.stringify(config) + await using fixture = await tmpdir() + const connector = await Registry.get("sqlite_predicates") + const execute = spyOn(connector, "execute") + const get = spyOn(Registry, "get") + const precedence = spyOn(Precedence, "check").mockResolvedValue({}) + const asked: unknown[] = [] + try { + await Instance.provide({ + directory: fixture.path, + fn: async () => { + const tool = await initTool(SqlCheckTool) + const result = await tool.execute( + { warehouse: "sqlite_predicates", check: "predicate_effect", base_sql: baseSql, predicates: ["id > 0"] }, + { + sessionID: "test-session", + messageID: "test-message", + agent: "test", + abort: new AbortController().signal, + messages: [], + ask: async (request: unknown) => { + asked.push(request) + }, + }, + ) + const error = + "predicate_effect is disabled by default because it executes caller-supplied population and predicate SQL whose server-side definitions (views, casts, operators) cannot be fully validated; enable it in config (sql_check.predicate_effect: true) for trusted databases." + expect(result).toMatchObject({ + title: "SQL check: predicate_effect ERROR", + metadata: { check: "predicate_effect", error, disabled: true }, + output: `predicate_effect: no verdict — ${error}`, + }) + expect(asked).toEqual([]) + expect(precedence).not.toHaveBeenCalled() + expect(get).not.toHaveBeenCalled() + expect(execute).not.toHaveBeenCalled() + }, + }) + } finally { + execute.mockRestore() + get.mockRestore() + precedence.mockRestore() + process.env.OPENCODE_CONFIG_CONTENT = enabledConfig + await Instance.provide({ directory: fixture.path, fn: () => Instance.dispose() }) + } + }, +) + +test("predicate_effect opt-in executes the registered tool after permission approval", async () => { + const connector = await Registry.get("sqlite_predicates") + const execute = spyOn(connector, "execute") + const precedence = spyOn(Precedence, "check").mockResolvedValue({}) + const asked: unknown[] = [] + try { + const tool = await initTool(SqlCheckTool) + const result = await tool.execute( + { warehouse: "sqlite_predicates", check: "predicate_effect", base_sql: baseSql, predicates: ["id > 2"] }, + { + sessionID: "test-session", + messageID: "test-message", + agent: "test", + abort: new AbortController().signal, + messages: [], + ask: async (request: unknown) => { + asked.push(request) + }, + }, + ) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + expect(asked).toHaveLength(1) + expect(precedence).toHaveBeenCalledTimes(1) + expect(execute).toHaveBeenCalledTimes(1) + } finally { + execute.mockRestore() + precedence.mockRestore() + } }) for (const warehouse of warehouses) { @@ -839,6 +948,131 @@ test("predicate_effect accepts FILTER and OVER clauses in populations and still ).rejects.toThrow(/refuses function filterx/) }) +test.each([ + "row_number()", + "rank()", + "dense_rank()", + "percent_rank()", + "cume_dist()", + "ntile(2)", + "lag(id)", + "lead(id)", + "first_value(id)", + "last_value(id)", + "nth_value(id, 2)", +])("SQLite predicate_effect accepts general-purpose window function %s", async (expression) => { + const result = await checkSql({ + warehouse: "sqlite_predicates", + check: "predicate_effect", + base_sql: `SELECT *, ${expression} OVER (PARTITION BY kind ORDER BY id) AS rn FROM orders`, + predicates: ["rn > 0"], + }) + expect(result.metadata.total_rows).toBe(5) + if (expression === "row_number()") expect(result.metadata.combined_kept_rows).toBe(5) +}) + +test.each(["[orders]", "`orders`", "main.[orders]", "main.`orders`"])( + "SQLite predicate_effect accepts quoted bare population %s and its table qualifier", + async (base_sql) => { + const table = base_sql.replace(/^main\./, "") + const result = await checkSql({ + warehouse: "sqlite_predicates", + check: "predicate_effect", + base_sql, + predicates: ["id > 2", `${table}.kind = 'a'`], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 2 }) + }, +) + +test.each(["[order details]", "`order details`"])( + "SQLite predicate_effect preserves spaces in quoted bare population %s", + async (base_sql) => { + const connector = await Registry.get("sqlite_predicates") + await connector.execute(`CREATE TABLE ${base_sql} AS SELECT * FROM orders`) + const result = await checkSql({ + warehouse: "sqlite_predicates", + check: "predicate_effect", + base_sql, + predicates: [`${base_sql}.id > 2`], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + }, +) + +test.each(["AS v(id)", "v(id)", 'AS "v"("id")'])( + "DuckDB predicate_effect accepts derived-table column aliases %s", + async (alias) => { + const result = await checkSql({ + warehouse: "scratch", + check: "predicate_effect", + base_sql: `SELECT id FROM (VALUES (1), (2)) ${alias}`, + predicates: ["id > 1"], + }) + expect(result.metadata).toMatchObject({ total_rows: 2, combined_kept_rows: 1 }) + }, +) + +test.each(["AS v(id)", "v(id)"])( + "SQLite predicate_effect passes derived-table column aliases %s through to the engine", + async (alias) => { + const connector = await Registry.get("sqlite_predicates") + const execute = spyOn(connector, "execute") + try { + // SQLite itself does not support column alias lists; the validator must not misread them as calls. + await expect( + checkSql({ + warehouse: "sqlite_predicates", + check: "predicate_effect", + base_sql: `SELECT id FROM (VALUES (1), (2)) ${alias}`, + predicates: ["id > 1"], + }), + ).rejects.toThrow(/syntax error/i) + expect(execute).toHaveBeenCalledTimes(1) + } finally { + execute.mockRestore() + } + }, +) + +test.each(["AS v(id)", "v(id)"])("DuckDB predicate_effect accepts table-function column aliases %s", async (alias) => { + const result = await checkSql({ + warehouse: "scratch", + check: "predicate_effect", + base_sql: `SELECT id FROM range(1, 3) ${alias}`, + predicates: ["id > 1"], + }) + expect(result.metadata).toMatchObject({ total_rows: 2, combined_kept_rows: 1 }) +}) + +test.each([...warehouses])("predicate_effect still refuses real calls sharing derived alias v on %s", async (warehouse) => { + const connector = await Registry.get(warehouse) + const execute = spyOn(connector, "execute") + try { + for (const alias of ["AS v(id)", "v(id)"]) { + await expect( + checkSql({ + warehouse, + check: "predicate_effect", + base_sql: `SELECT v(1) FROM (VALUES (1), (2)) ${alias}`, + predicates: ["id > 1"], + }), + ).rejects.toThrow(/refuses function v/) + await expect( + checkSql({ + warehouse, + check: "predicate_effect", + base_sql: `SELECT id FROM (VALUES (1), (2)) ${alias}`, + predicates: ["v(1) > 0"], + }), + ).rejects.toThrow(/refuses function v/) + } + expect(execute).not.toHaveBeenCalled() + } finally { + execute.mockRestore() + } +}) + test("SQLite predicate_effect enables query_only and restores the previous setting after success and failure", async () => { const connector = await Registry.get("sqlite_predicates") const guarded = await connector.execute("SELECT query_only FROM pragma_query_only()", undefined, undefined, { diff --git a/packages/opencode/test/config/config.test.ts b/packages/opencode/test/config/config.test.ts index cd2a3a9d6..f3130b6c9 100644 --- a/packages/opencode/test/config/config.test.ts +++ b/packages/opencode/test/config/config.test.ts @@ -308,6 +308,34 @@ it.instance("loads config with defaults when no files exist", () => }), ) +// altimate_change start — predicate-effect execution requires an explicit config opt-in +describe("sql_check.predicate_effect config", () => { + it.instance("leaves predicate_effect disabled when sql_check config is unset", () => + Effect.gen(function* () { + const config = yield* Config.use.get() + expect(config.sql_check?.predicate_effect ?? false).toBe(false) + }), + ) + + for (const enabled of [true, false]) { + it.instance( + `loads sql_check.predicate_effect: ${enabled} from project config`, + Effect.gen(function* () { + const config = yield* Config.use.get() + expect(config.sql_check?.predicate_effect).toBe(enabled) + }), + { config: { sql_check: { predicate_effect: enabled } } }, + ) + } + + test("rejects non-boolean sql_check.predicate_effect config", () => { + expect(() => + ConfigParse.schema(ConfigV1.Info, { sql_check: { predicate_effect: "true" } }, "test:config"), + ).toThrow() + }) +}) +// altimate_change end + it.instance("falls back to generic username when system user info is unavailable", () => Effect.gen(function* () { const userInfo = spyOn(os, "userInfo").mockImplementation(() => { From 9055a9102dfa0e950df6f1ed51d5c0d5ca0d71b3 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Wed, 14 Oct 2026 12:02:19 -0700 Subject: [PATCH 21/22] fix: authorize every query `predicate_effect` runs and check keyword-shaped calls - The permission request now includes the probe's AND-combined predicate expression and the fixed catalog lookups (`pg_operator`, `pg_proc`, `pg_type`, `duckdb_functions()`), so rules scoped to a joint condition or to catalog reads apply before anything executes. - Only reserved keywords that cannot name an unquoted function are skipped by call detection. `FILTER` and `OVER` count as clauses only right after a call's closing parenthesis. Unreserved names such as `range`, `rows`, `partition` and `within` are validated as calls, and `range` joins the pure allowlist so the catalog check still refuses a user-defined overload. Co-Authored-By: Claude Opus 5.5 --- .../opencode/src/altimate/tools/sql-check.ts | 81 ++++++++++++++----- .../test/altimate/sql-check-postgres.test.ts | 2 + .../sql-check-predicate-effect.test.ts | 49 ++++++++++- 3 files changed, 108 insertions(+), 24 deletions(-) diff --git a/packages/opencode/src/altimate/tools/sql-check.ts b/packages/opencode/src/altimate/tools/sql-check.ts index 4a21a67b8..b9deee061 100644 --- a/packages/opencode/src/altimate/tools/sql-check.ts +++ b/packages/opencode/src/altimate/tools/sql-check.ts @@ -418,6 +418,7 @@ const PURE_PREDICATE_FUNCTIONS = new Set([ "quote_literal", "quote_nullable", "radians", + "range", "rank", "regexp_count", "regexp_instr", @@ -920,16 +921,16 @@ function pureFunctions(sql: string, dialect?: string): string[] { const parts = call.match(new RegExp(identifier, "g"))! const quoted = /^["`\[]/.test(parts[0]) const name = quoted ? parts[0].slice(1, -1) : parts[0].replace(/[A-Z]/g, (char) => char.toLowerCase()) - // These unquoted SQL keywords introduce grouped expressions or clauses (`FILTER (WHERE …)`, - // `OVER (PARTITION BY …)`, `WITHIN GROUP (…)`, `USING (…)`), not function calls. - if ( - parts.length === 1 && - !quoted && - /^(all|and|any|as|between|distinct|else|exists|filter|from|groups|having|in|lateral|not|on|or|ordinality|over|partition|range|rows|select|some|then|using|values|when|where|within)$/.test( + // Reserved SQL keywords that introduce grouped expressions (`USING (…)`, `VALUES (…)`, + // `IN (…)`) cannot name an unquoted function, so they are not calls. + // `FILTER` and `OVER` are unreserved and DuckDB has a `filter()` function: they are clauses + // only right after a call's closing parenthesis. Every other name is checked as a call. + const clause = + /^(all|and|any|as|between|distinct|else|exists|from|having|in|lateral|not|on|or|select|some|then|using|values|when|where)$/.test( name, - ) - ) - continue + ) || + (/^(filter|over)$/.test(name) && /\)\s*$/.test(functions.slice(0, match.index))) + if (parts.length === 1 && !quoted && clause) continue if (parts.length !== 1 || !PURE_PREDICATE_FUNCTIONS.has(name)) { throw new Error(`sql_check refuses function ${call}: predicates are limited to built-in pure functions.`) } @@ -954,6 +955,28 @@ function readPredicate(predicate: string, dialect?: string): string { } /** Reject server-defined overloads even when a call's spelling is on the pure allowlist. */ +// Fixed, parameterized catalog lookups run during predicate validation. They are +// authorized together with the caller's SQL, so a rule denying catalog reads applies. +const POSTGRES_OPERATOR_CATALOG = `SELECT o.oprname, n.nspname + FROM pg_catalog.pg_operator o JOIN pg_catalog.pg_namespace n ON n.oid = o.oprnamespace + WHERE o.oprname = ANY($1) AND n.nspname <> 'pg_catalog'` +const POSTGRES_FUNCTION_CATALOG = `SELECT p.proname, n.nspname + FROM pg_catalog.pg_proc p JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE p.proname = ANY($1) AND p.pronamespace <> 'pg_catalog'::regnamespace` +const POSTGRES_TYPE_CATALOG = `SELECT t.typname, n.nspname + FROM pg_catalog.pg_type t JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace + WHERE t.typname = ANY($1) AND n.nspname <> 'pg_catalog'` +const duckdbFunctionCatalog = (count: number) => `SELECT function_name, schema_name FROM duckdb_functions() + WHERE internal = false AND function_name COLLATE NOCASE IN (${Array(count).fill("?").join(", ")})` + +/** SQL that predicate validation executes against the catalog for this dialect. */ +function predicateCatalogQueries(population: string, predicates: string[], dialect?: string): string[] { + if (dialect === "postgres") return [POSTGRES_OPERATOR_CATALOG, POSTGRES_FUNCTION_CATALOG, POSTGRES_TYPE_CATALOG] + if (dialect !== "duckdb") return [] + const functions = new Set([population, ...predicates].flatMap((sql) => pureFunctions(sql, dialect))) + return functions.size ? [duckdbFunctionCatalog(functions.size)] : [] +} + async function checkPredicateCatalog( population: string, predicates: string[], @@ -972,9 +995,7 @@ async function checkPredicateCatalog( const operators = [...new Set(operatorNames.flat())] if (operators.length) { const [operator] = await run( - `SELECT o.oprname, n.nspname - FROM pg_catalog.pg_operator o JOIN pg_catalog.pg_namespace n ON n.oid = o.oprnamespace - WHERE o.oprname = ANY($1) AND n.nspname <> 'pg_catalog'`, + POSTGRES_OPERATOR_CATALOG, [operators], ) if (operator) { @@ -985,9 +1006,7 @@ async function checkPredicateCatalog( } if (functions.length) { const [fn] = await run( - `SELECT p.proname, n.nspname - FROM pg_catalog.pg_proc p JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace - WHERE p.proname = ANY($1) AND p.pronamespace <> 'pg_catalog'::regnamespace`, + POSTGRES_FUNCTION_CATALOG, [functions], ) if (fn) { @@ -1001,9 +1020,7 @@ async function checkPredicateCatalog( const types = [...new Set(typeNames.flat())] if (types.length) { const [type] = await run( - `SELECT t.typname, n.nspname - FROM pg_catalog.pg_type t JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace - WHERE t.typname = ANY($1) AND n.nspname <> 'pg_catalog'`, + POSTGRES_TYPE_CATALOG, [types], ) if (type) { @@ -1016,8 +1033,7 @@ async function checkPredicateCatalog( } if (functions.length) { const [fn] = await run( - `SELECT function_name, schema_name FROM duckdb_functions() - WHERE internal = false AND function_name COLLATE NOCASE IN (${functions.map(() => "?").join(", ")})`, + duckdbFunctionCatalog(functions.length), functions, ) if (fn) { @@ -1398,15 +1414,36 @@ export const SqlCheckTool = Tool.define("sql_check", { // Permission rules are matched per pattern, so a rule scoped to a column-and-table // combination must also see the probe that joins them: authorize the combined // population + predicates in the shape the probe executes, not only the parts. + // The probe also counts the AND of all predicates, so a rule scoped to a joint + // condition must see that expression too, in both the supplied and the executed form. + const probeShape = (predicates: string[], population: string) => + `SELECT ${predicates.map((predicate) => `COUNT(*) FILTER (WHERE ${predicate})`).join(", ")}, ` + + `COUNT(*) FILTER (WHERE ${predicates.join(" AND ")}) FROM (${population}) AS t` const combined = input.base_sql !== undefined && input.predicates?.length ? [ - `SELECT ${input.predicates.map((predicate) => `COUNT(*) FILTER (WHERE ${predicate})`).join(", ")} FROM (${readBase(input.base_sql, dialect).sql}) AS t`, + ...new Set([ + probeShape(input.predicates, readBase(input.base_sql, dialect).sql), + probeShape( + input.predicates.map((predicate) => readPredicate(predicate, dialect)), + readBase(input.base_sql, dialect).sql, + ), + ]), ] : [] await ctx.ask({ permission: "sql_execute", - patterns: [...queries, ...combined], + patterns: [ + ...queries, + ...combined, + ...(input.base_sql !== undefined && input.predicates?.length + ? predicateCatalogQueries( + readBase(input.base_sql, dialect).sql, + input.predicates.map((predicate) => readPredicate(predicate, dialect)), + dialect, + ) + : []), + ], always: ["*"], metadata: { warehouse: input.warehouse }, }) diff --git a/packages/opencode/test/altimate/sql-check-postgres.test.ts b/packages/opencode/test/altimate/sql-check-postgres.test.ts index b26e695ce..151d7bb19 100644 --- a/packages/opencode/test/altimate/sql-check-postgres.test.ts +++ b/packages/opencode/test/altimate/sql-check-postgres.test.ts @@ -86,6 +86,8 @@ describe("predicate_effect with mocked Postgres", () => { test.each([ { catalog: "pg_operator" as const, name: ">", predicate: "id > 0", message: "uses operator `>`" }, { catalog: "pg_proc" as const, name: "length", predicate: "length(kind) > 0", message: "calls `length`" }, + // `range` is a keyword-shaped name: it must still reach the catalog check. + { catalog: "pg_proc" as const, name: "range", predicate: "range(id) > 0", message: "calls `range`" }, ])( "seeded $catalog rows refuse matching custom definitions before the probe", async ({ catalog, name, predicate, message }) => { diff --git a/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts b/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts index e736fb8bc..9983d6d4c 100644 --- a/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts +++ b/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts @@ -849,8 +849,9 @@ async function checkWithRules( input: { base_sql: string; predicates: string[] }, ruleset: PermissionNext.Rule[], sessionName: string, + warehouse = "sqlite_predicates", ) { - const connector = await Registry.get("sqlite_predicates") + const connector = await Registry.get(warehouse) const execute = spyOn(connector, "execute") const precedence = spyOn(Precedence, "check").mockResolvedValue({}) const asked: PermissionNext.Request[] = [] @@ -865,7 +866,7 @@ async function checkWithRules( const tool = await initTool(SqlCheckTool) const sessionID = SessionID.make(sessionName) const result = await tool.execute( - { warehouse: "sqlite_predicates", check: "predicate_effect", ...input }, + { warehouse, check: "predicate_effect", ...input }, { sessionID, messageID: "test-message", @@ -934,6 +935,50 @@ it.instance("predicate_effect authorizes the combined probe so a table-and-colum }), ) +it.instance("predicate_effect authorizes the AND-combined predicate so a joint-condition deny rule applies", () => + inInstance(async () => { + const ruleset: PermissionNext.Rule[] = [ + { permission: "sql_execute", pattern: "*", action: "ask" }, + { permission: "sql_execute", pattern: "*kind = 'a' AND amount > 4*FROM*orders*", action: "deny" }, + ] + // Each predicate alone misses the rule; only the probe's joint AND expression matches it. + const denied = await checkWithRules( + { base_sql: baseSql, predicates: ["kind = 'a'", "amount > 4"] }, + ruleset, + "session_joint_deny", + ) + expect(denied.result.metadata.error).toContain("rule which prevents") + expect(denied.executed).toBe(0) + expect(denied.precedenceChecked).toBe(0) + }), +) + +it.instance("predicate_effect authorizes its catalog lookups so a catalog deny rule applies", () => + inInstance(async () => { + const denied = await checkWithRules( + { base_sql: "SELECT id FROM range(1, 3) AS v(id)", predicates: ["abs(id) > 1"] }, + [ + { permission: "sql_execute", pattern: "*", action: "ask" }, + { permission: "sql_execute", pattern: "*duckdb_functions()*", action: "deny" }, + ], + "session_catalog_deny", + "scratch", + ) + expect(denied.result.metadata.error).toContain("rule which prevents") + expect(denied.executed).toBe(0) + expect(denied.precedenceChecked).toBe(0) + }), +) + +test.each(["rows(1)", "groups(1)", "partition(1)", "ordinality(1)", "within(1)", "filter(1)", "over(1)"])( + "predicate_effect checks keyword-shaped call %s as a function", + async (call) => { + await expect( + checkSql({ warehouse: "sqlite_predicates", check: "predicate_effect", base_sql: baseSql, predicates: [`${call} = 1`] }), + ).rejects.toThrow(/refuses function/) + }, +) + test("predicate_effect accepts FILTER and OVER clauses in populations and still refuses unknown functions", async () => { const accepted = await checkSql({ warehouse: "sqlite_predicates", From 6b16bdb2c9816f53b82fd8a70dd4f7008881f07d Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Wed, 14 Oct 2026 12:21:23 -0700 Subject: [PATCH 22/22] fix: authorize the row-estimate lookup and only the catalog queries `predicate_effect` runs The permission request now mirrors execution: the PostgreSQL `pg_class` row estimate for a bare table is included, and each catalog lookup (operators, functions, casts, `duckdb_functions()`) is requested only when the SQL contains what that lookup checks. The estimate statement is shared between the request and the probe. Co-Authored-By: Claude Opus 5.5 --- .../opencode/src/altimate/tools/sql-check.ts | 31 +++++++++++++------ .../test/altimate/sql-check-postgres.test.ts | 25 ++++++++++++++- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/packages/opencode/src/altimate/tools/sql-check.ts b/packages/opencode/src/altimate/tools/sql-check.ts index b9deee061..86e1f022e 100644 --- a/packages/opencode/src/altimate/tools/sql-check.ts +++ b/packages/opencode/src/altimate/tools/sql-check.ts @@ -966,15 +966,28 @@ const POSTGRES_FUNCTION_CATALOG = `SELECT p.proname, n.nspname const POSTGRES_TYPE_CATALOG = `SELECT t.typname, n.nspname FROM pg_catalog.pg_type t JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace WHERE t.typname = ANY($1) AND n.nspname <> 'pg_catalog'` +const postgresEstimateCatalog = (table: string) => `SELECT reltuples AS estimate FROM pg_catalog.pg_class + WHERE oid = pg_catalog.to_regclass(E${literal(table.replace(/\\/g, "\\\\"))}) + AND relkind IN ('r', 'm') AND reltuples > 0` const duckdbFunctionCatalog = (count: number) => `SELECT function_name, schema_name FROM duckdb_functions() WHERE internal = false AND function_name COLLATE NOCASE IN (${Array(count).fill("?").join(", ")})` -/** SQL that predicate validation executes against the catalog for this dialect. */ -function predicateCatalogQueries(population: string, predicates: string[], dialect?: string): string[] { - if (dialect === "postgres") return [POSTGRES_OPERATOR_CATALOG, POSTGRES_FUNCTION_CATALOG, POSTGRES_TYPE_CATALOG] - if (dialect !== "duckdb") return [] - const functions = new Set([population, ...predicates].flatMap((sql) => pureFunctions(sql, dialect))) - return functions.size ? [duckdbFunctionCatalog(functions.size)] : [] +/** @internal Catalog SQL that predicate validation and the row estimate will execute, mirroring their conditions. */ +export function predicateCatalogQueries( + population: { sql: string; table?: string }, + predicates: string[], + dialect?: string, +): string[] { + const queries = [population.sql, ...predicates] + const functions = new Set(queries.flatMap((sql) => pureFunctions(sql, dialect))) + if (dialect === "duckdb") return functions.size ? [duckdbFunctionCatalog(functions.size)] : [] + if (dialect !== "postgres") return [] + return [ + ...(queries.some((sql) => postgresOperators(sql).length) ? [POSTGRES_OPERATOR_CATALOG] : []), + ...(functions.size ? [POSTGRES_FUNCTION_CATALOG] : []), + ...(queries.some((sql) => postgresCasts(sql).length) ? [POSTGRES_TYPE_CATALOG] : []), + ...(population.table !== undefined ? [postgresEstimateCatalog(population.table)] : []), + ] } async function checkPredicateCatalog( @@ -1120,9 +1133,7 @@ export async function checkSql(input: SqlCheckPublicInput) { ${ localTable ? `SELECT COUNT(*) AS estimate FROM ${scope}source` - : `SELECT reltuples AS estimate FROM pg_catalog.pg_class - WHERE oid = pg_catalog.to_regclass(E${literal(population.table.replace(/\\/g, "\\\\"))}) - AND relkind IN ('r', 'm') AND reltuples > 0` + : postgresEstimateCatalog(population.table) } ), ${scope}probe AS ( ${probe} @@ -1438,7 +1449,7 @@ export const SqlCheckTool = Tool.define("sql_check", { ...combined, ...(input.base_sql !== undefined && input.predicates?.length ? predicateCatalogQueries( - readBase(input.base_sql, dialect).sql, + readBase(input.base_sql, dialect), input.predicates.map((predicate) => readPredicate(predicate, dialect)), dialect, ) diff --git a/packages/opencode/test/altimate/sql-check-postgres.test.ts b/packages/opencode/test/altimate/sql-check-postgres.test.ts index 151d7bb19..44b7f1b9e 100644 --- a/packages/opencode/test/altimate/sql-check-postgres.test.ts +++ b/packages/opencode/test/altimate/sql-check-postgres.test.ts @@ -3,7 +3,7 @@ import { Database } from "bun:sqlite" import { rejects } from "node:assert/strict" import type { Connector } from "@altimateai/drivers" import * as Registry from "../../src/altimate/native/connections/registry" -import { checkSql } from "../../src/altimate/tools/sql-check" +import { checkSql, predicateCatalogQueries } from "../../src/altimate/tools/sql-check" const warehouse = "postgres_predicates" const unknownEstimate = "row_estimate: unknown — probe runs under the query timeout" @@ -561,3 +561,26 @@ describe("predicate_effect with mocked Postgres", () => { } }) }) + +describe("predicate_effect authorizes exactly the catalog SQL it runs", () => { + const kinds = (queries: string[]) => + queries.map((sql) => /pg_catalog\.(pg_operator|pg_proc|pg_type|pg_class)\b/.exec(sql)?.[1]) + + test("a SELECT population with a constant predicate needs no catalog lookup", () => { + expect(predicateCatalogQueries({ sql: "SELECT 1 AS ok" }, ["TRUE"], "postgres")).toEqual([]) + }) + + test("a bare table adds its row-estimate lookup", () => { + const queries = predicateCatalogQueries({ sql: "SELECT * FROM orders", table: "orders" }, ["TRUE"], "postgres") + expect(kinds(queries)).toEqual(["pg_class"]) + expect(queries[0]).toContain("to_regclass(E'orders')") + }) + + test("operators, functions and casts each add their own lookup", () => { + expect(kinds(predicateCatalogQueries({ sql: "SELECT * FROM t" }, ["id > 0"], "postgres"))).toEqual(["pg_operator"]) + expect(kinds(predicateCatalogQueries({ sql: "SELECT * FROM t" }, ["length(kind)::int"], "postgres"))).toEqual([ + "pg_proc", + "pg_type", + ]) + }) +})