diff --git a/packages/core/src/v1/config/config.ts b/packages/core/src/v1/config/config.ts index b6fe79e97a..c9a39e21d2 100644 --- a/packages/core/src/v1/config/config.ts +++ b/packages/core/src/v1/config/config.ts @@ -169,6 +169,16 @@ export const Info = Schema.Struct({ description: "Thresholds for truncating tool output. When output exceeds either limit, the full text is written to the truncation directory and a preview is returned.", }), + // altimate_change start — require an explicit opt-in for the predicate-effect SQL probe + sql_check: Schema.optional( + Schema.Struct({ + predicate_effect: Schema.optional(Schema.Boolean).annotate({ + description: + "Enable predicate_effect for trusted databases (default: false). Executes caller-supplied SQL whose server-side definitions cannot be fully validated.", + }), + }), + ).annotate({ description: "SQL check tool options" }), + // altimate_change end compaction: Schema.optional( Schema.Struct({ auto: Schema.optional(Schema.Boolean).annotate({ diff --git a/packages/drivers/src/duckdb.ts b/packages/drivers/src/duckdb.ts index 48ddc00956..a3367904c1 100644 --- a/packages/drivers/src/duckdb.ts +++ b/packages/drivers/src/duckdb.ts @@ -166,11 +166,12 @@ export async function connect( params: any[] | undefined, timeoutMs: number | undefined, maxResultBytes?: number, + readOnly?: boolean, onDispatch?: () => void, ): Promise { if (!shared) throw new Error("DuckDB connection not open") try { - return await shared.engine.call("all", { sql, params, connectionId, maxResultBytes }, timeoutMs, onDispatch) + return await shared.engine.call("all", { sql, params, connectionId, maxResultBytes, readOnly }, timeoutMs, onDispatch) } catch (err) { throw err instanceof Error ? wrapDuckDBError(err) : err } @@ -321,6 +322,7 @@ export async function connect( binds?.length ? binds : undefined, resolveQueryTimeout(config, options).ms, options?.maxResultBytes, + options?.readOnly, options?.onDispatch, ) // altimate_change end diff --git a/packages/drivers/src/local-engine-child-source.ts b/packages/drivers/src/local-engine-child-source.ts index 8fcdf69559..8c4f1729c0 100644 --- a/packages/drivers/src/local-engine-child-source.ts +++ b/packages/drivers/src/local-engine-child-source.ts @@ -222,10 +222,41 @@ async function close() { return null; } +// Guard setup, result consumption and cleanup share one serialized request. +// Separate requests would let another caller run between setup and cleanup. +async function withReadOnly(p, execute) { + if (engine === "sqlite") { + const previous = sqliteDb.query("PRAGMA query_only").get().query_only; + sqliteDb.exec("PRAGMA query_only = ON"); + try { + return await execute(); + } finally { + sqliteDb.exec("PRAGMA query_only = " + (previous ? "ON" : "OFF")); + } + } + const control = (sql) => all({ sql, connectionId: p.connectionId }); + // The legacy binding has no autocommit getter. Implicit transactions receive + // different IDs; an explicit transaction keeps the same ID across statements. + // Refuse before BEGIN: a nested BEGIN would abort the caller's transaction. + const first = await control("SELECT txid_current() AS id"); + const second = await control("SELECT txid_current() AS id"); + if (first[0].id === second[0].id) { + throw new Error("DuckDB read-only execution cannot run inside an active transaction; finish it before running the probe."); + } + await control("BEGIN TRANSACTION READ ONLY"); + try { + return await execute(); + } finally { + await control("ROLLBACK"); + } +} + async function handle(req) { if (req.op === "open") return open(req.payload); - if (req.op === "all") return all(req.payload); - if (req.op === "run") return run(req.payload); + if (req.op === "all" || req.op === "run") { + const execute = () => req.op === "all" ? all(req.payload) : run(req.payload); + return req.payload.readOnly ? withReadOnly(req.payload, execute) : execute(); + } if (req.op === "disconnect") return disconnect(req.payload); if (req.op === "close") return close(); throw new Error("unknown op " + req.op); diff --git a/packages/drivers/src/postgres.ts b/packages/drivers/src/postgres.ts index 69eda18517..27544f1eb0 100644 --- a/packages/drivers/src/postgres.ts +++ b/packages/drivers/src/postgres.ts @@ -95,7 +95,7 @@ export async function connect(config: ConnectionConfig): Promise { pool = new Pool(poolConfig) }, - async execute(sql: string, limit?: number, _binds?: any[], options?: ExecuteOptions): Promise { + async execute(sql: string, limit?: number, binds?: any[], options?: ExecuteOptions): Promise { const client = await acquireClient(pool, options?.signal) // Tighter of the explicit statement_timeout and the common execution // deadline: a configured limit stricter than the default is preserved. @@ -184,7 +184,7 @@ export async function connect(config: ConnectionConfig): Promise { const consume = createResultByteBudget(options.maxResultBytes) const Query = pg.default?.Query ?? pg.Query return await new Promise((resolve, reject) => { - const stream = new Query({ text: query, rowMode: "array" }) + const stream = new Query({ text: query, values: binds, rowMode: "array" }) const rows: unknown[][] = [] let columns: string[] = [] let truncated = false @@ -221,7 +221,9 @@ export async function connect(config: ConnectionConfig): Promise { }) } // altimate_change end - const result = await client.query(query) + // altimate_change start — preserve bound catalog names and other query parameters + const result = await client.query(query, binds) + // altimate_change end const columns = result.fields?.map((f: any) => f.name) ?? [] const truncated = effectiveLimit > 0 && result.rows.length > effectiveLimit const rows = truncated diff --git a/packages/drivers/src/sqlite.ts b/packages/drivers/src/sqlite.ts index 85c6224ded..6ee5a6491d 100644 --- a/packages/drivers/src/sqlite.ts +++ b/packages/drivers/src/sqlite.ts @@ -77,7 +77,7 @@ export async function connect(config: ConnectionConfig): Promise { // altimate_change start — report dispatch so callers can order follow-up work const info = await engine.call<{ changes: number; lastInsertRowid: number | bigint }>( "run", - { sql }, + { sql, readOnly: options?.readOnly }, timeoutMs, options?.onDispatch, ) @@ -91,7 +91,12 @@ export async function connect(config: ConnectionConfig): Promise { } // altimate_change start — forward the budget before the helper collects rows - const rows = await engine.call("all", { sql: query, maxResultBytes: options?.maxResultBytes }, timeoutMs, options?.onDispatch) + const rows = await engine.call( + "all", + { sql: query, maxResultBytes: options?.maxResultBytes, readOnly: options?.readOnly }, + timeoutMs, + options?.onDispatch, + ) // altimate_change end const columns = rows.length > 0 ? Object.keys(rows[0]) : [] const truncated = effectiveLimit > 0 && rows.length > effectiveLimit diff --git a/packages/drivers/src/types.ts b/packages/drivers/src/types.ts index 281e838cf8..7aa18db86f 100644 --- a/packages/drivers/src/types.ts +++ b/packages/drivers/src/types.ts @@ -26,7 +26,7 @@ export interface ExecuteOptions { maxResultBytes?: number // altimate_change end // altimate_change start — PostgreSQL source extraction uses a pinned, rolled-back transaction - /** PostgreSQL only: execute in a READ ONLY transaction and roll it back before releasing the client. */ + /** PostgreSQL/DuckDB: rolled-back READ ONLY transaction; SQLite: scoped query_only pragma. */ readOnly?: boolean // altimate_change end /** PostgreSQL only: abandon pool acquisition and prevent SQL from starting after cancellation. */ diff --git a/packages/drivers/test/postgres-readonly.test.ts b/packages/drivers/test/postgres-readonly.test.ts index dc8dfd971e..eb6f77ef36 100644 --- a/packages/drivers/test/postgres-readonly.test.ts +++ b/packages/drivers/test/postgres-readonly.test.ts @@ -1,4 +1,5 @@ import { expect, mock, test } from "bun:test" +import { EventEmitter } from "node:events" import { connect } from "../src/postgres" import { QueryTimeoutError } from "../src/types" @@ -183,6 +184,64 @@ test("Postgres never starts user SQL when cancellation arrives during session se } }) +for (const maxResultBytes of [undefined, 1024]) { + test(`Postgres forwards bound catalog arrays in ${maxResultBytes === undefined ? "regular" : "byte-limited"} read-only queries`, async () => { + const calls: { sql: string; values?: unknown[] }[] = [] + const releases: unknown[] = [] + class Query extends EventEmitter { + constructor(readonly config: { text: string; values?: unknown[]; rowMode: string }) { + super() + } + } + mock.module("pg", () => ({ + default: { + Query, + Pool: class { + async connect() { + return { + query(query: string | Query, values?: unknown[]) { + const sql = typeof query === "string" ? query : query.config.text + calls.push({ sql, values: typeof query === "string" ? values : query.config.values }) + const fields = [{ name: "proname" }] + if (query instanceof Query) { + queueMicrotask(() => { + query.emit("row", ["lower"], { fields }) + query.emit("end", { fields }) + }) + return query + } + return Promise.resolve({ fields, rows: [{ proname: "lower" }] }) + }, + release(discard?: boolean) { releases.push(discard) }, + } + } + async end() {} + }, + }, + })) + const connector = await connect({ type: "postgres", query_timeout_ms: 0 }) + await connector.connect() + const sql = "SELECT proname FROM pg_catalog.pg_proc WHERE proname = ANY($1)" + const values = [["lower", "upper"]] + try { + const result = await connector.execute(sql, undefined, values, { + noLimit: true, + readOnly: true, + maxResultBytes, + }) + expect(result.rows).toEqual([["lower"]]) + expect(calls).toEqual([ + { sql: "BEGIN READ ONLY", values: undefined }, + { sql, values }, + { sql: "ROLLBACK", values: undefined }, + ]) + expect(releases).toEqual([undefined]) + } finally { + await connector.close() + } + }) +} + async function probeDatabase(config: Record = { query_timeout_ms: 1000 }) { const queries: string[] = [] const releases: unknown[] = [] diff --git a/packages/opencode/src/altimate/prompts/analyst.txt b/packages/opencode/src/altimate/prompts/analyst.txt index 71c9006ddd..ab915c58bd 100644 --- a/packages/opencode/src/altimate/prompts/analyst.txt +++ b/packages/opencode/src/altimate/prompts/analyst.txt @@ -1,6 +1,6 @@ You are altimate-code: a read-only analyst. Do not modify source data or files. When available, `sql_materialize` may write session-local `scratch` tables. -Inspect connections, schemas and actual rows before joining or parsing; key formats may differ across databases. Establish population, grain, denominator and assumptions. Every clause of the question must bite: a filter that removes nothing usually means the wrong column. When a phrase admits two readings (which population, denominator, transform or owner column), compute both, then choose by the question's own definitions and the data's shape; keep that reasoning out of the final message. Use `sql_check` for keys, fan-out, grain, nulls or extraction coverage when they affect the answer; trivial queries need no ritual. `sql_analyze` is optional. +Inspect connections, schemas and actual rows before joining or parsing; key formats may differ across databases. Establish population, grain, denominator and assumptions. Every clause of the question must bite: a filter that removes nothing usually means the wrong column. (`sql_check` `predicate_effect` reports what each filter removes; a filter that removes nothing at the right population is a reason to profile an alternative column, not to invent an exclusion). When a phrase admits two readings (which population, denominator, transform or owner column), compute both, then choose by the question's own definitions and the data's shape; keep that reasoning out of the final message. Use `sql_check` for keys, fan-out, grain, nulls or extraction coverage when they affect the answer; trivial queries need no ritual. `sql_analyze` is optional. Derive large intermediate sets in-engine. For a cross-engine join, move the smaller side: a few hundred keys go into an IN list or VALUES against the big table; materialize into `scratch` only the side that is small after filtering, never a multi-million-row table when a filter or aggregate could shrink it first. Preserve requested weighting: group means weighted by contributing counts can be exact; unweighted averages change the weighting. For many similar tables, prefer catalog-driven SQL (information_schema, DuckDB query_table/UNION ALL BY NAME) after confirming dialect support. diff --git a/packages/opencode/src/altimate/tools/sql-check.ts b/packages/opencode/src/altimate/tools/sql-check.ts index 78fb47ed7d..4a21a67b8f 100644 --- a/packages/opencode/src/altimate/tools/sql-check.ts +++ b/packages/opencode/src/altimate/tools/sql-check.ts @@ -1,10 +1,468 @@ import z from "zod" +import { isQueryTimeoutError } from "@altimateai/drivers/types" import { Tool } from "../../tool/tool" +import { Config } from "../../config/config" import * as Precedence from "../workspace/precedence" import * as Registry from "../native/connections/registry" import { classifyAndCheck, isSingleQuery } from "./sql-classify" import { maskSql, tokenizeSql } from "./sql-tokenize" +// Function calls that can change state outside the read-only transaction. PostgreSQL administrative, +// WAL, replication, statistics-reset, large-object, file and dblink functions are denied as whole classes: +// every `pg_*` call is refused unless it is a known pure read (type/size/catalog inspection, stat getters, +// LSN and snapshot reads), so a new server-control function cannot slip through one name at a time. +const PURE_PG_FUNCTIONS = + "typeof|size_pretty|column_size|\\w*_size|backend_pid|is_in_recovery|get_\\w*|describe_object|identify_object\\w*|" + + "input_is_valid|trigger_depth|has_role|\\w*_is_visible|current_\\w*_lsn|last_\\w*_lsn|wal_lsn_diff|walfile_name\\w*|" + + "split_walfile_name|postmaster_start_time|conf_load_time|my_temp_schema|is_other_temp_schema|jit_available|" + + "collation_for|tablespace_\\w*|relation_filepath|relation_filenode|filenode_relation|char_to_encoding|" + + "encoding_to_char|client_encoding|column_compression|xact_commit_timestamp\\w*|current_xact_id\\w*|current_snapshot|" + + "snapshot_\\w*|visible_in_snapshot|notification_queue_usage|listening_channels|blocking_pids|" + + "safe_snapshot_blocking_pids|isolation_test_session_is_blocked|stat_(?!reset)\\w*|statio_\\w*|mcv_list_\\w*|" + + "index\\w*_has_property|options_to_table|settings_get_flags|size_bytes|lsn\\w*|partition_\\w*|" + + "table_is_visible|has_\\w*_privilege|database_collation_actual_version|" + + "collation_actual_version|available_extension\\w*|control_\\w*|cursor\\w*|event_trigger_\\w*" +const SIDE_EFFECT_FUNCTION_CALL = new RegExp( + "\\b(nextval|setval|set_config|load_extension|writefile|dblink\\w*|lo_\\w*|lowrite|loread|checkpoint|force_checkpoint|query|" + + `pg_(?!(?:${PURE_PG_FUNCTIONS})\\s*\\()\\w*)\\s*\\(`, + "i", +) + +// Predicate probes and their populations accept only these pure functions; keep this list flat and sorted. +const PURE_PREDICATE_FUNCTIONS = new Set([ + "abs", + "acos", + "age", + "array", + "array_agg", + "array_append", + "array_cat", + "array_dims", + "array_length", + "array_lower", + "array_ndims", + "array_position", + "array_positions", + "array_prepend", + "array_remove", + "array_replace", + "array_to_string", + "array_upper", + "ascii", + "asin", + "atan", + "atan2", + "avg", + "bit_length", + "bool_and", + "bool_or", + "btrim", + "cardinality", + "cast", + "cbrt", + "ceil", + "ceiling", + "char", + "char_length", + "character", + "character_length", + "chr", + "coalesce", + "concat", + "concat_ws", + "cos", + "count", + "cume_dist", + "current_date", + "current_time", + "current_timestamp", + "date", + "date_part", + "date_trunc", + "datetime", + "decimal", + "degrees", + "dense_rank", + "div", + "exp", + "extract", + "first_value", + "floor", + "format", + "greatest", + "initcap", + "json_array_length", + "json_extract_path", + "json_extract_path_text", + "json_typeof", + "jsonb_array_length", + "jsonb_extract_path", + "jsonb_extract_path_text", + "jsonb_typeof", + "julianday", + "lag", + "last_value", + "lead", + "least", + "left", + "length", + "ln", + "log", + "lower", + "lpad", + "ltrim", + "make_date", + "make_interval", + "make_time", + "make_timestamp", + "max", + "md5", + "min", + "mod", + "now", + "nth_value", + "ntile", + "nullif", + "numeric", + "octet_length", + "overlay", + "percent_rank", + "pg_available_extension_versions", + "pg_available_extensions", + "pg_backend_pid", + "pg_blocking_pids", + "pg_char_to_encoding", + "pg_client_encoding", + "pg_collation_actual_version", + "pg_collation_for", + "pg_collation_is_visible", + "pg_column_compression", + "pg_column_size", + "pg_conf_load_time", + "pg_control_checkpoint", + "pg_control_init", + "pg_control_recovery", + "pg_control_system", + "pg_conversion_is_visible", + "pg_current_snapshot", + "pg_current_wal_flush_lsn", + "pg_current_wal_insert_lsn", + "pg_current_wal_lsn", + "pg_current_xact_id", + "pg_current_xact_id_if_assigned", + "pg_cursor", + "pg_database_collation_actual_version", + "pg_database_size", + "pg_describe_object", + "pg_encoding_to_char", + "pg_event_trigger_ddl_commands", + "pg_event_trigger_dropped_objects", + "pg_event_trigger_table_rewrite_oid", + "pg_event_trigger_table_rewrite_reason", + "pg_filenode_relation", + "pg_function_is_visible", + "pg_get_acl", + "pg_get_aios", + "pg_get_backend_memory_contexts", + "pg_get_catalog_foreign_keys", + "pg_get_constraintdef", + "pg_get_expr", + "pg_get_function_arg_default", + "pg_get_function_arguments", + "pg_get_function_identity_arguments", + "pg_get_function_result", + "pg_get_function_sqlbody", + "pg_get_functiondef", + "pg_get_indexdef", + "pg_get_keywords", + "pg_get_loaded_modules", + "pg_get_multixact_members", + "pg_get_object_address", + "pg_get_partition_constraintdef", + "pg_get_partkeydef", + "pg_get_publication_tables", + "pg_get_replica_identity_index", + "pg_get_replication_slots", + "pg_get_ruledef", + "pg_get_sequence_data", + "pg_get_serial_sequence", + "pg_get_shmem_allocations", + "pg_get_shmem_allocations_numa", + "pg_get_statisticsobjdef", + "pg_get_statisticsobjdef_columns", + "pg_get_statisticsobjdef_expressions", + "pg_get_triggerdef", + "pg_get_userbyid", + "pg_get_viewdef", + "pg_get_wait_events", + "pg_get_wal_replay_pause_state", + "pg_get_wal_resource_managers", + "pg_get_wal_summarizer_state", + "pg_has_role", + "pg_identify_object", + "pg_identify_object_as_address", + "pg_index_column_has_property", + "pg_index_has_property", + "pg_indexam_has_property", + "pg_indexes_size", + "pg_input_is_valid", + "pg_is_in_recovery", + "pg_is_other_temp_schema", + "pg_isolation_test_session_is_blocked", + "pg_jit_available", + "pg_last_wal_receive_lsn", + "pg_last_wal_replay_lsn", + "pg_listening_channels", + "pg_lsn", + "pg_lsn_cmp", + "pg_lsn_eq", + "pg_lsn_ge", + "pg_lsn_gt", + "pg_lsn_hash", + "pg_lsn_hash_extended", + "pg_lsn_in", + "pg_lsn_larger", + "pg_lsn_le", + "pg_lsn_lt", + "pg_lsn_mi", + "pg_lsn_mii", + "pg_lsn_ne", + "pg_lsn_out", + "pg_lsn_pli", + "pg_lsn_recv", + "pg_lsn_send", + "pg_lsn_smaller", + "pg_mcv_list_in", + "pg_mcv_list_items", + "pg_mcv_list_out", + "pg_mcv_list_recv", + "pg_mcv_list_send", + "pg_my_temp_schema", + "pg_notification_queue_usage", + "pg_opclass_is_visible", + "pg_operator_is_visible", + "pg_opfamily_is_visible", + "pg_options_to_table", + "pg_partition_ancestors", + "pg_partition_root", + "pg_partition_tree", + "pg_postmaster_start_time", + "pg_relation_filenode", + "pg_relation_filepath", + "pg_relation_size", + "pg_safe_snapshot_blocking_pids", + "pg_settings_get_flags", + "pg_size_bytes", + "pg_size_pretty", + "pg_snapshot_in", + "pg_snapshot_out", + "pg_snapshot_recv", + "pg_snapshot_send", + "pg_snapshot_xip", + "pg_snapshot_xmax", + "pg_snapshot_xmin", + "pg_split_walfile_name", + "pg_stat_file", + "pg_stat_get_activity", + "pg_stat_get_analyze_count", + "pg_stat_get_archiver", + "pg_stat_get_autoanalyze_count", + "pg_stat_get_autovacuum_count", + "pg_stat_get_backend_activity", + "pg_stat_get_backend_activity_start", + "pg_stat_get_backend_client_addr", + "pg_stat_get_backend_client_port", + "pg_stat_get_backend_dbid", + "pg_stat_get_backend_idset", + "pg_stat_get_backend_io", + "pg_stat_get_backend_pid", + "pg_stat_get_backend_start", + "pg_stat_get_backend_subxact", + "pg_stat_get_backend_userid", + "pg_stat_get_backend_wait_event", + "pg_stat_get_backend_wait_event_type", + "pg_stat_get_backend_wal", + "pg_stat_get_backend_xact_start", + "pg_stat_get_bgwriter_buf_written_checkpoints", + "pg_stat_get_bgwriter_buf_written_clean", + "pg_stat_get_bgwriter_maxwritten_clean", + "pg_stat_get_bgwriter_requested_checkpoints", + "pg_stat_get_bgwriter_stat_reset_time", + "pg_stat_get_bgwriter_timed_checkpoints", + "pg_stat_get_blocks_fetched", + "pg_stat_get_blocks_hit", + "pg_stat_get_buf_alloc", + "pg_stat_get_buf_fsync_backend", + "pg_stat_get_buf_written_backend", + "pg_stat_get_checkpoint_sync_time", + "pg_stat_get_checkpoint_write_time", + "pg_stat_get_checkpointer_buffers_written", + "pg_stat_get_checkpointer_num_performed", + "pg_stat_get_checkpointer_num_requested", + "pg_stat_get_checkpointer_num_timed", + "pg_stat_get_checkpointer_restartpoints_performed", + "pg_stat_get_checkpointer_restartpoints_requested", + "pg_stat_get_checkpointer_restartpoints_timed", + "pg_stat_get_checkpointer_slru_written", + "pg_stat_get_checkpointer_stat_reset_time", + "pg_stat_get_checkpointer_sync_time", + "pg_stat_get_checkpointer_write_time", + "pg_stat_get_db_active_time", + "pg_stat_get_db_blk_read_time", + "pg_stat_get_db_blk_write_time", + "pg_stat_get_db_blocks_fetched", + "pg_stat_get_db_blocks_hit", + "pg_stat_get_db_checksum_failures", + "pg_stat_get_db_checksum_last_failure", + "pg_stat_get_db_conflict_all", + "pg_stat_get_db_conflict_bufferpin", + "pg_stat_get_db_conflict_lock", + "pg_stat_get_db_conflict_logicalslot", + "pg_stat_get_db_conflict_snapshot", + "pg_stat_get_db_conflict_startup_deadlock", + "pg_stat_get_db_conflict_tablespace", + "pg_stat_get_db_deadlocks", + "pg_stat_get_db_idle_in_transaction_time", + "pg_stat_get_db_numbackends", + "pg_stat_get_db_parallel_workers_launched", + "pg_stat_get_db_parallel_workers_to_launch", + "pg_stat_get_db_session_time", + "pg_stat_get_db_sessions", + "pg_stat_get_db_sessions_abandoned", + "pg_stat_get_db_sessions_fatal", + "pg_stat_get_db_sessions_killed", + "pg_stat_get_db_stat_reset_time", + "pg_stat_get_db_temp_bytes", + "pg_stat_get_db_temp_files", + "pg_stat_get_db_tuples_deleted", + "pg_stat_get_db_tuples_fetched", + "pg_stat_get_db_tuples_inserted", + "pg_stat_get_db_tuples_returned", + "pg_stat_get_db_tuples_updated", + "pg_stat_get_db_xact_commit", + "pg_stat_get_db_xact_rollback", + "pg_stat_get_dead_tuples", + "pg_stat_get_function_calls", + "pg_stat_get_function_self_time", + "pg_stat_get_function_total_time", + "pg_stat_get_ins_since_vacuum", + "pg_stat_get_io", + "pg_stat_get_last_analyze_time", + "pg_stat_get_last_autoanalyze_time", + "pg_stat_get_last_autovacuum_time", + "pg_stat_get_last_vacuum_time", + "pg_stat_get_lastscan", + "pg_stat_get_live_tuples", + "pg_stat_get_mod_since_analyze", + "pg_stat_get_numscans", + "pg_stat_get_progress_info", + "pg_stat_get_recovery_prefetch", + "pg_stat_get_replication_slot", + "pg_stat_get_slru", + "pg_stat_get_snapshot_timestamp", + "pg_stat_get_subscription", + "pg_stat_get_subscription_stats", + "pg_stat_get_total_analyze_time", + "pg_stat_get_total_autoanalyze_time", + "pg_stat_get_total_autovacuum_time", + "pg_stat_get_total_vacuum_time", + "pg_stat_get_tuples_deleted", + "pg_stat_get_tuples_fetched", + "pg_stat_get_tuples_hot_updated", + "pg_stat_get_tuples_inserted", + "pg_stat_get_tuples_newpage_updated", + "pg_stat_get_tuples_returned", + "pg_stat_get_tuples_updated", + "pg_stat_get_vacuum_count", + "pg_stat_get_wal", + "pg_stat_get_wal_receiver", + "pg_stat_get_wal_senders", + "pg_stat_get_xact_blocks_fetched", + "pg_stat_get_xact_blocks_hit", + "pg_stat_get_xact_function_calls", + "pg_stat_get_xact_function_self_time", + "pg_stat_get_xact_function_total_time", + "pg_stat_get_xact_numscans", + "pg_stat_get_xact_tuples_deleted", + "pg_stat_get_xact_tuples_fetched", + "pg_stat_get_xact_tuples_hot_updated", + "pg_stat_get_xact_tuples_inserted", + "pg_stat_get_xact_tuples_newpage_updated", + "pg_stat_get_xact_tuples_returned", + "pg_stat_get_xact_tuples_updated", + "pg_stat_have_stats", + "pg_statistics_obj_is_visible", + "pg_table_is_visible", + "pg_table_size", + "pg_tablespace_databases", + "pg_tablespace_location", + "pg_tablespace_size", + "pg_total_relation_size", + "pg_trigger_depth", + "pg_ts_config_is_visible", + "pg_ts_dict_is_visible", + "pg_ts_parser_is_visible", + "pg_ts_template_is_visible", + "pg_type_is_visible", + "pg_typeof", + "pg_visible_in_snapshot", + "pg_wal_lsn_diff", + "pg_walfile_name", + "pg_walfile_name_offset", + "pg_xact_commit_timestamp", + "pg_xact_commit_timestamp_origin", + "pi", + "position", + "power", + "quote_ident", + "quote_literal", + "quote_nullable", + "radians", + "rank", + "regexp_count", + "regexp_instr", + "regexp_like", + "regexp_match", + "regexp_matches", + "regexp_replace", + "regexp_split_to_array", + "regexp_substr", + "repeat", + "replace", + "reverse", + "right", + "round", + "row_number", + "rpad", + "rtrim", + "sign", + "sin", + "split_part", + "sqrt", + "starts_with", + "strftime", + "string_agg", + "strpos", + "substr", + "substring", + "sum", + "tan", + "time", + "timediff", + "timestamp", + "to_char", + "to_date", + "to_number", + "to_timestamp", + "translate", + "trim", + "trunc", + "unixepoch", + "upper", + "varchar", + "width_bucket", +]) + const column = z .string() .min(1) @@ -12,7 +470,25 @@ const column = z const parameters = z .object({ warehouse: z.string().min(1), - check: z.enum(["fanout", "keys", "grain", "nulls", "coverage"]), + check: z.enum(["fanout", "keys", "grain", "nulls", "coverage", "predicate_effect"]), + base_sql: z + .string() + .optional() + .describe("Population before filtering: read-only SELECT or bare table reference for predicate_effect"), + predicates: z + .array(z.string().min(1)) + .min(1) + .max(8) + .optional() + .describe( + "1–8 boolean expressions; predicates reference the population's columns; for a bare table you may qualify with the table name; for a SELECT population use bare column names or the alias `t`", + ), + max_rows_scanned: z + .number() + .int() + .positive() + .optional() + .describe("Bare-table row estimate cap for predicate_effect (default 5000000); not a SELECT row limit"), sql: z.string().optional().describe("Read-only SELECT for grain, nulls or coverage"), columns: z.array(column).min(1).max(20).optional().describe("Output column names for grain or nulls (up to 20)"), column: column.optional().describe("Output column name for coverage"), @@ -26,7 +502,9 @@ const parameters = z .optional() .describe("Required for fanout: pairs of [left output column, right output column] for equality joins"), join: z.enum(["inner", "left"]).optional().default("inner"), - left_col: column.optional().describe("Left output column for keys; compared as text, with type differences reported"), + left_col: column + .optional() + .describe("Left output column for keys; compared as text, with type differences reported"), right_col: column.optional().describe("Right output column for keys"), left_warehouse: z.string().optional().describe("Optional assertion: must equal warehouse"), right_warehouse: z.string().optional().describe("Optional assertion: must equal warehouse"), @@ -43,7 +521,24 @@ export type SqlCheckInput = z.input const publicParameters = z .object({ warehouse: z.string().min(1), - check: z.enum(["fanout", "keys", "grain", "nulls", "coverage"]), + check: z.enum(["fanout", "keys", "grain", "nulls", "coverage", "predicate_effect"]), + base_sql: z + .string() + .optional() + .describe("Population before filtering: read-only SELECT or bare table reference for predicate_effect"), + predicates: z + .array(z.string()) + .max(8) + .optional() + .describe( + "1–8 boolean expressions; predicates reference the population's columns; for a bare table you may qualify with the table name; for a SELECT population use bare column names or the alias `t`", + ), + max_rows_scanned: z + .number() + .int() + .nonnegative() + .optional() + .describe("Bare-table row estimate cap for predicate_effect (default 5000000); not a SELECT row limit"), sql: z.string().optional().describe("Read-only SELECT for grain, nulls or coverage"), columns: z.array(z.string()).max(20).optional().describe("Output column names for grain or nulls (up to 20)"), column: z.string().optional().describe("Output column name for coverage"), @@ -56,7 +551,10 @@ const publicParameters = z .optional() .describe("Required for fanout: pairs of [left output column, right output column] for equality joins"), join: z.string().optional().describe('"inner" (default) or "left"'), - left_col: z.string().optional().describe("Left output column for keys; compared as text, with type differences reported"), + left_col: z + .string() + .optional() + .describe("Left output column for keys; compared as text, with type differences reported"), right_col: z.string().optional().describe("Right output column for keys"), left_warehouse: z.string().optional().describe("Optional assertion: must equal warehouse"), right_warehouse: z.string().optional().describe("Optional assertion: must equal warehouse"), @@ -68,16 +566,22 @@ export type SqlCheckPublicInput = z.input /** * Treat "", [] and null as absent so strict-schema callers can leave unused fields empty. + * Blank predicates are dropped; a zero max_rows_scanned uses the default cap. * A coverage check's `pattern` is the one field where "" is a value (match empty strings). */ export function dropEmpty(input: SqlCheckPublicInput): SqlCheckInput { const out: Record = {} - for (const [key, value] of Object.entries(input)) { + for (const [key, raw] of Object.entries(input)) { + const value = + key === "predicates" && Array.isArray(raw) + ? raw.filter((predicate) => typeof predicate !== "string" || predicate.trim() !== "") + : raw if (key === "pattern" && input.check === "coverage" && value === "") { out[key] = value continue } if (value === "" || value === null || (Array.isArray(value) && value.length === 0)) continue + if (key === "max_rows_scanned" && value === 0) continue out[key] = value } return out as SqlCheckInput @@ -99,16 +603,239 @@ function required(value: T | undefined, name: string): T { return value } +const OPERATOR_TOKEN = /[+\-*/<>=~!@#%^&|`?]+/g +const POSTGRES_IDENTIFIER = /^(?:"(?:[^"\x00]|"")+"|[A-Za-z_\u0080-\uFFFF][\w$\u0080-\uFFFF]*)$/ + +/** Keep quoted tokens opaque and retain source positions for syntax-only exclusions. */ +function sqlSyntax(sql: string, dialect?: string) { + let offset = 0 + return tokenizeSql(sql, dialect).tokens.flatMap(({ kind, text }) => { + const start = offset + offset += text.length + if (kind === "comment") return [] + if (kind !== "code") return [{ text, index: start }] + return [ + ...text.matchAll( + /[A-Za-z_\u0080-\uFFFF][\w$\u0080-\uFFFF]*|(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?|::|[+\-*/<>=~!@#%^&|`?]+|[^\s]/g, + ), + ].map((match) => ({ text: match[0], index: start + match.index })) + }) +} + +function closingParen(tokens: ReturnType, start: number) { + let depth = 0 + for (let i = start; i < tokens.length; i++) { + if (tokens[i].text === "(") depth++ + if (tokens[i].text === ")" && --depth === 0) return i + } + return -1 +} + +function postgresCasts(sql: string) { + const tokens = sqlSyntax(sql, "postgres") + const targets: { name: string; index: number }[] = [] + function typeIdentifier(start: number) { + let end = start + let type = tokens[end]?.text ?? "" + if (/^u$/i.test(type) && tokens[end + 1]?.text === "&" && tokens[end + 2]?.text.startsWith('"')) { + end += 2 + type = tokens[end].text.slice(1, -1).replaceAll('""', '"') + let escape = "\\" + if (/^uescape$/i.test(tokens[end + 1]?.text ?? "")) { + const literal = tokens[end + 2]?.text ?? "" + if (!/^'[^']'$/.test(literal)) throw new Error("sql_check cannot inspect this Unicode cast type escape") + escape = literal[1] + end += 2 + } + let decoded = "" + for (let i = 0; i < type.length; i++) { + if (type[i] !== escape) decoded += type[i] + else if (type[i + 1] === escape) decoded += type[++i] + else { + const length = type[i + 1] === "+" ? 6 : 4 + const start = i + (length === 6 ? 2 : 1) + const hex = type.slice(start, start + length) + const point = parseInt(hex, 16) + if (hex.length !== length || !/^[\da-f]+$/i.test(hex) || point === 0 || point > 0x10ffff) + throw new Error("sql_check cannot inspect this Unicode cast type escape") + decoded += String.fromCodePoint(point) + i = start + length - 1 + } + } + return { name: decoded, end } + } + if (!POSTGRES_IDENTIFIER.test(type)) return + return { name: type.startsWith('"') ? type.slice(1, -1).replaceAll('""', '"') : type.toLowerCase(), end } + } + for (let i = 0; i < tokens.length; i++) { + let target = -1 + if (tokens[i].text === "::") target = i + 1 + if (/^cast$/i.test(tokens[i].text) && tokens[i + 1]?.text === "(") { + for (let j = i + 2; j < tokens.length && tokens[j].text !== ")"; j++) { + if (tokens[j].text === "(") { + j = closingParen(tokens, j) + if (j < 0) break + } else if (/^as$/i.test(tokens[j].text)) { + target = j + 1 + break + } + } + } + if (target < 0) continue + let type = typeIdentifier(target) + if (!type) continue + while (tokens[type.end + 1]?.text === ".") { + const qualified = typeIdentifier(type.end + 2) + if (!qualified) break + type = qualified + } + targets.push({ name: type.name, index: tokens[target].index }) + } + return targets +} + +function sqlDeclarations(sql: string, dialect?: string) { + const tokens = sqlSyntax(sql, dialect) + const declarations = new Set() + for (let i = 0; i < tokens.length; i++) { + if (!/^with$/i.test(tokens[i].text)) continue + let j = i + 1 + if (/^recursive$/i.test(tokens[j]?.text ?? "")) j++ + while (POSTGRES_IDENTIFIER.test(tokens[j]?.text ?? "")) { + const name = tokens[j++] + if (tokens[j]?.text === "(") { + const end = closingParen(tokens, j) + if (end < 0) break + j = end + 1 + } + if (!/^as$/i.test(tokens[j++]?.text ?? "")) break + if (/^not$/i.test(tokens[j]?.text ?? "")) j++ + const materialized = /^materialized$/i.test(tokens[j]?.text ?? "") ? tokens[j++] : undefined + if (tokens[j]?.text !== "(") break + declarations.add(name.index) + if (materialized) declarations.add(materialized.index) + const end = closingParen(tokens, j) + if (end < 0 || tokens[end + 1]?.text !== ",") break + j = end + 2 + } + } + // A derived table or table function may declare output columns after its alias. + // Exclude only this source position, so calls sharing the alias name remain checked. + for (let i = 0; i < tokens.length; i++) { + if (tokens[i].text !== ")") continue + let j = i + 1 + if (/^as$/i.test(tokens[j]?.text ?? "")) j++ + const alias = tokens[j++] + if (!alias || !POSTGRES_IDENTIFIER.test(alias.text) || tokens[j++]?.text !== "(") continue + while (POSTGRES_IDENTIFIER.test(tokens[j]?.text ?? "")) { + j++ + if (tokens[j]?.text === ")") { + declarations.add(alias.index) + break + } + if (tokens[j++]?.text !== ",") break + } + } + return declarations +} + +function postgresOperators(sql: string): string[] { + const code = maskSql(tokenizeSql(sql, "postgres").tokens) + const tokens = sqlSyntax(sql, "postgres") + const selectList = [false] + const names = new Set() + for (let i = 0; i < tokens.length; i++) { + const token = tokens[i].text + if (token === "(") selectList.push(false) + else if (token === ")") selectList.pop() + else if (/^select$/i.test(token)) selectList[selectList.length - 1] = true + else if (/^(from|where|group|having|order|limit|offset|fetch|union|intersect|except)$/i.test(token)) + selectList[selectList.length - 1] = false + if (!/^[+\-*/<>=~!@#%^&|`?]+$/.test(token)) continue + const previous = tokens[i - 1]?.text ?? "" + let afterDistinctOn = false + if (token === "*" && previous === ")" && selectList.at(-1)) { + let depth = 0 + for (let j = i - 1; j >= 0; j--) { + if (tokens[j].text === ")") depth++ + if (tokens[j].text === "(" && --depth === 0) { + afterDistinctOn = /^select distinct on$/i.test( + tokens + .slice(j - 3, j) + .map((token) => token.text) + .join(" "), + ) + break + } + } + } + if ( + token === "*" && + (previous === "." || + afterDistinctOn || + (selectList.at(-1) && /^(select|all|distinct|,)$/i.test(previous)) || + (previous === "(" && /^(count|"count")$/i.test(tokens[i - 2]?.text ?? ""))) + ) + continue + names.add(token) + } + for (const operator of [...names]) { + // PostgreSQL splits trailing unary signs from ordinary operator runs: =- is = and -. + if (!/[~!@#%^&|`?]/.test(operator)) { + const signs = /[+-]+$/.exec(operator)?.[0] + if (signs) { + const head = operator.slice(0, -signs.length) + if (head) names.add(head) + for (const sign of signs) names.add(sign) + } + } + } + // The parser rewrites these spellings to catalog operator names. + if (names.has("!=")) names.add("<>") + for (const [keyword, operators] of [ + [/\bLIKE\b/i, ["~~", "!~~"]], + [/\bILIKE\b/i, ["~~*", "!~~*"]], + [/\bSIMILAR\s+TO\b/i, ["~", "!~"]], + [/\bBETWEEN\b/i, ["<", ">", "<=", ">="]], + [/\bIN\b/i, ["=", "<>"]], + [/\bIS\s+(?:NOT\s+)?DISTINCT\s+FROM\b/i, ["="]], + ] as const) { + if (keyword.test(code)) for (const operator of operators) names.add(operator) + } + return [...names] +} + /** Ignore quoted content while checking structure; preserve the original SELECT and its LIMIT. */ -export function readSql(sql: string, dialect?: string): string { - const { queryType, blocked } = classifyAndCheck(sql, dialect) +export function readSql(sql: string, dialect?: string, catalogOperators = false): string { // Independently reject write CTEs, SELECT INTO and statement/CTE escapes before // embedding user SQL, even when the parser reports a query category. const { tokens, complete } = tokenizeSql(sql, dialect) + // The structural parser does not know arbitrary PostgreSQL operator names. Only + // catalog-checked probes may substitute their spelling for structural parsing; + // all lexical checks and the executed SQL still use the original text. + const structuralSql = + dialect === "postgres" && catalogOperators + ? tokens + .map(({ kind, text }) => + kind === "code" ? text.replace(OPERATOR_TOKEN, (operator) => (operator === "*" ? "*" : "+")) : text, + ) + .join("") + : sql + const { queryType, blocked } = classifyAndCheck(structuralSql, dialect) const masked = maskSql(tokens) // SELECT alone does not make sequence setters, extension loaders or file writers read-only. // Preserve quoted function identifiers while hiding string literals and comments. const functions = maskSql(tokens, true) + // The tokenizer does not decode PostgreSQL U&"..." function names. Refuse + // those calls so Unicode escapes cannot hide a side-effect function. + const escapedFunction = tokens.some( + (token, i) => + token.kind === "identifier" && + token.text.startsWith('"') && + tokens[i - 1]?.kind === "code" && + /(?:^|[^\w$])U&$/i.test(tokens[i - 1].text) && + /^\s*(?:UESCAPE\s+)?\(/i.test(maskSql(tokens.slice(i + 1))), + ) const body = masked.trim().replace(/;$/, "").trim() let depth = 0 let balanced = true @@ -119,8 +846,9 @@ export function readSql(sql: string, dialect?: string): string { if ( blocked || queryType !== "read" || - !isSingleQuery(sql, dialect) || + !isSingleQuery(structuralSql, dialect) || !complete || + escapedFunction || !balanced || depth !== 0 || !/^(SELECT|WITH)\b/i.test(body) || @@ -129,9 +857,7 @@ export function readSql(sql: string, dialect?: string): string { body, ) || /\bREPLACE\b(?!\s*\()/i.test(body) || - /\b(nextval|setval|set_config|load_extension|writefile|dblink_exec|lo_import|lo_export|lo_create|lo_unlink|lo_put|pg_write_file|pg_append_file|pg_file_write|pg_file_rename|pg_file_unlink|pg_advisory_lock|pg_advisory_xact_lock)\s*\(/i.test( - functions, - ) + SIDE_EFFECT_FUNCTION_CALL.test(functions) ) { throw new Error( "sql_check only accepts a single read-only SELECT (or WITH ... SELECT); it never writes or fixes data.", @@ -145,9 +871,172 @@ function result(check: string, metadata: Row, lines: string[]): { title: string; return { title: `SQL check: ${check}`, metadata: { check, ...metadata }, output: lines.join("\n") } } +/** Only identifiers count as bare tables; SELECT populations are never rewritten. */ +function readBase(sql: string, dialect?: string) { + const identifier = + dialect === "sqlite" + ? '(?:"(?:[^"]|"")+"|`(?:[^`]|``)+`|\\[[^\\]]+\\]|[A-Za-z_][A-Za-z0-9_$]*)' + : '(?:"(?:[^"]|"")+"|[A-Za-z_][A-Za-z0-9_$]*)' + const bare = new RegExp(`^${identifier}(?:\\s*\\.\\s*${identifier}){0,2}$`).test(sql.trim()) + const table = bare ? sql.trim() : undefined + // Preserve quoting and case folding, including dots inside a quoted table name. + const alias = table?.match(new RegExp(identifier, "g"))?.at(-1) ?? "t" + const population = readSql(table ? `SELECT * FROM ${table}` : sql, dialect, true) + pureFunctions(population, dialect) + return { sql: population, table, alias } +} + +function populationLimit(sql: string, dialect: Dialect): string | undefined { + // Hide subqueries and CTE bodies, retaining the parentheses for TOP (n). + let depth = 0 + let topLevel = "" + for (const char of maskSql(tokenizeSql(sql, dialect).tokens)) { + if (char === ")") depth-- + topLevel += depth === 0 ? char : " " + if (char === "(") depth++ + } + if (/(?:^|[^\w$])LIMIT(?![\w$])/i.test(topLevel)) return "LIMIT" + if (/(?:^|[^\w$])FETCH\s+FIRST(?![\w$])[\s\S]*?\bROWS?\s+ONLY(?![\w$])/i.test(topLevel)) return "FETCH FIRST" + if (/(?:^|[^\w$])SELECT\s+(?:(?:ALL|DISTINCT)\s+)?TOP\b\s*(?:\d|\()/i.test(topLevel)) return "TOP" +} + +function pureFunctions(sql: string, dialect?: string): string[] { + const { tokens } = tokenizeSql(sql, dialect) + // Retain identifier boundaries and case; hide strings and comments, which cannot call functions. + const functions = tokens + .map(({ kind, text }) => (kind === "code" || kind === "identifier" ? text : text.replace(/[^\r\n]/g, " "))) + .join("") + const identifier = /^(postgres|duckdb)$/i.test(dialect ?? "") + ? '(?:"(?:[^"\\x00]|"")+"|[A-Za-z_\\u0080-\\uFFFF][\\w$\\u0080-\\uFFFF]*)' + : '(?:"(?:[^"\\x00]|"")+"|`(?:[^`]|``)+`|\\[(?:[^\\]]|\\]\\])+\\]|[A-Za-z_\\u0080-\\uFFFF][\\w$\\u0080-\\uFFFF]*)' + const calls = new RegExp(`${identifier}(?:\\s*\\.\\s*${identifier})*`, "g") + const declarations = sqlDeclarations(sql, dialect) + if (dialect === "postgres") for (const target of postgresCasts(sql)) declarations.add(target.index) + const names = new Set() + for (const match of functions.matchAll(calls)) { + if (!/^\s*\(/.test(functions.slice(match.index + match[0].length))) continue + if (declarations.has(match.index)) continue + const call = match[0].trim() + const parts = call.match(new RegExp(identifier, "g"))! + const quoted = /^["`\[]/.test(parts[0]) + const name = quoted ? parts[0].slice(1, -1) : parts[0].replace(/[A-Z]/g, (char) => char.toLowerCase()) + // These unquoted SQL keywords introduce grouped expressions or clauses (`FILTER (WHERE …)`, + // `OVER (PARTITION BY …)`, `WITHIN GROUP (…)`, `USING (…)`), not function calls. + if ( + parts.length === 1 && + !quoted && + /^(all|and|any|as|between|distinct|else|exists|filter|from|groups|having|in|lateral|not|on|or|ordinality|over|partition|range|rows|select|some|then|using|values|when|where|within)$/.test( + name, + ) + ) + continue + if (parts.length !== 1 || !PURE_PREDICATE_FUNCTIONS.has(name)) { + throw new Error(`sql_check refuses function ${call}: predicates are limited to built-in pure functions.`) + } + names.add(name) + } + return [...names] +} + +function readPredicate(predicate: string, dialect?: string): string { + pureFunctions(predicate, dialect) + const { tokens } = tokenizeSql(predicate, dialect) + readSql(`SELECT 1 WHERE (\n${predicate}\n)`, dialect, true) + // A read-only wrapper alone can still accept an expression that closes it and + // introduces another SELECT. Require the expression's own delimiters to balance. + let depth = 0 + for (const char of maskSql(tokens)) { + if (char === "(") depth++ + if (char === ")" && --depth < 0) throw new Error("sql_check predicates must be bare boolean expressions") + } + if (depth !== 0) throw new Error("sql_check predicates must be bare boolean expressions") + return `(\n${predicate}\n)` +} + +/** Reject server-defined overloads even when a call's spelling is on the pure allowlist. */ +async function checkPredicateCatalog( + population: string, + predicates: string[], + dialect: Dialect, + run: (sql: string, binds?: unknown[]) => Promise, +) { + // SQLite SQL cannot persist functions; custom functions must be registered by the application. + if (dialect === "sqlite") return + const queries = [population, ...predicates] + const functionNames = queries.map((sql) => pureFunctions(sql, dialect)) + const functions = [...new Set(functionNames.flat())] + const origin = (name: string, names: string[][]) => + names.slice(1).some((query) => query.includes(name)) ? "predicate" : "population" + if (dialect === "postgres") { + const operatorNames = queries.map(postgresOperators) + const operators = [...new Set(operatorNames.flat())] + if (operators.length) { + const [operator] = await run( + `SELECT o.oprname, n.nspname + FROM pg_catalog.pg_operator o JOIN pg_catalog.pg_namespace n ON n.oid = o.oprnamespace + WHERE o.oprname = ANY($1) AND n.nspname <> 'pg_catalog'`, + [operators], + ) + if (operator) { + throw new Error( + `sql_check ${origin(operator.oprname, operatorNames)} uses operator \`${operator.oprname}\`, which is not a built-in operator`, + ) + } + } + if (functions.length) { + const [fn] = await run( + `SELECT p.proname, n.nspname + FROM pg_catalog.pg_proc p JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE p.proname = ANY($1) AND p.pronamespace <> 'pg_catalog'::regnamespace`, + [functions], + ) + if (fn) { + throw new Error( + `sql_check ${origin(fn.proname, functionNames)} calls \`${fn.proname}\`, which has a user-defined overload in schema \`${fn.nspname}\``, + ) + } + } + // Text cannot determine whether a cast's source column has a user-defined type. + const typeNames = queries.map((sql) => postgresCasts(sql).map((target) => target.name)) + const types = [...new Set(typeNames.flat())] + if (types.length) { + const [type] = await run( + `SELECT t.typname, n.nspname + FROM pg_catalog.pg_type t JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace + WHERE t.typname = ANY($1) AND n.nspname <> 'pg_catalog'`, + [types], + ) + if (type) { + throw new Error( + `sql_check ${origin(type.typname, typeNames)} casts to \`${type.typname}\`, which is not a built-in type`, + ) + } + } + return + } + if (functions.length) { + const [fn] = await run( + `SELECT function_name, schema_name FROM duckdb_functions() + WHERE internal = false AND function_name COLLATE NOCASE IN (${functions.map(() => "?").join(", ")})`, + functions, + ) + if (fn) { + throw new Error( + `sql_check ${origin(fn.function_name.toLowerCase(), functionNames)} calls \`${fn.function_name}\`, which has a user-defined overload in schema \`${fn.schema_name}\``, + ) + } + } +} + /** Counts run in the warehouse; only aggregate rows and at most five samples per side leave it. */ export async function checkSql(input: SqlCheckPublicInput) { - const args = parameters.parse(dropEmpty(input)) + const parsed = parameters.safeParse(dropEmpty(input)) + if (!parsed.success) { + throw new Error( + `sql_check invalid input: ${parsed.error.issues.map((issue) => `${issue.path.join(".")}: ${issue.message}`).join("; ")}`, + ) + } + const args = parsed.data if ([args.left_warehouse, args.right_warehouse].some((w) => w !== undefined && w !== args.warehouse)) { throw new Error("sql_check runs in one warehouse; use sql_materialize to put both sides in scratch first.") } @@ -156,9 +1045,17 @@ export async function checkSql(input: SqlCheckPublicInput) { const sql = args.sql === undefined ? undefined : readSql(args.sql, sourceDialect) const left = args.left_sql === undefined ? undefined : readSql(args.left_sql, sourceDialect) const right = args.right_sql === undefined ? undefined : readSql(args.right_sql, sourceDialect) + const base = args.base_sql === undefined ? undefined : readBase(args.base_sql, sourceDialect) + const predicates = args.predicates?.map((predicate) => readPredicate(predicate, sourceDialect)) // Keep internal CTEs from shadowing any relation in the supplied queries. let scope = "__sql_check_0_" - for (let i = 1; [sql, left, right].some((s) => s?.toLowerCase().includes(scope)); i++) scope = `__sql_check_${i}_` + for ( + let i = 1; + [sql, left, right, base?.sql, ...(predicates ?? [])].some((s) => s?.toLowerCase().includes(scope)); + i++ + ) { + scope = `__sql_check_${i}_` + } const rawType = Registry.getConfig(args.warehouse)?.type.toLowerCase() const dialect = rawType === "postgresql" ? "postgres" : rawType if (dialect !== "sqlite" && dialect !== "duckdb" && dialect !== "postgres") { @@ -167,11 +1064,11 @@ export async function checkSql(input: SqlCheckPublicInput) { ) } const connector = await Registry.get(args.warehouse) - async function run(query: string): Promise { - const response = await connector.execute(query, undefined, undefined, { + async function run(query: string, binds?: unknown[]): Promise { + const response = await connector.execute(query, undefined, binds, { noLimit: true, + readOnly: true, queryTimeoutMs: args.query_timeout_ms, - ...(dialect === "postgres" ? { readOnly: true } : {}), }) if (response.truncated) throw new Error("sql_check received truncated aggregates; no verdict is available") return response.rows.map((row) => Object.fromEntries(response.columns.map((col, i) => [col, row[i]]))) @@ -180,6 +1077,86 @@ export async function checkSql(input: SqlCheckPublicInput) { const two = () => `WITH ${scope}lhs AS (\n${required(left, "left_sql")}\n), ${scope}rhs AS (\n${required(right, "right_sql")}\n)` + if (args.check === "predicate_effect") { + const population = required(base, "base_sql") + const expressions = required(predicates, "predicates") + try { + await checkPredicateCatalog(population.sql, expressions, dialect, run) + const cap = args.max_rows_scanned ?? 5_000_000 + const localTable = population.table !== undefined && dialect !== "postgres" + const count = (predicate: string) => `COUNT(*) FILTER (WHERE ${predicate})` + const probe = `SELECT COUNT(*) AS total, + ${expressions.map((predicate, i) => `${count(predicate)} AS keep_${i + 1}`).join(", ")}, + ${count(expressions.join(" AND "))} AS keep_all FROM (\n${localTable ? `SELECT * FROM ${scope}source` : population.sql}\n) ${population.alias}` + // Local engines reuse one bounded, materialized population for both the + // estimate and aggregate. A peer cannot swap the relation between them, + // and an exceeded cap never sends an unbounded relation to the aggregate. + const [row] = await run( + population.table === undefined + ? probe + : `WITH ${ + localTable + ? `${scope}source AS MATERIALIZED ( + ${population.sql} LIMIT ${BigInt(cap) + 1n} + ), ` + : "" + }${scope}est AS ( + ${ + localTable + ? `SELECT COUNT(*) AS estimate FROM ${scope}source` + : `SELECT reltuples AS estimate FROM pg_catalog.pg_class + WHERE oid = pg_catalog.to_regclass(E${literal(population.table.replace(/\\/g, "\\\\"))}) + AND relkind IN ('r', 'm') AND reltuples > 0` + } + ), ${scope}probe AS ( + ${probe} + WHERE COALESCE((SELECT estimate FROM ${scope}est) <= ${cap}, TRUE) + ) + SELECT ${scope}probe.*, (SELECT estimate FROM ${scope}est) AS estimate, + COALESCE((SELECT estimate FROM ${scope}est) > ${cap}, FALSE) AS estimate_exceeded + FROM ${scope}probe`, + ) + const estimate = row.estimate == null ? undefined : number(row.estimate) + if (row.estimate_exceeded) { + return result(args.check, { skipped: "estimate" }, [ + `predicate_effect: skipped: estimate; population estimate ${estimate} exceeds max_rows_scanned ${cap}; use a narrower base_sql.`, + ]) + } + const total = number(row.total) + const effects = args.predicates!.map((predicate, i) => { + const kept = number(row[`keep_${i + 1}`]) + return { + predicate, + kept_rows: kept, + removed_rows: total - kept, + ...(total > 0 && total === kept ? { warning: "this predicate removes no rows at this population" } : {}), + ...(total > 0 && kept === 0 ? { warning: "this predicate removes all rows" } : {}), + } + }) + const combined = number(row.keep_all) + const limit = populationLimit(population.sql, dialect) + let effectIndex = 0 + return result(args.check, { total_rows: total, predicates: effects, combined_kept_rows: combined }, [ + `predicate_effect: ${total ? `${total} rows in population` : "empty population"}`, + ...(estimate === undefined ? ["row_estimate: unknown — probe runs under the query timeout"] : []), + ...(limit ? [`counts are over the limited population (base_sql includes ${limit})`] : []), + ...input.predicates!.map((predicate, i) => { + const label = `predicate ${i + 1} ${display(predicate)}` + if (!predicate.trim()) return `${label}: skipped (blank)` + const effect = effects[effectIndex++] + return `${label}: kept ${effect.kept_rows}; removed ${effect.removed_rows}${effect.warning ? `; ${effect.warning}` : ""}` + }), + `combined (AND): kept ${combined}; removed ${total - combined}${total > 0 && combined === 0 ? "; this combination removes all rows" : ""}`, + "A predicate that returns NULL counts as not kept.", + ]) + } catch (error) { + if (!isQueryTimeoutError(error) || (error as { stateLost?: boolean }).stateLost === true) throw error + return result(args.check, { skipped: "timeout" }, [ + "predicate_effect: skipped: timeout; use a narrower base_sql.", + ]) + } + } + if (args.check === "fanout") { const on = required(args.on, "on") const names = on.map((_pair, i) => `k${i}`) @@ -390,22 +1367,46 @@ async function checkKeys( export const SqlCheckTool = Tool.define("sql_check", { description: - "Check SQL result invariants: fanout (join defaults to inner), keys, grain, nulls, or extraction coverage. " + + "Check SQL result invariants: fanout (join defaults to inner), keys, grain, nulls, extraction coverage, or predicate_effect. " + "Read-only SELECTs run in one SQLite, DuckDB or Postgres warehouse, with their original LIMITs preserved; samples cap at five. " + "Use sql_materialize to put cross-warehouse or other-engine sources into scratch first. " + "Key normalization is diagnostic and may collapse unrelated values; no automatic fixes. Coverage uses regex except SQLite's explicit LIKE fallback. " + + "predicate_effect is opt-in via sql_check.predicate_effect (default false); takes base_sql before filtering and 1–8 explicit predicates and reports individual and combined removals, with NULL not kept. " + + "predicates reference the population's columns; for a bare table you may qualify with the table name; for a SELECT population use bare column names or the alias `t`. " + + "Bare-table estimates above max_rows_scanned (default 5000000) are refused; unknown estimates use the query timeout and expiry is reported as skipped. " + "Leave fields a check does not use empty.", parameters: publicParameters, async execute(args, ctx) { try { // altimate_change start — validate and authorize reads before workspace redirects + if (args.check === "predicate_effect" && (await Config.get()).sql_check?.predicate_effect !== true) { + const error = + "predicate_effect is disabled by default because it executes caller-supplied population and predicate SQL whose server-side definitions (views, casts, operators) cannot be fully validated; enable it in config (sql_check.predicate_effect: true) for trusted databases." + return { + title: "SQL check: predicate_effect ERROR", + metadata: { check: args.check, error, disabled: true }, + output: `predicate_effect: no verdict — ${error}`, + } + } const input = dropEmpty(args) const dialect = Registry.canonicalType(Registry.getConfig(input.warehouse)?.type) ?? undefined const queries = [input.sql, input.left_sql, input.right_sql].filter((sql): sql is string => sql !== undefined) + if (input.base_sql !== undefined) queries.push(readBase(input.base_sql, dialect).sql) + for (const predicate of input.predicates ?? []) + queries.push(`SELECT 1 WHERE ${readPredicate(predicate, dialect)}`) for (const sql of queries) readSql(sql, dialect) + // Permission rules are matched per pattern, so a rule scoped to a column-and-table + // combination must also see the probe that joins them: authorize the combined + // population + predicates in the shape the probe executes, not only the parts. + const combined = + input.base_sql !== undefined && input.predicates?.length + ? [ + `SELECT ${input.predicates.map((predicate) => `COUNT(*) FILTER (WHERE ${predicate})`).join(", ")} FROM (${readBase(input.base_sql, dialect).sql}) AS t`, + ] + : [] await ctx.ask({ permission: "sql_execute", - patterns: queries, + patterns: [...queries, ...combined], always: ["*"], metadata: { warehouse: input.warehouse }, }) diff --git a/packages/opencode/src/altimate/tools/sql-tokenize.ts b/packages/opencode/src/altimate/tools/sql-tokenize.ts index d1cd50a5d4..15dc9053fd 100644 --- a/packages/opencode/src/altimate/tools/sql-tokenize.ts +++ b/packages/opencode/src/altimate/tools/sql-tokenize.ts @@ -6,6 +6,9 @@ export function tokenizeSql(sql: string, dialect?: string): { tokens: SqlToken[] const tokens: SqlToken[] = [] const mysqlComments = !dialect || /^(mysql|mariadb)$/i.test(dialect) const dollarQuotes = /^(postgres|redshift|duckdb|snowflake)$/i.test(dialect ?? "") + // PostgreSQL backticks are operators; PostgreSQL/DuckDB brackets contain array expressions. + const backtickIdentifiers = !/^(postgres|duckdb)$/i.test(dialect ?? "") + const bracketIdentifiers = !/^(postgres|duckdb)$/i.test(dialect ?? "") let complete = true let i = 0 let code = 0 @@ -16,8 +19,11 @@ export function tokenizeSql(sql: string, dialect?: string): { tokens: SqlToken[] // to the write-clause checks even when another dialect treats it as a comment. if (sql.startsWith("--", i) && (!mysqlComments || i + 2 === sql.length || /[\s\x00-\x1f]/.test(sql[i + 2]))) { kind = "comment" - i = sql.indexOf("\n", i + 2) - if (i < 0) i = sql.length + // PostgreSQL also ends line comments at CR. Keep following code visible + // when the dialect is unknown; other dialects retain their LF handling. + const newline = !dialect || /^postgres$/i.test(dialect) ? /[\r\n]/ : /\n/ + const end = sql.slice(i + 2).search(newline) + i = end < 0 ? sql.length : i + 2 + end } else if (sql.startsWith("/*", i)) { // MySQL/MariaDB execute these comments. Generic SQL parsers discard them, // so they cannot establish that a query is read-only or scratch-confined. @@ -38,7 +44,7 @@ export function tokenizeSql(sql: string, dialect?: string): { tokens: SqlToken[] } else { const escaped = /[eE]/.test(sql[i]) && sql[i + 1] === "'" && !/[\w$]/.test(sql[i - 1] ?? "") const quote = escaped ? "'" : sql[i] - if (quote === "'" || quote === '"' || quote === "`" || quote === "[") { + if (quote === "'" || quote === '"' || (quote === "`" && backtickIdentifiers) || (quote === "[" && bracketIdentifiers)) { kind = quote === "'" ? "literal" : "identifier" const end = quote === "[" ? "]" : quote i += escaped ? 2 : 1 diff --git a/packages/opencode/src/permission/next.ts b/packages/opencode/src/permission/next.ts index edc2ca1d03..c67acb4c57 100644 --- a/packages/opencode/src/permission/next.ts +++ b/packages/opencode/src/permission/next.ts @@ -135,6 +135,10 @@ export namespace PermissionNext { async (input) => { const s = await state() const { ruleset, ...request } = input + // altimate_change start — upstream_fix: evaluate every pattern before prompting so a + // later explicit deny cannot be bypassed by approving an earlier pattern + let needsAsk = false + // altimate_change end for (const pattern of request.patterns ?? []) { const rule = evaluate(request.permission, pattern, ruleset, s.approved) log.info("evaluated", { permission: request.permission, pattern, action: rule }) @@ -149,23 +153,25 @@ export namespace PermissionNext { }) throw new DeniedError(ruleset.filter((r) => Wildcard.match(request.permission, r.permission))) } - if (rule.action === "ask") { - const id = input.id ?? PermissionID.ascending() - return new Promise((resolve, reject) => { - const info: Request = { - id, - ...request, - } - s.pending.set(id, { - info, - resolve, - reject, - }) - Bus.publish(Event.Asked, info) - }) - } - if (rule.action === "allow") continue + // altimate_change start — upstream_fix: defer the prompt until all patterns are evaluated + if (rule.action === "ask") needsAsk = true } + // A later pattern can deny the whole request; evaluate all of them before prompting. + if (!needsAsk) return + // altimate_change end + const id = input.id ?? PermissionID.ascending() + return new Promise((resolve, reject) => { + const info: Request = { + id, + ...request, + } + s.pending.set(id, { + info, + resolve, + reject, + }) + Bus.publish(Event.Asked, info) + }) }, ) diff --git a/packages/opencode/test/agent/analyst-data-explorer.test.ts b/packages/opencode/test/agent/analyst-data-explorer.test.ts index b91db472c2..b8dc3a47da 100644 --- a/packages/opencode/test/agent/analyst-data-explorer.test.ts +++ b/packages/opencode/test/agent/analyst-data-explorer.test.ts @@ -75,6 +75,16 @@ it.instance("analyst can load skills", () => }), ) +it.instance("analyst explains how to use predicate_effect without inventing exclusions", () => + Effect.gen(function* () { + const analyst = yield* Agent.use.get("analyst") + expect(analyst?.prompt).toContain("a filter that removes nothing usually means the wrong column") + expect(analyst?.prompt).toContain("`sql_check` `predicate_effect` reports what each filter removes") + expect(analyst?.prompt).toContain("a reason to profile an alternative column, not to invent an exclusion") + expect(analyst?.prompt).toContain("not to invent an exclusion).") + }), +) + it.instance("analyst may delegate only to data-explorer", () => Effect.gen(function* () { const analyst = yield* Agent.use.get("analyst") diff --git a/packages/opencode/test/altimate/sql-check-postgres.test.ts b/packages/opencode/test/altimate/sql-check-postgres.test.ts new file mode 100644 index 0000000000..b26e695ced --- /dev/null +++ b/packages/opencode/test/altimate/sql-check-postgres.test.ts @@ -0,0 +1,561 @@ +import { afterEach, beforeEach, describe, expect, mock, spyOn, test } from "bun:test" +import { Database } from "bun:sqlite" +import { rejects } from "node:assert/strict" +import type { Connector } from "@altimateai/drivers" +import * as Registry from "../../src/altimate/native/connections/registry" +import { checkSql } from "../../src/altimate/tools/sql-check" + +const warehouse = "postgres_predicates" +const unknownEstimate = "row_estimate: unknown — probe runs under the query timeout" +let database: Database +let connectorSpy: { mockRestore(): void } + +beforeEach(() => { + Registry.setConfigs({ [warehouse]: { type: "postgres" } }) + database = new Database(":memory:") + database.run("CREATE TABLE pg_class (oid INTEGER, relkind TEXT, reltuples REAL)") + database.run("CREATE TABLE orders (id INTEGER, kind TEXT, amount INTEGER)") + database.run("INSERT INTO orders VALUES (1, 'a', 5), (2, 'b', 7), (3, 'a', 1), (4, 'z', 3), (5, 'a', NULL)") +}) + +afterEach(() => { + connectorSpy?.mockRestore() + database.close() + Registry.reset() +}) + +function postgres( + relkind = "r", + reltuples = 5, + translate = (sql: string, _options?: Parameters[3]) => sql, + catalog: Partial> = {}, +) { + database.run("INSERT INTO pg_class VALUES (1, ?, ?)", [relkind, reltuples]) + const execute = mock(async (...[sql, , binds, options]: Parameters) => { + for (const [table, column] of [ + ["pg_operator", "oprname"], + ["pg_proc", "proname"], + ["pg_type", "typname"], + ] as const) { + if (!sql.includes(`pg_catalog.${table} `)) continue + expect(sql).toContain("ANY($1)") + expect(binds).toHaveLength(1) + const names = binds![0] as string[] + const rows = (catalog[table] ?? []).filter(([name, schema]) => names.includes(name) && schema !== "pg_catalog") + return { columns: [column, "nspname"], rows, row_count: rows.length, truncated: false } + } + // Keep the generated WHERE and FILTER clauses executable: removing either + // guard condition must change results, rather than leave a canned mock row. + const query = translate(sql, options) + .replace("pg_catalog.pg_class", "pg_class") + .replace(/pg_catalog\.to_regclass\(E'(?:[^']|'')*'\)/, "1") + const statement = database.query(query) + const rows = statement.values(...(binds ?? [])) + return { columns: statement.columnNames, rows, row_count: rows.length, truncated: false } + }) + connectorSpy = spyOn(Registry, "get").mockResolvedValue({ execute } as unknown as Connector) + return execute +} + +// Keep assertions about estimates and population SQL separate from safety-catalog lookups. +function dataCalls(execute: ReturnType) { + return execute.mock.calls.filter(([sql]) => !/pg_catalog\.(?:pg_operator|pg_proc|pg_type)\b/.test(sql)) +} + +describe("predicate_effect with mocked Postgres", () => { + test("an ordinary population and id > 0 pass the catalog checks and reach the probe", async () => { + const execute = postgres() + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "SELECT * FROM orders", + predicates: ["id > 0"], + query_timeout_ms: 4321, + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 5 }) + const [catalog, probe] = execute.mock.calls + expect(catalog[0]).toContain("pg_catalog.pg_operator") + expect(catalog[2]).toEqual([[">"]]) + expect(probe[0]).toContain("COUNT(*) AS total") + expect(dataCalls(execute)).toHaveLength(1) + for (const call of execute.mock.calls) { + expect(call[3]).toMatchObject({ noLimit: true, queryTimeoutMs: 4321, readOnly: true }) + } + }) + + test.each([ + { catalog: "pg_operator" as const, name: ">", predicate: "id > 0", message: "uses operator `>`" }, + { catalog: "pg_proc" as const, name: "length", predicate: "length(kind) > 0", message: "calls `length`" }, + ])( + "seeded $catalog rows refuse matching custom definitions before the probe", + async ({ catalog, name, predicate, message }) => { + const execute = postgres("r", 5, undefined, { [catalog]: [[name, "custom"]] }) + await expect( + checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "SELECT * FROM orders", + predicates: [predicate], + }), + ).rejects.toThrow(message) + expect(dataCalls(execute)).toEqual([]) + expect(execute.mock.calls.at(-1)?.[2]?.[0]).toContain(name) + }, + ) + + test("the Postgres fixture preserves bind arguments for data SQL", async () => { + const execute = postgres() + const result = await execute("SELECT ? AS value", undefined, [42]) + expect(result.rows).toEqual([[42]]) + expect(execute.mock.calls[0][2]).toEqual([42]) + }) + + test.each([ + { name: "predicates", base_sql: "orders", predicates: ["custom_write_udf() > 0"] as string[] }, + { + name: "populations", + base_sql: "SELECT * FROM orders WHERE custom_write_udf() > 0", + predicates: ["id > 0"] as string[], + }, + ])("rejects writing UDFs in $name before query execution", async ({ base_sql, predicates }) => { + let writes = 0 + const execute = postgres("r", 5, (sql, options) => { + if (!sql.includes("custom_write_udf()")) return sql + // A missing function guard would reach this simulated write-capable call. + if (options?.readOnly) throw new Error("cannot execute INSERT in a read-only transaction") + writes++ + return sql.replaceAll("custom_write_udf()", "1") + }) + await rejects( + checkSql({ warehouse, check: "predicate_effect", base_sql, predicates, query_timeout_ms: 4321 }), + /sql_check refuses function custom_write_udf: predicates are limited to built-in pure functions/, + ) + expect(writes).toBe(0) + expect(execute).not.toHaveBeenCalled() + expect(dataCalls(execute)).toEqual([]) + }) + + for (const [kind, calls] of [ + [ + "WAL-message", + [ + "pg_logical_emit_message(false, 'probe', 'message')", + "PG_CATALOG.PG_LOGICAL_EMIT_MESSAGE(false, 'probe', 'message')", + `pg_catalog."pg_logical_emit_message" /* WAL */ (false, 'probe', 'message')`, + ], + ], + [ + "synchronous-dblink", + [ + "dblink('remote', 'DELETE FROM important_table RETURNING id')", + "PG_CATALOG.DBLINK('remote', 'DELETE FROM important_table RETURNING id')", + `public."dblink" /* remote */ ('remote', 'DELETE FROM important_table RETURNING id')`, + ], + ], + ...[ + "pg_advisory_lock", + "pg_advisory_lock_shared", + "pg_try_advisory_lock", + "pg_try_advisory_lock_shared", + "pg_advisory_xact_lock", + "pg_advisory_xact_lock_shared", + "pg_try_advisory_xact_lock", + "pg_try_advisory_xact_lock_shared", + "pg_advisory_unlock", + "pg_advisory_unlock_shared", + "pg_advisory_unlock_all", + ].map((name): [string, string[]] => [ + "advisory-lock", + [`${name}(1)`, `PG_CATALOG.${name.toUpperCase()}(1, 2)`, `pg_catalog."${name}" /* lock */ (1)`], + ]), + [ + "server-control", + [ + "pg_promote()", + 'pg_catalog."pg_promote" /* promote */ ()', + "PG_RELOAD_CONF()", + 'pg_catalog."pg_reload_conf" -- reload\n()', + ], + ], + [ + "dblink-write", + [ + "dblink_send_query('remote', 'DELETE FROM important_table')", + `public."dblink_send_query" /* send */ ('remote', 'DELETE FROM important_table')`, + "DBLINK_CONNECT('remote', 'dbname=remote')", + `public."dblink_connect" -- connect\n('remote', 'dbname=remote')`, + "dblink_connect_u('remote', 'dbname=remote')", + `public."dblink_connect_u" /* connect */ ('remote', 'dbname=remote')`, + "dblink_open('remote', 'c', 'WITH w AS (DELETE FROM important_table RETURNING 1) SELECT 1')", + `public."dblink_open" /* open */ ('remote', 'c', 'SELECT 1')`, + "dblink_fetch('remote', 'c', 10)", + "DBLINK_CLOSE('remote', 'c')", + `public."dblink_close" -- close\n('remote', 'c')`, + ], + ], + [ + "large-object-write", + [ + "lo_from_bytea(0, '\\x00'::bytea)", + "pg_catalog.\"lo_from_bytea\" /* write */ (0, '\\x00'::bytea)", + "LO_TRUNCATE(1, 0)", + "lo_truncate64(1, 0)", + "lowrite(lo_open(1, 131072), '\\x00'::bytea)", + ], + ], + [ + "class-denied", + [ + "lo_import_with_oid('/tmp/x', 1)", + "lo_creat(131072)", + "pg_log_standby_snapshot()", + "PG_READ_FILE('/etc/passwd')", + "pg_sleep(30)", + "checkpoint()", + "force_checkpoint(current_database())", + "pg_some_future_control_function(1)", + "query('SELECT force_checkpoint(current_database())')", + "pg_ls_dir('.')", + "pg_ls_waldir()", + ], + ], + [ + "recovery-control", + [ + "pg_wal_replay_pause()", + 'pg_catalog."pg_wal_replay_pause" /* pause */ ()', + "PG_WAL_REPLAY_RESUME()", + "pg_create_restore_point('probe-' || id::text)", + "pg_catalog.\"pg_create_restore_point\" /* mark */ ('p')", + "PG_SWITCH_WAL()", + "pg_drop_replication_slot('slot')", + "pg_create_logical_replication_slot('slot', 'pgoutput')", + "pg_rotate_logfile()", + "pg_logical_slot_get_changes('slot', NULL, NULL)", + "pg_sync_replication_slots()", + "pg_replication_origin_create('origin')", + "pg_replication_slot_advance('slot', '0/1')", + "pg_stat_reset()", + "PG_STAT_RESET_SHARED('bgwriter')", + 'pg_catalog."pg_stat_reset_single_table_counters" /* reset */ (1)', + ], + ], + ] satisfies [string, string[]][]) { + test.each(calls)(`rejects ${kind} predicate %s before query execution`, async (call) => { + const predicate = `${call} IS NOT NULL` + const execute = postgres("r", 5, (sql) => sql.replaceAll(call, "1")) + await rejects( + checkSql({ warehouse, check: "predicate_effect", base_sql: "orders", predicates: ["id > 0", predicate] }), + /sql_check refuses function .*: predicates are limited to built-in pure functions/, + ) + expect(execute).not.toHaveBeenCalled() + }) + + test.each(calls)(`rejects ${kind} population %s before query execution`, async (call) => { + const execute = postgres("r", 5, (sql) => sql.replaceAll(call, "1")) + await rejects( + checkSql({ + warehouse, + check: "predicate_effect", + base_sql: `SELECT ${call} AS value FROM orders`, + predicates: ["value IS NOT NULL"], + }), + /sql_check only accepts a single read-only SELECT/, + ) + expect(execute).not.toHaveBeenCalled() + }) + } + + test.each([ + "pg_typeof(id)", + "PG_SIZE_PRETTY(id::bigint)", + "pg_column_size(id)", + "pg_get_expr('1', 1)", + "pg_stat_get_numscans(1)", + "pg_relation_size('orders')", + "pg_current_wal_lsn()", + "pg_has_role('x', 'USAGE')", + ])("accepts the pure read function %s in predicates and populations", async (call) => { + const execute = postgres("r", 5, (sql) => sql.replaceAll(call, "1")) + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 0", `${call} IS NOT NULL`], + }) + expect(result.metadata).toMatchObject({ total_rows: 5 }) + // The pure call reached execution (a bare table probe also includes the row-estimate lookup). + expect(execute).toHaveBeenCalled() + }) + + test.each([ + "pg_promote", + "pg_reload_conf", + "dblink_send_query", + "dblink_connect", + "dblink_connect_u", + "pg_logical_emit_message", + "dblink", + "pg_advisory_lock_shared", + "pg_try_advisory_lock_shared", + "pg_advisory_xact_lock_shared", + "pg_try_advisory_xact_lock_shared", + ])("accepts %s in literals, comments and column identifiers", async (name) => { + const execute = postgres() + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: `SELECT id AS "${name}" FROM orders`, + predicates: [`"${name}" > 2 -- ${name}()`, `length('${name}()') > 0 /* ${name}() */`], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + expect(dataCalls(execute)).toHaveLength(1) + }) + + test.each([ + "pg_terminate_backend(id)", + "PG_TERMINATE_BACKEND(id)", + 'pg_catalog."pg_terminate_backend" /* signal */ (id)', + "pg_cancel_backend(id)", + 'pg_catalog."pg_cancel_backend" -- signal\n(id)', + "EXISTS (SELECT 1 WHERE pg_catalog.pg_terminate_backend(id))", + String.raw`U&"pg\005fterminate_backend"(id)`, + String.raw`pg_catalog.U&"pg_terminate_\+000062ackend"(id)`, + `U&"pg!005fcancel_backend" /* escape */ UESCAPE '!' (id)`, + "true -- comment\rAND pg_terminate_backend(id)", + "true -- comment\rAND pg_cancel_backend(id)", + ])("rejects backend-signaling predicate %s before query execution", async (predicate) => { + // Substitute only at the connector so an unguarded probe succeeds instead + // of failing because SQLite does not provide PostgreSQL's signaling functions. + const execute = postgres("r", 5, (sql) => sql.replaceAll(predicate, "id > 0")) + await rejects( + checkSql({ warehouse, check: "predicate_effect", base_sql: "orders", predicates: [predicate] }), + /sql_check refuses function .*: predicates are limited to built-in pure functions/, + ) + expect(execute).not.toHaveBeenCalled() + }) + + test.each(["pg_terminate_backend", "pg_cancel_backend"])( + "rejects backend-signaling population %s before query execution", + async (name) => { + const execute = postgres("r", 5, (sql) => sql.replaceAll(`${name}(id)`, "id > 0")) + await rejects( + checkSql({ + warehouse, + check: "predicate_effect", + base_sql: `SELECT ${name}(id) AS signaled FROM orders`, + predicates: ["signaled"], + }), + /sql_check only accepts a single read-only SELECT/, + ) + expect(execute).not.toHaveBeenCalled() + }, + ) + + test("accepts backend-signaling names in literals, comments and column identifiers", async () => { + const escapedColumn = String.raw`U&"pg\005fterminate_backend"` + const execute = postgres("r", 5, (sql) => sql.replaceAll(escapedColumn, '"pg_terminate_backend"')) + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: 'SELECT id AS "pg_terminate_backend" FROM orders', + predicates: [ + '"pg_terminate_backend" > 2 -- pg_terminate_backend(id)', + "length('pg_cancel_backend(id)') > 0 /* pg_terminate_backend(id) */", + `${escapedColumn} > 2`, + ], + }) + expect(result.metadata).toMatchObject({ + total_rows: 5, + predicates: [{ kept_rows: 3 }, { kept_rows: 5 }, { kept_rows: 3 }], + combined_kept_rows: 3, + }) + expect(dataCalls(execute)).toHaveLength(1) + }) + + test("qualifies a bare table using its quoted unqualified name with a dot and escaped quote", async () => { + database.run('CREATE TABLE "Odd.orders""quoted" AS SELECT * FROM orders') + const execute = postgres() + const base_sql = 'main."Odd.orders""quoted"' + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql, + predicates: ['"Odd.orders""quoted".kind = \'a\''], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + expect(execute.mock.calls.at(-1)?.[0]).toContain(`FROM (\nSELECT * FROM ${base_sql}\n) "Odd.orders""quoted"`) + }) + + test("retains the t qualifier for SELECT populations", async () => { + const execute = postgres() + const base_sql = "SELECT * FROM orders" + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql, + predicates: ["t.kind = 'a'"], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + expect(dataCalls(execute)[0][0]).toContain(`FROM (\n${base_sql}\n) t`) + }) + + test.each([ + { name: "FETCH FIRST", population: "SELECT * FROM orders ORDER BY id FETCH FIRST 2 ROWS ONLY" }, + { name: "TOP", population: "SELECT TOP 2 * FROM orders ORDER BY id" }, + ])("notes a top-level $name while preserving the population SQL", async ({ population }) => { + const execute = postgres("r", 5, (sql) => sql.replace(population, "SELECT * FROM orders ORDER BY id LIMIT 2")) + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: population, + predicates: ["id > 1"], + }) + expect(result.metadata).toMatchObject({ total_rows: 2, combined_kept_rows: 1 }) + expect(dataCalls(execute)[0][0]).toContain(`FROM (\n${population}\n) t`) + expect(result.output).toContain("counts are over the limited population") + }) + + test.each([ + { name: "FETCH FIRST", inner: "SELECT * FROM orders ORDER BY id FETCH FIRST 2 ROWS ONLY" }, + { name: "TOP", inner: "SELECT TOP 2 * FROM orders ORDER BY id" }, + ])("does not note $name confined to a subquery", async ({ inner }) => { + const execute = postgres("r", 5, (sql) => sql.replace(inner, "SELECT * FROM orders ORDER BY id LIMIT 2")) + const base_sql = `SELECT * FROM (${inner}) limited_orders` + const result = await checkSql({ warehouse, check: "predicate_effect", base_sql, predicates: ["id > 1"] }) + expect(result.metadata).toMatchObject({ total_rows: 2, combined_kept_rows: 1 }) + expect(dataCalls(execute)[0][0]).toContain(`FROM (\n${base_sql}\n) t`) + expect(result.output).not.toContain("limited population") + }) + + test("uses independent FILTER counts and a parenthesized AND probe with newline-safe comments", async () => { + const execute = postgres() + const base_sql = "SELECT * FROM orders -- population comment" + const predicates = ["kind = 'a' OR kind = 'b' -- first predicate", "amount > 4 -- second predicate"] + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql, + predicates, + query_timeout_ms: 4321, + }) + expect(result.metadata).toMatchObject({ + total_rows: 5, + predicates: [ + { kept_rows: 4, removed_rows: 1 }, + { kept_rows: 2, removed_rows: 3 }, + ], + combined_kept_rows: 2, + }) + expect(dataCalls(execute)).toHaveLength(1) + const sql = dataCalls(execute)[0][0] + expect(sql).toContain("COUNT(*) AS total") + expect(sql).toContain(`COUNT(*) FILTER (WHERE (\n${predicates[0]}\n)) AS keep_1`) + expect(sql).toContain(`COUNT(*) FILTER (WHERE (\n${predicates[1]}\n)) AS keep_2`) + expect(sql).toContain(`COUNT(*) FILTER (WHERE (\n${predicates[0]}\n) AND (\n${predicates[1]}\n)) AS keep_all`) + expect(sql).toContain(`FROM (\n${base_sql}\n) t`) + expect(dataCalls(execute)[0][3]).toMatchObject({ noLimit: true, queryTimeoutMs: 4321 }) + }) + + test.each([ + ["orders", "E'orders'"], + ['"Foo"."Bar"', 'E\'"Foo"."Bar"\''], + [String.raw`"Foo"."O'rd\\ers"`, String.raw`E'"Foo"."O''rd\\\\ers"'`], + ])("resolves %s through an escaped to_regclass literal before enforcing reltuples", async (base_sql, literal) => { + if (base_sql !== "orders") { + database.run(`ATTACH DATABASE ':memory:' AS "Foo"`) + database.run(`CREATE TABLE ${base_sql} AS SELECT * FROM orders`) + } + const execute = postgres("r", 6) + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql, + predicates: ["id > 0"], + max_rows_scanned: 5, + }) + expect(result.metadata).toMatchObject({ skipped: "estimate" }) + expect(result.output).toContain("population estimate 6 exceeds max_rows_scanned 5") + expect(dataCalls(execute)).toHaveLength(1) + expect(dataCalls(execute)[0][0]).toContain("SELECT reltuples AS estimate FROM pg_catalog.pg_class") + expect(dataCalls(execute)[0][0]).toContain(`oid = pg_catalog.to_regclass(${literal})`) + expect(dataCalls(execute)[0][0]).toContain("WHERE COALESCE((SELECT estimate FROM __sql_check_0_est) <= 5, TRUE)") + }) + + test("a lost helper session during the estimate lookup reaches the caller instead of becoming an unknown estimate", async () => { + const execute = postgres("r", 5, (sql) => { + if (/pg_class/.test(sql)) + throw Object.assign(new Error("session state was lost"), { code: "SESSION_STATE_LOST", stateLost: true }) + return sql + }) + await expect( + checkSql({ warehouse, check: "predicate_effect", base_sql: "orders", predicates: ["id > 2"] }), + ).rejects.toMatchObject({ stateLost: true }) + expect(dataCalls(execute)).toHaveLength(1) + }) + + test("a missing-catalog error in the combined estimate and probe query reaches the caller", async () => { + const execute = postgres("r", 5, (sql) => { + if (/pg_class/.test(sql)) throw new Error('relation "pg_catalog.pg_class" does not exist') + return sql + }) + await expect( + checkSql({ warehouse, check: "predicate_effect", base_sql: "orders", predicates: ["id > 2"] }), + ).rejects.toThrow('relation "pg_catalog.pg_class" does not exist') + expect(dataCalls(execute)).toHaveLength(1) + }) + + test("allows a known estimate equal to the cap and does not label it unknown", async () => { + const execute = postgres() + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 2"], + max_rows_scanned: 5, + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + expect(result.output).not.toContain(unknownEstimate) + expect(dataCalls(execute)).toHaveLength(1) + }) + + test("guards materialized views using their positive reltuples estimate", async () => { + const execute = postgres("m", 6) + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 0"], + max_rows_scanned: 5, + }) + expect(result.metadata).toMatchObject({ skipped: "estimate" }) + expect(result.output).toContain("population estimate 6 exceeds max_rows_scanned 5") + expect(dataCalls(execute)).toHaveLength(1) + }) + + test.each([ + ["unanalyzed table with -1 reltuples", "r", -1], + ["unanalyzed table with 0 reltuples", "r", 0], + ["view with 0 reltuples", "v", 0], + ["view with positive reltuples", "v", 6], + ["partitioned parent with 0 reltuples", "p", 0], + ["partitioned parent with positive reltuples", "p", 6], + ])("reports an unknown estimate for %s and probes under the timeout", async (_name, relkind, reltuples) => { + const execute = postgres(String(relkind), Number(reltuples)) + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 2"], + max_rows_scanned: 2, + query_timeout_ms: 4321, + }) + expect(result.output).toContain(unknownEstimate) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + expect(dataCalls(execute)).toHaveLength(1) + expect(dataCalls(execute)[0][0]).toContain("pg_catalog.to_regclass(E'orders')") + for (const call of execute.mock.calls) { + expect(call[3]).toMatchObject({ noLimit: true, queryTimeoutMs: 4321 }) + } + }) +}) diff --git a/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts b/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts new file mode 100644 index 0000000000..e736fb8bc9 --- /dev/null +++ b/packages/opencode/test/altimate/sql-check-predicate-effect.test.ts @@ -0,0 +1,1130 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, spyOn, test } from "bun:test" +import { QueryTimeoutError } from "@altimateai/drivers/types" +import * as Registry from "../../src/altimate/native/connections/registry" +import { closeScratch } from "../../src/altimate/native/connections/scratch" +import * as Precedence from "../../src/altimate/workspace/precedence" +import { checkSql, SqlCheckTool } from "../../src/altimate/tools/sql-check" +import { initTool as initTestTool } from "./tool-fixture" +import { Bus } from "../../src/bus" +import { PermissionNext } from "../../src/permission/next" +import { SessionID } from "../../src/session/schema" +import { Effect, Layer } from "effect" +import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner" +import { Instance } from "../../src/project/instance" +import { TestInstance, testInstanceStoreLayer, tmpdir } from "../fixture/fixture" +import { testEffect } from "../lib/effect" + +const it = testEffect(Layer.mergeAll(CrossSpawnSpawner.defaultLayer, testInstanceStoreLayer)) + +const warehouses = ["sqlite_predicates", "scratch"] as const +const baseSql = "SELECT * FROM orders" +const noRemoval = "this predicate removes no rows at this population" +const allRemoval = "this predicate removes all rows" +const unknownEstimate = "row_estimate: unknown — probe runs under the query timeout" +let previousTelemetry: string | undefined +let previousConfig: string | undefined +let configFixture: Awaited> + +beforeAll(async () => { + previousTelemetry = process.env.ALTIMATE_TELEMETRY_DISABLED + process.env.ALTIMATE_TELEMETRY_DISABLED = "true" + previousConfig = process.env.OPENCODE_CONFIG_CONTENT + process.env.OPENCODE_CONFIG_CONTENT = JSON.stringify({ sql_check: { predicate_effect: true } }) + configFixture = await tmpdir() +}) + +beforeEach(async () => { + Registry.setConfigs({ sqlite_predicates: { type: "sqlite", path: ":memory:" } }) + await closeScratch() + for (const warehouse of warehouses) { + const connector = await Registry.get(warehouse) + await connector.execute("CREATE TABLE orders (id INTEGER, kind TEXT, amount INTEGER, note TEXT)") + await connector.execute( + "INSERT INTO orders VALUES (1, 'a', 5, 'first'), (2, 'b', 7, NULL), (3, 'a', 1, NULL), (4, 'z', 3, 'tail'), (5, 'a', NULL, NULL)", + ) + } +}) + +afterEach(async () => { + await Registry.closeAll() + await closeScratch() +}) + +afterAll(async () => { + if (previousTelemetry === undefined) delete process.env.ALTIMATE_TELEMETRY_DISABLED + else process.env.ALTIMATE_TELEMETRY_DISABLED = previousTelemetry + if (previousConfig === undefined) delete process.env.OPENCODE_CONFIG_CONTENT + else process.env.OPENCODE_CONFIG_CONTENT = previousConfig + await Instance.provide({ directory: configFixture.path, fn: () => Instance.dispose() }) + await configFixture[Symbol.asyncDispose]() +}) + +// Tool config is instance-scoped; retain an existing permission fixture's instance when present. +async function initTool(tool: typeof SqlCheckTool) { + const initialized = await initTestTool(tool) + return { + ...initialized, + execute: (args: Parameters[0], ctx: Parameters[1]) => { + let directory = configFixture.path + try { + directory = Instance.directory + } catch {} + return Instance.provide({ directory, fn: () => initialized.execute(args, ctx) }) + }, + } +} + +test.each([ + { label: "unset", config: {} }, + { label: "false", config: { sql_check: { predicate_effect: false } } }, +])( + "predicate_effect disabled with flag $label asks no permission and performs no routing or SQL", + async ({ config }) => { + const enabledConfig = process.env.OPENCODE_CONFIG_CONTENT + process.env.OPENCODE_CONFIG_CONTENT = JSON.stringify(config) + await using fixture = await tmpdir() + const connector = await Registry.get("sqlite_predicates") + const execute = spyOn(connector, "execute") + const get = spyOn(Registry, "get") + const precedence = spyOn(Precedence, "check").mockResolvedValue({}) + const asked: unknown[] = [] + try { + await Instance.provide({ + directory: fixture.path, + fn: async () => { + const tool = await initTool(SqlCheckTool) + const result = await tool.execute( + { warehouse: "sqlite_predicates", check: "predicate_effect", base_sql: baseSql, predicates: ["id > 0"] }, + { + sessionID: "test-session", + messageID: "test-message", + agent: "test", + abort: new AbortController().signal, + messages: [], + ask: async (request: unknown) => { + asked.push(request) + }, + }, + ) + const error = + "predicate_effect is disabled by default because it executes caller-supplied population and predicate SQL whose server-side definitions (views, casts, operators) cannot be fully validated; enable it in config (sql_check.predicate_effect: true) for trusted databases." + expect(result).toMatchObject({ + title: "SQL check: predicate_effect ERROR", + metadata: { check: "predicate_effect", error, disabled: true }, + output: `predicate_effect: no verdict — ${error}`, + }) + expect(asked).toEqual([]) + expect(precedence).not.toHaveBeenCalled() + expect(get).not.toHaveBeenCalled() + expect(execute).not.toHaveBeenCalled() + }, + }) + } finally { + execute.mockRestore() + get.mockRestore() + precedence.mockRestore() + process.env.OPENCODE_CONFIG_CONTENT = enabledConfig + await Instance.provide({ directory: fixture.path, fn: () => Instance.dispose() }) + } + }, +) + +test("predicate_effect opt-in executes the registered tool after permission approval", async () => { + const connector = await Registry.get("sqlite_predicates") + const execute = spyOn(connector, "execute") + const precedence = spyOn(Precedence, "check").mockResolvedValue({}) + const asked: unknown[] = [] + try { + const tool = await initTool(SqlCheckTool) + const result = await tool.execute( + { warehouse: "sqlite_predicates", check: "predicate_effect", base_sql: baseSql, predicates: ["id > 2"] }, + { + sessionID: "test-session", + messageID: "test-message", + agent: "test", + abort: new AbortController().signal, + messages: [], + ask: async (request: unknown) => { + asked.push(request) + }, + }, + ) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + expect(asked).toHaveLength(1) + expect(precedence).toHaveBeenCalledTimes(1) + expect(execute).toHaveBeenCalledTimes(1) + } finally { + execute.mockRestore() + precedence.mockRestore() + } +}) + +for (const warehouse of warehouses) { + describe(`predicate_effect with ${warehouse === "scratch" ? "DuckDB" : "SQLite"}`, () => { + test("reports independent removals and flags only predicates that remove no rows", async () => { + const connector = await Registry.get(warehouse) + const execute = spyOn(connector, "execute") + try { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: baseSql, + predicates: ["id > 0", "kind = 'a'"], + }) + expect(result.metadata).toMatchObject({ + check: "predicate_effect", + total_rows: 5, + predicates: [ + { predicate: "id > 0", kept_rows: 5, removed_rows: 0, warning: noRemoval }, + { predicate: "kind = 'a'", kept_rows: 3, removed_rows: 2 }, + ], + combined_kept_rows: 3, + }) + expect(result.metadata.predicates[1].warning).toBeUndefined() + expect(result.output).toContain(noRemoval) + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][3]).toMatchObject({ noLimit: true, readOnly: true }) + } finally { + execute.mockRestore() + } + }) + + test("counts NULL-valued predicates as not kept and explains that rule", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: baseSql, + predicates: ["amount > 0", "note = 'first'"], + }) + expect(result.metadata).toMatchObject({ + total_rows: 5, + predicates: [ + { kept_rows: 4, removed_rows: 1 }, + { kept_rows: 1, removed_rows: 4 }, + ], + combined_kept_rows: 1, + }) + expect(result.output).toMatch(/NULL.*not kept/i) + }) + + test("reports an empty population without flagging predicates", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: `${baseSql} WHERE 1 = 0`, + predicates: ["id > 0", "amount > 10"], + }) + expect(result.metadata).toMatchObject({ + total_rows: 0, + predicates: [ + { kept_rows: 0, removed_rows: 0 }, + { kept_rows: 0, removed_rows: 0 }, + ], + combined_kept_rows: 0, + }) + expect(result.metadata.predicates.every((predicate: { warning?: string }) => !predicate.warning)).toBe(true) + expect(result.output).toContain("empty population") + expect(result.output).not.toContain(noRemoval) + expect(result.output).not.toContain("removes all rows") + }) + + test("warns when a predicate removes all rows", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: baseSql, + predicates: ["id > 100", "id > 2"], + }) + expect(result.metadata.predicates[0]).toMatchObject({ kept_rows: 0, removed_rows: 5, warning: allRemoval }) + expect(result.metadata.predicates[1].warning).toBeUndefined() + expect(result.output).toContain(`predicate 1 "id > 100": kept 0; removed 5; ${allRemoval}`) + }) + + test("warns when the combination removes all rows but individual predicates do not", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: baseSql, + predicates: ["id < 3", "id > 3"], + }) + expect(result.metadata.predicates.every((predicate: { warning?: string }) => !predicate.warning)).toBe(true) + expect(result.output).toContain("combined (AND): kept 0; removed 5; this combination removes all rows") + }) + + test("combined count matches direct AND filtering while preserving OR precedence", async () => { + const predicates = ["kind = 'a' OR kind = 'b'", "amount > 4"] + const result = await checkSql({ warehouse, check: "predicate_effect", base_sql: baseSql, predicates }) + const connector = await Registry.get(warehouse) + const direct = await connector.execute( + `SELECT COUNT(*) AS kept FROM (${baseSql}) t WHERE (${predicates[0]}) AND (${predicates[1]})`, + ) + expect(result.metadata.combined_kept_rows).toBe(Number(direct.rows[0][0])) + expect(result.metadata).toMatchObject({ + predicates: [ + { kept_rows: 4, removed_rows: 1 }, + { kept_rows: 2, removed_rows: 3 }, + ], + combined_kept_rows: 2, + }) + }) + + test("accepts a bare table reference as the population", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 2"], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + }) + + test.each(["orders", "main.orders"])("accepts table-qualified predicates for bare population %s", async (base_sql) => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql, + predicates: ["orders.kind = 'a'"], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + }) + + test("preserves quoted unqualified table names containing dots and escaped quotes", async () => { + const connector = await Registry.get(warehouse) + await connector.execute('CREATE TABLE "Order.Items""Archive" AS SELECT * FROM orders') + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: 'main."Order.Items""Archive"', + predicates: ['"Order.Items""Archive".kind = \'a\''], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + }) + + test("keeps alias t for SELECT populations", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: baseSql, + predicates: ["t.kind = 'a'"], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + }) + + test("skips a bare table whose bounded estimate exceeds the custom cap", async () => { + const connector = await Registry.get(warehouse) + if (warehouse === "sqlite_predicates") await connector.execute("ANALYZE orders") + const execute = spyOn(connector, "execute") + try { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 0"], + max_rows_scanned: 4, + }) + expect(result.metadata).toMatchObject({ skipped: "estimate" }) + expect(result.output).toMatch(/narrower.*base_sql/i) + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][0]).toContain("LIMIT 5") + } finally { + execute.mockRestore() + } + }) + + test("bounds view populations and skips an exceeded cap under the timeout", async () => { + const connector = await Registry.get(warehouse) + await connector.execute("CREATE VIEW order_view AS SELECT * FROM orders") + const execute = spyOn(connector, "execute") + try { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "order_view", + predicates: ["id > 2"], + max_rows_scanned: 1, + query_timeout_ms: 4321, + }) + expect(result.metadata).toMatchObject({ skipped: "estimate" }) + expect(result.output).toContain("2 exceeds max_rows_scanned 1") + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls.at(-1)?.[3]).toMatchObject({ noLimit: true, queryTimeoutMs: 4321 }) + } finally { + execute.mockRestore() + } + }) + + test("does not report a bounded bare-table estimate as unknown", async () => { + const connector = await Registry.get(warehouse) + if (warehouse === "sqlite_predicates") await connector.execute("ANALYZE orders") + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 2"], + }) + expect(result.metadata.total_rows).toBe(5) + expect(result.output).not.toContain(unknownEstimate) + }) + + test("applies the estimate guard only to bare table populations", async () => { + const connector = await Registry.get(warehouse) + const execute = spyOn(connector, "execute") + try { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: baseSql, + predicates: ["id > 0"], + max_rows_scanned: 1, + }) + expect(result.metadata.total_rows).toBe(5) + expect(result.output).toContain(unknownEstimate) + expect(execute).toHaveBeenCalledTimes(1) + } finally { + execute.mockRestore() + } + }) + + test("accepts eight predicates and preserves an explicit population LIMIT", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: `${baseSql} ORDER BY id LIMIT 2`, + predicates: Array.from({ length: 8 }, (_, i) => `id > ${i}`), + }) + expect(result.metadata.total_rows).toBe(2) + expect(result.metadata.predicates).toHaveLength(8) + expect(result.metadata.combined_kept_rows).toBe(0) + expect(result.output).toContain("counts are over the limited population (base_sql includes LIMIT)") + }) + + test("does not mistake quoted or commented LIMIT text for a population limit", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: `SELECT id, 'LIMIT 1' AS "LIMIT" FROM orders -- LIMIT 2`, + predicates: ["id > 2"], + }) + expect(result.metadata.total_rows).toBe(5) + expect(result.output).not.toContain("limited population") + }) + + test("does not mistake a LIMIT suffix in an unquoted identifier for a population limit", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: 'SELECT row$LIMIT FROM (SELECT id AS "row$LIMIT" FROM orders) subset', + predicates: ['"row$LIMIT" > 2'], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + expect(result.output).not.toContain("limited population") + }) + + test("does not mistake column top2 for a population TOP clause", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "SELECT top2 FROM (SELECT id AS top2 FROM orders) subset", + predicates: ["top2 > 2"], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + expect(result.output).not.toContain("limited population") + }) + + test.each([ + "SELECT * FROM (SELECT * FROM orders ORDER BY id LIMIT 2) subset", + "WITH subset AS (SELECT * FROM orders ORDER BY id LIMIT 2) SELECT * FROM subset", + ])("does not label an inner LIMIT as a population limit: %s", async (base_sql) => { + const result = await checkSql({ warehouse, check: "predicate_effect", base_sql, predicates: ["id > 0"] }) + expect(result.metadata).toMatchObject({ total_rows: 2, combined_kept_rows: 2 }) + expect(result.output).not.toContain("limited population") + }) + + test("preserves caller positions and lists leading, middle and trailing blank predicates", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: baseSql, + predicates: ["", "id > 2", " \t ", "kind = 'a'", " "], + }) + expect(result.output.match(/^predicate \d+.*$/gm)).toEqual([ + 'predicate 1 "": skipped (blank)', + 'predicate 2 "id > 2": kept 3; removed 2', + 'predicate 3 " \\t ": skipped (blank)', + `predicate 4 "kind = 'a'": kept 3; removed 2`, + 'predicate 5 " ": skipped (blank)', + ]) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 2 }) + }) + + test("strict-schema zero cap and blank predicates are treated as absent by the registered tool", async () => { + const precedence = spyOn(Precedence, "check").mockResolvedValue({}) + try { + const tool = await initTool(SqlCheckTool) + const result = await tool.execute( + { + warehouse, + check: "predicate_effect", + base_sql: "SELECT 1 AS id UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5", + predicates: ["", "id > 2", " "], + max_rows_scanned: 0, + }, + { + sessionID: "test-session", + messageID: "test-message", + agent: "test", + abort: new AbortController().signal, + messages: [], + }, + ) + expect(result.metadata.error).toBeUndefined() + expect(result.metadata).toMatchObject({ + total_rows: 5, + predicates: [{ predicate: "id > 2", kept_rows: 3 }], + combined_kept_rows: 3, + }) + expect(result.metadata.predicates).toHaveLength(1) + expect(result.output).toContain('predicate 1 "": skipped (blank)') + expect(result.output).toContain('predicate 2 "id > 2": kept 3; removed 2') + expect(result.output).toContain('predicate 3 " ": skipped (blank)') + } finally { + precedence.mockRestore() + } + }) + + test.each([ + { base_sql: baseSql, predicates: [""], missing: "predicates" }, + { base_sql: baseSql, predicates: ["", " \t "], missing: "predicates" }, + { base_sql: "", predicates: ["id > 0"], missing: "base_sql" }, + ])("strict-schema empty inputs return the tool's requires $missing message", async ({ base_sql, predicates, missing }) => { + const connector = await Registry.get(warehouse) + const execute = spyOn(connector, "execute") + const precedence = spyOn(Precedence, "check").mockResolvedValue({}) + try { + const tool = await initTool(SqlCheckTool) + const result = await tool.execute( + { warehouse, check: "predicate_effect", base_sql, predicates }, + { + sessionID: "test-session", + messageID: "test-message", + agent: "test", + abort: new AbortController().signal, + messages: [], + }, + ) + expect(result.output).toBe(`predicate_effect: no verdict — sql_check requires ${missing} for this check`) + expect(result.metadata.error).toBe(`sql_check requires ${missing} for this check`) + expect(execute).not.toHaveBeenCalled() + } finally { + execute.mockRestore() + precedence.mockRestore() + } + }) + + test("rejects nine predicates before query execution", async () => { + const connector = await Registry.get(warehouse) + const execute = spyOn(connector, "execute") + try { + await expect( + checkSql({ warehouse, check: "predicate_effect", base_sql: baseSql, predicates: Array(9).fill("id > 0") }), + ).rejects.toThrow(/sql_check.*predicates.*8/) + expect(execute).not.toHaveBeenCalled() + } finally { + execute.mockRestore() + } + }) + + test("requires at least one predicate before query execution", async () => { + const connector = await Registry.get(warehouse) + const execute = spyOn(connector, "execute") + try { + await expect( + checkSql({ warehouse, check: "predicate_effect", base_sql: baseSql, predicates: [] }), + ).rejects.toThrow(/predicate/i) + expect(execute).not.toHaveBeenCalled() + } finally { + execute.mockRestore() + } + }) + + test("rejects writes and statement escapes inside predicates before query execution", async () => { + const connector = await Registry.get(warehouse) + const execute = spyOn(connector, "execute") + try { + for (const predicate of [ + "1=1); DELETE FROM orders; --", + "1=1) UNION SELECT 1 --", + "1=1) UNION SELECT (1", + "1=1) OR (1=1", + ]) { + await expect( + checkSql({ warehouse, check: "predicate_effect", base_sql: baseSql, predicates: [predicate] }), + ).rejects.toThrow(/read|SELECT|expression|predicate/i) + } + expect(execute).not.toHaveBeenCalled() + } finally { + execute.mockRestore() + } + }) + + test("rejects a write population before query execution", async () => { + const connector = await Registry.get(warehouse) + const execute = spyOn(connector, "execute") + try { + for (const base_sql of ["DELETE FROM orders", "SELECT * FROM orders; DELETE FROM orders"]) { + await expect( + checkSql({ warehouse, check: "predicate_effect", base_sql, predicates: ["id > 0"] }), + ).rejects.toThrow(/read|SELECT/i) + } + expect(execute).not.toHaveBeenCalled() + } finally { + execute.mockRestore() + } + }) + + test("accepts quoted semicolons and a read-only scalar expression", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "SELECT 'a;b' AS value", + predicates: ["REPLACE(value, ';', '') = 'ab'"], + }) + expect(result.metadata).toMatchObject({ total_rows: 1, combined_kept_rows: 1 }) + }) + + test("preserves trailing comments in the population and predicate", async () => { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: `${baseSql} -- population comment`, + predicates: ["id > 2 -- predicate comment"], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + }) + + test("propagates an unexpected query error instead of reporting a timeout", async () => { + const connector = await Registry.get(warehouse) + const execute = spyOn(connector, "execute").mockRejectedValue(new Error("unexpected query failure")) + try { + await expect( + checkSql({ warehouse, check: "predicate_effect", base_sql: baseSql, predicates: ["id > 0"] }), + ).rejects.toThrow("unexpected query failure") + expect(execute).toHaveBeenCalledTimes(1) + } finally { + execute.mockRestore() + } + }) + + test("skips without a verdict when the bounded estimate statement times out", async () => { + const connector = await Registry.get(warehouse) + const execute = spyOn(connector, "execute").mockRejectedValue( + new QueryTimeoutError({ engine: warehouse === "scratch" ? "DuckDB" : "SQLite", timeoutMs: 4321 }), + ) + try { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 0"], + query_timeout_ms: 4321, + }) + expect(result.metadata).toMatchObject({ skipped: "timeout" }) + expect(result.metadata.total_rows).toBeUndefined() + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][0]).toMatch(/COUNT\(\*\)\s+FILTER/i) + expect(execute.mock.calls[0][3]).toMatchObject({ noLimit: true, queryTimeoutMs: 4321 }) + } finally { + execute.mockRestore() + } + }) + + test("returns skipped timeout without a verdict when the query expires", async () => { + const connector = await Registry.get(warehouse) + const execute = spyOn(connector, "execute").mockRejectedValue( + new QueryTimeoutError({ engine: warehouse === "scratch" ? "DuckDB" : "SQLite", timeoutMs: 4321 }), + ) + try { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: baseSql, + predicates: ["id > 0"], + query_timeout_ms: 4321, + }) + expect(result.metadata).toMatchObject({ check: "predicate_effect", skipped: "timeout" }) + expect(result.metadata.error).toBeUndefined() + expect(result.metadata.total_rows).toBeUndefined() + expect(result.output).toContain("skipped: timeout") + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][3]).toMatchObject({ noLimit: true, queryTimeoutMs: 4321 }) + } finally { + execute.mockRestore() + } + }) + }) +} + +test("DuckDB predicate_effect respects a view shadowing a larger table on the schema search path", async () => { + const connector = await Registry.get("scratch") + await connector.execute("CREATE SCHEMA smaller") + await connector.execute("CREATE VIEW smaller.orders AS SELECT * FROM main.orders LIMIT 1") + await connector.execute("SET schema = 'smaller'") + const result = await checkSql({ + warehouse: "scratch", + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 0"], + max_rows_scanned: 2, + }) + expect(result.metadata).toMatchObject({ total_rows: 1, combined_kept_rows: 1 }) + expect(result.output).not.toContain(unknownEstimate) +}) + +test("DuckDB predicate_effect guards a two-part catalog and table reference", async () => { + const connector = await Registry.get("scratch") + const database = await connector.execute("SELECT current_database()") + const catalog = String(database.rows[0][0]).replaceAll('"', '""') + const execute = spyOn(connector, "execute") + try { + const result = await checkSql({ + warehouse: "scratch", + check: "predicate_effect", + base_sql: `"${catalog}".orders`, + predicates: ["id > 0"], + max_rows_scanned: 2, + }) + expect(result.metadata).toMatchObject({ skipped: "estimate" }) + expect(result.output).toMatch(/narrower.*base_sql/i) + expect(execute).toHaveBeenCalledTimes(1) + } finally { + execute.mockRestore() + } +}) + +test("DuckDB predicate_effect resolves mixed-case schema names on the search path", async () => { + const connector = await Registry.get("scratch") + await connector.execute('CREATE SCHEMA "Foo"') + await connector.execute('CREATE TABLE "Foo"."Orders" AS SELECT * FROM main.orders') + await connector.execute("DROP TABLE main.orders") + await connector.execute("SET schema = 'foo'") + const execute = spyOn(connector, "execute") + try { + const result = await checkSql({ + warehouse: "scratch", + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 0"], + max_rows_scanned: 2, + }) + expect(result.metadata).toMatchObject({ skipped: "estimate" }) + expect(result.output).toMatch(/narrower.*base_sql/i) + expect(execute).toHaveBeenCalledTimes(1) + } finally { + execute.mockRestore() + } +}) + +test("SQLite predicate_effect respects a temporary table shadowing larger main-table statistics", async () => { + const connector = await Registry.get("sqlite_predicates") + await connector.execute("ANALYZE main.orders") + await connector.execute("CREATE TEMP TABLE orders AS SELECT * FROM main.orders LIMIT 1") + const result = await checkSql({ + warehouse: "sqlite_predicates", + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 0"], + max_rows_scanned: 2, + }) + expect(result.metadata).toMatchObject({ total_rows: 1, combined_kept_rows: 1 }) +}) + +test("SQLite predicate_effect defaults the bare-table estimate cap to five million", async () => { + const connector = await Registry.get("sqlite_predicates") + await connector.execute("ANALYZE orders") + await connector.execute("UPDATE sqlite_stat1 SET stat = '5000001' WHERE tbl = 'orders'") + const execute = spyOn(connector, "execute") + try { + const result = await checkSql({ + warehouse: "sqlite_predicates", + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 0"], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 5 }) + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][0]).toContain("LIMIT 5000001") + } finally { + execute.mockRestore() + } +}) + +test("SQLite predicate_effect enforces the bounded estimate without ANALYZE", async () => { + const result = await checkSql({ + warehouse: "sqlite_predicates", + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 2"], + max_rows_scanned: 1, + }) + expect(result.metadata).toMatchObject({ skipped: "estimate" }) + expect(result.output).toContain("2 exceeds max_rows_scanned 1") +}) + +test("SQLite predicate_effect uses the default estimate cap when max_rows_scanned is zero", async () => { + const connector = await Registry.get("sqlite_predicates") + await connector.execute("ANALYZE orders") + await connector.execute("UPDATE sqlite_stat1 SET stat = '5000001' WHERE tbl = 'orders'") + const execute = spyOn(connector, "execute") + try { + const result = await checkSql({ + warehouse: "sqlite_predicates", + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 0"], + max_rows_scanned: 0, + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 5 }) + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][0]).toContain("LIMIT 5000001") + } finally { + execute.mockRestore() + } +}) + +test("predicate_effect returns the existing unsupported-dialect message before connecting", async () => { + Registry.setConfigs({ unsupported: { type: "mysql", host: "unused" } }) + const get = spyOn(Registry, "get") + try { + await expect( + checkSql({ + warehouse: "unsupported", + check: "predicate_effect", + base_sql: "SELECT 1 AS id", + predicates: ["id > 0"], + }), + ).rejects.toThrow("sql_check supports SQLite, DuckDB and Postgres; use sql_materialize") + expect(get).not.toHaveBeenCalled() + } finally { + get.mockRestore() + } +}) + +test("sql_execute denied prevents predicate_effect routing and query execution", async () => { + const connector = await Registry.get("sqlite_predicates") + const execute = spyOn(connector, "execute") + const precedence = spyOn(Precedence, "check") + const asked: { permission: string; patterns: string[] }[] = [] + try { + const tool = await initTool(SqlCheckTool) + const result = await tool.execute( + { warehouse: "sqlite_predicates", check: "predicate_effect", base_sql: baseSql, predicates: ["id > 0"] }, + { + sessionID: "test-session", + messageID: "test-message", + callID: "test-call", + agent: "test", + abort: new AbortController().signal, + messages: [], + metadata: () => {}, + ask: async (request: { permission: string; patterns: string[] }) => { + asked.push(request) + throw new Error("sql_execute denied") + }, + }, + ) + expect(result.output).toContain("sql_execute denied") + expect(asked).toHaveLength(1) + expect(asked[0]).toMatchObject({ permission: "sql_execute" }) + expect(asked[0].patterns.join("\n")).toContain(baseSql) + expect(execute).not.toHaveBeenCalled() + expect(precedence).not.toHaveBeenCalled() + } finally { + execute.mockRestore() + precedence.mockRestore() + } +}) + +/** Run predicate_effect through the tool boundary with a permission ruleset; returns the result and the asks that reached the bus. */ +async function checkWithRules( + input: { base_sql: string; predicates: string[] }, + ruleset: PermissionNext.Rule[], + sessionName: string, +) { + const connector = await Registry.get("sqlite_predicates") + const execute = spyOn(connector, "execute") + const precedence = spyOn(Precedence, "check").mockResolvedValue({}) + const asked: PermissionNext.Request[] = [] + const publish = spyOn(Bus, "publish").mockImplementation(async (event, properties) => { + if (event.type !== PermissionNext.Event.Asked.type) return + const request = properties as PermissionNext.Request + asked.push(request) + // Approving one pattern must never authorize a later deny. + await PermissionNext.reply({ requestID: request.id, reply: "once" }) + }) + try { + const tool = await initTool(SqlCheckTool) + const sessionID = SessionID.make(sessionName) + const result = await tool.execute( + { warehouse: "sqlite_predicates", check: "predicate_effect", ...input }, + { + sessionID, + messageID: "test-message", + agent: "test", + abort: new AbortController().signal, + messages: [], + ask: (request: Omit) => + PermissionNext.ask({ ...request, sessionID, ruleset }), + }, + ) + return { result, asked, executed: execute.mock.calls.length, precedenceChecked: precedence.mock.calls.length } + } finally { + publish.mockRestore() + execute.mockRestore() + precedence.mockRestore() + } +} + +/** Run promise-based tool code inside the fixture's instance (the tool reads the instance through async context). */ +const inInstance = (fn: () => Promise) => + Effect.gen(function* () { + const { directory } = yield* TestInstance + yield* Effect.promise(() => Instance.provide({ directory, fn })) + }) + +it.instance("predicate_effect refuses a later denied predicate before requesting population approval", () => + inInstance(async () => { + const { result, asked, executed, precedenceChecked } = await checkWithRules( + { base_sql: baseSql, predicates: ["id > 0", "amount > 4"] }, + [ + { permission: "sql_execute", pattern: "*", action: "ask" }, + { permission: "sql_execute", pattern: "*amount*", action: "deny" }, + ], + "session_predicate_permissions", + ) + expect(result.metadata.error).toContain("rule which prevents") + expect(asked).toEqual([]) + expect(await PermissionNext.list()).toEqual([]) + expect(precedenceChecked).toBe(0) + expect(executed).toBe(0) + }), +) + +it.instance("predicate_effect authorizes the combined probe so a table-and-column deny rule applies", () => + inInstance(async () => { + const ruleset: PermissionNext.Rule[] = [ + { permission: "sql_execute", pattern: "*", action: "ask" }, + { permission: "sql_execute", pattern: "*amount*FROM*orders*", action: "deny" }, + ] + // Neither part matches the rule on its own; the combined probe (predicate before FROM) does. + const denied = await checkWithRules({ base_sql: baseSql, predicates: ["amount > 4"] }, ruleset, "session_combined_deny") + expect(denied.result.metadata.error).toContain("rule which prevents") + expect(denied.executed).toBe(0) + expect(denied.precedenceChecked).toBe(0) + // The same rule does not touch a probe over another table. + const allowed = await checkWithRules( + { base_sql: "SELECT id, amount FROM orders AS archive", predicates: ["id > 0"] }, + [ + { permission: "sql_execute", pattern: "*", action: "allow" }, + { permission: "sql_execute", pattern: "*amount*FROM*sensitive_table*", action: "deny" }, + ], + "session_combined_allow", + ) + expect(allowed.result.metadata.error).toBeUndefined() + expect(allowed.result.metadata).toMatchObject({ check: "predicate_effect" }) + }), +) + +test("predicate_effect accepts FILTER and OVER clauses in populations and still refuses unknown functions", async () => { + const accepted = await checkSql({ + warehouse: "sqlite_predicates", + check: "predicate_effect", + base_sql: "SELECT id, COUNT(*) FILTER (WHERE amount > 0) OVER (PARTITION BY kind) AS positives FROM orders", + predicates: ["positives >= 0"], + }) + expect(accepted.metadata.error).toBeUndefined() + expect(accepted.metadata).toMatchObject({ check: "predicate_effect" }) + await expect( + checkSql({ warehouse: "sqlite_predicates", check: "predicate_effect", base_sql: baseSql, predicates: ["filterx(id) = 1"] }), + ).rejects.toThrow(/refuses function filterx/) +}) + +test.each([ + "row_number()", + "rank()", + "dense_rank()", + "percent_rank()", + "cume_dist()", + "ntile(2)", + "lag(id)", + "lead(id)", + "first_value(id)", + "last_value(id)", + "nth_value(id, 2)", +])("SQLite predicate_effect accepts general-purpose window function %s", async (expression) => { + const result = await checkSql({ + warehouse: "sqlite_predicates", + check: "predicate_effect", + base_sql: `SELECT *, ${expression} OVER (PARTITION BY kind ORDER BY id) AS rn FROM orders`, + predicates: ["rn > 0"], + }) + expect(result.metadata.total_rows).toBe(5) + if (expression === "row_number()") expect(result.metadata.combined_kept_rows).toBe(5) +}) + +test.each(["[orders]", "`orders`", "main.[orders]", "main.`orders`"])( + "SQLite predicate_effect accepts quoted bare population %s and its table qualifier", + async (base_sql) => { + const table = base_sql.replace(/^main\./, "") + const result = await checkSql({ + warehouse: "sqlite_predicates", + check: "predicate_effect", + base_sql, + predicates: ["id > 2", `${table}.kind = 'a'`], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 2 }) + }, +) + +test.each(["[order details]", "`order details`"])( + "SQLite predicate_effect preserves spaces in quoted bare population %s", + async (base_sql) => { + const connector = await Registry.get("sqlite_predicates") + await connector.execute(`CREATE TABLE ${base_sql} AS SELECT * FROM orders`) + const result = await checkSql({ + warehouse: "sqlite_predicates", + check: "predicate_effect", + base_sql, + predicates: [`${base_sql}.id > 2`], + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + }, +) + +test.each(["AS v(id)", "v(id)", 'AS "v"("id")'])( + "DuckDB predicate_effect accepts derived-table column aliases %s", + async (alias) => { + const result = await checkSql({ + warehouse: "scratch", + check: "predicate_effect", + base_sql: `SELECT id FROM (VALUES (1), (2)) ${alias}`, + predicates: ["id > 1"], + }) + expect(result.metadata).toMatchObject({ total_rows: 2, combined_kept_rows: 1 }) + }, +) + +test.each(["AS v(id)", "v(id)"])( + "SQLite predicate_effect passes derived-table column aliases %s through to the engine", + async (alias) => { + const connector = await Registry.get("sqlite_predicates") + const execute = spyOn(connector, "execute") + try { + // SQLite itself does not support column alias lists; the validator must not misread them as calls. + await expect( + checkSql({ + warehouse: "sqlite_predicates", + check: "predicate_effect", + base_sql: `SELECT id FROM (VALUES (1), (2)) ${alias}`, + predicates: ["id > 1"], + }), + ).rejects.toThrow(/syntax error/i) + expect(execute).toHaveBeenCalledTimes(1) + } finally { + execute.mockRestore() + } + }, +) + +test.each(["AS v(id)", "v(id)"])("DuckDB predicate_effect accepts table-function column aliases %s", async (alias) => { + const result = await checkSql({ + warehouse: "scratch", + check: "predicate_effect", + base_sql: `SELECT id FROM range(1, 3) ${alias}`, + predicates: ["id > 1"], + }) + expect(result.metadata).toMatchObject({ total_rows: 2, combined_kept_rows: 1 }) +}) + +test.each([...warehouses])("predicate_effect still refuses real calls sharing derived alias v on %s", async (warehouse) => { + const connector = await Registry.get(warehouse) + const execute = spyOn(connector, "execute") + try { + for (const alias of ["AS v(id)", "v(id)"]) { + await expect( + checkSql({ + warehouse, + check: "predicate_effect", + base_sql: `SELECT v(1) FROM (VALUES (1), (2)) ${alias}`, + predicates: ["id > 1"], + }), + ).rejects.toThrow(/refuses function v/) + await expect( + checkSql({ + warehouse, + check: "predicate_effect", + base_sql: `SELECT id FROM (VALUES (1), (2)) ${alias}`, + predicates: ["v(1) > 0"], + }), + ).rejects.toThrow(/refuses function v/) + } + expect(execute).not.toHaveBeenCalled() + } finally { + execute.mockRestore() + } +}) + +test("SQLite predicate_effect enables query_only and restores the previous setting after success and failure", async () => { + const connector = await Registry.get("sqlite_predicates") + const guarded = await connector.execute("SELECT query_only FROM pragma_query_only()", undefined, undefined, { + readOnly: true, + }) + expect(guarded.rows).toEqual([[1]]) + const result = await checkSql({ + warehouse: "sqlite_predicates", + check: "predicate_effect", + base_sql: "orders", + predicates: ["id > 0"], + }) + expect(result.metadata.combined_kept_rows).toBe(5) + expect((await connector.execute("PRAGMA query_only")).rows).toEqual([[0]]) + await expect(connector.execute("INSERT INTO orders (id) VALUES (6)", undefined, undefined, { readOnly: true })) + .rejects.toThrow(/readonly/i) + expect((await connector.execute("SELECT COUNT(*) FROM orders")).rows).toEqual([[5]]) + expect((await connector.execute("PRAGMA query_only")).rows).toEqual([[0]]) + await connector.execute("INSERT INTO orders (id) VALUES (6)") + await connector.execute("PRAGMA query_only = ON") + await checkSql({ warehouse: "sqlite_predicates", check: "predicate_effect", base_sql: baseSql, predicates: ["id > 0"] }) + await expect(checkSql({ warehouse: "sqlite_predicates", check: "predicate_effect", base_sql: "SELECT * FROM missing", predicates: ["id > 0"] })) + .rejects.toThrow(/missing/) + expect((await connector.execute("PRAGMA query_only")).rows).toEqual([[1]]) +}) + +test.each([ + { location: "population view", base_sql: "sequence_writer", predicates: ["id > 0"] }, + { location: "predicate subquery", base_sql: baseSql, predicates: ["(SELECT id FROM sequence_writer) > 0"] }, +])("DuckDB predicate_effect blocks hidden sequence writes in $location and restores the session", async ({ base_sql, predicates }) => { + const connector = await Registry.get("scratch") + await connector.execute("CREATE SEQUENCE probe_sequence") + await connector.execute("CREATE VIEW sequence_writer AS SELECT nextval('probe_sequence') AS id") + await expect(checkSql({ warehouse: "scratch", check: "predicate_effect", base_sql, predicates: [...predicates] })) + .rejects.toThrow(/read.only/i) + expect((await connector.execute("SELECT last_value FROM duckdb_sequences() WHERE sequence_name = 'probe_sequence'")).rows).toEqual([[null]]) + await connector.execute("INSERT INTO orders (id) VALUES (6)") + const result = await checkSql({ warehouse: "scratch", check: "predicate_effect", base_sql: baseSql, predicates: ["id > 0"] }) + expect(result.metadata.total_rows).toBe(6) +}) + +test("DuckDB predicate_effect refuses an active transaction without aborting caller changes", async () => { + const connector = await Registry.get("scratch") + await connector.execute("BEGIN TRANSACTION") + try { + await connector.execute("INSERT INTO orders (id) VALUES (6)") + await expect(checkSql({ warehouse: "scratch", check: "predicate_effect", base_sql: baseSql, predicates: ["id > 0"] })) + .rejects.toThrow(/DuckDB.*active transaction/i) + await connector.execute("COMMIT") + expect(Number((await connector.execute("SELECT COUNT(*) FROM orders")).rows[0][0])).toBe(6) + } catch (error) { + await connector.execute("ROLLBACK").catch(() => {}) + throw error + } +}) diff --git a/packages/opencode/test/altimate/tools/sql-check.test.ts b/packages/opencode/test/altimate/tools/sql-check.test.ts index ee02e74997..4ace0bd361 100644 --- a/packages/opencode/test/altimate/tools/sql-check.test.ts +++ b/packages/opencode/test/altimate/tools/sql-check.test.ts @@ -592,6 +592,47 @@ for (const warehouse of ["sqlite_fixture", "scratch"] as const) { expect((await checkSql({ warehouse, check: "grain", sql: leftSql, columns: ["id"] })).metadata.total_rows).toBe(5) }) + for (const [probe, baseSql, query] of [ + ["bounded estimate", "orders", /LIMIT 5000001[\s\S]*SELECT COUNT\(\*\) AS estimate/], + ["predicate query", leftSql, /SELECT COUNT\(\*\) AS total/], + ] as const) { + for (const stateLost of [true, undefined]) { + test(`predicate_effect ${stateLost ? "rethrows" : "skips"} ${probe} timeouts ${stateLost ? "with" : "without"} state loss`, async () => { + const connector = await Registry.get(warehouse) + const error = Object.assign(new Error("query deadline exceeded"), { + name: "QueryTimeoutError", + code: "QUERY_TIMEOUT", + engine: warehouse === "scratch" ? "duckdb" : "sqlite", + timeoutMs: 4321, + cancelled: true, + ...(stateLost ? { stateLost: true } : {}), + }) + const execute = spyOn(connector, "execute").mockRejectedValue(error) + try { + const pending = checkSql({ + warehouse, + check: "predicate_effect", + base_sql: baseSql, + predicates: ["amount > 0"], + query_timeout_ms: 4321, + }) + if (stateLost) { + await expect(pending).rejects.toBe(error) + await expect(pending).rejects.toMatchObject({ stateLost: true }) + } else { + const result = await pending + expect(result.metadata.skipped).toBe("timeout") + expect(result.output).toContain("skipped: timeout") + } + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][0]).toMatch(query) + } finally { + execute.mockRestore() + } + }) + } + } + test("forwards the query timeout and suppresses connector LIMIT injection", async () => { const connector = await Registry.get(warehouse) const execute = spyOn(connector, "execute") @@ -686,6 +727,198 @@ test("refuses cross-warehouse checks and explains the materialization prerequisi ).rejects.toThrow(/sql_materialize/) }) +function emptyPredicateCatalog(sql: string) { + const columns = sql.includes("pg_catalog.pg_operator") + ? ["oprname", "nspname"] + : sql.includes("pg_catalog.pg_proc") + ? ["proname", "nspname"] + : sql.includes("pg_catalog.pg_type") + ? ["typname", "nspname"] + : sql.includes("duckdb_functions()") + ? ["function_name", "schema_name"] + : undefined + return columns ? { columns, rows: [], row_count: 0, truncated: false } : undefined +} + +async function predicateCatalogFixture( + type: string, + catalog: { operators?: string[][]; functions?: string[][]; types?: string[][] } = {}, +) { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute").mockImplementation(async (sql, _limit, binds) => { + const response = emptyPredicateCatalog(sql) + if (response) { + const entries = sql.includes("pg_catalog.pg_operator") + ? catalog.operators + : sql.includes("pg_catalog.pg_type") + ? catalog.types + : catalog.functions + const names = sql.includes("duckdb_functions()") ? binds : (binds?.[0] as string[] | undefined) + const rows = (entries ?? []).filter(([name, schema]) => schema !== "pg_catalog" && names?.includes(name)) + return { ...response, rows, row_count: rows.length } + } + return { + columns: ["total", "keep_1", "keep_all"], + rows: [[1, 1, 1]], + row_count: 1, + truncated: false, + } + }) + const get = spyOn(Registry, "get").mockResolvedValue(connector) + Registry.setConfigs({ catalog_fixture: { type } }) + return { + execute, + restore() { + get.mockRestore() + execute.mockRestore() + }, + } +} + +test("PostgreSQL predicate_effect checks the estimate and population in one statement", async () => { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute").mockImplementation( + async (sql) => + emptyPredicateCatalog(sql) ?? + (sql.includes("COUNT(*)") + ? { + columns: ["total", "keep_1", "keep_all", "estimate", "estimate_exceeded"], + rows: [[5, 3, 3, 5, false]], + row_count: 1, + truncated: false, + } + : { columns: ["estimate"], rows: [[5]], row_count: 1, truncated: false }), + ) + const get = spyOn(Registry, "get").mockResolvedValue(connector) + try { + for (const type of ["postgres", "postgresql"]) { + Registry.setConfigs({ pg_fixture: { type } }) + execute.mockClear() + const result = await checkSql({ + warehouse: "pg_fixture", + check: "predicate_effect", + base_sql: '"orders"', + predicates: ["amount > 0"], + max_rows_scanned: 10, + query_timeout_ms: 4321, + }) + expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 3 }) + expect(execute).toHaveBeenCalledTimes(2) + const [sql, , , options] = execute.mock.calls[1] + expect(sql).toContain("pg_catalog.pg_class") + expect(sql).toContain(`pg_catalog.to_regclass(E'"orders"')`) + expect(sql).toContain("COUNT(*) AS total") + expect(sql).toMatch(/WHERE COALESCE\(\(SELECT estimate FROM \w+\) <= 10, TRUE\)/) + expect(sql).toContain("AS estimate_exceeded") + expect(options).toMatchObject({ readOnly: true, noLimit: true, queryTimeoutMs: 4321 }) + } + } finally { + get.mockRestore() + execute.mockRestore() + } +}) + +test("PostgreSQL predicate_effect skips an exceeded estimate with the population scan gated", async () => { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute").mockImplementation( + async (sql) => + emptyPredicateCatalog(sql) ?? + (sql.includes("COUNT(*)") + ? { + columns: ["total", "keep_1", "keep_all", "estimate", "estimate_exceeded"], + rows: [[0, 0, 0, 1000, true]], + row_count: 1, + truncated: false, + } + : { columns: ["estimate"], rows: [[1000]], row_count: 1, truncated: false }), + ) + const get = spyOn(Registry, "get").mockResolvedValue(connector) + Registry.setConfigs({ pg_fixture: { type: "postgres" } }) + try { + const result = await checkSql({ + warehouse: "pg_fixture", + check: "predicate_effect", + base_sql: "orders", + predicates: ["amount > 0"], + max_rows_scanned: 10, + }) + expect(result.metadata).toMatchObject({ skipped: "estimate" }) + expect(result.metadata.total_rows).toBeUndefined() + expect(result.output).toContain("1000 exceeds max_rows_scanned 10") + expect(execute).toHaveBeenCalledTimes(2) + const sql = execute.mock.calls[1][0] + // The uncorrelated catalog guard belongs below the aggregate, so it can + // reject the population before any row or predicate is evaluated. + expect(sql).toMatch( + /FROM \(\s*SELECT \* FROM orders\s*\) orders\s+WHERE COALESCE\(\(SELECT estimate FROM \w+\) <= 10, TRUE\)/, + ) + expect(sql).toContain("pg_catalog.pg_class") + } finally { + get.mockRestore() + execute.mockRestore() + } +}) + +test("PostgreSQL predicate_effect keeps an unknown atomic estimate and empty population distinct", async () => { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute").mockImplementation( + async (sql) => + emptyPredicateCatalog(sql) ?? { + columns: ["total", "keep_1", "keep_all", "estimate", "estimate_exceeded"], + rows: [[0, 0, 0, null, false]], + row_count: 1, + truncated: false, + }, + ) + const get = spyOn(Registry, "get").mockResolvedValue(connector) + Registry.setConfigs({ pg_fixture: { type: "postgres" } }) + try { + const result = await checkSql({ + warehouse: "pg_fixture", + check: "predicate_effect", + base_sql: "orders", + predicates: ["amount > 0"], + }) + expect(result.metadata).toMatchObject({ total_rows: 0, combined_kept_rows: 0 }) + expect(result.metadata.skipped).toBeUndefined() + expect(result.output).toContain("row_estimate: unknown") + expect(result.output).toContain("empty population") + expect(execute).toHaveBeenCalledTimes(2) + } finally { + get.mockRestore() + execute.mockRestore() + } +}) + +test("PostgreSQL predicate_effect avoids catalog CTE collisions with its population table", async () => { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute").mockImplementation( + async (sql) => + emptyPredicateCatalog(sql) ?? { + columns: ["total", "keep_1", "keep_all", "estimate", "estimate_exceeded"], + rows: [[1, 1, 1, 1, false]], + row_count: 1, + truncated: false, + }, + ) + const get = spyOn(Registry, "get").mockResolvedValue(connector) + Registry.setConfigs({ pg_fixture: { type: "postgres" } }) + try { + await checkSql({ + warehouse: "pg_fixture", + check: "predicate_effect", + base_sql: "__sql_check_0_est", + predicates: ["amount > 0"], + }) + expect(execute).toHaveBeenCalledTimes(2) + expect(execute.mock.calls[1][0]).toMatch(/^WITH __sql_check_1_est AS/) + expect(execute.mock.calls[1][0]).toContain("SELECT * FROM __sql_check_0_est") + } finally { + get.mockRestore() + execute.mockRestore() + } +}) + test("PostgreSQL checks execute with readOnly for postgres and postgresql connectors", async () => { const connector = await Registry.get("sqlite_fixture") const execute = spyOn(connector, "execute").mockResolvedValue({ @@ -755,6 +988,147 @@ test("PostgreSQL checks report a write-performing SELECT as an error under readO } }) +test("predicate_effect refuses emit_wrapper before SQL execution on every filtered dialect", async () => { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute").mockResolvedValue({ + columns: ["total", "keep_1", "keep_all"], + rows: [[1, 1, 1]], + row_count: 1, + truncated: false, + }) + const get = spyOn(Registry, "get").mockResolvedValue(connector) + try { + for (const type of ["postgres", "postgresql", "duckdb", "sqlite"]) { + Registry.setConfigs({ filtered_fixture: { type } }) + await expect( + checkSql({ + warehouse: "filtered_fixture", + check: "predicate_effect", + base_sql: "SELECT 1 AS id", + predicates: ["emit_wrapper(id) IS NOT NULL"], + }), + ).rejects.toThrow(/emit_wrapper.*predicates are limited to built-in pure functions/i) + } + expect(get).not.toHaveBeenCalled() + expect(execute).not.toHaveBeenCalled() + } finally { + get.mockRestore() + execute.mockRestore() + } +}) + +test("predicate_effect refuses qualified, quoted, and pg-prefixed wrappers before SQL execution", async () => { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute").mockResolvedValue({ + columns: ["total", "keep_1", "keep_all"], + rows: [[1, 1, 1]], + row_count: 1, + truncated: false, + }) + const get = spyOn(Registry, "get").mockResolvedValue(connector) + Registry.setConfigs({ pg_fixture: { type: "postgres" } }) + try { + for (const fn of [ + "public.my_fn", + "public.lower", + '"public" /* gap */ . "my_fn"', + '"emit_wrapper"', + '"LOWER"', + '"custom.lower"', + "pg_get_wrapper", + "émit_wrapper", + "width_bucKet", + "wrapper$fn", + ]) { + const pending = checkSql({ + warehouse: "pg_fixture", + check: "predicate_effect", + base_sql: "SELECT 2 AS x", + predicates: [`${fn}(x) IS NOT NULL`], + }) + await expect(pending).rejects.toThrow(/predicates are limited to built-in pure functions/i) + if (fn === "public.my_fn") await expect(pending).rejects.toThrow("public.my_fn") + } + expect(get).not.toHaveBeenCalled() + expect(execute).not.toHaveBeenCalled() + } finally { + get.mockRestore() + execute.mockRestore() + } +}) + +test("predicate_effect accepts coalesce and lower on every filtered dialect", async () => { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute").mockImplementation( + async (sql) => + emptyPredicateCatalog(sql) ?? { + columns: ["total", "keep_1", "keep_2", "keep_all"], + rows: [[1, 1, 1, 1]], + row_count: 1, + truncated: false, + }, + ) + const get = spyOn(Registry, "get").mockResolvedValue(connector) + try { + for (const type of ["postgres", "postgresql", "duckdb", "sqlite"]) { + Registry.setConfigs({ filtered_fixture: { type } }) + const result = await checkSql({ + warehouse: "filtered_fixture", + check: "predicate_effect", + base_sql: "SELECT 2 AS x, 'A' AS name", + predicates: ["coalesce(x, 0) > 1", "lower(name) = 'a'"], + }) + expect(result.metadata).toMatchObject({ total_rows: 1, combined_kept_rows: 1 }) + } + expect(execute.mock.calls.filter(([sql]) => !emptyPredicateCatalog(sql))).toHaveLength(4) + } finally { + get.mockRestore() + execute.mockRestore() + } +}) + +test("predicate_effect accepts scalar built-ins, casts, and boolean grouping", async () => { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute").mockImplementation( + async (sql) => + emptyPredicateCatalog(sql) ?? { + columns: ["total", "keep_1", "keep_all"], + rows: [[1, 1, 1]], + row_count: 1, + truncated: false, + }, + ) + const get = spyOn(Registry, "get").mockResolvedValue(connector) + Registry.setConfigs({ pg_fixture: { type: "postgres" } }) + const predicates = [ + "CAST(x AS numeric(10, 2)) > 0 AND (x IN (1, 2) OR NOT (x = 0))", + "x::numeric(10, 2) > 0 AND CAST(name AS varchar(10)) IS NOT NULL", + "extract(year FROM current_date) > 2000 AND position('a' IN lower(name)) > 0", + "substring(name FROM 1 FOR 1) = 'a' AND trim(name) <> ''", + "pg_typeof(x) IS NOT NULL AND pg_column_size(x) > 0", + "\"lower\"(name) = 'a' AND x = ANY (ARRAY[1, 2])", + "name = 'emit_wrapper(x)' /* public.my_fn(x) */ AND lower(name) <> ''", + "CASE WHEN (x > 0) THEN (abs(x) > 0) ELSE (false) END", + '"emit_wrapper(id)" IS NOT NULL', + "EXISTS (SELECT 1 WHERE (x > 0))", + ] + try { + for (const predicate of predicates) { + const result = await checkSql({ + warehouse: "pg_fixture", + check: "predicate_effect", + base_sql: "SELECT 2 AS x, 'a' AS name", + predicates: [predicate], + }) + expect(result.metadata.total_rows).toBe(1) + } + expect(execute.mock.calls.filter(([sql]) => !emptyPredicateCatalog(sql))).toHaveLength(predicates.length) + } finally { + get.mockRestore() + execute.mockRestore() + } +}) + test("the registered tool reports SQL refusal as an error result", async () => { const tool = await initTool(SqlCheckTool) const result = await tool.execute( @@ -773,3 +1147,684 @@ test("the registered tool reports SQL refusal as an error result", async () => { expect(result.metadata.error).toBeDefined() expect(result.output).toMatch(/read|SELECT/i) }) + +for (const type of ["postgres", "postgresql", "duckdb", "sqlite"]) { + test(`predicate_effect refuses wrapper population functions before execution on ${type}`, async () => { + const fixture = await predicateCatalogFixture(type) + try { + await expect( + checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: "SELECT emit_wrapper(id) AS id FROM orders", + predicates: ["id > 0"], + }), + ).rejects.toThrow(/emit_wrapper.*built-in pure functions/i) + expect(fixture.execute).not.toHaveBeenCalled() + } finally { + fixture.restore() + } + }) +} + +test("predicate_effect refuses DuckDB postgres_execute population before execution", async () => { + const fixture = await predicateCatalogFixture("duckdb") + try { + await expect( + checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: "SELECT * FROM postgres_execute('remote', 'DELETE FROM orders', use_transaction=false)", + predicates: ["id > 0"], + }), + ).rejects.toThrow(/postgres_execute.*built-in pure functions/i) + expect(fixture.execute).not.toHaveBeenCalled() + } finally { + fixture.restore() + } +}) + +for (const field of ["predicate", "population"] as const) { + test(`PostgreSQL predicate_effect refuses a user-defined ${field} operator before the probe`, async () => { + const fixture = await predicateCatalogFixture("postgres", { operators: [["##", "public"]] }) + try { + await expect( + checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: field === "population" ? "SELECT id ## 0 AS id FROM orders" : "SELECT 1 AS id", + predicates: [field === "predicate" ? "id ## 0" : "id > 0"], + query_timeout_ms: 4321, + }), + ).rejects.toThrow(new RegExp(`${field} uses operator .##., which is not a built-in operator`)) + const calls = fixture.execute.mock.calls + const catalogs = calls.filter(([sql]) => sql.includes("pg_catalog.pg_operator")) + expect(catalogs).toHaveLength(1) + expect(calls.every(([sql]) => !!emptyPredicateCatalog(sql))).toBe(true) + const [sql, limit, binds, options] = catalogs[0] + expect(sql).toContain("pg_catalog.pg_namespace") + expect(sql).toMatch(/oprname\s*=\s*ANY\(\$1\)/) + expect(sql).toMatch(/(?:nspname\s*<>\s*'pg_catalog'|oprnamespace\s*<>\s*'pg_catalog'::regnamespace)/) + expect(limit).toBeUndefined() + expect(binds).toEqual([expect.arrayContaining(["##"])]) + expect(options).toMatchObject({ readOnly: true, noLimit: true, queryTimeoutMs: 4321 }) + } finally { + fixture.restore() + } + }) +} + +for (const field of ["predicate", "population"] as const) { + for (const cast of ["x::custom_type", "CAST(x AS custom_type)"]) { + test(`PostgreSQL predicate_effect refuses ${field} cast ${cast} before the probe`, async () => { + const fixture = await predicateCatalogFixture("postgres", { types: [["custom_type", "public"]] }) + try { + await expect( + checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: field === "population" ? `SELECT ${cast} AS x FROM orders` : "SELECT 1 AS x", + predicates: [field === "predicate" ? `${cast} > 0` : "x > 0"], + query_timeout_ms: 4321, + }), + ).rejects.toThrow(`${field} casts to \`custom_type\`, which is not a built-in type`) + const calls = fixture.execute.mock.calls + expect(calls.every(([sql]) => !!emptyPredicateCatalog(sql))).toBe(true) + const catalogs = calls.filter(([sql]) => sql.includes("pg_catalog.pg_type")) + expect(catalogs).toHaveLength(1) + const [sql, limit, binds, options] = catalogs[0] + expect(sql).toContain("pg_catalog.pg_namespace") + expect(sql).toMatch(/typname\s*=\s*ANY\(\$1\)/) + expect(sql).toMatch(/(?:nspname\s*<>\s*'pg_catalog'|typnamespace\s*<>\s*'pg_catalog'::regnamespace)/) + expect(limit).toBeUndefined() + expect(binds).toEqual([["custom_type"]]) + expect(options).toMatchObject({ readOnly: true, noLimit: true, queryTimeoutMs: 4321 }) + } finally { + fixture.restore() + } + }) + } +} + +test("PostgreSQL predicate_effect accepts built-in cast targets from population and predicates", async () => { + const fixture = await predicateCatalogFixture("postgres", { types: [["integer", "pg_catalog"]] }) + try { + const result = await checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: "SELECT CAST(id AS integer) AS x FROM orders", + predicates: ["x::integer > 0 AND CAST(x AS integer) > 0"], + query_timeout_ms: 4321, + }) + expect(result.metadata.total_rows).toBe(1) + const calls = fixture.execute.mock.calls + const catalogs = calls.filter(([sql]) => sql.includes("pg_catalog.pg_type")) + expect(catalogs).toHaveLength(1) + expect(catalogs[0][2]).toEqual([["integer"]]) + expect(catalogs[0][3]).toMatchObject({ readOnly: true, noLimit: true, queryTimeoutMs: 4321 }) + expect(calls.slice(0, -1).every(([sql]) => !!emptyPredicateCatalog(sql))).toBe(true) + expect(calls.at(-1)![0]).toContain("COUNT(*) AS total") + } finally { + fixture.restore() + } +}) + +test("PostgreSQL predicate_effect resolves qualified, quoted, array, nested, and typmod cast targets", async () => { + const fixture = await predicateCatalogFixture("postgres", { + types: [ + ["custom_type", "public"], + ['custom"type', "public"], + ], + }) + const casts = [ + ["x::public.custom_type", "custom_type", ["custom_type"]], + ['CAST(x AS "public"."custom""type")', 'custom"type', ['custom"type']], + ["x::custom_type[]", "custom_type", ["custom_type"]], + ["x::custom_type::integer", "custom_type", ["custom_type", "integer"]], + ["CAST(CAST(x AS custom_type) AS integer)", "custom_type", ["custom_type", "integer"]], + ["CAST(x AS custom_type(12))", "custom_type", ["custom_type"]], + ] as const + try { + for (const [cast, target, names] of casts) { + fixture.execute.mockClear() + await expect( + checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: "SELECT 1 AS x", + predicates: [`${cast} IS NOT NULL`], + }), + ).rejects.toThrow(`predicate casts to \`${target}\`, which is not a built-in type`) + const calls = fixture.execute.mock.calls + expect(calls.every(([sql]) => !!emptyPredicateCatalog(sql))).toBe(true) + const catalogs = calls.filter(([sql]) => sql.includes("pg_catalog.pg_type")) + expect(catalogs).toHaveLength(1) + expect(catalogs[0][2]?.[0]).toEqual(expect.arrayContaining([...names])) + expect(catalogs[0][2]?.[0]).toHaveLength(names.length) + } + } finally { + fixture.restore() + } +}) + +test("PostgreSQL predicate_effect ignores cast syntax inside literals, comments, and column identifiers", async () => { + const fixture = await predicateCatalogFixture("postgres", { types: [["custom_type", "public"]] }) + try { + const result = await checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: "SELECT 1 AS x, 'x::custom_type' AS note, 1 AS \"x::custom_type\"", + predicates: ["x > 0 /* CAST(x AS custom_type) */ AND note <> 'CAST(x AS custom_type)' -- x::custom_type\n"], + }) + expect(result.metadata.total_rows).toBe(1) + expect(fixture.execute.mock.calls.some(([sql]) => sql.includes("pg_catalog.pg_type"))).toBe(false) + expect(fixture.execute.mock.calls.at(-1)![0]).toContain("COUNT(*) AS total") + } finally { + fixture.restore() + } +}) + +for (const [description, cast] of [ + ["default", String.raw`x::U&"custom_\0074ype"`], + ["custom", String.raw`x::U&"custom_!0074ype" UESCAPE '!'`], +]) { + test(`PostgreSQL predicate_effect refuses ${description} Unicode escaped cast targets before the probe`, async () => { + const fixture = await predicateCatalogFixture("postgres", { types: [["custom_type", "public"]] }) + try { + await expect( + checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: "SELECT 1 AS x", + predicates: [`${cast} IS NOT NULL`], + }), + ).rejects.toThrow( + description === "custom" + ? "sql_check only accepts a single read-only SELECT" + : "predicate casts to `custom_type`, which is not a built-in type", + ) + const calls = fixture.execute.mock.calls + expect(calls.every(([sql]) => !!emptyPredicateCatalog(sql))).toBe(true) + // The structural parser already refuses UESCAPE; ordinary Unicode names need the type catalog. + if (description === "custom") expect(calls).toHaveLength(0) + else expect(calls.find(([sql]) => sql.includes("pg_catalog.pg_type"))?.[2]).toEqual([["custom_type"]]) + } finally { + fixture.restore() + } + }) +} + +test("PostgreSQL predicate_effect accepts CTE column lists in populations", async () => { + const fixture = await predicateCatalogFixture("postgres") + const populations = [ + "WITH population(id) AS (SELECT 1) SELECT id FROM population", + 'WITH first(id) AS (SELECT 1), "population"(id) AS (SELECT id FROM first) SELECT id FROM "population"', + "WITH RECURSIVE population(id) AS (SELECT 1) SELECT id FROM population", + "WITH population(id) AS MATERIALIZED (SELECT 1) SELECT id FROM population", + ] + try { + for (const base_sql of populations) { + const result = await checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql, + predicates: ["id > 0"], + }) + expect(result.metadata.total_rows).toBe(1) + } + expect(fixture.execute.mock.calls.filter(([sql]) => !emptyPredicateCatalog(sql))).toHaveLength(populations.length) + expect(fixture.execute.mock.calls.some(([sql]) => sql.includes("pg_catalog.pg_proc"))).toBe(false) + } finally { + fixture.restore() + } +}) + +test("PostgreSQL predicate_effect refuses actual function calls that share a CTE declaration name", async () => { + const fixture = await predicateCatalogFixture("postgres") + try { + for (const base_sql of [ + "WITH population(id) AS (SELECT population(1)) SELECT id FROM population", + "WITH population(id) AS (SELECT 1) SELECT population(id) FROM population", + ]) { + await expect( + checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql, + predicates: ["id > 0"], + }), + ).rejects.toThrow(/refuses function population.*built-in pure functions/) + } + expect(fixture.execute).not.toHaveBeenCalled() + } finally { + fixture.restore() + } +}) + +test("PostgreSQL predicate_effect excludes projection and count wildcards from the operator catalog", async () => { + const fixture = await predicateCatalogFixture("postgres", { operators: [["*", "public"]] }) + const populations = [ + "SELECT * FROM orders", + "SELECT t.* FROM orders t", + "SELECT id, * FROM orders", + "SELECT count(*) AS id FROM orders", + "SELECT ALL * FROM orders", + "SELECT DISTINCT * FROM orders", + "SELECT DISTINCT ON (id) * FROM orders", + "SELECT count(/* rows */ *) AS id FROM orders", + ] + try { + for (const base_sql of populations) { + const result = await checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql, + predicates: ["id > 0"], + }) + expect(result.metadata.total_rows).toBe(1) + } + const calls = fixture.execute.mock.calls + expect(calls.filter(([sql]) => !emptyPredicateCatalog(sql))).toHaveLength(populations.length) + const catalogs = calls.filter(([sql]) => sql.includes("pg_catalog.pg_operator")) + expect(catalogs).toHaveLength(populations.length) + for (const [, , binds] of catalogs) expect(binds).toEqual([[">"]]) + } finally { + fixture.restore() + } +}) + +test("PostgreSQL predicate_effect still checks multiplication beside projection wildcards", async () => { + const fixture = await predicateCatalogFixture("postgres", { operators: [["*", "public"]] }) + try { + for (const base_sql of ["SELECT *, a * b AS id FROM orders", "SELECT 1.*2 AS id"]) { + await expect( + checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql, + predicates: ["id > 0"], + }), + ).rejects.toThrow("population uses operator `*`, which is not a built-in operator") + } + const calls = fixture.execute.mock.calls + expect(calls.every(([sql]) => !!emptyPredicateCatalog(sql))).toBe(true) + const catalogs = calls.filter(([sql]) => sql.includes("pg_catalog.pg_operator")) + expect(catalogs).toHaveLength(2) + for (const [, , binds] of catalogs) expect(binds).toEqual([expect.arrayContaining(["*", ">"])]) + } finally { + fixture.restore() + } +}) + +test("PostgreSQL predicate_effect accepts built-in-only operator names after one read-only catalog query", async () => { + const fixture = await predicateCatalogFixture("postgres", { operators: [["##", "pg_catalog"]] }) + try { + const result = await checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: "SELECT 1 AS id", + predicates: ["id ## 0"], + }) + expect(result.metadata.total_rows).toBe(1) + expect(fixture.execute).toHaveBeenCalledTimes(2) + const [catalog, probe] = fixture.execute.mock.calls + expect(catalog[0]).toContain("pg_catalog.pg_operator") + expect(catalog[2]).toEqual([["##"]]) + expect(catalog[3]).toMatchObject({ readOnly: true }) + expect(probe[0]).toContain("COUNT(*) AS total") + expect(probe[3]).toMatchObject({ readOnly: true }) + } finally { + fixture.restore() + } +}) + +test("PostgreSQL predicate_effect masks non-code operator text and deduplicates catalog names", async () => { + const fixture = await predicateCatalogFixture("postgres") + try { + await checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: "SELECT 1 + 1 AS id, '##' AS name, 1 AS \"@@\"", + predicates: ["id >= 1 AND id >= 0 AND name = '~~' AND \"@@\" = 1 /* !! */ -- ??\n"], + }) + const catalogs = fixture.execute.mock.calls.filter(([sql]) => sql.includes("pg_catalog.pg_operator")) + expect(catalogs).toHaveLength(1) + expect(catalogs[0][2]?.[0]).toEqual(expect.arrayContaining(["+", ">=", "="])) + expect(catalogs[0][2]?.[0]).toHaveLength(3) + expect(fixture.execute.mock.calls.at(-1)![0]).toContain("COUNT(*) AS total") + } finally { + fixture.restore() + } +}) + +for (const field of ["predicate", "population"] as const) { + test(`PostgreSQL predicate_effect refuses a user-defined ${field} overload before the probe`, async () => { + const fixture = await predicateCatalogFixture("postgres", { functions: [["lower", "public"]] }) + try { + await expect( + checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: field === "population" ? "SELECT lower(id) AS id FROM orders" : "SELECT 1 AS id", + predicates: [field === "predicate" ? "lower(id) = 'x'" : "id = 'x'"], + query_timeout_ms: 4321, + }), + ).rejects.toThrow(new RegExp(`${field} calls .lower., which has a user-defined overload in schema .public.`)) + const calls = fixture.execute.mock.calls + expect(calls.every(([sql]) => !!emptyPredicateCatalog(sql))).toBe(true) + const catalogs = calls.filter(([sql]) => sql.includes("pg_catalog.pg_proc")) + expect(catalogs).toHaveLength(1) + const [sql, limit, binds, options] = catalogs[0] + expect(sql).toContain("pg_catalog.pg_namespace") + expect(sql).toMatch(/proname\s*=\s*ANY\(\$1\)/) + expect(sql).toMatch(/pronamespace\s*<>\s*'pg_catalog'::regnamespace/) + expect(limit).toBeUndefined() + expect(binds).toEqual([["lower"]]) + expect(options).toMatchObject({ readOnly: true, noLimit: true, queryTimeoutMs: 4321 }) + } finally { + fixture.restore() + } + }) +} + +test("PostgreSQL predicate_effect accepts lower after read-only function and operator catalogs before the probe", async () => { + const fixture = await predicateCatalogFixture("postgres", { functions: [["lower", "pg_catalog"]] }) + try { + const result = await checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: "SELECT lower('X') AS id", + predicates: ["lower(id) = 'x' AND lower(id) = 'x'"], + query_timeout_ms: 4321, + }) + expect(result.metadata.total_rows).toBe(1) + expect(fixture.execute).toHaveBeenCalledTimes(3) + const calls = fixture.execute.mock.calls + expect(calls.slice(0, -1).every(([sql]) => !!emptyPredicateCatalog(sql))).toBe(true) + expect(calls.at(-1)![0]).toContain("COUNT(*) AS total") + expect(calls.find(([sql]) => sql.includes("pg_catalog.pg_proc"))![2]).toEqual([["lower"]]) + expect(calls.find(([sql]) => sql.includes("pg_catalog.pg_operator"))![2]).toEqual([["="]]) + for (const [, , , options] of calls) + expect(options).toMatchObject({ readOnly: true, noLimit: true, queryTimeoutMs: 4321 }) + } finally { + fixture.restore() + } +}) + +for (const field of ["predicate", "population"] as const) { + test(`DuckDB predicate_effect refuses a user-defined ${field} macro before the probe`, async () => { + const fixture = await predicateCatalogFixture("duckdb", { functions: [["lower", "main"]] }) + try { + await expect( + checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: field === "population" ? "SELECT lower(id) AS id FROM orders" : "SELECT 1 AS id", + predicates: [field === "predicate" ? "lower(id) = 'x'" : "id = 'x'"], + query_timeout_ms: 4321, + }), + ).rejects.toThrow(new RegExp(`${field} calls .lower., which has a user-defined overload in schema .main.`)) + expect(fixture.execute).toHaveBeenCalledTimes(1) + const [sql, limit, binds, options] = fixture.execute.mock.calls[0] + expect(sql).toContain("duckdb_functions()") + expect(sql).toMatch(/internal\s*=\s*false/i) + expect(binds).toEqual(["lower"]) + expect(limit).toBeUndefined() + expect(options).toMatchObject({ readOnly: true, noLimit: true, queryTimeoutMs: 4321 }) + } finally { + fixture.restore() + } + }) +} + +for (const warehouse of ["sqlite_fixture", "scratch"] as const) { + const dialect = warehouse === "scratch" ? "DuckDB" : "SQLite" + for (const cap of [10, 2]) { + test(`${dialect} predicate_effect bounds the estimate and aggregate in one execute call${cap === 2 ? " and skips an exceeded cap" : ""}`, async () => { + const connector = await Registry.get(warehouse) + const execute = spyOn(connector, "execute") + try { + const result = await checkSql({ + warehouse, + check: "predicate_effect", + base_sql: "orders", + predicates: ["amount > 0"], + max_rows_scanned: cap, + }) + if (cap === 2) { + expect(result.metadata).toMatchObject({ skipped: "estimate" }) + expect(result.metadata.total_rows).toBeUndefined() + expect(result.output).toContain("3 exceeds max_rows_scanned 2") + } else expect(result.metadata).toMatchObject({ total_rows: 5, combined_kept_rows: 4 }) + expect(execute).toHaveBeenCalledTimes(1) + const [sql, , , options] = execute.mock.calls[0] + expect(sql).toMatch(new RegExp(`AS MATERIALIZED \\(\\s*SELECT \\* FROM orders LIMIT ${cap + 1}\\s*\\)`)) + expect(sql).toMatch(/SELECT COUNT\(\*\) AS estimate FROM \w+/) + expect(sql).toContain("COUNT(*) AS total") + expect(sql).toContain("AS estimate_exceeded") + expect(options).toMatchObject({ readOnly: true, noLimit: true }) + } finally { + execute.mockRestore() + } + }) + } +} + +test("SQLite predicate_effect accepts strftime without a function catalog lookup", async () => { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute") + try { + const result = await checkSql({ + warehouse: "sqlite_fixture", + check: "predicate_effect", + base_sql: "SELECT '2024-01-01' AS created_at UNION ALL SELECT '2023-12-31'", + predicates: ["strftime('%Y', created_at) = '2024'"], + }) + expect(result.metadata).toMatchObject({ total_rows: 2, combined_kept_rows: 1 }) + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][0]).toContain("COUNT(*) AS total") + } finally { + execute.mockRestore() + } +}) + +test("predicate_effect accepts pure date built-ins and the bare array constructor", async () => { + const fixture = await predicateCatalogFixture("sqlite") + try { + for (const predicate of [ + "strftime('%Y', created_at) = '2024'", + "julianday(created_at) > 0", + "unixepoch(created_at) > 0", + "date(created_at) IS NOT NULL", + "time(created_at) IS NOT NULL", + "datetime(created_at) IS NOT NULL", + "timediff(created_at, created_at) IS NOT NULL", + "array(SELECT 1) IS NOT NULL", + ]) { + const result = await checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: "SELECT '2024-01-01' AS created_at", + predicates: [predicate], + }) + expect(result.metadata.total_rows).toBe(1) + } + expect(fixture.execute).toHaveBeenCalledTimes(8) + } finally { + fixture.restore() + } +}) + +test("PostgreSQL predicate_effect refuses a wrapper inside an array constructor before execution", async () => { + const fixture = await predicateCatalogFixture("postgres") + try { + await expect( + checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: "SELECT 1 AS id", + predicates: ["ARRAY[emit_wrapper(id)] IS NOT NULL"], + }), + ).rejects.toThrow(/emit_wrapper.*built-in pure functions/i) + expect(fixture.execute).not.toHaveBeenCalled() + } finally { + fixture.restore() + } +}) + +for (const [description, operator, predicate] of [ + ["array", "##", "ARRAY[id ## 0] IS NOT NULL"], + ["backtick", "`", "id ` 0"], +]) { + test(`PostgreSQL predicate_effect resolves a user-defined ${description} operator before execution`, async () => { + const fixture = await predicateCatalogFixture("postgres", { operators: [[operator, "public"]] }) + try { + await expect( + checkSql({ + warehouse: "catalog_fixture", + check: "predicate_effect", + base_sql: "SELECT 1 AS id", + predicates: [predicate], + }), + ).rejects.toThrow(/uses operator .*not a built-in operator/) + expect(fixture.execute.mock.calls.every(([sql]) => !!emptyPredicateCatalog(sql))).toBe(true) + expect(fixture.execute.mock.calls.find(([sql]) => sql.includes("pg_catalog.pg_operator"))?.[2]).toEqual([ + [operator], + ]) + } finally { + fixture.restore() + } + }) +} + +test("DuckDB predicate_effect refuses a mixed-case macro from the live function catalog", async () => { + const connector = await Registry.get("scratch") + await connector.execute("CREATE MACRO LoWeR(value) AS value") + const execute = spyOn(connector, "execute") + try { + await expect( + checkSql({ + warehouse: "scratch", + check: "predicate_effect", + base_sql: "SELECT 'X' AS id", + predicates: ["lower(id) = 'x'"], + }), + ).rejects.toThrow(/predicate calls .lower., which has a user-defined overload in schema .main./i) + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][0]).toContain("duckdb_functions()") + expect(execute.mock.calls[0][0]).toContain("COLLATE NOCASE") + expect(execute.mock.calls[0][2]).toEqual(["lower"]) + expect(execute.mock.calls[0][3]).toMatchObject({ readOnly: true }) + } finally { + execute.mockRestore() + } +}) + +for (const [run, predicate] of [ + ["=-", "id=-1"], + [">+-", "id>+-1"], +]) { + test(`PostgreSQL predicate_effect checks overloads after splitting trailing signs from ${run}`, async () => { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute").mockImplementation(async (sql, _limit, binds) => { + const catalog = emptyPredicateCatalog(sql) + if (catalog) { + const names = binds?.[0] as string[] | undefined + const name = run[0] + const rows = sql.includes("pg_catalog.pg_operator") && names?.includes(name) ? [[name, "public"]] : [] + return { ...catalog, rows, row_count: rows.length } + } + return { columns: ["total", "keep_1", "keep_all"], rows: [[1, 1, 1]], row_count: 1, truncated: false } + }) + const get = spyOn(Registry, "get").mockResolvedValue(connector) + Registry.setConfigs({ pg_fixture: { type: "postgres" } }) + try { + await expect( + checkSql({ + warehouse: "pg_fixture", + check: "predicate_effect", + base_sql: "SELECT 1 AS id", + predicates: [predicate], + }), + ).rejects.toThrow(/predicate uses operator .*not a built-in operator/) + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][0]).toContain("pg_catalog.pg_operator") + expect(execute.mock.calls[0][2]?.[0]).toEqual(expect.arrayContaining([run[0], "-"])) + expect(execute.mock.calls[0][3]).toMatchObject({ readOnly: true }) + } finally { + get.mockRestore() + execute.mockRestore() + } + }) +} + +test("PostgreSQL predicate_effect checks the normalized <> overload for !=", async () => { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute").mockImplementation(async (sql, _limit, binds) => { + const catalog = emptyPredicateCatalog(sql) + if (catalog) { + const names = binds?.[0] as string[] | undefined + const rows = sql.includes("pg_catalog.pg_operator") && names?.includes("<>") ? [["<>", "public"]] : [] + return { ...catalog, rows, row_count: rows.length } + } + return { columns: ["total", "keep_1", "keep_all"], rows: [[1, 1, 1]], row_count: 1, truncated: false } + }) + const get = spyOn(Registry, "get").mockResolvedValue(connector) + Registry.setConfigs({ pg_fixture: { type: "postgres" } }) + try { + await expect( + checkSql({ + warehouse: "pg_fixture", + check: "predicate_effect", + base_sql: "SELECT 1 AS id", + predicates: ["id != 0"], + }), + ).rejects.toThrow(/predicate uses operator .<>., which is not a built-in operator/) + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][0]).toContain("pg_catalog.pg_operator") + expect(execute.mock.calls[0][2]?.[0]).toEqual(expect.arrayContaining(["<>"])) + expect(execute.mock.calls[0][3]).toMatchObject({ readOnly: true }) + } finally { + get.mockRestore() + execute.mockRestore() + } +}) + +for (const [predicate, operator] of [ + ["id LIKE 'x'", "~~"], + ["id IN (0, 1)", "="], + ["id BETWEEN 0 AND 1", ">="], + ["id IS DISTINCT FROM 0", "="], +]) { + test(`PostgreSQL predicate_effect resolves implicit operator ${operator} in ${predicate}`, async () => { + const connector = await Registry.get("sqlite_fixture") + const execute = spyOn(connector, "execute").mockImplementation(async (sql, _limit, binds) => { + const catalog = emptyPredicateCatalog(sql) + if (catalog) { + const names = binds?.[0] as string[] | undefined + const rows = sql.includes("pg_catalog.pg_operator") && names?.includes(operator) ? [[operator, "public"]] : [] + return { ...catalog, rows, row_count: rows.length } + } + return { columns: ["total", "keep_1", "keep_all"], rows: [[1, 1, 1]], row_count: 1, truncated: false } + }) + const get = spyOn(Registry, "get").mockResolvedValue(connector) + Registry.setConfigs({ pg_fixture: { type: "postgres" } }) + try { + await expect( + checkSql({ + warehouse: "pg_fixture", + check: "predicate_effect", + base_sql: "SELECT 1 AS id", + predicates: [predicate], + }), + ).rejects.toThrow(/predicate uses operator .*not a built-in operator/) + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][0]).toContain("pg_catalog.pg_operator") + expect(execute.mock.calls[0][2]?.[0]).toEqual(expect.arrayContaining([operator])) + expect(execute.mock.calls[0][3]).toMatchObject({ readOnly: true }) + } finally { + get.mockRestore() + execute.mockRestore() + } + }) +} diff --git a/packages/opencode/test/config/config.test.ts b/packages/opencode/test/config/config.test.ts index cd2a3a9d6d..f3130b6c98 100644 --- a/packages/opencode/test/config/config.test.ts +++ b/packages/opencode/test/config/config.test.ts @@ -308,6 +308,34 @@ it.instance("loads config with defaults when no files exist", () => }), ) +// altimate_change start — predicate-effect execution requires an explicit config opt-in +describe("sql_check.predicate_effect config", () => { + it.instance("leaves predicate_effect disabled when sql_check config is unset", () => + Effect.gen(function* () { + const config = yield* Config.use.get() + expect(config.sql_check?.predicate_effect ?? false).toBe(false) + }), + ) + + for (const enabled of [true, false]) { + it.instance( + `loads sql_check.predicate_effect: ${enabled} from project config`, + Effect.gen(function* () { + const config = yield* Config.use.get() + expect(config.sql_check?.predicate_effect).toBe(enabled) + }), + { config: { sql_check: { predicate_effect: enabled } } }, + ) + } + + test("rejects non-boolean sql_check.predicate_effect config", () => { + expect(() => + ConfigParse.schema(ConfigV1.Info, { sql_check: { predicate_effect: "true" } }, "test:config"), + ).toThrow() + }) +}) +// altimate_change end + it.instance("falls back to generic username when system user info is unavailable", () => Effect.gen(function* () { const userInfo = spyOn(os, "userInfo").mockImplementation(() => {