diff --git a/CHANGELOG.md b/CHANGELOG.md index 26b8c46..2dc4f45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,33 @@ +## [0.8.0] — 2026-10-21 (Stage 3: tree-sitter symbols + watchRoots + structured search) + +### Added +- **tree-sitter WASM symbol extraction** — `extractSymbolsTree`/`symbolsFor` parse Swift / + TypeScript / TSX / JS / Python via `web-tree-sitter` (0.25.4) + the `tree-sitter-wasms` pack. + Grammar policy is CALLER-supplied (`SymbolGrammars`: ext → pack name or absolute .wasm path); + `defaultSymbolGrammars()` ships butler's preset. No grammar configured → v1 TS/JS regex + fallback; grammar load failure → empty result, never fatal. + **Vendored `grammars/tree-sitter-swift.wasm`** (from tree-sitter-wasms@0.1.8): the pack's + swift build ≥ 0.1.10 aborts Node ≥ 24 (`Fatal process out of memory: Zone`, V8 turboshaft + wasm tier-up — background thread, process-fatal; `--liftoff-only` avoids it but is banned in + NODE_OPTIONS). See grammars/README.md; re-extract when upstream fixes it. +- **Per-symbol chunks (F1)** — `NormalizedDocument.chunks?` passthrough: ingest stores + pre-chunked docs verbatim. `repoConnector(symbols: true)` now emits one chunk per symbol with + `heading_path "symbol: ()"` (v1 concatenated all symbols into doc content, + unretrievable by heading). Doc content stays a browsable summary. +- **`repoDocumentFor(cfg, absPath)`** (F3) — per-file normalize (file doc + symbol doc) honoring + the same skip rules; the freshness watcher path re-ingests one changed file without paying the + walk. `symbolDocFor` exported alongside. +- **`watchRoots(roots, opts, onBatch)`** — debounced changed-path batches, one OS watch per root + (no polling), caller-supplied ignore policy, `healthy()`/`describe()` surface; watcher errors + mark the root dead, never throw into the caller's loop. +- **`EsCompatRetriever.search({ query, headingPrefix?, size })`** (F2) — structured search: + BM25 + optional heading_path prefix filter (match_phrase_prefix over the analyzed field). + Same throw contract as `retrieve()`. +- Hermetic tests: WASM extraction per language, per-file normalize, chunks passthrough + + idempotence, watcher batches/ignore/health (real fs events, tmp dirs), structured search + (fake endpoint). Full suite 226 passed; `node:module` + `web-tree-sitter` added to the + boundary allowlist. + ## [0.7.0] — 2026-10-21 (Stage 2b: ES-compatible retrieval + xerj sidecar runner) ### Added diff --git a/grammars/README.md b/grammars/README.md new file mode 100644 index 0000000..bbfe31f --- /dev/null +++ b/grammars/README.md @@ -0,0 +1,22 @@ +# Vendored grammar WASMs + +## tree-sitter-swift.wasm — from tree-sitter-wasms@0.1.8 + +The swift grammar shipped in tree-sitter-wasms >= 0.1.10 (identical bytes through +0.1.13) aborts Node >= 24 during background wasm tier-up: + + # Fatal process out of memory: Zone + v8::internal::compiler::turboshaft::WasmLoweringPhase + +The parse itself succeeds; the crash fires in the background compile thread, so +any process that loads the module eventually dies (observed on Node 24.12, +web-tree-sitter 0.25.x, macOS arm64; `node --liftoff-only` avoids it but is not +allowed in NODE_OPTIONS). The 0.1.8 build (older, smaller swift grammar) tier-ups +cleanly and extracts the same declaration set. tree-sitter/typescript/tsx/python +from the current pack are unaffected and resolve from `tree-sitter-wasms` at +runtime. + +Re-extract when the upstream pack ships a fixed swift build: + npm pack tree-sitter-wasms@ && tar -xzf … package/out/tree-sitter-swift.wasm +then re-run test/tree-sitter-wasm.test.ts WITHOUT --liftoff-only to prove the +crash is gone before deleting this vendored copy. diff --git a/grammars/tree-sitter-swift.wasm b/grammars/tree-sitter-swift.wasm new file mode 100755 index 0000000..5e18be5 Binary files /dev/null and b/grammars/tree-sitter-swift.wasm differ diff --git a/package.json b/package.json index e492880..a842036 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,8 @@ { "name": "@mainactordev/butler", - "version": "0.7.0", + "version": "0.8.0", "private": true, - "description": "Butler \u2014 reusable agent runtime: provider, loop, tool registry, layered memory, source ingestion, conversation API + MCP transports (plan 20261001T035356Z)", + "description": "Butler — reusable agent runtime: provider, loop, tool registry, layered memory, source ingestion, conversation API + MCP transports (plan 20261001T035356Z)", "type": "module", "main": "src/index.ts", "exports": { @@ -18,9 +18,11 @@ "@fastify/websocket": "^11.3.1", "@modelcontextprotocol/fastify": "^2.0.0", "@modelcontextprotocol/sdk": "^1.31.0", + "better-sqlite3": "^13.0.3", "pg": "^8.23.0", - "zod": "^4.6.5", - "better-sqlite3": "^13.0.3" + "tree-sitter-wasms": "0.1.13", + "web-tree-sitter": "0.25.4", + "zod": "^4.6.5" }, "devDependencies": { "@types/better-sqlite3": "^7.6.13", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ccf8a5e..6afec2a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,6 +26,12 @@ importers: pg: specifier: ^8.23.0 version: 8.23.0 + tree-sitter-wasms: + specifier: 0.1.13 + version: 0.1.13 + web-tree-sitter: + specifier: 0.25.4 + version: 0.25.4 zod: specifier: ^4.6.5 version: 4.6.5 @@ -1469,6 +1475,9 @@ packages: resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} engines: {node: '>=0.6'} + tree-sitter-wasms@0.1.13: + resolution: {integrity: sha512-wT+cR6DwaIz80/vho3AvSF0N4txuNx/5bcRKoXouOfClpxh/qqrF4URNLQXbbt8MaAxeksZcZd1j8gcGjc+QxQ==} + tsx@4.23.15: resolution: {integrity: sha512-Yiex1Ovn8z2xPpOWckIiysV1SSyRMY9BkLF++q0yKiDxCqRhosKfMg3janKkiLBwZ5c/YryloKwGZcrEmtwxKw==} engines: {node: '>=18.0.0'} @@ -1627,6 +1636,9 @@ packages: typescript: optional: true + web-tree-sitter@0.25.4: + resolution: {integrity: sha512-BNbAL2tgeuqpufHOM4TapD8evwq+J+X514rTL7zMgDSc+RN9LlByE3zVtTheucwQFkQ+FPU0Pz8Wn4Emz9itRg==} + which@2.0.2: resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} engines: {node: '>= 8'} @@ -2964,6 +2976,8 @@ snapshots: toidentifier@1.0.1: {} + tree-sitter-wasms@0.1.13: {} + tsx@4.23.15: dependencies: esbuild: 0.28.2 @@ -3094,6 +3108,8 @@ snapshots: optionalDependencies: typescript: 5.9.3 + web-tree-sitter@0.25.4: {} + which@2.0.2: dependencies: isexe: 2.0.0 diff --git a/src/index.ts b/src/index.ts index 0f49a37..9fa458e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -30,7 +30,7 @@ export type { SourceConnector, NormalizedDocument, IngestResult } from './source export { fileConnector } from './sources/file.ts'; export { jiraConnector, githubConnector } from './sources/jira-github.ts'; export type { JiraSourceConfig, GithubSourceConfig } from './sources/jira-github.ts'; -export { repoConnector } from './sources/repo.ts'; +export { repoConnector, repoDocumentFor, symbolDocFor } from './sources/repo.ts'; export type { RepoSourceConfig } from './sources/repo.ts'; // Stage 4 — transports (conversation API + MCP; consumers mount these) @@ -39,9 +39,14 @@ export type { ConversationDeps, TurnOutcome } from './transport/conversation.ts' export { mountMcp } from './transport/mcp.ts'; export type { McpMountOpts } from './transport/mcp.ts'; -// Roadmap stage G — symbol chunks (repo sources opt-in) -export { extractSymbols, symbolChunks } from './sources/tree-sitter.ts'; -export type { SymbolInfo, SymbolChunk } from './sources/tree-sitter.ts'; +// Roadmap stage G → Stage-3 v2 — symbol chunks (tree-sitter WASM; repo +// sources opt in with a grammar map, regex fallback without) +export { extractSymbols, extractSymbolsTree, symbolsFor, symbolChunksFor, prepareGrammars, defaultSymbolGrammars } from './sources/tree-sitter.ts'; +export type { SymbolInfo, SymbolChunk, SymbolGrammars } from './sources/tree-sitter.ts'; + +// Stage 3 — watchRoots mechanism (freshness; policy lives with the consumer) +export { watchRoots } from './watch.ts'; +export type { WatchHandle, WatchOptions } from './watch.ts'; // Roadmap stage F — postgres storage driver export { PostgresStorage } from './memory/postgres.ts'; diff --git a/src/retrieval/es-compat.ts b/src/retrieval/es-compat.ts index 8a04d20..534112e 100644 --- a/src/retrieval/es-compat.ts +++ b/src/retrieval/es-compat.ts @@ -48,6 +48,32 @@ export class EsCompatRetriever implements Retriever { return (body.hits?.hits ?? []).map((h) => this.map(h)); } + /** Structured search (Stage-3 F2): BM25 over `query` with an optional + * heading_path prefix filter. The mapping's heading_path is text+keyword; + * match_phrase_prefix matches the ANALYZED field, so a prefix like + * "symbol: CliError" hits "symbol: CliError (class)" exactly without + * wildcard cost. Same failure contract as retrieve(): throws. */ + async search(opts: { query: string; headingPrefix?: string; size?: number }): Promise { + const must = opts.query.length > 0 ? [{ match: { [this.field]: opts.query } }] : [{ match_all: {} }]; + const filter = opts.headingPrefix ? [{ bool: { should: [ + { match_phrase_prefix: { heading_path: opts.headingPrefix } }, + { match_phrase: { heading_path: opts.headingPrefix } }, + ] } }] : []; + const res = await this.fetchImpl(`${this.baseUrl}/${this.index}/_search`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + size: opts.size ?? 10, + query: { bool: { must, filter } }, + }), + }); + if (!res.ok) { + throw new Error(`es-compat structured search failed (${res.status}): ${(await res.text().catch(() => '')).slice(0, 200)}`); + } + const body = (await res.json()) as { hits?: { hits?: EsHit[] } }; + return (body.hits?.hits ?? []).map((h) => this.map(h)); + } + private map(h: EsHit): RetrievedChunk { const src = h._source ?? {}; if (h._id.startsWith('note:')) { diff --git a/src/sources/connector.ts b/src/sources/connector.ts index e2f9b8f..5d3b519 100644 --- a/src/sources/connector.ts +++ b/src/sources/connector.ts @@ -4,7 +4,7 @@ * unchanged content skips re-chunking. Caps per D12. */ import { createHash } from 'node:crypto'; import type { Storage, DocumentRow, ChunkRow } from '../memory/contract.ts'; -import { structuralChunks } from '../chunk.ts'; +import { structuralChunks, type StructuralChunk } from '../chunk.ts'; export interface NormalizedDocument { uri: string; @@ -14,6 +14,11 @@ export interface NormalizedDocument { kind: string; /** binary bytes when content is null → stored at bytes_path */ bytes?: Uint8Array; + /** PRE-CHUNKED content (Stage-3 F1 passthrough): when present, ingest + * stores these chunks verbatim instead of re-chunking `content`. Lets a + * connector emit structured chunks (e.g. per-symbol heading_path) while + * `content` stays a browsable summary. Ordinals as given, 0-based. */ + chunks?: StructuralChunk[]; } export interface SourceConnector { @@ -86,7 +91,7 @@ export async function ingest( let chunks: Array> = []; if (isText) { - chunks = structuralChunks(doc.content!, doc.kind).map((c) => ({ ordinal: c.ordinal, heading_path: c.headingPath, content: c.content, tokens: c.tokens })); + chunks = (doc.chunks ?? structuralChunks(doc.content!, doc.kind)).map((c) => ({ ordinal: c.ordinal, heading_path: c.headingPath, content: c.content, tokens: c.tokens })); if (totalChunks + chunks.length > MAX_CHUNKS_PER_SOURCE) { chunks = chunks.slice(0, Math.max(0, MAX_CHUNKS_PER_SOURCE - totalChunks)); if (chunks.length === 0) { out.skipped.push({ uri: doc.uri, reason: 'chunk cap reached for source' }); continue; } diff --git a/src/sources/repo.ts b/src/sources/repo.ts index bb2d6c4..e1c2f1d 100644 --- a/src/sources/repo.ts +++ b/src/sources/repo.ts @@ -1,13 +1,15 @@ /** repo source connector (plan Stage 3): a codebase as structured source — - * file tree + grep + explicit file fetch (v1; tree-sitter depth deferred per - * ROADMAP). Local path or shallow clone when handed a URL. */ + * file tree + per-file docs + per-symbol chunks (Stage-3 v2: tree-sitter + * WASM when the consumer configures grammars; regex fallback otherwise). + * Local path or shallow clone when handed a URL. */ import { readdirSync, readFileSync, statSync } from 'node:fs'; -import { join, relative } from 'node:path'; +import { join, relative, isAbsolute } from 'node:path'; import { execFileSync } from 'node:child_process'; import { mkdtempSync } from 'node:fs'; import { tmpdir } from 'node:os'; import type { NormalizedDocument, SourceConnector } from './connector.ts'; -import { symbolChunks } from './tree-sitter.ts'; +import { symbolsFor, symbolChunksFor, type SymbolGrammars } from './tree-sitter.ts'; +import type { StructuralChunk } from '../chunk.ts'; const SKIP_DIRS = new Set(['.git', 'node_modules', 'dist', 'build', '.venv', '__pycache__', 'DerivedData', '.derived-data', '.build', 'Pods', '.swiftpm', 'buck-out']); const TEXT_EXT = new Set(['.ts', '.tsx', '.js', '.mjs', '.cjs', '.json', '.md', '.yaml', '.yml', '.txt', '.css', '.scss', '.html', '.swift', '.kt', '.go', '.rs', '.py', '.rb', '.java', '.sql', '.sh']); @@ -20,8 +22,13 @@ export interface RepoSourceConfig { maxFiles?: number; /** include the tree doc (default true) */ treeDoc?: boolean; - /** emit symbol chunks for TS/JS-family files (plan Stage G) */ + /** emit a per-symbol-chunks doc per file (plan Stage G). With + * `symbolGrammars`, extraction is tree-sitter; without, the TS/JS regex + * fallback. */ symbols?: boolean; + /** extension→grammar policy for tree-sitter extraction (caller-owned; + * see defaultSymbolGrammars). Pure mechanism: butler picks nothing. */ + symbolGrammars?: SymbolGrammars; /** ADDITIONAL directory names to skip (consumer-level: workspace clones, * artifact dirs — anything that would eat the walk budget or pollute * context with foreign source). Unioned with the builtin SKIP_DIRS. */ @@ -49,6 +56,50 @@ function listRepoFiles(root: string, maxFiles: number, maxBytes: number, skipDir return out; } +/** file + symbol docs for ONE file (Stage-3: watcher batches re-ingest a + * single changed file without paying the walk). Same skip rules as fetch(); + * null when the path is outside the root, skipped, missing, or oversized. */ +export async function repoDocumentFor(cfg: RepoSourceConfig, absPath: string): Promise { + if (/^https?:\/\//.test(cfg.origin)) return null; // URL origins have no stable local path + const root = cfg.origin; + if (!isAbsolute(absPath) || !absPath.startsWith(root + '/') && absPath !== root) return null; + const rel = relative(root, absPath); + if (rel.startsWith('..') || rel === '') return null; + for (const seg of rel.split('/').slice(0, -1)) { + if (cfg.skipDirs?.includes(seg) || SKIP_DIRS.has(seg)) return null; + } + const ext = rel.slice(rel.lastIndexOf('.')).toLowerCase(); + if (!TEXT_EXT.has(ext)) return null; + let st; + try { st = statSync(absPath); } catch { return null; } // deleted/moved — batch handles + if (st.size > (cfg.maxFileBytes ?? 256 * 1024)) return null; + let content: string; + try { content = readFileSync(absPath, 'utf8'); } catch { return null; } + const docs: NormalizedDocument[] = [{ uri: `repo://${rel}`, title: rel, content, kind: 'plain' }]; + if (cfg.symbols) { + const symDoc = await symbolDocFor(content, rel, cfg.symbolGrammars); + if (symDoc) docs.push(symDoc); + } + return docs; +} + +/** per-symbol doc: content = one summary line per symbol (browsable), + * chunks = one chunk per symbol with heading_path "symbol: ()" + * (Stage-3 shape — structured filtering; v1 concatenated all symbols into + * the content body, unretrievable by heading). */ +export async function symbolDocFor(content: string, rel: string, grammars?: SymbolGrammars): Promise { + const symbols = await symbolsFor(content, rel, grammars ?? {}); + if (symbols.length === 0) return null; + const chunks: StructuralChunk[] = symbolChunksFor(symbols, content, rel); + return { + uri: `repo://${rel}#symbols`, + title: `${rel} (symbols)`, + content: symbols.map((s) => `symbol: ${s.name} (${s.kind}) — ${rel}:${s.line}`).join('\n'), + kind: 'symbols', + chunks, + }; +} + export function repoConnector(sourceId: string, cfg: RepoSourceConfig): SourceConnector { const maxFiles = cfg.maxFiles ?? 2_000; const maxBytes = cfg.maxFileBytes ?? 256 * 1024; @@ -72,12 +123,11 @@ export function repoConnector(sourceId: string, cfg: RepoSourceConfig): SourceCo } for (const f of files) { const rel = relative(root, f); - out.push({ uri: `repo://${rel}`, title: rel, content: readFileSync(f, 'utf8'), kind: 'plain' }); + const content = readFileSync(f, 'utf8'); + out.push({ uri: `repo://${rel}`, title: rel, content, kind: 'plain' }); if (cfg.symbols) { - // symbol chunks ride the SAME pipeline: they're just more chunks - // for this doc's kind, retrievable by "symbol: name (kind)" - const sym = symbolChunks(readFileSync(f, 'utf8'), rel); - if (sym.length > 0) out.push({ uri: `repo://${rel}#symbols`, title: `${rel} (symbols)`, content: sym.map((s) => `${s.headingPath}\n${s.content}`).join('\n\n'), kind: 'symbols' }); + const symDoc = await symbolDocFor(content, rel, cfg.symbolGrammars); + if (symDoc) out.push(symDoc); } } return out; diff --git a/src/sources/tree-sitter.ts b/src/sources/tree-sitter.ts index 97cfc1b..04249a0 100644 --- a/src/sources/tree-sitter.ts +++ b/src/sources/tree-sitter.ts @@ -1,23 +1,167 @@ -/** tree-sitter symbol chunks (plan Stage G, condition 7): repo sources may - * opt into symbol indexing — functions/types/classes become chunks with - * heading_path "symbol: ()". Grammars are WASM loaded at BUILD - * time (no runtime network); missing grammar → plain file chunks, logged. +/** tree-sitter symbol chunks (plan Stage G, condition 7 → Stage-3 v2): repo + * sources may opt into symbol indexing — functions/types/classes become + * chunks with heading_path "symbol: ()". * - * v1 ships a DEPENDENCY-FREE structural fallback (regex-based symbol scan for - * TS/JS-family files) so the chunk SHAPE is proven without pulling - * web-tree-sitter grammars per language — the web-tree-sitter upgrade path - * swaps `extractSymbols()` internals only. This keeps the plan's hermetic-CI - * and no-runtime-network invariants intact while delivering the chunk shape. */ + * v2 (2026-10-21): real parsing via web-tree-sitter WASM grammars + * (tree-sitter-wasms pack — no native modules, no runtime network; the + * grammar set is CALLER-supplied policy, this module is pure mechanism). + * No grammar configured for a file → v1 dependency-free regex fallback for + * TS/JS-family files, [] otherwise (caller keeps plain file chunks). + * Grammar WASM fails to load → empty result, never fatal. */ +import { createRequire } from 'node:module'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { Parser, Language, Node as TsNode } from 'web-tree-sitter'; export interface SymbolInfo { name: string; - kind: 'function' | 'class' | 'interface' | 'type' | 'method' | 'const'; + kind: 'function' | 'class' | 'interface' | 'type' | 'method' | 'const' | 'struct' | 'protocol' | 'enum'; line: number; } -/** Structural symbol scan for TS/JS-family sources. Honest scope: regex over - * declaration forms; not a parser. Returns [] for non-matching files so the - * caller falls back to plain chunks. */ +/** caller policy: file extension → grammar. Value is either a grammar name + * in the tree-sitter-wasms pack ('typescript', 'python', …) or an ABSOLUTE + * path to a .wasm file (see grammars/README.md for the vendored-swift + * story). e.g. { swift: '/…/grammars/tree-sitter-swift.wasm', ts: 'typescript' } */ +export type SymbolGrammars = Record; + +const require = createRequire(import.meta.url); + +let parserInit: Promise | null = null; +const grammarCache = new Map>(); + +function wasmSource(value: string): { key: string; src: string; family: string } { + // absolute path → load as-is (family from the file name); bare name → pack + if (value.startsWith('/')) { + const base = value.split('/').pop() ?? value; + return { key: value, src: value, family: base.replace(/^tree-sitter-/, '').replace(/\.wasm$/, '') }; + } + const packDir = dirname(require.resolve('tree-sitter-wasms/package.json')); + return { key: value, src: join(packDir, 'out', `tree-sitter-${value}.wasm`), family: value }; +} + +/** butler's shipped default: swift via the vendored 0.1.8 wasm (see + * grammars/README.md — pack swift >= 0.1.10 crashes Node 24 tier-up), the + * rest from the pack. Consumers pass their own map to override. */ +export function defaultSymbolGrammars(): SymbolGrammars { + return { + swift: join(dirname(fileURLToPath(import.meta.url)), '../../grammars/tree-sitter-swift.wasm'), + ts: 'typescript', + mts: 'typescript', + cts: 'typescript', + tsx: 'tsx', + js: 'typescript', + mjs: 'typescript', + cjs: 'typescript', + py: 'python', + }; +} + +async function loadParser(): Promise { + parserInit ??= Parser.init(); + return parserInit; +} + +async function loadGrammar(key: string, src: string): Promise { + if (!grammarCache.has(key)) { + grammarCache.set(key, (async () => { + try { + await loadParser(); + return await Language.load(src); + } catch { + return null; // empty result per file; never fatal to the walk + } + })()); + } + return grammarCache.get(key)!; +} + +/** node.type → SymbolInfo kind, per grammar family. Verified against the + * tree-sitter-wasms grammars by parsing fixtures (test/tree-sitter-wasm.test.ts). */ +const NODE_KINDS: Record> = { + swift: { + function_declaration: 'function', + class_declaration: 'class', + struct_declaration: 'struct', + protocol_declaration: 'protocol', + enum_declaration: 'enum', + type_alias: 'type', + property_declaration: 'const', + }, + typescript: { + function_declaration: 'function', + class_declaration: 'class', + interface_declaration: 'interface', + type_alias_declaration: 'type', + method_definition: 'method', + enum_declaration: 'enum', + lexical_declaration: 'const', + }, + tsx: { + function_declaration: 'function', + class_declaration: 'class', + interface_declaration: 'interface', + type_alias_declaration: 'type', + method_definition: 'method', + enum_declaration: 'enum', + lexical_declaration: 'const', + }, + python: { + function_definition: 'function', + class_definition: 'class', + }, +}; + +/** Tree-sitter symbols for one file. Returns [] when no grammar is configured + * for the extension or the grammar fails to load — the caller falls back. */ +export async function extractSymbolsTree(source: string, filePath: string, grammars: SymbolGrammars): Promise { + const ext = /(?:^|\.)([a-z0-9]+)$/i.exec(filePath)?.[1]?.toLowerCase() ?? ''; + const value = grammars[ext]; + if (!value) return []; + const { key, src, family } = wasmSource(value); + const lang = await loadGrammar(key, src); + if (!lang) return []; + const kinds = NODE_KINDS[family]; + if (!kinds) return []; + await loadParser(); + const parser = new Parser(); + parser.setLanguage(lang); + const tree = parser.parse(source); + const out: SymbolInfo[] = []; + const visit = (node: TsNode): void => { + let kind = kinds[node.type]; + if (kind) { + // swift models struct/class/extension all as class_declaration — the + // keyword token child says which (verified against the 0.1.8 grammar) + if (family === 'swift' && node.type === 'class_declaration') { + const toks = [...Array(node.childCount).keys()].map((i) => node.child(i)?.type); + kind = toks.includes('struct') ? 'struct' : toks.includes('enum') ? 'enum' : 'class'; + } + // typescript: `const NAME = …` carries its identifier on the + // variable_declarator child, not on the declaration itself + if ((family === 'typescript' || family === 'tsx') && node.type === 'lexical_declaration') { + for (let i = 0; i < node.childCount; i++) { + const d = node.child(i)!; + if (d.type === 'variable_declarator') { + const dn = d.childForFieldName('name')?.text?.trim(); + if (dn) out.push({ name: dn, kind: 'const', line: (d.startPosition.row) + 1 }); + } + } + kind = undefined; // handled per-declarator above + } + // these grammars expose the declared identifier as the `name` field + const nameNode = node.childForFieldName('name'); + const name = nameNode?.text?.trim(); + if (kind && name) out.push({ name, kind, line: (nameNode?.startPosition.row ?? node.startPosition.row) + 1 }); + } + for (let i = 0; i < node.childCount; i++) visit(node.child(i)!); + } + if (tree) visit(tree.rootNode); + return out; +} + +/** v1 structural symbol scan for TS/JS-family sources. Honest scope: regex + * over declaration forms; not a parser. Returns [] for non-matching files. */ export function extractSymbols(source: string, filePath: string): SymbolInfo[] { const isTsFamily = /\.(ts|tsx|js|jsx|mjs|cjs)$/.test(filePath); if (!isTsFamily) return []; @@ -51,12 +195,12 @@ export interface SymbolChunk { tokens: number; } -/** file source → symbol chunks (the repo connector merges these with file chunks) */ -export function symbolChunks(source: string, filePath: string): SymbolChunk[] { - const symbols = extractSymbols(source, filePath); +/** one chunk per symbol — the Stage-3 shape (heading_path carries the symbol + * for structured filtering; v1 concatenated all symbols into doc content) */ +export function symbolChunksFor(symbols: SymbolInfo[], source: string, filePath: string, contextLines = 6): SymbolChunk[] { const lines = source.split('\n'); return symbols.map((s, i) => { - const head = lines.slice(s.line - 1, s.line + 6).join('\n'); // decl + 6 lines of context + const head = lines.slice(s.line - 1, s.line + contextLines).join('\n'); // decl + context return { ordinal: i, headingPath: `symbol: ${s.name} (${s.kind})`, @@ -65,3 +209,19 @@ export function symbolChunks(source: string, filePath: string): SymbolChunk[] { }; }); } + +/** best-effort symbols for a file: tree-sitter when a grammar is configured + * for the extension (even if it yields nothing — a truly declaration-free + * file must not fall through to the TS regex), else the v1 regex fallback. */ +export async function symbolsFor(source: string, filePath: string, grammars: SymbolGrammars): Promise { + const ext = /(?:^|\.)([a-z0-9]+)$/i.exec(filePath)?.[1]?.toLowerCase() ?? ''; + if (grammars[ext]) return extractSymbolsTree(source, filePath, grammars); + return extractSymbols(source, filePath); +} + +/** pre-warm the WASM parser + configured grammars (one-time; else lazy per + * first file of each language) */ +export async function prepareGrammars(grammars: SymbolGrammars): Promise { + const sources = [...new Set(Object.values(grammars))].map((v) => wasmSource(v)); + await Promise.all(sources.map((s) => loadGrammar(s.key, s.src))); +} diff --git a/src/watch.ts b/src/watch.ts new file mode 100644 index 0000000..4607f07 --- /dev/null +++ b/src/watch.ts @@ -0,0 +1,109 @@ +/** watchRoots (Stage 3, plan F3): debounced changed-path batches over one or + * more directory roots. Pure mechanism — the consumer owns policy (which + * roots, what to ignore, what to do with batches). + * + * One OS watch per root (fs.watch recursive — FSEvents on macOS); no + * polling. A watcher error (root deleted, EMFILE) marks that root dead and + * surfaces via healthy(); it does NOT throw into the caller's loop. No + * mid-process resurrection — restart on next boot keeps the contract simple + * (the stale-trigger full walk remains the fallback leg). */ +import { watch, type FSWatcher } from 'node:fs'; +import { join } from 'node:path'; + +export interface WatchHandle { + /** stop every watcher; pending batch is dropped (shutdown, not flush) */ + stop(): void; + /** false once every root has died (caller degrades to its fallback leg) */ + healthy(): boolean; + /** human-readable state for status notes */ + describe(): string; +} + +export interface WatchOptions { + /** quiet period before a batch fires (default 400ms — one editor save is + * several filesystem events) */ + debounceMs?: number; + /** policy: return true to IGNORE a changed path (abs) */ + ignore?: (absPath: string) => boolean; +} + +interface RootState { + root: string; + watcher: FSWatcher | null; + alive: boolean; + lastError: string | null; +} + +export function watchRoots(roots: string[], opts: WatchOptions, onBatch: (changedAbsPaths: string[]) => void): WatchHandle { + const debounceMs = opts.debounceMs ?? 400; + const pending = new Map(); // absPath → seq + let timer: ReturnType | null = null; + const states: RootState[] = roots.map((root) => ({ root, watcher: null, alive: false, lastError: null })); + + const flush = (): void => { + timer = null; + if (pending.size === 0) return; + const batch = [...pending.keys()]; + pending.clear(); + try { + onBatch(batch); + } catch { + // consumer errors never kill the watcher; their problem to log + } + }; + + const schedule = (absPath: string): void => { + pending.set(absPath, (pending.get(absPath) ?? 0) + 1); + if (timer) clearTimeout(timer); + timer = setTimeout(flush, debounceMs); + }; + + const startRoot = (state: RootState): void => { + try { + state.watcher = watch(state.root, { recursive: true }, (_event, filename) => { + if (!state.alive) return; + if (typeof filename !== 'string') { schedule(state.root); return; } // whole-root event + const abs = join(state.root, filename); + if (opts.ignore?.(abs)) return; + schedule(abs); + }); + state.alive = true; + state.lastError = null; + state.watcher.on('error', (err: Error) => { + // root deleted / perms / EMFILE — mark dead, surface, keep others up + state.alive = false; + state.lastError = err.message; + try { state.watcher?.close(); } catch { /* already gone */ } + state.watcher = null; + }); + state.watcher.on('close', () => { state.alive = false; }); + } catch (err) { + state.alive = false; + state.lastError = err instanceof Error ? err.message : String(err); + } + }; + + for (const state of states) startRoot(state); + + return { + stop(): void { + if (timer) { clearTimeout(timer); timer = null; } + pending.clear(); + for (const state of states) { + try { state.watcher?.close(); } catch { /* already gone */ } + state.watcher = null; + state.alive = false; + } + }, + healthy(): boolean { + return states.some((s) => s.alive); + }, + describe(): string { + const alive = states.filter((s) => s.alive).length; + const dead = states.filter((s) => !s.alive && s.lastError); + return dead.length > 0 + ? `${alive}/${states.length} roots watching (dead: ${dead.map((d) => `${d.root.split('/').pop()}: ${d.lastError}`).join('; ')})` + : `${alive}/${states.length} roots watching`; + }, + }; +} diff --git a/test/boundary.test.ts b/test/boundary.test.ts index bb0d02f..1c67fec 100644 --- a/test/boundary.test.ts +++ b/test/boundary.test.ts @@ -20,8 +20,11 @@ const ALLOWED_PACKAGES = new Set([ 'yaml', 'pg', 'zod', + // Stage 3 — symbol extraction (web-tree-sitter runs the grammar WASMs; + // tree-sitter-wasms is assets-only, reached via createRequire) + 'web-tree-sitter', 'node:fs', 'node:path', 'node:os', 'node:crypto', 'node:util', 'node:events', - 'node:child_process', 'node:url', 'node:buffer', 'node:stream', 'node:abort-controller', + 'node:child_process', 'node:url', 'node:buffer', 'node:stream', 'node:abort-controller', 'node:module', ]); function tsFiles(dir: string): string[] { diff --git a/test/es-compat-search.test.ts b/test/es-compat-search.test.ts new file mode 100644 index 0000000..9798cb2 --- /dev/null +++ b/test/es-compat-search.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from 'vitest'; +import { EsCompatRetriever } from '../src/retrieval/es-compat.ts'; + +/** Stage-3 F2: structured search — heading_path prefix filter + BM25, same + * failure contract as retrieve(). Hermetic fake endpoint. */ + +type Call = { path: string; body: unknown }; + +function fakeEndpoint(hits: unknown[], log: Call[]) { + return (async (url: string | URL | Request, init?: RequestInit) => { + log.push({ path: String(url), body: JSON.parse(String(init?.body ?? '{}')) }); + return new Response(JSON.stringify({ hits: { hits, total: { value: hits.length } } }), { status: 200, headers: { 'content-type': 'application/json' } }); + }) as unknown as typeof fetch; +} + +describe('EsCompatRetriever.search', () => { + it('filter-only query (empty q) with heading_prefix — symbol lookups', async () => { + const calls: Call[] = []; + const r = new EsCompatRetriever({ + baseUrl: 'http://x', index: 'corpus', + fetchImpl: fakeEndpoint([{ _id: 'abc#2', _score: 3.1, _source: { content: 'src/a.swift:5\nfunc validate', heading_path: 'symbol: validate (function)', ordinal: 2 } }], calls), + }); + const hits = await r.search({ query: '', headingPrefix: 'symbol: validate', size: 5 }); + expect(hits[0]?.headingPath).toBe('symbol: validate (function)'); + expect(hits[0]?.ordinal).toBe(2); + const body = calls[0]!.body as { size: number; query: { bool: { must: unknown[]; filter: unknown[] } } }; + expect(body.size).toBe(5); + expect(body.query.bool.must[0]).toEqual({ match_all: {} }); + expect(JSON.stringify(body.query.bool.filter)).toContain('match_phrase_prefix'); + }); + + it('query + heading_prefix combine (must + filter)', async () => { + const calls: Call[] = []; + const r = new EsCompatRetriever({ baseUrl: 'http://x', index: 'corpus', fetchImpl: fakeEndpoint([], calls) }); + await r.search({ query: 'validator', headingPrefix: 'symbol: Validator' }); + const body = calls[0]!.body as { query: { bool: { must: unknown[]; filter: unknown[] } } }; + expect(JSON.stringify(body.query.bool.must)).toContain('"validator"'); + expect(body.query.bool.filter.length).toBe(1); + }); + + it('no headingPrefix → no filter clause', async () => { + const calls: Call[] = []; + const r = new EsCompatRetriever({ baseUrl: 'http://x', index: 'corpus', fetchImpl: fakeEndpoint([], calls) }); + await r.search({ query: 'anything' }); + const body = calls[0]!.body as { query: { bool: { must: unknown[]; filter: unknown[] } } }; + expect(body.query.bool.filter).toEqual([]); + }); + + it('non-2xx throws (same contract as retrieve)', async () => { + const r = new EsCompatRetriever({ + baseUrl: 'http://x', index: 'corpus', + fetchImpl: (async () => new Response('boom', { status: 500 })) as unknown as typeof fetch, + }); + await expect(r.search({ query: 'x' })).rejects.toThrow(/structured search failed \(500\)/); + }); +}); diff --git a/test/symbols.test.ts b/test/symbols.test.ts index cdab189..9698b06 100644 --- a/test/symbols.test.ts +++ b/test/symbols.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { extractSymbols, symbolChunks } from '../src/sources/tree-sitter.ts'; +import { extractSymbols, symbolChunksFor } from '../src/sources/tree-sitter.ts'; import { repoConnector } from '../src/sources/repo.ts'; import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; @@ -52,13 +52,13 @@ describe('extractSymbols (TS-family structural scan)', () => { expect(extractSymbols('def f(): pass', 'script.py')).toEqual([]); expect(extractChunksPlain()).toBe(true); function extractChunksPlain(): boolean { - const chunks = symbolChunks('def f(): pass', 'script.py'); + const chunks = symbolChunksFor(extractSymbols('def f(): pass', 'script.py'), 'def f(): pass', 'script.py'); return chunks.length === 0; } }); it('symbolChunks carry heading_path + file:line + context', () => { - const chunks = symbolChunks(SAMPLE, 'src/sample.ts'); + const chunks = symbolChunksFor(extractSymbols(SAMPLE, 'src/sample.ts'), SAMPLE, 'src/sample.ts'); const rl = chunks.find((c) => c.headingPath === 'symbol: runLoop (function)')!; expect(rl.content).toContain('src/sample.ts:'); expect(rl.content).toContain('export async function runLoop'); @@ -80,6 +80,8 @@ describe('repo connector symbols opt-in', () => { expect(symDoc).toBeTruthy(); expect(symDoc!.content).toContain('symbol: alpha (function)'); expect(symDoc!.content).toContain('symbol: beta (function)'); + // Stage-3 shape: per-symbol chunks with structured heading_path (F1) + expect(symDoc!.chunks?.map((c) => c.headingPath)).toContain('symbol: beta (function)'); rmSync(dir, { recursive: true, force: true }); }); }); diff --git a/test/tree-sitter-wasm.test.ts b/test/tree-sitter-wasm.test.ts new file mode 100644 index 0000000..2cc9bf7 --- /dev/null +++ b/test/tree-sitter-wasm.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, it } from 'vitest'; +import { extractSymbolsTree, symbolsFor, defaultSymbolGrammars, prepareGrammars, symbolChunksFor } from '../src/sources/tree-sitter.ts'; +import { repoDocumentFor } from '../src/sources/repo.ts'; +import { SqliteStorage } from '../src/memory/sqlite.ts'; +import { ingest } from '../src/sources/connector.ts'; +import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +/** Stage-3 v2: tree-sitter WASM extraction over the shipped grammars + * (vendored swift — see grammars/README.md — + pack typescript/tsx/python). + * Hermetic: grammars load from node_modules, no network. */ + +const SWIFT = `import Foundation + +public struct PostalRule { let code: String } + +public final class AddressValidator { + public func validate(_ code: String) -> Bool { return !code.isEmpty } +} + +protocol Validating { func validate(_ code: String) -> Bool } + +enum Region { case intl, domestic } +`; + +const TS = `export interface Rule { code: string } +export type Alias = string; +export class Validator { method(x: string): boolean { return true } } +export function validate(code: string): boolean { return true } +export const MAX = 10; +`; + +const PY = `def top_level(): pass + +class Thing: + def method(self): pass +`; + +describe('tree-sitter WASM extraction (defaultSymbolGrammars)', () => { + it('swift: struct/class/protocol/enum + functions with lines', async () => { + const g = defaultSymbolGrammars(); + const syms = await extractSymbolsTree(SWIFT, 'a.swift', g); + const names = syms.map((s) => `${s.name}:${s.kind}`); + expect(names).toContain('PostalRule:struct'); + expect(names).toContain('AddressValidator:class'); + expect(names).toContain('validate:function'); + expect(names).toContain('Validating:protocol'); + expect(names).toContain('Region:enum'); + const v = syms.find((s) => s.name === 'validate')!; + expect(v.line).toBeGreaterThan(3); + }); + + it('typescript: interface/type/class/method/function/const', async () => { + const g = defaultSymbolGrammars(); + const syms = await extractSymbolsTree(TS, 'a.ts', g); + const names = syms.map((s) => `${s.name}:${s.kind}`); + expect(names).toContain('Rule:interface'); + expect(names).toContain('Alias:type'); + expect(names).toContain('Validator:class'); + expect(names).toContain('method:method'); + expect(names).toContain('validate:function'); + expect(names).toContain('MAX:const'); + }); + + it('python: functions and classes', async () => { + const g = defaultSymbolGrammars(); + const syms = await extractSymbolsTree(PY, 'a.py', g); + const names = syms.map((s) => `${s.name}:${s.kind}`); + expect(names).toContain('top_level:function'); + expect(names).toContain('Thing:class'); + expect(names).toContain('method:function'); + }); + + it('symbolsFor: grammar-configured extension uses tree-sitter even when the file has no decls', async () => { + const g = defaultSymbolGrammars(); + // a decl-free swift body (property/func/class-free): plain statements + expect(await symbolsFor('print("hi")\n', 'a.swift', g)).toEqual([]); // swift path, NOT the regex + expect((await symbolsFor('export function f() {}\n', 'a.ts', g)).length).toBeGreaterThan(0); + }); + + it('prepareGrammars warms the set (idempotent)', async () => { + const g = defaultSymbolGrammars(); + await prepareGrammars(g); + await prepareGrammars(g); + }); + + it('symbolChunksFor emits one chunk per symbol with structured heading', async () => { + const g = defaultSymbolGrammars(); + const syms = await extractSymbolsTree(SWIFT, 'a.swift', g); + const chunks = symbolChunksFor(syms, SWIFT, 'a.swift'); + expect(chunks.length).toBe(syms.length); + const rule = chunks.find((c) => c.headingPath === 'symbol: PostalRule (struct)')!; + expect(rule.content).toContain('a.swift:'); + expect(rule.content).toContain('public struct PostalRule'); + }); +}); + +describe('repoDocumentFor (per-file normalize, F3)', () => { + it('one changed file → file doc + per-symbol doc, no walk', async () => { + const dir = mkdtempSync(join(tmpdir(), 'butler-docfor-')); + writeFileSync(join(dir, 'a.swift'), SWIFT); + const cfg = { origin: dir, symbols: true, symbolGrammars: defaultSymbolGrammars() }; + const docs = await repoDocumentFor(cfg, join(dir, 'a.swift')); + expect(docs?.map((d) => d.uri)).toContain('repo://a.swift'); + const sym = docs?.find((d) => d.uri === 'repo://a.swift#symbols'); + expect(sym?.chunks?.length).toBeGreaterThan(0); + rmSync(dir, { recursive: true, force: true }); + }); + + it('outside root / skipped dir / missing file → null', async () => { + const dir = mkdtempSync(join(tmpdir(), 'butler-docfor2-')); + const cfg = { origin: dir, symbols: true }; + expect(await repoDocumentFor(cfg, '/etc/hosts')).toBeNull(); + expect(await repoDocumentFor(cfg, join(dir, 'nope.swift'))).toBeNull(); + rmSync(dir, { recursive: true, force: true }); + }); +}); + +describe('ingest chunks passthrough (F1)', () => { + it('pre-chunked docs land verbatim — heading_path per symbol, idempotent re-ingest', async () => { + const dir = mkdtempSync(join(tmpdir(), 'butler-passthru-')); + const storage = new SqliteStorage(join(dir, 't.db')); + const repoDir = mkdtempSync(join(tmpdir(), 'butler-passthru-repo-')); + writeFileSync(join(repoDir, 'a.swift'), SWIFT); + const connector = { + kind: 'repo' as const, + sourceId: 't-passthru', + fetch: async () => (await repoDocumentFor({ origin: repoDir, symbols: true, symbolGrammars: defaultSymbolGrammars() }, join(repoDir, 'a.swift')))!, + }; + const r1 = await ingest(connector, storage, { artifactsDir: dir }); + expect(r1.documents.some((d) => d.uri.endsWith('#symbols'))).toBe(true); + const sha = r1.documents.find((d) => d.uri.endsWith('#symbols'))!.sha256; + const chunks = await storage.documentChunks(sha); + expect(chunks.some((c) => c.heading_path === 'symbol: PostalRule (struct)')).toBe(true); + // content-addressed idempotence: re-ingest → cached, chunks unchanged + const r2 = await ingest(connector, storage, { artifactsDir: dir }); + expect(r2.documents.find((d) => d.uri.endsWith('#symbols'))!.cached).toBe(true); + await storage.close?.(); + rmSync(dir, { recursive: true, force: true }); + rmSync(repoDir, { recursive: true, force: true }); + }); +}); diff --git a/test/watch.test.ts b/test/watch.test.ts new file mode 100644 index 0000000..8e22104 --- /dev/null +++ b/test/watch.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from 'vitest'; +import { watchRoots } from '../src/watch.ts'; +import { mkdtempSync, mkdirSync, writeFileSync, appendFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { vi } from 'vitest'; + +/** Stage-3 watchRoots mechanism: debounced batches, ignore policy, health. + * Real fs events on a tmp dir (FSEvents on macOS) with a short debounce. */ + +describe('watchRoots', () => { + it('coalesces a save into one debounced batch of changed paths', async () => { + const dir = mkdtempSync(join(tmpdir(), 'butler-watch-')); + const batches: string[][] = []; + const h = watchRoots([dir], { debounceMs: 60 }, (b) => batches.push(b)); + try { + expect(h.healthy()).toBe(true); + writeFileSync(join(dir, 'a.ts'), 'export const a = 1;\n'); + // one editor save = several events; debounce must merge them + appendFileSync(join(dir, 'a.ts'), 'export const b = 2;\n'); + await vi.waitFor(() => expect(batches.length).toBeGreaterThan(0)); + const flat = batches.flat(); + expect(flat).toContain(join(dir, 'a.ts')); + expect(flat.filter((p) => p === join(dir, 'a.ts')).length).toBe(1); + } finally { + h.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('ignore policy filters paths from batches', async () => { + const dir = mkdtempSync(join(tmpdir(), 'butler-watch2-')); + mkdirSync(join(dir, 'noise')); + const batches: string[][] = []; + const h = watchRoots([dir], { debounceMs: 60, ignore: (p) => p.includes('/noise/') }, (b) => batches.push(b)); + try { + writeFileSync(join(dir, 'noise', 'junk.log'), 'x'); + writeFileSync(join(dir, 'keep.ts'), 'export const k = 1;\n'); + await vi.waitFor(() => expect(batches.flat().length).toBeGreaterThan(0)); + expect(batches.flat()).not.toContain(join(dir, 'noise', 'junk.log')); + expect(batches.flat()).toContain(join(dir, 'keep.ts')); + } finally { + h.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('stop() silences future events and reports unhealthy', async () => { + const dir = mkdtempSync(join(tmpdir(), 'butler-watch3-')); + let fired = 0; + const h = watchRoots([dir], { debounceMs: 50 }, () => { fired += 1; }); + h.stop(); + expect(h.healthy()).toBe(false); + writeFileSync(join(dir, 'late.ts'), 'late'); + await new Promise((r) => setTimeout(r, 250)); + expect(fired).toBe(0); + rmSync(dir, { recursive: true, force: true }); + }); + + it('a dead root surfaces via describe(); other roots keep watching', async () => { + const dirA = mkdtempSync(join(tmpdir(), 'butler-watchA-')); + const dirB = mkdtempSync(join(tmpdir(), 'butler-watchB-')); + const batches: string[][] = []; + // dirA does not exist → its watcher dies at start; dirB must survive + const gone = join(dirA, 'missing-root'); + const h = watchRoots([gone, dirB], { debounceMs: 60 }, (b) => batches.push(b)); + try { + expect(h.healthy()).toBe(true); // B alive + expect(h.describe()).toContain('1/2 roots watching'); + writeFileSync(join(dirB, 'b.ts'), 'export const b = 1;\n'); + await vi.waitFor(() => expect(batches.flat()).toContain(join(dirB, 'b.ts'))); + } finally { + h.stop(); + rmSync(dirA, { recursive: true, force: true }); + rmSync(dirB, { recursive: true, force: true }); + } + }); +});