feat(symbols,watch): Stage 3 — tree-sitter WASM symbols, per-symbol chunks, watchRoots, structured search

CHANGELOG.md

@@ -1,3 +1,33 @@
+## [0.8.0] — 2026-10-21 (Stage 3: tree-sitter symbols + watchRoots + structured search)
+
+### Added
+- **tree-sitter WASM symbol extraction** — `extractSymbolsTree`/`symbolsFor` parse Swift /
+  TypeScript / TSX / JS / Python via `web-tree-sitter` (0.25.4) + the `tree-sitter-wasms` pack.
+  Grammar policy is CALLER-supplied (`SymbolGrammars`: ext → pack name or absolute .wasm path);
+  `defaultSymbolGrammars()` ships butler's preset. No grammar configured → v1 TS/JS regex
+  fallback; grammar load failure → empty result, never fatal.
+  **Vendored `grammars/tree-sitter-swift.wasm`** (from tree-sitter-wasms@0.1.8): the pack's
+  swift build ≥ 0.1.10 aborts Node ≥ 24 (`Fatal process out of memory: Zone`, V8 turboshaft
+  wasm tier-up — background thread, process-fatal; `--liftoff-only` avoids it but is banned in
+  NODE_OPTIONS). See grammars/README.md; re-extract when upstream fixes it.
+- **Per-symbol chunks (F1)** — `NormalizedDocument.chunks?` passthrough: ingest stores
+  pre-chunked docs verbatim. `repoConnector(symbols: true)` now emits one chunk per symbol with
+  `heading_path "symbol: <name> (<kind>)"` (v1 concatenated all symbols into doc content,
+  unretrievable by heading). Doc content stays a browsable summary.
+- **`repoDocumentFor(cfg, absPath)`** (F3) — per-file normalize (file doc + symbol doc) honoring
+  the same skip rules; the freshness watcher path re-ingests one changed file without paying the
+  walk. `symbolDocFor` exported alongside.
+- **`watchRoots(roots, opts, onBatch)`** — debounced changed-path batches, one OS watch per root
+  (no polling), caller-supplied ignore policy, `healthy()`/`describe()` surface; watcher errors
+  mark the root dead, never throw into the caller's loop.
+- **`EsCompatRetriever.search({ query, headingPrefix?, size })`** (F2) — structured search:
+  BM25 + optional heading_path prefix filter (match_phrase_prefix over the analyzed field).
+  Same throw contract as `retrieve()`.
+- Hermetic tests: WASM extraction per language, per-file normalize, chunks passthrough +
+  idempotence, watcher batches/ignore/health (real fs events, tmp dirs), structured search
+  (fake endpoint). Full suite 226 passed; `node:module` + `web-tree-sitter` added to the
+  boundary allowlist.
+
 ## [0.7.0] — 2026-10-21 (Stage 2b: ES-compatible retrieval + xerj sidecar runner)
 
 ### Added

grammars/README.md

@@ -0,0 +1,22 @@
+# Vendored grammar WASMs
+
+## tree-sitter-swift.wasm — from tree-sitter-wasms@0.1.8
+
+The swift grammar shipped in tree-sitter-wasms >= 0.1.10 (identical bytes through
+0.1.13) aborts Node >= 24 during background wasm tier-up:
+
+    # Fatal process out of memory: Zone
+    v8::internal::compiler::turboshaft::WasmLoweringPhase
+
+The parse itself succeeds; the crash fires in the background compile thread, so
+any process that loads the module eventually dies (observed on Node 24.12,
+web-tree-sitter 0.25.x, macOS arm64; `node --liftoff-only` avoids it but is not
+allowed in NODE_OPTIONS). The 0.1.8 build (older, smaller swift grammar) tier-ups
+cleanly and extracts the same declaration set. tree-sitter/typescript/tsx/python
+from the current pack are unaffected and resolve from `tree-sitter-wasms` at
+runtime.
+
+Re-extract when the upstream pack ships a fixed swift build:
+    npm pack tree-sitter-wasms@<ver> && tar -xzf … package/out/tree-sitter-swift.wasm
+then re-run test/tree-sitter-wasm.test.ts WITHOUT --liftoff-only to prove the
+crash is gone before deleting this vendored copy.

package.json

@@ -1,8 +1,8 @@
 {
   "name": "@mainactordev/butler",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "private": true,
-  "description": "Butler \u2014 reusable agent runtime: provider, loop, tool registry, layered memory, source ingestion, conversation API + MCP transports (plan 20261001T035356Z)",
+  "description": "Butler — reusable agent runtime: provider, loop, tool registry, layered memory, source ingestion, conversation API + MCP transports (plan 20261001T035356Z)",
   "type": "module",
   "main": "src/index.ts",
   "exports": {
@@ -18,9 +18,11 @@
     "@fastify/websocket": "^11.3.1",
     "@modelcontextprotocol/fastify": "^2.0.0",
     "@modelcontextprotocol/sdk": "^1.31.0",
+    "better-sqlite3": "^13.0.3",
     "pg": "^8.23.0",
-    "zod": "^4.6.5",
-    "better-sqlite3": "^13.0.3"
+    "tree-sitter-wasms": "0.1.13",
+    "web-tree-sitter": "0.25.4",
+    "zod": "^4.6.5"
   },
   "devDependencies": {
     "@types/better-sqlite3": "^7.6.13",

src/index.ts

@@ -30,7 +30,7 @@ export type { SourceConnector, NormalizedDocument, IngestResult } from './source
 export { fileConnector } from './sources/file.ts';
 export { jiraConnector, githubConnector } from './sources/jira-github.ts';
 export type { JiraSourceConfig, GithubSourceConfig } from './sources/jira-github.ts';
-export { repoConnector } from './sources/repo.ts';
+export { repoConnector, repoDocumentFor, symbolDocFor } from './sources/repo.ts';
 export type { RepoSourceConfig } from './sources/repo.ts';
 
 // Stage 4 — transports (conversation API + MCP; consumers mount these)
@@ -39,9 +39,14 @@ export type { ConversationDeps, TurnOutcome } from './transport/conversation.ts'
 export { mountMcp } from './transport/mcp.ts';
 export type { McpMountOpts } from './transport/mcp.ts';
 
-// Roadmap stage G — symbol chunks (repo sources opt-in)
-export { extractSymbols, symbolChunks } from './sources/tree-sitter.ts';
-export type { SymbolInfo, SymbolChunk } from './sources/tree-sitter.ts';
+// Roadmap stage G → Stage-3 v2 — symbol chunks (tree-sitter WASM; repo
+// sources opt in with a grammar map, regex fallback without)
+export { extractSymbols, extractSymbolsTree, symbolsFor, symbolChunksFor, prepareGrammars, defaultSymbolGrammars } from './sources/tree-sitter.ts';
+export type { SymbolInfo, SymbolChunk, SymbolGrammars } from './sources/tree-sitter.ts';
+
+// Stage 3 — watchRoots mechanism (freshness; policy lives with the consumer)
+export { watchRoots } from './watch.ts';
+export type { WatchHandle, WatchOptions } from './watch.ts';
 
 // Roadmap stage F — postgres storage driver
 export { PostgresStorage } from './memory/postgres.ts';

src/retrieval/es-compat.ts

@@ -48,6 +48,32 @@ export class EsCompatRetriever implements Retriever {
     return (body.hits?.hits ?? []).map((h) => this.map(h));
   }
 
+  /** Structured search (Stage-3 F2): BM25 over `query` with an optional
+   *  heading_path prefix filter. The mapping's heading_path is text+keyword;
+   *  match_phrase_prefix matches the ANALYZED field, so a prefix like
+   *  "symbol: CliError" hits "symbol: CliError (class)" exactly without
+   *  wildcard cost. Same failure contract as retrieve(): throws. */
+  async search(opts: { query: string; headingPrefix?: string; size?: number }): Promise<RetrievedChunk[]> {
+    const must = opts.query.length > 0 ? [{ match: { [this.field]: opts.query } }] : [{ match_all: {} }];
+    const filter = opts.headingPrefix ? [{ bool: { should: [
+      { match_phrase_prefix: { heading_path: opts.headingPrefix } },
+      { match_phrase: { heading_path: opts.headingPrefix } },
+    ] } }] : [];
+    const res = await this.fetchImpl(`${this.baseUrl}/${this.index}/_search`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({
+        size: opts.size ?? 10,
+        query: { bool: { must, filter } },
+      }),
+    });
+    if (!res.ok) {
+      throw new Error(`es-compat structured search failed (${res.status}): ${(await res.text().catch(() => '')).slice(0, 200)}`);
+    }
+    const body = (await res.json()) as { hits?: { hits?: EsHit[] } };
+    return (body.hits?.hits ?? []).map((h) => this.map(h));
+  }
+
   private map(h: EsHit): RetrievedChunk {
     const src = h._source ?? {};
     if (h._id.startsWith('note:')) {

src/sources/connector.ts

@@ -4,7 +4,7 @@
  *  unchanged content skips re-chunking. Caps per D12. */
 import { createHash } from 'node:crypto';
 import type { Storage, DocumentRow, ChunkRow } from '../memory/contract.ts';
-import { structuralChunks } from '../chunk.ts';
+import { structuralChunks, type StructuralChunk } from '../chunk.ts';
 
 export interface NormalizedDocument {
   uri: string;
@@ -14,6 +14,11 @@ export interface NormalizedDocument {
   kind: string;
   /** binary bytes when content is null → stored at bytes_path */
   bytes?: Uint8Array;
+  /** PRE-CHUNKED content (Stage-3 F1 passthrough): when present, ingest
+   *  stores these chunks verbatim instead of re-chunking `content`. Lets a
+   *  connector emit structured chunks (e.g. per-symbol heading_path) while
+   *  `content` stays a browsable summary. Ordinals as given, 0-based. */
+  chunks?: StructuralChunk[];
 }
 
 export interface SourceConnector {
@@ -86,7 +91,7 @@ export async function ingest(
 
     let chunks: Array<Omit<ChunkRow, 'id' | 'doc_sha256'>> = [];
     if (isText) {
-      chunks = structuralChunks(doc.content!, doc.kind).map((c) => ({ ordinal: c.ordinal, heading_path: c.headingPath, content: c.content, tokens: c.tokens }));
+      chunks = (doc.chunks ?? structuralChunks(doc.content!, doc.kind)).map((c) => ({ ordinal: c.ordinal, heading_path: c.headingPath, content: c.content, tokens: c.tokens }));
       if (totalChunks + chunks.length > MAX_CHUNKS_PER_SOURCE) {
         chunks = chunks.slice(0, Math.max(0, MAX_CHUNKS_PER_SOURCE - totalChunks));
         if (chunks.length === 0) { out.skipped.push({ uri: doc.uri, reason: 'chunk cap reached for source' }); continue; }

src/sources/repo.ts

@@ -1,13 +1,15 @@
 /** repo source connector (plan Stage 3): a codebase as structured source —
- *  file tree + grep + explicit file fetch (v1; tree-sitter depth deferred per
- *  ROADMAP). Local path or shallow clone when handed a URL. */
+ *  file tree + per-file docs + per-symbol chunks (Stage-3 v2: tree-sitter
+ *  WASM when the consumer configures grammars; regex fallback otherwise).
+ *  Local path or shallow clone when handed a URL. */
 import { readdirSync, readFileSync, statSync } from 'node:fs';
-import { join, relative } from 'node:path';
+import { join, relative, isAbsolute } from 'node:path';
 import { execFileSync } from 'node:child_process';
 import { mkdtempSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import type { NormalizedDocument, SourceConnector } from './connector.ts';
-import { symbolChunks } from './tree-sitter.ts';
+import { symbolsFor, symbolChunksFor, type SymbolGrammars } from './tree-sitter.ts';
+import type { StructuralChunk } from '../chunk.ts';
 
 const SKIP_DIRS = new Set(['.git', 'node_modules', 'dist', 'build', '.venv', '__pycache__', 'DerivedData', '.derived-data', '.build', 'Pods', '.swiftpm', 'buck-out']);
 const TEXT_EXT = new Set(['.ts', '.tsx', '.js', '.mjs', '.cjs', '.json', '.md', '.yaml', '.yml', '.txt', '.css', '.scss', '.html', '.swift', '.kt', '.go', '.rs', '.py', '.rb', '.java', '.sql', '.sh']);
@@ -20,8 +22,13 @@ export interface RepoSourceConfig {
   maxFiles?: number;
   /** include the tree doc (default true) */
   treeDoc?: boolean;
-  /** emit symbol chunks for TS/JS-family files (plan Stage G) */
+  /** emit a per-symbol-chunks doc per file (plan Stage G). With
+   *  `symbolGrammars`, extraction is tree-sitter; without, the TS/JS regex
+   *  fallback. */
   symbols?: boolean;
+  /** extension→grammar policy for tree-sitter extraction (caller-owned;
+   *  see defaultSymbolGrammars). Pure mechanism: butler picks nothing. */
+  symbolGrammars?: SymbolGrammars;
   /** ADDITIONAL directory names to skip (consumer-level: workspace clones,
    *  artifact dirs — anything that would eat the walk budget or pollute
    *  context with foreign source). Unioned with the builtin SKIP_DIRS. */
@@ -49,6 +56,50 @@ function listRepoFiles(root: string, maxFiles: number, maxBytes: number, skipDir
   return out;
 }
 
+/** file + symbol docs for ONE file (Stage-3: watcher batches re-ingest a
+ *  single changed file without paying the walk). Same skip rules as fetch();
+ *  null when the path is outside the root, skipped, missing, or oversized. */
+export async function repoDocumentFor(cfg: RepoSourceConfig, absPath: string): Promise<NormalizedDocument[] | null> {
+  if (/^https?:\/\//.test(cfg.origin)) return null; // URL origins have no stable local path
+  const root = cfg.origin;
+  if (!isAbsolute(absPath) || !absPath.startsWith(root + '/') && absPath !== root) return null;
+  const rel = relative(root, absPath);
+  if (rel.startsWith('..') || rel === '') return null;
+  for (const seg of rel.split('/').slice(0, -1)) {
+    if (cfg.skipDirs?.includes(seg) || SKIP_DIRS.has(seg)) return null;
+  }
+  const ext = rel.slice(rel.lastIndexOf('.')).toLowerCase();
+  if (!TEXT_EXT.has(ext)) return null;
+  let st;
+  try { st = statSync(absPath); } catch { return null; } // deleted/moved — batch handles
+  if (st.size > (cfg.maxFileBytes ?? 256 * 1024)) return null;
+  let content: string;
+  try { content = readFileSync(absPath, 'utf8'); } catch { return null; }
+  const docs: NormalizedDocument[] = [{ uri: `repo://${rel}`, title: rel, content, kind: 'plain' }];
+  if (cfg.symbols) {
+    const symDoc = await symbolDocFor(content, rel, cfg.symbolGrammars);
+    if (symDoc) docs.push(symDoc);
+  }
+  return docs;
+}
+
+/** per-symbol doc: content = one summary line per symbol (browsable),
+ *  chunks = one chunk per symbol with heading_path "symbol: <name> (<kind>)"
+ *  (Stage-3 shape — structured filtering; v1 concatenated all symbols into
+ *  the content body, unretrievable by heading). */
+export async function symbolDocFor(content: string, rel: string, grammars?: SymbolGrammars): Promise<NormalizedDocument | null> {
+  const symbols = await symbolsFor(content, rel, grammars ?? {});
+  if (symbols.length === 0) return null;
+  const chunks: StructuralChunk[] = symbolChunksFor(symbols, content, rel);
+  return {
+    uri: `repo://${rel}#symbols`,
+    title: `${rel} (symbols)`,
+    content: symbols.map((s) => `symbol: ${s.name} (${s.kind}) — ${rel}:${s.line}`).join('\n'),
+    kind: 'symbols',
+    chunks,
+  };
+}
+
 export function repoConnector(sourceId: string, cfg: RepoSourceConfig): SourceConnector {
   const maxFiles = cfg.maxFiles ?? 2_000;
   const maxBytes = cfg.maxFileBytes ?? 256 * 1024;
@@ -72,12 +123,11 @@ export function repoConnector(sourceId: string, cfg: RepoSourceConfig): SourceCo
         }
         for (const f of files) {
           const rel = relative(root, f);
-          out.push({ uri: `repo://${rel}`, title: rel, content: readFileSync(f, 'utf8'), kind: 'plain' });
+          const content = readFileSync(f, 'utf8');
+          out.push({ uri: `repo://${rel}`, title: rel, content, kind: 'plain' });
           if (cfg.symbols) {
-            // symbol chunks ride the SAME pipeline: they're just more chunks
-            // for this doc's kind, retrievable by "symbol: name (kind)"
-            const sym = symbolChunks(readFileSync(f, 'utf8'), rel);
-            if (sym.length > 0) out.push({ uri: `repo://${rel}#symbols`, title: `${rel} (symbols)`, content: sym.map((s) => `${s.headingPath}\n${s.content}`).join('\n\n'), kind: 'symbols' });
+            const symDoc = await symbolDocFor(content, rel, cfg.symbolGrammars);
+            if (symDoc) out.push(symDoc);
           }
         }
         return out;