From b939d9ffa6226dd399cbcf5edbc4d0d5c3c4d75f Mon Sep 17 00:00:00 2001 From: issacboy123 Date: Mon, 5 Oct 2026 04:36:28 -0700 Subject: [PATCH 1/3] fix(coding-agent): retry failed evidence scans --- .../src/core/evolution-runtime.ts | 73 ++++++++++----- .../interactive/components/evolution-menu.ts | 6 ++ .../coding-agent/test/evolution-menu.test.ts | 25 ++++++ .../test/evolution-runtime.test.ts | 62 +++++++++++-- .../test/invocation-tracing.test.ts | 16 ++-- packages/evolution/src/control-plane.ts | 17 +++- packages/evolution/src/control-service.ts | 15 ++-- packages/evolution/src/index.ts | 2 + packages/evolution/src/ports.ts | 18 ++-- packages/evolution/test/control-plane.test.ts | 90 +++++++++++++++++-- .../evolution/test/control-service.test.ts | 44 ++++++++- 11 files changed, 303 insertions(+), 65 deletions(-) diff --git a/packages/coding-agent/src/core/evolution-runtime.ts b/packages/coding-agent/src/core/evolution-runtime.ts index 9f02f6713..09e0ba201 100644 --- a/packages/coding-agent/src/core/evolution-runtime.ts +++ b/packages/coding-agent/src/core/evolution-runtime.ts @@ -25,6 +25,7 @@ import { EvolutionControlPlane, EvolutionControlService, type EvolutionEvidenceCatalog, + type EvolutionEvidenceScan, EvolutionExecutionError, type EvolutionHarnessVersionInfo, type EvolutionJobDetail, @@ -492,7 +493,10 @@ export class TraceEvidenceCatalog implements EvolutionEvidenceCatalog { readonly #scope: EvolutionScope; readonly #materializeInvocation: typeof materializeJsonlInvocation; readonly #isTraceActive: (path: string) => Promise; - readonly #cache = new Map>(); + readonly #cache = new Map< + string, + { readonly ingestionCursor: number; readonly materialization: Promise } + >(); constructor(options: { agentDir: string; @@ -700,40 +704,67 @@ export class TraceEvidenceCatalog implements EvolutionEvidenceCatalog { return (await this.#paths()).length; } - async listSuccessful( - scope: EvolutionScope, - afterCursor: number, - throughCursor: number, - ): Promise { + async scan(scope: EvolutionScope, afterCursor: number, throughCursor: number): Promise { this.#assertScope(scope); const paths = await this.#paths(); const selected = paths.slice(afterCursor, Math.min(throughCursor, paths.length)); - const materializations = await Promise.all( - selected.map((path, index) => { - const ingestionCursor = afterCursor + index + 1; - const key = `${ingestionCursor}:${path}`; - const cached = this.#cache.get(key); - if (cached) return cached; - const created = this.#materialize(path, ingestionCursor).catch(() => null); - this.#cache.set(key, created); - return created; - }), - ); - return materializations.filter((entry): entry is SuccessfulMaterialization => entry !== null); + const materializations: SuccessfulMaterialization[] = []; + for (const [index, path] of selected.entries()) { + const ingestionCursor = afterCursor + index + 1; + const key = `${ingestionCursor}:${path}`; + let cached = this.#cache.get(key); + if (cached === undefined) { + const materialization = this.#materialize(path, ingestionCursor); + cached = { ingestionCursor, materialization }; + this.#cache.set(key, cached); + void materialization.catch(() => { + if (this.#cache.get(key) === cached) this.#cache.delete(key); + }); + } + try { + const materialization = await cached.materialization; + if (materialization !== null) materializations.push(materialization); + } catch { + return { + materializations, + firstFailure: { + ingestionCursor, + traceName: basename(path), + notice: "Trace evidence could not be read and will be retried.", + }, + }; + } + } + return { materializations, firstFailure: null }; } async excludedThroughCursor(scope: EvolutionScope, afterCursor: number, throughCursor: number): Promise { this.#assertScope(scope); - await this.listSuccessful(scope, afterCursor, throughCursor); const excludedNames = new Set((await this.#readExclusions()).map((exclusion) => exclusion.traceName)); const paths = await this.#paths(); let cursor = afterCursor; - for (const path of paths.slice(afterCursor, Math.min(throughCursor, paths.length))) { - if (!excludedNames.has(basename(path))) break; + for (const [index, path] of paths.slice(afterCursor, Math.min(throughCursor, paths.length)).entries()) { + const ingestionCursor = afterCursor + index + 1; + const cached = this.#cache.get(`${ingestionCursor}:${path}`); + if (cached === undefined && !excludedNames.has(basename(path))) break; + if (cached !== undefined) { + try { + await cached.materialization; + } catch { + break; + } + } cursor += 1; } return cursor; } + + async discardProcessedThrough(scope: EvolutionScope, cursor: number): Promise { + this.#assertScope(scope); + for (const [key, cached] of this.#cache) { + if (cached.ingestionCursor <= cursor) this.#cache.delete(key); + } + } } export function createEvolutionSkillSnapshots(skills: readonly Skill[]): readonly EvolutionSkillSnapshotSource[] { diff --git a/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts b/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts index 2fa0dde03..3b0d3bad5 100644 --- a/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts +++ b/packages/coding-agent/src/modes/interactive/components/evolution-menu.ts @@ -681,6 +681,12 @@ export class EvolutionMenuComponent extends Container implements Focusable { const evolutionEnabled = this.#client.getEnabled(); this.#addLine(`Evolution tokens: ${overview.unprocessedTokens.toLocaleString()}`); this.#addLine(`Evolution invocations: ${overview.settledInvocationCount}`); + if (overview.firstEvidenceFailure !== null && overview.firstEvidenceFailure !== undefined) { + this.#addLine( + `Evidence retry pending at cursor ${overview.firstEvidenceFailure.ingestionCursor} (${overview.firstEvidenceFailure.traceName}): ${overview.firstEvidenceFailure.notice}`, + "warning", + ); + } this.#addLine(`Last evolution: ${formatEvolutionTimestamp(overview.lastEvolutionAt)}`); this.#addLine( !evolutionEnabled diff --git a/packages/coding-agent/test/evolution-menu.test.ts b/packages/coding-agent/test/evolution-menu.test.ts index 4fb0a5dd3..8ba90a33b 100644 --- a/packages/coding-agent/test/evolution-menu.test.ts +++ b/packages/coding-agent/test/evolution-menu.test.ts @@ -115,6 +115,7 @@ function overview(overrides: Partial = {}): EvolutionOverview ingestionHighWaterCursor: 8, unprocessedTokens: 1_100_001, settledInvocationCount: 6, + firstEvidenceFailure: null, automaticTokenThreshold: AUTOMATIC_EVOLUTION_TOKEN_THRESHOLD, automaticMinimumInvocations: 5, lastEvolutionAt: null, @@ -437,6 +438,30 @@ describe("EvolutionMenuComponent", () => { expect(output).toContain("Cancel [unavailable]"); }); + it("shows a generic retry notice for the first unreadable evidence trace", async () => { + const fixture = client( + overview({ + firstEvidenceFailure: { + ingestionCursor: 7, + traceName: "unreadable.jsonl", + notice: "Trace evidence could not be read and will be retried.", + }, + }), + ); + const component = new EvolutionMenuComponent( + fixture.client, + () => {}, + () => {}, + () => {}, + ); + await vi.waitFor(() => expect(stripAnsi(component.render(180).join("\n"))).toContain("Browse")); + + const output = stripAnsi(component.render(180).join("\n")); + expect(output).toContain("Evidence retry pending at cursor 7 (unreadable.jsonl)"); + expect(output).toContain("Trace evidence could not be read and will be retried."); + expect(output).not.toContain("simulated incompatible trace"); + }); + it("cancels a resumable job by its durable job ID", async () => { const resumable = job({ jobId: "job-resumable", diff --git a/packages/coding-agent/test/evolution-runtime.test.ts b/packages/coding-agent/test/evolution-runtime.test.ts index c1b3f9f72..af82b99a7 100644 --- a/packages/coding-agent/test/evolution-runtime.test.ts +++ b/packages/coding-agent/test/evolution-runtime.test.ts @@ -661,30 +661,80 @@ describe("coding-agent evolution runtime", () => { const unreadablePath = join(traceDirectory, "m-unreadable.jsonl"); writeFileSync(unreadablePath, "unreadable\n"); const scope: EvolutionScope = { repositoryId: "repository", scopeId: "repository" }; + let shouldFail = true; + let materializationAttempts = 0; const catalog = new TraceEvidenceCatalog({ agentDir: root, scope, async materializeInvocation(path) { - if (path === unreadablePath) throw new Error("simulated incompatible trace"); + materializationAttempts += 1; + if (path === unreadablePath && shouldFail) throw new Error("simulated incompatible trace"); return materializedInvocation(basename(path, ".jsonl"), sessionPath); }, }); expect(await catalog.highWaterCursor(scope)).toBe(3); - expect(await catalog.listSuccessful(scope, 0, 3)).toMatchObject([ + const failedScan = await catalog.scan(scope, 0, 3); + expect(failedScan.materializations).toMatchObject([ { materializationId: "z-first", ingestionCursor: 1 }, { materializationId: "a-second", ingestionCursor: 2 }, ]); + expect(failedScan.firstFailure).toEqual({ + ingestionCursor: 3, + traceName: "m-unreadable.jsonl", + notice: "Trace evidence could not be read and will be retried.", + }); + shouldFail = false; + expect((await catalog.scan(scope, 2, 3)).materializations).toMatchObject([ + { materializationId: "m-unreadable", ingestionCursor: 3 }, + ]); + expect(materializationAttempts).toBe(4); + await catalog.discardProcessedThrough(scope, 3); + expect((await catalog.scan(scope, 0, 2)).materializations).toHaveLength(2); + expect(materializationAttempts).toBe(6); await new Promise((resolveDelay) => setTimeout(resolveDelay, 5)); const latePath = join(traceDirectory, "00-late.jsonl"); writeFileSync(latePath, "late\n"); expect(await catalog.highWaterCursor(scope)).toBe(4); - expect(await catalog.listSuccessful(scope, 3, 4)).toMatchObject([ + expect((await catalog.scan(scope, 3, 4)).materializations).toMatchObject([ { materializationId: "00-late", ingestionCursor: 4 }, ]); }); + it("reuses an in-flight successful materialization across concurrent scans", async () => { + const root = join(tmpdir(), `diffex-inflight-evidence-${Date.now()}-${Math.random().toString(36).slice(2)}`); + const traceDirectory = join(root, "traces"); + mkdirSync(traceDirectory, { recursive: true }); + temporaryDirectories.push(root); + writeFileSync(join(traceDirectory, "trace.jsonl"), "trace\n"); + const scope: EvolutionScope = { repositoryId: "repository", scopeId: "repository" }; + let materializationCalls = 0; + let releaseMaterialization: (() => void) | undefined; + const materialized = new Promise((resolve) => { + releaseMaterialization = resolve; + }); + const catalog = new TraceEvidenceCatalog({ + agentDir: root, + scope, + async materializeInvocation() { + materializationCalls += 1; + await materialized; + return materializedInvocation("trace", "/session.jsonl"); + }, + }); + + const first = catalog.scan(scope, 0, 1); + await vi.waitFor(() => expect(materializationCalls).toBe(1)); + const second = catalog.scan(scope, 0, 1); + releaseMaterialization?.(); + expect(await Promise.all([first, second])).toEqual([ + { materializations: [expect.objectContaining({ materializationId: "trace" })], firstFailure: null }, + { materializations: [expect.objectContaining({ materializationId: "trace" })], firstFailure: null }, + ]); + expect(materializationCalls).toBe(1); + }); + it("indexes settled traces without letting an older active invocation block them", async () => { const root = join(tmpdir(), `diffex-global-evidence-${Date.now()}-${Math.random().toString(36).slice(2)}`); const traceDirectory = join(root, "traces"); @@ -714,14 +764,14 @@ describe("coding-agent evolution runtime", () => { }); expect(await Promise.all([catalog.highWaterCursor(scope), peerCatalog.highWaterCursor(scope)])).toEqual([2, 2]); - expect(await catalog.listSuccessful(scope, 0, 3)).toMatchObject([ + expect((await catalog.scan(scope, 0, 3)).materializations).toMatchObject([ { materializationId: "first", ingestionCursor: 1 }, { materializationId: "later", ingestionCursor: 2 }, ]); active.delete(activePath); expect(await Promise.all([catalog.highWaterCursor(scope), peerCatalog.highWaterCursor(scope)])).toEqual([3, 3]); - expect(await peerCatalog.listSuccessful(scope, 2, 3)).toMatchObject([ + expect((await peerCatalog.scan(scope, 2, 3)).materializations).toMatchObject([ { materializationId: "active", ingestionCursor: 3 }, ]); }); @@ -743,7 +793,7 @@ describe("coding-agent evolution runtime", () => { }), }); - expect(await catalog.listSuccessful(scope, 0, await catalog.highWaterCursor(scope))).toEqual([]); + expect((await catalog.scan(scope, 0, await catalog.highWaterCursor(scope))).materializations).toEqual([]); expect(JSON.parse(readFileSync(join(traceDirectory, ".evolution-exclusions.json"), "utf8"))).toEqual({ version: 1, exclusions: [ diff --git a/packages/coding-agent/test/invocation-tracing.test.ts b/packages/coding-agent/test/invocation-tracing.test.ts index 256244f17..70de4c1e7 100644 --- a/packages/coding-agent/test/invocation-tracing.test.ts +++ b/packages/coding-agent/test/invocation-tracing.test.ts @@ -293,7 +293,7 @@ describe("CLI invocation tracing", () => { await expect(lockfile.check(firstPath, { realpath: false })).resolves.toBe(false); expect(await catalog.highWaterCursor(scope)).toBe(1); - expect(await catalog.listSuccessful(scope, 0, 1)).toHaveLength(1); + expect((await catalog.scan(scope, 0, 1)).materializations).toHaveLength(1); expect(footer(await readTrace(firstPath))?.terminationStatus).toBe("completed"); expect((await readTrace(firstPath)).find((record) => record.type === "session_segment")).toMatchObject({ taskLineageId: "operation-1", @@ -325,7 +325,7 @@ describe("CLI invocation tracing", () => { }); expect(await catalog.highWaterCursor(scope)).toBe(2); - expect(await catalog.listSuccessful(scope, 1, 2)).toHaveLength(1); + expect((await catalog.scan(scope, 1, 2)).materializations).toHaveLength(1); expect(tracePaths(directories.agentDir)).toHaveLength(2); await trace.close(); expect(tracePaths(directories.agentDir)).toHaveLength(2); @@ -460,7 +460,7 @@ describe("CLI invocation tracing", () => { expect(await Promise.all([firstCatalog.highWaterCursor(scope), secondCatalog.highWaterCursor(scope)])).toEqual([ 2, 2, ]); - const materializations = await firstCatalog.listSuccessful(scope, 0, 2); + const materializations = (await firstCatalog.scan(scope, 0, 2)).materializations; expect(materializations).toHaveLength(2); expect(new Set(materializations.map((entry) => entry.materializationId)).size).toBe(2); await Promise.all([firstTrace.close(), secondTrace.close()]); @@ -510,7 +510,7 @@ describe("CLI invocation tracing", () => { expect(tracePaths(directories.agentDir)).toEqual([path]); const scope: EvolutionScope = { repositoryId: "global", scopeId: "global" }; const catalog = new TraceEvidenceCatalog({ agentDir: directories.agentDir, scope }); - const [materialization] = await catalog.listSuccessful(scope, 0, await catalog.highWaterCursor(scope)); + const [materialization] = (await catalog.scan(scope, 0, await catalog.highWaterCursor(scope))).materializations; const sessionEvidence = materialization?.invocation.evidence.filter((entry) => entry.kind === "session_item") ?? []; expect(sessionEvidence).toHaveLength(2); @@ -749,7 +749,7 @@ describe("CLI invocation tracing", () => { const catalog = new TraceEvidenceCatalog({ agentDir: directories.agentDir, scope }); expect(tracePaths(directories.agentDir)).toEqual([]); expect(await catalog.highWaterCursor(scope)).toBe(0); - expect(await catalog.listSuccessful(scope, 0, 0)).toEqual([]); + expect((await catalog.scan(scope, 0, 0)).materializations).toEqual([]); }); it.each([ @@ -778,7 +778,7 @@ describe("CLI invocation tracing", () => { const scope: EvolutionScope = { repositoryId: "global", scopeId: "global" }; const catalog = new TraceEvidenceCatalog({ agentDir: directories.agentDir, scope }); expect(tracePaths(directories.agentDir)).toHaveLength(expectedRawTraceCount); - expect(await catalog.listSuccessful(scope, 0, await catalog.highWaterCursor(scope))).toEqual([]); + expect((await catalog.scan(scope, 0, await catalog.highWaterCursor(scope))).materializations).toEqual([]); expect(await catalog.excludedThroughCursor(scope, 0, await catalog.highWaterCursor(scope))).toBe( expectedRawTraceCount, ); @@ -848,7 +848,7 @@ describe("CLI invocation tracing", () => { const scope: EvolutionScope = { repositoryId: "global", scopeId: "global" }; const catalog = new TraceEvidenceCatalog({ agentDir: directories.agentDir, scope }); - const materializations = await catalog.listSuccessful(scope, 0, await catalog.highWaterCursor(scope)); + const materializations = (await catalog.scan(scope, 0, await catalog.highWaterCursor(scope))).materializations; expect(materializations).toHaveLength(1); expect( materializations[0]?.invocation.evidence.some((entry) => JSON.stringify(entry).includes("exit-operation")), @@ -891,7 +891,7 @@ describe("CLI invocation tracing", () => { expect(footer(records)?.terminationStatus).toBe("failed"); const scope: EvolutionScope = { repositoryId: "global", scopeId: "global" }; const catalog = new TraceEvidenceCatalog({ agentDir: traceStorageDirectory(tracePath), scope }); - expect(await catalog.listSuccessful(scope, 0, await catalog.highWaterCursor(scope))).toHaveLength(1); + expect((await catalog.scan(scope, 0, await catalog.highWaterCursor(scope))).materializations).toHaveLength(1); }); it.each([ diff --git a/packages/evolution/src/control-plane.ts b/packages/evolution/src/control-plane.ts index 9b1e97a43..49c717bf2 100644 --- a/packages/evolution/src/control-plane.ts +++ b/packages/evolution/src/control-plane.ts @@ -469,11 +469,12 @@ export class EvolutionControlPlane implements EvolutionJobRunner { retryable: false, }); } - const availableMaterializations = await this.#dependencies.evidence.listSuccessful( + const scan = await this.#dependencies.evidence.scan( request.scope, processedCursor.sequence, availableHighWaterCursor, ); + const availableMaterializations = scan.materializations; const seenMaterializations = new Set(); let previousCursor = processedCursor.sequence; for (const materialization of availableMaterializations) { @@ -497,7 +498,7 @@ export class EvolutionControlPlane implements EvolutionJobRunner { (await this.#dependencies.evidence.excludedThroughCursor?.( request.scope, processedCursor.sequence, - availableHighWaterCursor, + scan.firstFailure === null ? availableHighWaterCursor : scan.firstFailure.ingestionCursor - 1, )) ?? processedCursor.sequence; if (excludedThroughCursor > processedCursor.sequence) { const commitId = `excluded-${computeEvolutionContentDigest({ @@ -519,13 +520,14 @@ export class EvolutionControlPlane implements EvolutionJobRunner { retryable: false, }); } + await this.#dependencies.evidence.discardProcessedThrough?.(request.scope, excludedThroughCursor); } return { status: "not-created", reason: "no-new-evidence" }; } const scopeJobs = await this.#dependencies.jobs.list(request.scope); const availableEvidenceRange = { afterCursor: processedCursor.sequence, - throughCursor: availableHighWaterCursor, + throughCursor: availableMaterializations.at(-1)?.ingestionCursor ?? processedCursor.sequence, }; const blocking = scopeJobs.find( (job) => @@ -559,7 +561,13 @@ export class EvolutionControlPlane implements EvolutionJobRunner { const ingestionHighWaterCursor = request.trigger === "automatic" ? (materializations.at(-1)?.ingestionCursor ?? processedCursor.sequence) - : availableHighWaterCursor; + : ((await this.#dependencies.evidence.excludedThroughCursor?.( + request.scope, + processedCursor.sequence, + scan.firstFailure === null ? availableHighWaterCursor : scan.firstFailure.ingestionCursor - 1, + )) ?? + materializations.at(-1)?.ingestionCursor ?? + processedCursor.sequence); const evidenceRange = { afterCursor: processedCursor.sequence, throughCursor: ingestionHighWaterCursor, @@ -788,6 +796,7 @@ export class EvolutionControlPlane implements EvolutionJobRunner { retryable: false, }); } + await this.#dependencies.evidence.discardProcessedThrough?.(job.metadata.scope, job.evidenceRange.throughCursor); const completion: EvolutionJobCompletion = { kind: directive === "complete-no-op" ? "no-op" : "activated", resultingHarnessVersion: directive === "complete-no-op" ? null : resultingHarnessVersion, diff --git a/packages/evolution/src/control-service.ts b/packages/evolution/src/control-service.ts index 79b4f35ee..8d0722dff 100644 --- a/packages/evolution/src/control-service.ts +++ b/packages/evolution/src/control-service.ts @@ -44,6 +44,7 @@ import type { EvolutionCursorStore, EvolutionEventSink, EvolutionEvidenceCatalog, + EvolutionEvidenceScanFailure, EvolutionJobStore, EvolutionLedger, EvolutionProgressEvent, @@ -121,6 +122,7 @@ export interface EvolutionOverview { readonly ingestionHighWaterCursor: number; readonly unprocessedTokens: number; readonly settledInvocationCount: number; + readonly firstEvidenceFailure?: EvolutionEvidenceScanFailure | null; readonly automaticTokenThreshold: number; readonly automaticMinimumInvocations: number; readonly lastEvolutionAt: string | null; @@ -658,12 +660,8 @@ export class EvolutionControlService { this.#dependencies.jobs.list(this.#dependencies.scope), this.#dependencies.harnesses.listVersions(), ]); - const materializations = await this.#dependencies.evidence.listSuccessful( - this.#dependencies.scope, - cursor.sequence, - highWater, - ); - const tokenCount = countEvolutionTokens(materializations.map((entry) => entry.invocation)); + const scan = await this.#dependencies.evidence.scan(this.#dependencies.scope, cursor.sequence, highWater); + const tokenCount = countEvolutionTokens(scan.materializations.map((entry) => entry.invocation)); const lastAutomaticRunAt = jobs .filter((job) => !isCancelledJob(job) && job.trigger === "automatic" && job.terminalAt !== null) @@ -687,7 +685,7 @@ export class EvolutionControlService { active !== null ? ({ eligible: false, reason: "blocking-job" } as const) : evaluateAutomaticEvolutionEligibility({ - invocationCount: materializations.length, + invocationCount: scan.materializations.length, tokens: tokenCount.totalTokens, now: new Date(currentTime).toISOString(), previousAutomaticTerminalAt: lastAutomaticRunAt, @@ -703,7 +701,8 @@ export class EvolutionControlService { processedEvidenceCursor: cursor.sequence, ingestionHighWaterCursor: highWater, unprocessedTokens: tokenCount.totalTokens, - settledInvocationCount: materializations.length, + settledInvocationCount: scan.materializations.length, + firstEvidenceFailure: scan.firstFailure, automaticTokenThreshold: AUTOMATIC_EVOLUTION_TOKEN_THRESHOLD, automaticMinimumInvocations: AUTOMATIC_EVOLUTION_MINIMUM_INVOCATIONS, lastEvolutionAt, diff --git a/packages/evolution/src/index.ts b/packages/evolution/src/index.ts index 806f5b43b..4412b4897 100644 --- a/packages/evolution/src/index.ts +++ b/packages/evolution/src/index.ts @@ -313,6 +313,8 @@ export type { EvolutionCursorStore, EvolutionEventSink, EvolutionEvidenceCatalog, + EvolutionEvidenceScan, + EvolutionEvidenceScanFailure, EvolutionFailureRecord, EvolutionFailureReporter, EvolutionIdGenerator, diff --git a/packages/evolution/src/ports.ts b/packages/evolution/src/ports.ts index ccc02407f..74399adb6 100644 --- a/packages/evolution/src/ports.ts +++ b/packages/evolution/src/ports.ts @@ -604,14 +604,22 @@ export interface SuccessfulMaterialization { readonly invocation: MaterializedInvocation; } +export interface EvolutionEvidenceScanFailure { + readonly ingestionCursor: number; + readonly traceName: string; + readonly notice: string; +} + +export interface EvolutionEvidenceScan { + readonly materializations: readonly SuccessfulMaterialization[]; + readonly firstFailure: EvolutionEvidenceScanFailure | null; +} + export interface EvolutionEvidenceCatalog { highWaterCursor(scope: EvolutionScope): Promise; - listSuccessful( - scope: EvolutionScope, - afterCursor: number, - throughCursor: number, - ): Promise; + scan(scope: EvolutionScope, afterCursor: number, throughCursor: number): Promise; excludedThroughCursor?(scope: EvolutionScope, afterCursor: number, throughCursor: number): Promise; + discardProcessedThrough?(scope: EvolutionScope, cursor: number): Promise; } export type EvolutionSkillSnapshotSource = Omit; diff --git a/packages/evolution/test/control-plane.test.ts b/packages/evolution/test/control-plane.test.ts index 2a19daf90..c63827cf1 100644 --- a/packages/evolution/test/control-plane.test.ts +++ b/packages/evolution/test/control-plane.test.ts @@ -222,6 +222,7 @@ interface ControlPlaneFixture( ], highWaterCursor?: number, capturedSnapshot: EvolutionSnapshotContext = snapshotContext(), + firstFailure: { + readonly ingestionCursor: number; + readonly traceName: string; + readonly notice: string; + } | null = null, ): Promise> { const root = await mkdtemp(join(tmpdir(), "diffex-evolution-control-")); temporaryDirectories.push(root); @@ -241,6 +247,7 @@ async function controlPlaneFixture( const clock = new AdvancingClock(); const events: EvolutionProgressEvent[] = []; const failureRecords: EvolutionFailureRecord[] = []; + const prunedCursors: number[] = []; const eventSink: EvolutionEventSink = { emit(event) { events.push(event); @@ -267,10 +274,22 @@ async function controlPlaneFixture( async highWaterCursor() { return highWaterCursor ?? materializations.at(-1)?.ingestionCursor ?? 0; }, - async listSuccessful(_scope, afterCursor, throughCursor) { - return materializations.filter( - (item) => item.ingestionCursor > afterCursor && item.ingestionCursor <= throughCursor, - ); + async scan(_scope, afterCursor, throughCursor) { + return { + materializations: materializations.filter( + (item) => + item.ingestionCursor > afterCursor && + item.ingestionCursor <= throughCursor && + (firstFailure === null || item.ingestionCursor < firstFailure.ingestionCursor), + ), + firstFailure, + }; + }, + async discardProcessedThrough(_scope, cursor) { + prunedCursors.push(cursor); + }, + async excludedThroughCursor(_scope, _afterCursor, throughCursor) { + return throughCursor; }, }, snapshots: { @@ -282,7 +301,7 @@ async function controlPlaneFixture( }, { leaseTtlMs: 5_000, heartbeatIntervalMs: 1_000, retryBaseDelayMs: 0, retryMaxDelayMs: 0 }, ); - return { root, clock, controlPlane, events, executor, failureRecords, persistence }; + return { root, clock, controlPlane, events, executor, failureRecords, prunedCursors, persistence }; } async function createManualJob(fixture: ControlPlaneFixture) { @@ -294,6 +313,7 @@ async function createManualJob(fixture: ControlPlaneFixture) { describe("durable evolution control plane", () => { it("durably consumes an excluded-only evidence range", async () => { + const prunedCursors: number[] = []; const fixture = await controlPlaneFixture(new RecordingExecutor(), []); const controlPlane = new EvolutionControlPlane( { @@ -311,12 +331,15 @@ describe("durable evolution control plane", () => { async highWaterCursor() { return 3; }, - async listSuccessful() { - return []; + async scan() { + return { materializations: [], firstFailure: null }; }, async excludedThroughCursor() { return 3; }, + async discardProcessedThrough(_scope, cursor) { + prunedCursors.push(cursor); + }, }, snapshots: { async capture() { @@ -333,6 +356,55 @@ describe("durable evolution control plane", () => { reason: "no-new-evidence", }); expect(await fixture.persistence.load(scope)).toMatchObject({ sequence: 3 }); + expect(prunedCursors).toEqual([3]); + }); + + it("processes only the successful prefix before a retryable evidence failure", async () => { + const fixture = await controlPlaneFixture( + new RecordingExecutor({ noOp: true }), + [materialization(1), materialization(2), materialization(3)], + 3, + snapshotContext(), + { + ingestionCursor: 2, + traceName: "failed.jsonl", + notice: "Trace evidence could not be read and will be retried.", + }, + ); + + const created = await createManualJob(fixture); + expect(created.evidenceRange).toEqual({ afterCursor: 0, throughCursor: 1 }); + await fixture.controlPlane.runJob(created.jobId); + expect(await fixture.persistence.load(scope)).toMatchObject({ sequence: 1 }); + expect(fixture.prunedCursors).toEqual([1]); + await expect(fixture.controlPlane.createJob({ scope, trigger: "manual" })).resolves.toEqual({ + status: "not-created", + reason: "no-new-evidence", + }); + }); + + it("does not count evidence after a failure toward automatic eligibility", async () => { + const fixture = await controlPlaneFixture( + new RecordingExecutor(), + [ + materialization(1, 200_000, 0), + materialization(2, 200_000, 0), + materialization(3, 200_000, 0), + materialization(4, 200_000, 0), + ], + 4, + snapshotContext(), + { + ingestionCursor: 4, + traceName: "unreadable.jsonl", + notice: "Trace evidence could not be read and will be retried.", + }, + ); + + await expect(fixture.controlPlane.createJob({ scope, trigger: "automatic" })).resolves.toMatchObject({ + status: "created", + job: { evidenceRange: { afterCursor: 0, throughCursor: 2 } }, + }); }); it("commits a mixed range through excluded evidence after eligible work completes", async () => { @@ -741,8 +813,8 @@ describe("durable evolution control plane", () => { async highWaterCursor() { return 0; }, - async listSuccessful() { - return []; + async scan() { + return { materializations: [], firstFailure: null }; }, }, snapshots: { diff --git a/packages/evolution/test/control-service.test.ts b/packages/evolution/test/control-service.test.ts index a1a3d87f3..3a63ad9d0 100644 --- a/packages/evolution/test/control-service.test.ts +++ b/packages/evolution/test/control-service.test.ts @@ -125,6 +125,11 @@ async function serviceFixture( evidenceOptions: { readonly highWaterCursor?: number; readonly materializations?: readonly SuccessfulMaterialization[]; + readonly firstFailure?: { + readonly ingestionCursor: number; + readonly traceName: string; + readonly notice: string; + } | null; } = {}, ): Promise { const root = await mkdtemp(join(tmpdir(), "diffex-evolution-service-")); @@ -174,10 +179,18 @@ async function serviceFixture( async highWaterCursor() { return evidenceOptions.highWaterCursor ?? 3; }, - async listSuccessful(_scope: EvolutionScope, afterCursor: number, throughCursor: number) { - return materializations.filter( - (entry) => entry.ingestionCursor > afterCursor && entry.ingestionCursor <= throughCursor, - ); + async scan(_scope: EvolutionScope, afterCursor: number, throughCursor: number) { + return { + materializations: materializations.filter( + (entry) => + entry.ingestionCursor > afterCursor && + entry.ingestionCursor <= throughCursor && + (evidenceOptions.firstFailure === undefined || + evidenceOptions.firstFailure === null || + entry.ingestionCursor < evidenceOptions.firstFailure.ingestionCursor), + ), + firstFailure: evidenceOptions.firstFailure ?? null, + }; }, }; const controlPlane = new EvolutionControlPlane( @@ -531,6 +544,29 @@ describe("evolution control service", () => { await fixture.service.cancelJob(started.job.jobId); }); + it("reports only the valid evidence prefix before the first materialization failure", async () => { + const eligible = materialization(1); + const fixture = await serviceFixture(undefined, { + highWaterCursor: 3, + materializations: [eligible, materialization(2), materialization(3)], + firstFailure: { + ingestionCursor: 2, + traceName: "unreadable.jsonl", + notice: "Trace evidence could not be read and will be retried.", + }, + }); + + expect(await fixture.service.overview()).toMatchObject({ + settledInvocationCount: 1, + unprocessedTokens: countEvolutionTokens([eligible.invocation]).totalTokens, + firstEvidenceFailure: { + ingestionCursor: 2, + traceName: "unreadable.jsonl", + notice: "Trace evidence could not be read and will be retried.", + }, + }); + }); + it("projects eligibility and attaches a concurrent manual trigger", async () => { const fixture = await serviceFixture(); const events: string[] = []; From 2dde1138330d95675b067db4dd4bf55fa4c325f3 Mon Sep 17 00:00:00 2001 From: issacboy123 Date: Mon, 5 Oct 2026 04:44:41 -0700 Subject: [PATCH 2/3] test(evolution): cover first failed evidence cursor --- packages/evolution/test/control-plane.test.ts | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/packages/evolution/test/control-plane.test.ts b/packages/evolution/test/control-plane.test.ts index c63827cf1..7d7806be6 100644 --- a/packages/evolution/test/control-plane.test.ts +++ b/packages/evolution/test/control-plane.test.ts @@ -383,6 +383,28 @@ describe("durable evolution control plane", () => { }); }); + it("leaves the cursor unchanged when the first evidence trace is retryable", async () => { + const fixture = await controlPlaneFixture( + new RecordingExecutor(), + [materialization(1), materialization(2)], + 2, + snapshotContext(), + { + ingestionCursor: 1, + traceName: "failed-first.jsonl", + notice: "Trace evidence could not be read and will be retried.", + }, + ); + + await expect(fixture.controlPlane.createJob({ scope, trigger: "manual" })).resolves.toEqual({ + status: "not-created", + reason: "no-new-evidence", + }); + expect(await fixture.persistence.load(scope)).toMatchObject({ sequence: 0 }); + expect(await fixture.persistence.list(scope)).toEqual([]); + expect(fixture.prunedCursors).toEqual([]); + }); + it("does not count evidence after a failure toward automatic eligibility", async () => { const fixture = await controlPlaneFixture( new RecordingExecutor(), From ee3f1174c71eda50aeb97e60f7310859b13452d4 Mon Sep 17 00:00:00 2001 From: issacboy123 Date: Fri, 9 Oct 2026 05:39:21 -0700 Subject: [PATCH 3/3] fix(coding-agent): bound evidence materialization retries --- .../src/core/evolution-runtime.ts | 31 +++++++++++- .../test/evolution-runtime.test.ts | 49 +++++++++++++++++++ 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/packages/coding-agent/src/core/evolution-runtime.ts b/packages/coding-agent/src/core/evolution-runtime.ts index 09e0ba201..dc1d088a5 100644 --- a/packages/coding-agent/src/core/evolution-runtime.ts +++ b/packages/coding-agent/src/core/evolution-runtime.ts @@ -88,6 +88,7 @@ const TRACE_EVIDENCE_INDEX_NAME = ".evolution-index.json"; const TRACE_EVIDENCE_INDEX_VERSION = 1; const TRACE_EXCLUSION_INDEX_NAME = ".evolution-exclusions.json"; const TRACE_EXCLUSION_INDEX_VERSION = 1; +const MAX_TRACE_MATERIALIZATION_RETRIES = 2; const RESPONSE_TOOL_NAME = "submit_evolution_response"; const MAX_MODEL_ROUNDS = 10; const PROVIDER_FIRST_BYTE_TIMEOUT_MS = 60_000; @@ -497,6 +498,7 @@ export class TraceEvidenceCatalog implements EvolutionEvidenceCatalog { string, { readonly ingestionCursor: number; readonly materialization: Promise } >(); + readonly #failedMaterializationAttempts = new Map(); constructor(options: { agentDir: string; @@ -713,8 +715,30 @@ export class TraceEvidenceCatalog implements EvolutionEvidenceCatalog { const ingestionCursor = afterCursor + index + 1; const key = `${ingestionCursor}:${path}`; let cached = this.#cache.get(key); + if ( + cached === undefined && + (this.#failedMaterializationAttempts.get(key) ?? 0) >= MAX_TRACE_MATERIALIZATION_RETRIES + 1 + ) { + return { + materializations, + firstFailure: { + ingestionCursor, + traceName: basename(path), + notice: "Trace evidence could not be read after 2 retries.", + }, + }; + } if (cached === undefined) { - const materialization = this.#materialize(path, ingestionCursor); + const materialization = this.#materialize(path, ingestionCursor).then( + (result) => { + this.#failedMaterializationAttempts.delete(key); + return result; + }, + (error: unknown) => { + this.#failedMaterializationAttempts.set(key, (this.#failedMaterializationAttempts.get(key) ?? 0) + 1); + throw error; + }, + ); cached = { ingestionCursor, materialization }; this.#cache.set(key, cached); void materialization.catch(() => { @@ -730,7 +754,10 @@ export class TraceEvidenceCatalog implements EvolutionEvidenceCatalog { firstFailure: { ingestionCursor, traceName: basename(path), - notice: "Trace evidence could not be read and will be retried.", + notice: + (this.#failedMaterializationAttempts.get(key) ?? 0) >= MAX_TRACE_MATERIALIZATION_RETRIES + 1 + ? "Trace evidence could not be read after 2 retries." + : "Trace evidence could not be read and will be retried.", }, }; } diff --git a/packages/coding-agent/test/evolution-runtime.test.ts b/packages/coding-agent/test/evolution-runtime.test.ts index af82b99a7..62c7d91c7 100644 --- a/packages/coding-agent/test/evolution-runtime.test.ts +++ b/packages/coding-agent/test/evolution-runtime.test.ts @@ -702,6 +702,55 @@ describe("coding-agent evolution runtime", () => { ]); }); + it("stops retrying unreadable evidence after two retries", async () => { + const root = join(tmpdir(), `diffex-bounded-evidence-retry-${Date.now()}-${Math.random().toString(36).slice(2)}`); + const traceDirectory = join(root, "traces"); + mkdirSync(traceDirectory, { recursive: true }); + temporaryDirectories.push(root); + const unreadablePath = join(traceDirectory, "unreadable.jsonl"); + writeFileSync(unreadablePath, "unreadable\n"); + const scope: EvolutionScope = { repositoryId: "repository", scopeId: "repository" }; + let materializationAttempts = 0; + const catalog = new TraceEvidenceCatalog({ + agentDir: root, + scope, + async materializeInvocation() { + materializationAttempts += 1; + throw new Error("simulated incompatible trace"); + }, + }); + + const retryableFailure = { + materializations: [], + firstFailure: { + ingestionCursor: 1, + traceName: "unreadable.jsonl", + notice: "Trace evidence could not be read and will be retried.", + }, + }; + expect(await catalog.scan(scope, 0, 1)).toEqual(retryableFailure); + expect(await catalog.scan(scope, 0, 1)).toEqual(retryableFailure); + expect(await catalog.scan(scope, 0, 1)).toEqual({ + materializations: [], + firstFailure: { + ingestionCursor: 1, + traceName: "unreadable.jsonl", + notice: "Trace evidence could not be read after 2 retries.", + }, + }); + expect(materializationAttempts).toBe(3); + expect(await catalog.scan(scope, 0, 1)).toEqual({ + materializations: [], + firstFailure: { + ingestionCursor: 1, + traceName: "unreadable.jsonl", + notice: "Trace evidence could not be read after 2 retries.", + }, + }); + expect(materializationAttempts).toBe(3); + expect(await catalog.excludedThroughCursor(scope, 0, 1)).toBe(0); + }); + it("reuses an in-flight successful materialization across concurrent scans", async () => { const root = join(tmpdir(), `diffex-inflight-evidence-${Date.now()}-${Math.random().toString(36).slice(2)}`); const traceDirectory = join(root, "traces");