fix(evolution): preserve failed evidence for retry

packages/coding-agent/src/core/evolution-runtime.ts

@@ -25,6 +25,7 @@ import {
 	EvolutionControlPlane,
 	EvolutionControlService,
 	type EvolutionEvidenceCatalog,
+	type EvolutionEvidenceScan,
 	EvolutionExecutionError,
 	type EvolutionHarnessVersionInfo,
 	type EvolutionJobDetail,
@@ -87,6 +88,7 @@ const TRACE_EVIDENCE_INDEX_NAME = ".evolution-index.json";
 const TRACE_EVIDENCE_INDEX_VERSION = 1;
 const TRACE_EXCLUSION_INDEX_NAME = ".evolution-exclusions.json";
 const TRACE_EXCLUSION_INDEX_VERSION = 1;
+const MAX_TRACE_MATERIALIZATION_RETRIES = 2;
 const RESPONSE_TOOL_NAME = "submit_evolution_response";
 const MAX_MODEL_ROUNDS = 10;
 const PROVIDER_FIRST_BYTE_TIMEOUT_MS = 60_000;
@@ -492,7 +494,11 @@ export class TraceEvidenceCatalog implements EvolutionEvidenceCatalog {
 	readonly #scope: EvolutionScope;
 	readonly #materializeInvocation: typeof materializeJsonlInvocation;
 	readonly #isTraceActive: (path: string) => Promise<boolean>;
-	readonly #cache = new Map<string, Promise<SuccessfulMaterialization | null>>();
+	readonly #cache = new Map<
+		string,
+		{ readonly ingestionCursor: number; readonly materialization: Promise<SuccessfulMaterialization | null> }
+	>();
+	readonly #failedMaterializationAttempts = new Map<string, number>();
 
 	constructor(options: {
 		agentDir: string;
@@ -700,40 +706,92 @@ export class TraceEvidenceCatalog implements EvolutionEvidenceCatalog {
 		return (await this.#paths()).length;
 	}
 
-	async listSuccessful(
-		scope: EvolutionScope,
-		afterCursor: number,
-		throughCursor: number,
-	): Promise<readonly SuccessfulMaterialization[]> {
+	async scan(scope: EvolutionScope, afterCursor: number, throughCursor: number): Promise<EvolutionEvidenceScan> {
 		this.#assertScope(scope);
 		const paths = await this.#paths();
 		const selected = paths.slice(afterCursor, Math.min(throughCursor, paths.length));
-		const materializations = await Promise.all(
-			selected.map((path, index) => {
-				const ingestionCursor = afterCursor + index + 1;
-				const key = `${ingestionCursor}:${path}`;
-				const cached = this.#cache.get(key);
-				if (cached) return cached;
-				const created = this.#materialize(path, ingestionCursor).catch(() => null);
-				this.#cache.set(key, created);
-				return created;
-			}),
-		);
-		return materializations.filter((entry): entry is SuccessfulMaterialization => entry !== null);
+		const materializations: SuccessfulMaterialization[] = [];
+		for (const [index, path] of selected.entries()) {
+			const ingestionCursor = afterCursor + index + 1;
+			const key = `${ingestionCursor}:${path}`;
+			let cached = this.#cache.get(key);
+			if (
+				cached === undefined &&
+				(this.#failedMaterializationAttempts.get(key) ?? 0) >= MAX_TRACE_MATERIALIZATION_RETRIES + 1
+			) {
+				return {
+					materializations,
+					firstFailure: {
+						ingestionCursor,
+						traceName: basename(path),
+						notice: "Trace evidence could not be read after 2 retries.",
+					},
+				};
+			}
+			if (cached === undefined) {
+				const materialization = this.#materialize(path, ingestionCursor).then(
+					(result) => {
+						this.#failedMaterializationAttempts.delete(key);
+						return result;
+					},
+					(error: unknown) => {
+						this.#failedMaterializationAttempts.set(key, (this.#failedMaterializationAttempts.get(key) ?? 0) + 1);
+						throw error;
+					},
+				);
+				cached = { ingestionCursor, materialization };
+				this.#cache.set(key, cached);
+				void materialization.catch(() => {
+					if (this.#cache.get(key) === cached) this.#cache.delete(key);
+				});
+			}
+			try {
+				const materialization = await cached.materialization;
+				if (materialization !== null) materializations.push(materialization);
+			} catch {
+				return {
+					materializations,
+					firstFailure: {
+						ingestionCursor,
+						traceName: basename(path),
+						notice:
+							(this.#failedMaterializationAttempts.get(key) ?? 0) >= MAX_TRACE_MATERIALIZATION_RETRIES + 1
+								? "Trace evidence could not be read after 2 retries."
+								: "Trace evidence could not be read and will be retried.",
+					},
+				};
+			}
+		}
+		return { materializations, firstFailure: null };
 	}
 
 	async excludedThroughCursor(scope: EvolutionScope, afterCursor: number, throughCursor: number): Promise<number> {
 		this.#assertScope(scope);
-		await this.listSuccessful(scope, afterCursor, throughCursor);
 		const excludedNames = new Set((await this.#readExclusions()).map((exclusion) => exclusion.traceName));
 		const paths = await this.#paths();
 		let cursor = afterCursor;
-		for (const path of paths.slice(afterCursor, Math.min(throughCursor, paths.length))) {
-			if (!excludedNames.has(basename(path))) break;
+		for (const [index, path] of paths.slice(afterCursor, Math.min(throughCursor, paths.length)).entries()) {
+			const ingestionCursor = afterCursor + index + 1;
+			const cached = this.#cache.get(`${ingestionCursor}:${path}`);
+			if (cached === undefined && !excludedNames.has(basename(path))) break;
+			if (cached !== undefined) {
+				try {
+					await cached.materialization;
+				} catch {
+					break;
+				}
+			}
 			cursor += 1;
 		}
 		return cursor;
 	}
+
+	async discardProcessedThrough(scope: EvolutionScope, cursor: number): Promise<void> {
+		this.#assertScope(scope);
+		for (const [key, cached] of this.#cache) {
+			if (cached.ingestionCursor <= cursor) this.#cache.delete(key);
+		}
+	}
 }
 
 export function createEvolutionSkillSnapshots(skills: readonly Skill[]): readonly EvolutionSkillSnapshotSource[] {

packages/coding-agent/src/modes/interactive/components/evolution-menu.ts

@@ -681,6 +681,12 @@ export class EvolutionMenuComponent extends Container implements Focusable {
 		const evolutionEnabled = this.#client.getEnabled();
 		this.#addLine(`Evolution tokens: ${overview.unprocessedTokens.toLocaleString()}`);
 		this.#addLine(`Evolution invocations: ${overview.settledInvocationCount}`);
+		if (overview.firstEvidenceFailure !== null && overview.firstEvidenceFailure !== undefined) {
+			this.#addLine(
+				`Evidence retry pending at cursor ${overview.firstEvidenceFailure.ingestionCursor} (${overview.firstEvidenceFailure.traceName}): ${overview.firstEvidenceFailure.notice}`,
+				"warning",
+			);
+		}
 		this.#addLine(`Last evolution: ${formatEvolutionTimestamp(overview.lastEvolutionAt)}`);
 		this.#addLine(
 			!evolutionEnabled

packages/coding-agent/test/evolution-menu.test.ts

@@ -115,6 +115,7 @@ function overview(overrides: Partial<EvolutionOverview> = {}): EvolutionOverview
 		ingestionHighWaterCursor: 8,
 		unprocessedTokens: 1_100_001,
 		settledInvocationCount: 6,
+		firstEvidenceFailure: null,
 		automaticTokenThreshold: AUTOMATIC_EVOLUTION_TOKEN_THRESHOLD,
 		automaticMinimumInvocations: 5,
 		lastEvolutionAt: null,
@@ -437,6 +438,30 @@ describe("EvolutionMenuComponent", () => {
 		expect(output).toContain("Cancel [unavailable]");
 	});
 
+	it("shows a generic retry notice for the first unreadable evidence trace", async () => {
+		const fixture = client(
+			overview({
+				firstEvidenceFailure: {
+					ingestionCursor: 7,
+					traceName: "unreadable.jsonl",
+					notice: "Trace evidence could not be read and will be retried.",
+				},
+			}),
+		);
+		const component = new EvolutionMenuComponent(
+			fixture.client,
+			() => {},
+			() => {},
+			() => {},
+		);
+		await vi.waitFor(() => expect(stripAnsi(component.render(180).join("\n"))).toContain("Browse"));
+
+		const output = stripAnsi(component.render(180).join("\n"));
+		expect(output).toContain("Evidence retry pending at cursor 7 (unreadable.jsonl)");
+		expect(output).toContain("Trace evidence could not be read and will be retried.");
+		expect(output).not.toContain("simulated incompatible trace");
+	});
+
 	it("cancels a resumable job by its durable job ID", async () => {
 		const resumable = job({
 			jobId: "job-resumable",

packages/coding-agent/test/evolution-runtime.test.ts

@@ -661,30 +661,129 @@ describe("coding-agent evolution runtime", () => {
 		const unreadablePath = join(traceDirectory, "m-unreadable.jsonl");
 		writeFileSync(unreadablePath, "unreadable\n");
 		const scope: EvolutionScope = { repositoryId: "repository", scopeId: "repository" };
+		let shouldFail = true;
+		let materializationAttempts = 0;
 		const catalog = new TraceEvidenceCatalog({
 			agentDir: root,
 			scope,
 			async materializeInvocation(path) {
-				if (path === unreadablePath) throw new Error("simulated incompatible trace");
+				materializationAttempts += 1;
+				if (path === unreadablePath && shouldFail) throw new Error("simulated incompatible trace");
 				return materializedInvocation(basename(path, ".jsonl"), sessionPath);
 			},
 		});
 
 		expect(await catalog.highWaterCursor(scope)).toBe(3);
-		expect(await catalog.listSuccessful(scope, 0, 3)).toMatchObject([
+		const failedScan = await catalog.scan(scope, 0, 3);
+		expect(failedScan.materializations).toMatchObject([
 			{ materializationId: "z-first", ingestionCursor: 1 },
 			{ materializationId: "a-second", ingestionCursor: 2 },
 		]);
+		expect(failedScan.firstFailure).toEqual({
+			ingestionCursor: 3,
+			traceName: "m-unreadable.jsonl",
+			notice: "Trace evidence could not be read and will be retried.",
+		});
+		shouldFail = false;
+		expect((await catalog.scan(scope, 2, 3)).materializations).toMatchObject([
+			{ materializationId: "m-unreadable", ingestionCursor: 3 },
+		]);
+		expect(materializationAttempts).toBe(4);
+		await catalog.discardProcessedThrough(scope, 3);
+		expect((await catalog.scan(scope, 0, 2)).materializations).toHaveLength(2);
+		expect(materializationAttempts).toBe(6);
 
 		await new Promise((resolveDelay) => setTimeout(resolveDelay, 5));
 		const latePath = join(traceDirectory, "00-late.jsonl");
 		writeFileSync(latePath, "late\n");
 		expect(await catalog.highWaterCursor(scope)).toBe(4);
-		expect(await catalog.listSuccessful(scope, 3, 4)).toMatchObject([
+		expect((await catalog.scan(scope, 3, 4)).materializations).toMatchObject([
 			{ materializationId: "00-late", ingestionCursor: 4 },
 		]);
 	});
 
+	it("stops retrying unreadable evidence after two retries", async () => {
+		const root = join(tmpdir(), `diffex-bounded-evidence-retry-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+		const traceDirectory = join(root, "traces");
+		mkdirSync(traceDirectory, { recursive: true });
+		temporaryDirectories.push(root);
+		const unreadablePath = join(traceDirectory, "unreadable.jsonl");
+		writeFileSync(unreadablePath, "unreadable\n");
+		const scope: EvolutionScope = { repositoryId: "repository", scopeId: "repository" };
+		let materializationAttempts = 0;
+		const catalog = new TraceEvidenceCatalog({
+			agentDir: root,
+			scope,
+			async materializeInvocation() {
+				materializationAttempts += 1;
+				throw new Error("simulated incompatible trace");
+			},
+		});
+
+		const retryableFailure = {
+			materializations: [],
+			firstFailure: {
+				ingestionCursor: 1,
+				traceName: "unreadable.jsonl",
+				notice: "Trace evidence could not be read and will be retried.",
+			},
+		};
+		expect(await catalog.scan(scope, 0, 1)).toEqual(retryableFailure);
+		expect(await catalog.scan(scope, 0, 1)).toEqual(retryableFailure);
+		expect(await catalog.scan(scope, 0, 1)).toEqual({
+			materializations: [],
+			firstFailure: {
+				ingestionCursor: 1,
+				traceName: "unreadable.jsonl",
+				notice: "Trace evidence could not be read after 2 retries.",
+			},
+		});
+		expect(materializationAttempts).toBe(3);
+		expect(await catalog.scan(scope, 0, 1)).toEqual({
+			materializations: [],
+			firstFailure: {
+				ingestionCursor: 1,
+				traceName: "unreadable.jsonl",
+				notice: "Trace evidence could not be read after 2 retries.",
+			},
+		});
+		expect(materializationAttempts).toBe(3);
+		expect(await catalog.excludedThroughCursor(scope, 0, 1)).toBe(0);
+	});
+
+	it("reuses an in-flight successful materialization across concurrent scans", async () => {
+		const root = join(tmpdir(), `diffex-inflight-evidence-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+		const traceDirectory = join(root, "traces");
+		mkdirSync(traceDirectory, { recursive: true });
+		temporaryDirectories.push(root);
+		writeFileSync(join(traceDirectory, "trace.jsonl"), "trace\n");
+		const scope: EvolutionScope = { repositoryId: "repository", scopeId: "repository" };
+		let materializationCalls = 0;
+		let releaseMaterialization: (() => void) | undefined;
+		const materialized = new Promise<void>((resolve) => {
+			releaseMaterialization = resolve;
+		});
+		const catalog = new TraceEvidenceCatalog({
+			agentDir: root,
+			scope,
+			async materializeInvocation() {
+				materializationCalls += 1;
+				await materialized;
+				return materializedInvocation("trace", "/session.jsonl");
+			},
+		});
+
+		const first = catalog.scan(scope, 0, 1);
+		await vi.waitFor(() => expect(materializationCalls).toBe(1));
+		const second = catalog.scan(scope, 0, 1);
+		releaseMaterialization?.();
+		expect(await Promise.all([first, second])).toEqual([
+			{ materializations: [expect.objectContaining({ materializationId: "trace" })], firstFailure: null },
+			{ materializations: [expect.objectContaining({ materializationId: "trace" })], firstFailure: null },
+		]);
+		expect(materializationCalls).toBe(1);
+	});
+
 	it("indexes settled traces without letting an older active invocation block them", async () => {
 		const root = join(tmpdir(), `diffex-global-evidence-${Date.now()}-${Math.random().toString(36).slice(2)}`);
 		const traceDirectory = join(root, "traces");
@@ -714,14 +813,14 @@ describe("coding-agent evolution runtime", () => {
 		});
 
 		expect(await Promise.all([catalog.highWaterCursor(scope), peerCatalog.highWaterCursor(scope)])).toEqual([2, 2]);
-		expect(await catalog.listSuccessful(scope, 0, 3)).toMatchObject([
+		expect((await catalog.scan(scope, 0, 3)).materializations).toMatchObject([
 			{ materializationId: "first", ingestionCursor: 1 },
 			{ materializationId: "later", ingestionCursor: 2 },
 		]);
 
 		active.delete(activePath);
 		expect(await Promise.all([catalog.highWaterCursor(scope), peerCatalog.highWaterCursor(scope)])).toEqual([3, 3]);
-		expect(await peerCatalog.listSuccessful(scope, 2, 3)).toMatchObject([
+		expect((await peerCatalog.scan(scope, 2, 3)).materializations).toMatchObject([
 			{ materializationId: "active", ingestionCursor: 3 },
 		]);
 	});
@@ -743,7 +842,7 @@ describe("coding-agent evolution runtime", () => {
 			}),
 		});
 
-		expect(await catalog.listSuccessful(scope, 0, await catalog.highWaterCursor(scope))).toEqual([]);
+		expect((await catalog.scan(scope, 0, await catalog.highWaterCursor(scope))).materializations).toEqual([]);
 		expect(JSON.parse(readFileSync(join(traceDirectory, ".evolution-exclusions.json"), "utf8"))).toEqual({
 			version: 1,
 			exclusions: [