diff --git a/package-lock.json b/package-lock.json index 6b4540941..d98c1e29f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5772,6 +5772,7 @@ "@types/node": "24.12.4", "@types/proper-lockfile": "4.1.4", "shx": "0.4.0", + "typebox": "1.3.7", "vitest": "4.1.9" }, "engines": { diff --git a/packages/coding-agent-server/vitest.config.ts b/packages/coding-agent-server/vitest.config.ts index 5539adc2a..3a177c301 100644 --- a/packages/coding-agent-server/vitest.config.ts +++ b/packages/coding-agent-server/vitest.config.ts @@ -1,3 +1,42 @@ -import { defineConfig } from "vitest/config"; +import { fileURLToPath } from "node:url"; +import { defineConfig, mergeConfig } from "vitest/config"; +import baseConfig, { workspaceSourcePaths } from "../../vitest.base.ts"; -export default defineConfig({ test: { include: ["test/**/*.test.ts"] } }); +export default mergeConfig( + baseConfig, + defineConfig({ + test: { include: ["test/**/*.test.ts"], testTimeout: 30000 }, + resolve: { + alias: [ + { + find: /^@earendil-works\/diffex-client\/websocket$/, + replacement: fileURLToPath(new URL("../client/src/websocket.ts", import.meta.url)), + }, + { + find: /^@earendil-works\/diffex-client$/, + replacement: fileURLToPath(new URL("../client/src/index.ts", import.meta.url)), + }, + { + find: /^@earendil-works\/diffex-coding-agent$/, + replacement: workspaceSourcePaths.codingAgentIndex, + }, + { + find: /^@earendil-works\/diffex-protocol$/, + replacement: fileURLToPath(new URL("../protocol/src/index.ts", import.meta.url)), + }, + { + find: /^@earendil-works\/diffex-server\/testing$/, + replacement: fileURLToPath(new URL("../server/src/testing/index.ts", import.meta.url)), + }, + { + find: /^@earendil-works\/diffex-server\/websocket$/, + replacement: fileURLToPath(new URL("../server/src/transports/websocket/index.ts", import.meta.url)), + }, + { + find: /^@earendil-works\/diffex-server$/, + replacement: fileURLToPath(new URL("../server/src/index.ts", import.meta.url)), + }, + ], + }, + }), +); diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 27ee19bc1..fd2a2be4e 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,13 +2,8 @@ ## [Unreleased] -### Breaking Changes - -- SDK-provided `Skill` objects now carry stable catalog identity, provenance, captured content, digest, and version fields. Use `createSkill()` to construct custom skills. - ### Added -- Added a unified skill catalog shared by interactive selection, prompts, commands, SDK and RPC consumers, and frozen evolution inputs. - Added `pi auth check` provider/model auth preflight with optional credential output ([#7152](https://github.com/earendil-works/pi/issues/7152)). ### Changed diff --git a/packages/coding-agent/src/core/evolution-runtime.ts b/packages/coding-agent/src/core/evolution-runtime.ts index 72a6d0e9d..5bfe29713 100644 --- a/packages/coding-agent/src/core/evolution-runtime.ts +++ b/packages/coding-agent/src/core/evolution-runtime.ts @@ -137,7 +137,7 @@ function validateEvolutionOutput(request: EvolutionModelRequest, output: JsonObj const issue = [...Errors(schema, output)][0]; return issue?.message ?? "Response does not match the required schema"; } - if (request.stage !== "reflector" || request.validateOutput === undefined) return null; + if (request.validateOutput === undefined) return null; try { request.validateOutput(output); return null; @@ -343,7 +343,7 @@ export class CodingAgentEvolutionModelExecutor implements EvolutionModelExecutor } if (lastCorrection !== null) { throw new EvolutionExecutionError(`Evolution model did not submit a valid response: ${lastCorrection}`, { - reasonCode: "reflection-invalid-response", + reasonCode: request.stage === "curator" ? "curation-invalid-response" : "reflection-invalid-response", retryable: false, diagnostics: { lastCorrection }, }); diff --git a/packages/coding-agent/test/evolution-menu.test.ts b/packages/coding-agent/test/evolution-menu.test.ts index 8ffc4e0a5..6b6c8b1e5 100644 --- a/packages/coding-agent/test/evolution-menu.test.ts +++ b/packages/coding-agent/test/evolution-menu.test.ts @@ -587,7 +587,6 @@ describe("EvolutionMenuComponent", () => { reason: "it is imprecise", }, addition: { - destination: { section: "Verification", anchor: null }, content: "Use precise verification guidance.", intendedScope: ["repository changes"], }, diff --git a/packages/coding-agent/test/evolution-runtime.test.ts b/packages/coding-agent/test/evolution-runtime.test.ts index 5cc66d1ab..21f21025f 100644 --- a/packages/coding-agent/test/evolution-runtime.test.ts +++ b/packages/coding-agent/test/evolution-runtime.test.ts @@ -598,6 +598,116 @@ describe("coding-agent evolution runtime", () => { expect(validations).toBe(1); }); + it("corrects curator semantic errors within the active transcript", async () => { + const harness = await createHarness(); + harnesses.push(harness); + harness.setResponses([ + fauxAssistantMessage(fauxToolCall("submit_evolution_response", { decisions: ["invalid"] }), { + stopReason: "toolUse", + }), + (context) => { + expect(context.messages).toHaveLength(3); + expect(context.messages[2]).toMatchObject({ + role: "toolResult", + toolName: "submit_evolution_response", + isError: true, + }); + const correctionMessage = context.messages[2]; + const correction = + correctionMessage?.role === "toolResult" + ? correctionMessage.content + .filter((part): part is { type: "text"; text: string } => part.type === "text") + .map((part) => part.text) + .join("\n") + : ""; + expect(correction).toContain("decisions[0].proposal.exactDelta"); + expect(correction).toContain("decisions[1].relationship"); + return fauxAssistantMessage(fauxToolCall("submit_evolution_response", { decisions: [] }), { + stopReason: "toolUse", + }); + }, + ]); + const executor = new CodingAgentEvolutionModelExecutor(harness.session, harness.session.modelRuntime, { + async waitUntilBackgroundAllowed() {}, + }); + const model = harness.getModel(); + + await expect( + executor.execute({ + stage: "curator", + model: { + provider: model.provider, + model: model.id, + configurationDigest: `sha256:${"1".repeat(64)}`, + }, + promptVersion: "curator-v1", + idempotencyKey: "curator-semantic-correction", + input: {}, + outputSchema: { + type: "object", + additionalProperties: false, + required: ["decisions"], + properties: { decisions: { type: "array" } }, + }, + tools: [], + validateOutput(output) { + if (Array.isArray(output.decisions) && output.decisions.length === 0) return; + throw new Error( + 'Decision "first" at decisions[0].proposal.exactDelta: invalid delta\n' + + 'Decision "second" at decisions[1].relationship: invalid relationship', + ); + }, + }), + ).resolves.toMatchObject({ output: { decisions: [] }, submissionMethod: "response-tool" }); + }); + + it.each([ + ["reflector", "response-tool", "reflection-invalid-response"], + ["reflector", "json-text", "reflection-invalid-response"], + ["curator", "response-tool", "curation-invalid-response"], + ["curator", "json-text", "curation-invalid-response"], + ] as const)("classifies exhausted %s %s corrections", async (stage, submissionMethod, expectedReasonCode) => { + const harness = await createHarness(); + harnesses.push(harness); + const invalidResponse = + submissionMethod === "response-tool" + ? fauxAssistantMessage(fauxToolCall("submit_evolution_response", { complete: false }), { + stopReason: "toolUse", + }) + : fauxAssistantMessage(JSON.stringify({ complete: false })); + harness.setResponses(Array.from({ length: 10 }, () => invalidResponse)); + const executor = new CodingAgentEvolutionModelExecutor(harness.session, harness.session.modelRuntime, { + async waitUntilBackgroundAllowed() {}, + }); + const model = harness.getModel(); + + await expect( + executor.execute({ + stage, + model: { + provider: model.provider, + model: model.id, + configurationDigest: `sha256:${"1".repeat(64)}`, + }, + promptVersion: `${stage}-v1`, + idempotencyKey: `${stage}-${submissionMethod}-exhaustion`, + input: {}, + outputSchema: { + type: "object", + additionalProperties: false, + required: ["complete"], + properties: { complete: { enum: [true] } }, + }, + tools: [], + }), + ).rejects.toMatchObject({ + name: "EvolutionExecutionError", + reasonCode: expectedReasonCode, + retryable: false, + diagnostics: { lastCorrection: expect.any(String) }, + }); + }); + it("isolates transcripts between node executor invocations", async () => { const harness = await createHarness(); harnesses.push(harness); diff --git a/packages/evolution/package.json b/packages/evolution/package.json index 5358a66d0..80c724460 100644 --- a/packages/evolution/package.json +++ b/packages/evolution/package.json @@ -40,6 +40,7 @@ "@types/node": "24.12.4", "@types/proper-lockfile": "4.1.4", "shx": "0.4.0", + "typebox": "1.3.7", "vitest": "4.1.9" } } diff --git a/packages/evolution/src/contracts.ts b/packages/evolution/src/contracts.ts index f49bb1506..379faca5c 100644 --- a/packages/evolution/src/contracts.ts +++ b/packages/evolution/src/contracts.ts @@ -486,6 +486,7 @@ export interface EvolutionProposalInstructionRelationship { export interface EvolutionMemoryAddDelta { readonly operation: "ADD"; + /** A section must name one existing heading. Both fields null append after the final memory item. */ readonly destination: { readonly section: string | null; readonly anchor: string | null; @@ -504,7 +505,7 @@ export interface EvolutionMemoryRemoveDelta { export interface EvolutionMemoryReplaceDelta { readonly operation: "REPLACE"; readonly removal: Omit; - readonly addition: Omit; + readonly addition: Pick; } export type EvolutionMemoryDelta = EvolutionMemoryAddDelta | EvolutionMemoryRemoveDelta | EvolutionMemoryReplaceDelta; diff --git a/packages/evolution/src/curator.ts b/packages/evolution/src/curator.ts index fc07210b5..43ab27260 100644 --- a/packages/evolution/src/curator.ts +++ b/packages/evolution/src/curator.ts @@ -59,6 +59,47 @@ export function createCuratorOutputSchema(references: CuratorSchemaReferences): minItems, items: { type: "string", enum: values }, }); + const proposalRequired = [ + "target", + "action", + "exactDelta", + "expectedBenefit", + "acceptanceCriterion", + "risk", + "rollbackDescription", + "curatorConfidence", + "uncertainty", + "counterexamples", + "conflictingDecisionKeys", + "finding", + ]; + const proposalProperties = { + expectedBenefit: { type: "string", minLength: 1 }, + acceptanceCriterion: { type: "string", minLength: 1 }, + risk: { type: "string", minLength: 1 }, + rollbackDescription: { type: "string", minLength: 1 }, + curatorConfidence: { type: "number", minimum: 0, maximum: 1 }, + uncertainty: { type: ["string", "null"] }, + counterexamples: stringArray, + conflictingDecisionKeys: stringArray, + }; + const proposalVariant = ( + target: JsonObject, + action: ProposalDraft["action"], + exactDelta: JsonObject, + finding: JsonObject, + ): JsonObject => ({ + type: "object", + additionalProperties: false, + required: proposalRequired, + properties: { + ...proposalProperties, + target, + action: { enum: [action] }, + exactDelta, + finding, + }, + }); return { $defs: { relevance: { @@ -106,33 +147,29 @@ export function createCuratorOutputSchema(references: CuratorSchemaReferences): strengthening: { $ref: "#/$defs/strengthening" }, }, }, - target: { - anyOf: [ - { - type: "object", - additionalProperties: false, - required: ["kind"], - properties: { kind: { enum: ["evolution-memory"] } }, - }, - { - type: "object", - additionalProperties: false, - required: ["kind", "skillName"], - properties: { - kind: { enum: ["skill"] }, - skillName: { type: "string", minLength: 1 }, - }, - }, - { - type: "object", - additionalProperties: false, - required: ["kind", "sourcePath"], - properties: { - kind: { enum: ["orchestration-source"] }, - sourcePath: { type: "string", minLength: 1 }, - }, - }, - ], + evolutionMemoryTarget: { + type: "object", + additionalProperties: false, + required: ["kind"], + properties: { kind: { enum: ["evolution-memory"] } }, + }, + skillTarget: { + type: "object", + additionalProperties: false, + required: ["kind", "skillName"], + properties: { + kind: { enum: ["skill"] }, + skillName: { type: "string", minLength: 1 }, + }, + }, + orchestrationSourceTarget: { + type: "object", + additionalProperties: false, + required: ["kind", "sourcePath"], + properties: { + kind: { enum: ["orchestration-source"] }, + sourcePath: { type: "string", minLength: 1 }, + }, }, destination: { type: "object", @@ -142,16 +179,16 @@ export function createCuratorOutputSchema(references: CuratorSchemaReferences): section: { type: ["string", "null"] }, anchor: { enum: [null, ...references.memoryRefs], - description: "Existing memory item reference to insert after, or null to append within the section.", + description: + "Existing memory item reference to insert after, null to append within a named section, or null with a null section to append globally.", }, }, }, addition: { type: "object", additionalProperties: false, - required: ["destination", "content", "intendedScope"], + required: ["content", "intendedScope"], properties: { - destination: { $ref: "#/$defs/destination" }, content: { type: "string", minLength: 1, @@ -169,78 +206,75 @@ export function createCuratorOutputSchema(references: CuratorSchemaReferences): reason: { type: "string", minLength: 1 }, }, }, - exactDelta: { - anyOf: [ - { - type: "object", - additionalProperties: false, - required: ["operation", "destination", "content", "intendedScope"], - properties: { - operation: { enum: ["ADD"] }, - destination: { $ref: "#/$defs/destination" }, - content: { - type: "string", - minLength: 1, - description: "Lesson prose of at most three sentences. Omit the deterministic Lesson number.", - }, - intendedScope: stringArray, - }, - }, - { - type: "object", - additionalProperties: false, - required: ["operation", "memoryRef", "reason"], - properties: { - operation: { enum: ["REMOVE"] }, - memoryRef: { type: "string", enum: references.memoryRefs }, - reason: { type: "string", minLength: 1 }, - }, - }, - { - type: "object", - additionalProperties: false, - required: ["operation", "removal", "addition"], - properties: { - operation: { enum: ["REPLACE"] }, - removal: { $ref: "#/$defs/removal" }, - addition: { $ref: "#/$defs/addition" }, - }, + addDelta: { + type: "object", + additionalProperties: false, + required: ["operation", "destination", "content", "intendedScope"], + properties: { + operation: { enum: ["ADD"] }, + destination: { $ref: "#/$defs/destination" }, + content: { + type: "string", + minLength: 1, + description: "Lesson prose of at most three sentences. Omit the deterministic Lesson number.", }, - { type: "null" }, - ], + intendedScope: stringArray, + }, }, - proposal: { + removeDelta: { type: "object", additionalProperties: false, - required: [ - "target", - "action", - "exactDelta", - "expectedBenefit", - "acceptanceCriterion", - "risk", - "rollbackDescription", - "curatorConfidence", - "uncertainty", - "counterexamples", - "conflictingDecisionKeys", - "finding", - ], + required: ["operation", "memoryRef", "reason"], properties: { - target: { $ref: "#/$defs/target" }, - action: { enum: ["ADD", "DEFER", "REMOVE", "REPLACE"] }, - exactDelta: { $ref: "#/$defs/exactDelta" }, - expectedBenefit: { type: "string", minLength: 1 }, - acceptanceCriterion: { type: "string", minLength: 1 }, - risk: { type: "string", minLength: 1 }, - rollbackDescription: { type: "string", minLength: 1 }, - curatorConfidence: { type: "number", minimum: 0, maximum: 1 }, - uncertainty: { type: ["string", "null"] }, - counterexamples: stringArray, - conflictingDecisionKeys: stringArray, - finding: { type: ["string", "null"] }, + operation: { enum: ["REMOVE"] }, + memoryRef: { type: "string", enum: references.memoryRefs }, + reason: { type: "string", minLength: 1 }, }, }, + replaceDelta: { + type: "object", + additionalProperties: false, + required: ["operation", "removal", "addition"], + properties: { + operation: { enum: ["REPLACE"] }, + removal: { $ref: "#/$defs/removal" }, + addition: { $ref: "#/$defs/addition" }, + }, + }, + proposal: { + anyOf: [ + proposalVariant( + { $ref: "#/$defs/evolutionMemoryTarget" }, + "ADD", + { $ref: "#/$defs/addDelta" }, + { type: "null" }, + ), + proposalVariant( + { $ref: "#/$defs/evolutionMemoryTarget" }, + "REMOVE", + { $ref: "#/$defs/removeDelta" }, + { type: "null" }, + ), + proposalVariant( + { $ref: "#/$defs/evolutionMemoryTarget" }, + "REPLACE", + { $ref: "#/$defs/replaceDelta" }, + { type: "null" }, + ), + proposalVariant( + { $ref: "#/$defs/skillTarget" }, + "DEFER", + { type: "null" }, + { type: "string", minLength: 1 }, + ), + proposalVariant( + { $ref: "#/$defs/orchestrationSourceTarget" }, + "DEFER", + { type: "null" }, + { type: "string", minLength: 1 }, + ), + ], + }, }, type: "object", additionalProperties: false, @@ -303,6 +337,74 @@ function isRecord(value: unknown): value is Readonly> { return typeof value === "object" && value !== null && !Array.isArray(value); } +interface CuratorDecisionValidationIssue { + readonly decisionKey: string | null; + readonly path: string; + readonly message: string; + readonly invalidReferences: readonly string[]; + readonly permittedReferences: readonly string[]; +} + +class CuratorFieldValidationError extends Error { + readonly path: string; + readonly validationCause: unknown; + + constructor(path: string, validationCause: unknown) { + super(validationCause instanceof Error ? validationCause.message : "Response validation failed"); + this.name = "CuratorFieldValidationError"; + this.path = path; + this.validationCause = validationCause; + } +} + +class CuratorOutputValidationError extends Error { + readonly issues: readonly CuratorDecisionValidationIssue[]; + readonly invalidReferences: readonly string[]; + readonly permittedReferences: readonly string[]; + + constructor(issues: readonly CuratorDecisionValidationIssue[]) { + super( + issues + .map( + (issue) => + `${issue.decisionKey === null ? "Decision" : `Decision "${issue.decisionKey}"`} at ${issue.path}: ${issue.message}`, + ) + .join("\n"), + ); + this.name = "CuratorOutputValidationError"; + this.issues = issues; + this.invalidReferences = unique(issues.flatMap((issue) => issue.invalidReferences)); + this.permittedReferences = unique(issues.flatMap((issue) => issue.permittedReferences)); + } +} + +function atPath(path: string, parse: () => T): T { + try { + return parse(); + } catch (error) { + if (error instanceof CuratorFieldValidationError) { + throw new CuratorFieldValidationError(`${path}.${error.path}`, error.validationCause); + } + throw new CuratorFieldValidationError(path, error); + } +} + +function decisionValidationIssue( + error: unknown, + index: number, + decisionKey: string | null, +): CuratorDecisionValidationIssue { + const fieldError = error instanceof CuratorFieldValidationError ? error : null; + const cause = fieldError?.validationCause ?? error; + return { + decisionKey, + path: `decisions[${String(index)}]${fieldError === null ? "" : `.${fieldError.path}`}`, + message: cause instanceof Error ? cause.message : "Response validation failed", + invalidReferences: cause instanceof EvolutionModelReferenceError ? cause.invalidReferences : [], + permittedReferences: cause instanceof EvolutionModelReferenceError ? cause.permittedReferences : [], + }; +} + function unique(values: readonly T[]): T[] { return [...new Set(values)]; } @@ -463,21 +565,14 @@ function target(value: unknown, input: CuratorInput): EvolutionTarget { throw new Error(`Unsupported target kind "${String(value.kind)}"`); } -function addition(value: unknown, registry: CuratorToolRegistry, nested = false): EvolutionMemoryAddDelta { +function addition(value: unknown, registry: CuratorToolRegistry): EvolutionMemoryAddDelta { if (!isRecord(value)) throw new Error("ADD exactDelta must be an object"); - assertOnlyFields( - value, - nested ? ["destination", "content", "intendedScope"] : ["operation", "destination", "content", "intendedScope"], - "ADD exactDelta", - ); - if (!nested && value.operation !== "ADD") throw new Error("ADD exactDelta operation must be ADD"); + assertOnlyFields(value, ["operation", "destination", "content", "intendedScope"], "ADD exactDelta"); + if (value.operation !== "ADD") throw new Error("ADD exactDelta operation must be ADD"); if (!isRecord(value.destination)) throw new Error("ADD destination must be an object"); assertOnlyFields(value.destination, ["section", "anchor"], "ADD destination"); const section = nullableString(value.destination, "section"); const anchorReference = nullableString(value.destination, "anchor"); - if (section === null && anchorReference === null) { - throw new Error("ADD must identify a destination section or anchor"); - } const anchorBinding = anchorReference === null ? null : registry.memory.resolve(anchorReference, "anchor"); if (section !== null && anchorBinding !== null && anchorBinding.block.section !== section) { throw new Error(`ADD anchor "${anchorReference}" is outside section "${section}"`); @@ -515,16 +610,19 @@ function replacement(value: unknown, registry: CuratorToolRegistry): EvolutionMe assertOnlyFields(value, ["operation", "removal", "addition"], "REPLACE exactDelta"); if (value.operation !== "REPLACE") throw new Error("REPLACE exactDelta operation must be REPLACE"); const parsedRemoval = removal(value.removal, registry, true); - const parsedAddition = addition(value.addition, registry, true); - if ( - parsedAddition.destination.anchor !== null && - computeEvolutionContentDigest(parsedAddition.destination.anchor) === parsedRemoval.expectedContentDigest - ) { - throw new Error("REPLACE addition cannot anchor to the memory item being removed"); - } + if (!isRecord(value.addition)) throw new Error("REPLACE addition must be an object"); + assertOnlyFields(value.addition, ["content", "intendedScope"], "REPLACE addition"); + const content = requiredString(value.addition, "content"); + assertEvolutionMemoryLessonContent(content); const { operation: _removeOperation, ...removalDelta } = parsedRemoval; - const { operation: _addOperation, ...additionDelta } = parsedAddition; - return { operation: "REPLACE", removal: removalDelta, addition: additionDelta }; + return { + operation: "REPLACE", + removal: removalDelta, + addition: { + content, + intendedScope: stringArray(value.addition.intendedScope, "intendedScope", false), + }, + }; } function delta( @@ -568,39 +666,51 @@ function proposalDraft( ], "proposal", ); - const parsedTarget = target(value.target, input); - const action = value.action; - if (action !== "ADD" && action !== "DEFER" && action !== "REMOVE" && action !== "REPLACE") { - throw new Error(`Unsupported proposal action "${String(action)}"`); - } + const parsedTarget = atPath("target", () => target(value.target, input)); + const action = atPath("action", () => { + if ( + value.action !== "ADD" && + value.action !== "DEFER" && + value.action !== "REMOVE" && + value.action !== "REPLACE" + ) { + throw new Error(`Unsupported proposal action "${String(value.action)}"`); + } + return value.action; + }); if (parsedTarget.kind === "evolution-memory" && action === "DEFER") { throw new Error("Evolution memory proposals must contain an executable delta"); } if (parsedTarget.kind !== "evolution-memory" && action !== "DEFER") { throw new Error(`Target "${parsedTarget.kind}" must be deferred`); } - const finding = value.finding; - if (action === "DEFER") { - if (typeof finding !== "string" || finding.length === 0) - throw new Error("Deferred proposals must include a finding"); - } else if (finding !== null) { - throw new Error("Actionable proposal finding must be null"); - } + const finding = atPath("finding", () => { + if (action === "DEFER") { + if (typeof value.finding !== "string" || value.finding.length === 0) { + throw new Error("Deferred proposals must include a finding"); + } + } else if (value.finding !== null) { + throw new Error("Actionable proposal finding must be null"); + } + return value.finding; + }); return { decisionKey, lessonIds, relationship: parsedRelationship, target: parsedTarget, action, - exactDelta: delta(value.exactDelta, action, registry), - expectedBenefit: requiredString(value, "expectedBenefit"), - acceptanceCriterion: requiredString(value, "acceptanceCriterion"), - risk: requiredString(value, "risk"), - rollbackDescription: requiredString(value, "rollbackDescription"), - curatorConfidence: confidence(value.curatorConfidence), - uncertainty: nullableString(value, "uncertainty"), - counterexamples: stringArray(value.counterexamples, "counterexamples"), - conflictingDecisionKeys: stringArray(value.conflictingDecisionKeys, "conflictingDecisionKeys"), + exactDelta: atPath("exactDelta", () => delta(value.exactDelta, action, registry)), + expectedBenefit: atPath("expectedBenefit", () => requiredString(value, "expectedBenefit")), + acceptanceCriterion: atPath("acceptanceCriterion", () => requiredString(value, "acceptanceCriterion")), + risk: atPath("risk", () => requiredString(value, "risk")), + rollbackDescription: atPath("rollbackDescription", () => requiredString(value, "rollbackDescription")), + curatorConfidence: atPath("curatorConfidence", () => confidence(value.curatorConfidence)), + uncertainty: atPath("uncertainty", () => nullableString(value, "uncertainty")), + counterexamples: atPath("counterexamples", () => stringArray(value.counterexamples, "counterexamples")), + conflictingDecisionKeys: atPath("conflictingDecisionKeys", () => + stringArray(value.conflictingDecisionKeys, "conflictingDecisionKeys"), + ), finding: typeof finding === "string" ? finding : null, }; } @@ -673,85 +783,123 @@ function assertLessonEvidence(lessons: readonly Lesson[], index: ReadonlyMap(); const drafts: ProposalDraft[] = []; - for (const value of response.output.decisions) { - if (!isRecord(value)) throw new Error("Curator decision must be an object"); - assertOnlyFields( - value, - ["decisionKey", "lessonRefs", "relevance", "relationship", "proposal"], - "Curator decision", - ); - const decisionKey = requiredString(value, "decisionKey"); - if (!PROPOSAL_ID_PATTERN.test(decisionKey)) throw new Error(`Invalid decision key "${decisionKey}"`); - if (seenDecisionKeys.has(decisionKey)) throw new Error(`Duplicate decision key "${decisionKey}"`); - seenDecisionKeys.add(decisionKey); - if (!isRecord(value.relevance)) throw new Error("relevance must be an object"); - assertOnlyFields( - value.relevance, - [ - "userTaskOrFailure", - "causalAgentBehavior", - "futureBehaviorChange", - "verifiableExpectedResult", - "observerDocumentationOnly", - "telemetryFormatOnly", - ], - "relevance", - ); - for (const field of [ - "userTaskOrFailure", - "causalAgentBehavior", - "futureBehaviorChange", - "verifiableExpectedResult", - ] as const) { - requiredString(value.relevance, field); - } - if (value.relevance.observerDocumentationOnly !== false || value.relevance.telemetryFormatOnly !== false) { - throw new Error("Observer documentation and telemetry-format facts are not actionable evolution memory"); - } - const selectedLessons = referenceArray(value.lessonRefs, "lessonRefs", registry.lessons, { allowEmpty: false }); - const parsedRelationship = relationship(value.relationship, registry); - if (parsedRelationship.classification === "duplicate" || parsedRelationship.classification === "conflicts") { - if (value.proposal !== null) { - throw new Error(`${parsedRelationship.classification} decisions must not contain a proposal`); + const draftIndexes = new Map(); + const issues: CuratorDecisionValidationIssue[] = []; + for (const [index, value] of output.decisions.entries()) { + let decisionKey = isRecord(value) && typeof value.decisionKey === "string" ? value.decisionKey : null; + try { + if (!isRecord(value)) throw new Error("Curator decision must be an object"); + assertOnlyFields( + value, + ["decisionKey", "lessonRefs", "relevance", "relationship", "proposal"], + "Curator decision", + ); + decisionKey = atPath("decisionKey", () => { + const parsed = requiredString(value, "decisionKey"); + if (!PROPOSAL_ID_PATTERN.test(parsed)) throw new Error(`Invalid decision key "${parsed}"`); + if (seenDecisionKeys.has(parsed)) throw new Error(`Duplicate decision key "${parsed}"`); + return parsed; + }); + seenDecisionKeys.add(decisionKey); + atPath("relevance", () => { + if (!isRecord(value.relevance)) throw new Error("relevance must be an object"); + assertOnlyFields( + value.relevance, + [ + "userTaskOrFailure", + "causalAgentBehavior", + "futureBehaviorChange", + "verifiableExpectedResult", + "observerDocumentationOnly", + "telemetryFormatOnly", + ], + "relevance", + ); + for (const field of [ + "userTaskOrFailure", + "causalAgentBehavior", + "futureBehaviorChange", + "verifiableExpectedResult", + ] as const) { + requiredString(value.relevance, field); + } + if (value.relevance.observerDocumentationOnly !== false || value.relevance.telemetryFormatOnly !== false) { + throw new Error("Observer documentation and telemetry-format facts are not actionable evolution memory"); + } + }); + const selectedLessons = atPath("lessonRefs", () => + referenceArray(value.lessonRefs, "lessonRefs", registry.lessons, { allowEmpty: false }), + ); + const parsedRelationship = atPath("relationship", () => relationship(value.relationship, registry)); + if (parsedRelationship.classification === "duplicate" || parsedRelationship.classification === "conflicts") { + if (value.proposal !== null) { + throw new CuratorFieldValidationError( + "proposal", + new Error(`${parsedRelationship.classification} decisions must not contain a proposal`), + ); + } + continue; + } + if (value.proposal === null) { + throw new CuratorFieldValidationError( + "proposal", + new Error(`${parsedRelationship.classification} decisions require a proposal`), + ); } - continue; + const actionableRelationship: EvolutionProposalInstructionRelationship = { + classification: parsedRelationship.classification, + relevantEntries: parsedRelationship.relevantEntries, + strengthening: parsedRelationship.strengthening, + }; + const draft = atPath("proposal", () => + proposalDraft( + value.proposal, + decisionKey as string, + selectedLessons.map((lesson) => lesson.lessonId), + actionableRelationship, + input, + registry, + ), + ); + drafts.push(draft); + draftIndexes.set(draft.decisionKey, index); + } catch (error) { + issues.push(decisionValidationIssue(error, index, decisionKey)); } - if (value.proposal === null) throw new Error(`${parsedRelationship.classification} decisions require a proposal`); - const actionableRelationship: EvolutionProposalInstructionRelationship = { - classification: parsedRelationship.classification, - relevantEntries: parsedRelationship.relevantEntries, - strengthening: parsedRelationship.strengthening, - }; - drafts.push( - proposalDraft( - value.proposal, - decisionKey, - selectedLessons.map((lesson) => lesson.lessonId), - actionableRelationship, - input, - registry, - ), - ); } const emittedDecisionKeys = new Set(drafts.map((draft) => draft.decisionKey)); for (const draft of drafts) { - for (const conflictingDecisionKey of draft.conflictingDecisionKeys) { + for (const [conflictIndex, conflictingDecisionKey] of draft.conflictingDecisionKeys.entries()) { if (conflictingDecisionKey === draft.decisionKey) { - throw new Error(`Decision "${draft.decisionKey}" cannot conflict with itself`); + issues.push({ + decisionKey: draft.decisionKey, + path: `decisions[${String(draftIndexes.get(draft.decisionKey) ?? 0)}].proposal.conflictingDecisionKeys[${String(conflictIndex)}]`, + message: `Decision "${draft.decisionKey}" cannot conflict with itself`, + invalidReferences: [], + permittedReferences: [], + }); + continue; } if (!emittedDecisionKeys.has(conflictingDecisionKey)) { - throw new Error(`Decision "${draft.decisionKey}" cites unknown conflict "${conflictingDecisionKey}"`); + issues.push({ + decisionKey: draft.decisionKey, + path: `decisions[${String(draftIndexes.get(draft.decisionKey) ?? 0)}].proposal.conflictingDecisionKeys[${String(conflictIndex)}]`, + message: `Decision "${draft.decisionKey}" cites unknown conflict "${conflictingDecisionKey}"`, + invalidReferences: [], + permittedReferences: [], + }); } } } + if (issues.length > 0) throw new CuratorOutputValidationError(issues); return drafts; } @@ -963,7 +1111,7 @@ function modelInput(input: CuratorInput, registry: CuratorToolRegistry, response memoryOperations: ["ADD", "REMOVE", "REPLACE"], additionAnchorReferences: registry.memory.references, additionAnchorRule: - "Use an existing memoryRef to insert after that item, or null to append within the named section. Sentinels such as end are invalid.", + "Use an existing memoryRef to insert after that item, null to append within a named section, or null with a null section to append globally. Sentinels such as end are invalid.", deferredTargets: ["skill", "orchestration-source"], maxEvolutionMemoryLessonSentences: EVOLUTION_MEMORY_MAX_LESSON_SENTENCES, omitEvolutionMemoryLessonNumber: true, @@ -1038,6 +1186,9 @@ export class StructuredDeltaCurator implements Curator { lessonRefs: registry.lessons.references, memoryRefs: registry.memory.references, }); + const validateOutput = (output: JsonObject): void => { + parseDecisions(output, input, registry); + }; let lastError: unknown = null; let responseCorrection: string | null = null; const invalidAliases = new Set(); @@ -1054,17 +1205,21 @@ export class StructuredDeltaCurator implements Curator { input: modelInput(input, registry, responseCorrection), outputSchema: schema, tools: registry.tools, + validateOutput, signal: input.signal, }), input.signal, ); try { - const drafts = parseDecisions(response, input, registry); + const drafts = parseDecisions(response.output, input, registry); return proposalsFromDrafts(drafts, input, lessons, response, this.#dependencies.version); } catch (error) { lastError = error; responseCorrection = error instanceof Error ? error.message : "Response validation failed"; - if (error instanceof EvolutionModelReferenceError) { + if (error instanceof CuratorOutputValidationError) { + for (const reference of error.invalidReferences) invalidAliases.add(reference); + for (const reference of error.permittedReferences) permittedAliases.add(reference); + } else if (error instanceof EvolutionModelReferenceError) { for (const reference of error.invalidReferences) invalidAliases.add(reference); for (const reference of error.permittedReferences) permittedAliases.add(reference); } diff --git a/packages/evolution/src/evolution-memory-writer.ts b/packages/evolution/src/evolution-memory-writer.ts index e14933ef5..13fbf9bf2 100644 --- a/packages/evolution/src/evolution-memory-writer.ts +++ b/packages/evolution/src/evolution-memory-writer.ts @@ -13,6 +13,7 @@ import type { EvolutionMemoryAddDelta, EvolutionMemoryExactDiff, EvolutionMemoryItem, + EvolutionMemoryReplaceDelta, EvolutionMemoryWriteOutput, EvolutionMutation, HarnessActivationResult, @@ -386,18 +387,7 @@ function contentWithoutMetadata(batch: MutationBatch): Omit evolutionMemoryLessonContentMatches(block.content, delta.destination.anchor ?? ""), @@ -701,7 +691,6 @@ export class EvolutionMemoryWriter implements EvolutionTargetWriter { } let memory = base.state.memory; - const bootstrappingBlankMemory = memory.trim() === ""; let items = [...baseCatalog.items]; const activeOrder = [...baseCatalog.activeOrder]; const existingLessonNumbers = parseEvolutionMemory(memory) @@ -721,7 +710,15 @@ export class EvolutionMemoryWriter implements EvolutionTargetWriter { let nextLessonNumber = durableNextLessonNumber; const addedIds = new Set(); const removedIds = new Set(); - const remove = (memoryItemId: string, expectedContentDigest: string): void => { + const removable = ( + memoryItemId: string, + expectedContentDigest: string, + ): { + readonly activeIndex: number; + readonly block: MarkdownMemoryBlock; + readonly item: EvolutionMemoryItem; + readonly lessonNumber: number; + } => { const activeIndex = activeOrder.indexOf(memoryItemId); if (activeIndex === -1) throw new Error(`Mutation removes inactive memory item "${memoryItemId}"`); const parsed = parseEvolutionMemory(memory); @@ -738,8 +735,12 @@ export class EvolutionMemoryWriter implements EvolutionTargetWriter { ) { throw new Error(`Mutation cannot automatically remove memory item "${memoryItemId}"`); } - memory = compactLessonNumbers(removeMemoryBlock(memory, block), removedLessonNumber); - activeOrder.splice(activeIndex, 1); + return { activeIndex, block, item, lessonNumber: removedLessonNumber }; + }; + const remove = (memoryItemId: string, expectedContentDigest: string): void => { + const target = removable(memoryItemId, expectedContentDigest); + memory = compactLessonNumbers(removeMemoryBlock(memory, target.block), target.lessonNumber); + activeOrder.splice(target.activeIndex, 1); const compactedBlocks = parseEvolutionMemory(memory).blocks; const activeDigests = new Map( activeOrder.map((activeMemoryItemId, index) => [ @@ -754,21 +755,13 @@ export class EvolutionMemoryWriter implements EvolutionTargetWriter { nextLessonNumber -= 1; removedIds.add(memoryItemId); }; - const add = (mutation: EvolutionMutation, delta: EvolutionMemoryAddDelta): void => { - const addition = parseEvolutionMemory(delta.content); - if ( - addition.headings.length > 0 || - addition.blocks.length !== 1 || - addition.blocks[0]?.content !== delta.content - ) { + const validateAddition = (content: string): void => { + const addition = parseEvolutionMemory(content); + if (addition.headings.length > 0 || addition.blocks.length !== 1 || addition.blocks[0]?.content !== content) { throw new Error("Each EVOLVE.md addition must contain exactly one non-heading memory item"); } - const numberedContent = numberEvolutionMemoryLesson(delta.content, nextLessonNumber); - nextLessonNumber += 1; - const parsed = parseEvolutionMemory(memory); - const offset = insertionOffset(memory, parsed, delta, bootstrappingBlankMemory); - const insertionIndex = parsed.blocks.filter((block) => block.end <= offset).length; - memory = insertMemory(memory, offset, numberedContent); + }; + const createItem = (mutation: EvolutionMutation, content: string): string => { const memoryItemId = `memory-item-${createHash("sha256").update(mutation.mutationId).digest("hex").slice(0, 32)}`; if (items.some((item) => item.memoryItemId === memoryItemId)) { throw new Error(`Mutation reuses memory item ID "${memoryItemId}"`); @@ -776,7 +769,7 @@ export class EvolutionMemoryWriter implements EvolutionTargetWriter { items.push({ schemaVersion: EVOLUTION_MEMORY_ITEM_SCHEMA_VERSION, memoryItemId, - contentDigest: itemDigest(numberedContent), + contentDigest: itemDigest(content), creationRevision: batch.expectedBaseVersion, supersessionRevision: null, ledger: { @@ -785,8 +778,28 @@ export class EvolutionMemoryWriter implements EvolutionTargetWriter { supersessionEventId: null, }, }); - activeOrder.splice(insertionIndex, 0, memoryItemId); addedIds.add(memoryItemId); + return memoryItemId; + }; + const add = (mutation: EvolutionMutation, delta: EvolutionMemoryAddDelta): void => { + validateAddition(delta.content); + const numberedContent = numberEvolutionMemoryLesson(delta.content, nextLessonNumber); + nextLessonNumber += 1; + const parsed = parseEvolutionMemory(memory); + const offset = insertionOffset(memory, parsed, delta); + const insertionIndex = parsed.blocks.filter((block) => block.end <= offset).length; + memory = insertMemory(memory, offset, numberedContent); + const memoryItemId = createItem(mutation, numberedContent); + activeOrder.splice(insertionIndex, 0, memoryItemId); + }; + const replace = (mutation: EvolutionMutation, delta: EvolutionMemoryReplaceDelta): void => { + validateAddition(delta.addition.content); + const target = removable(delta.removal.memoryItemId, delta.removal.expectedContentDigest); + const numberedContent = numberEvolutionMemoryLesson(delta.addition.content, target.lessonNumber); + memory = `${memory.slice(0, target.block.start)}${numberedContent}${memory.slice(target.block.end)}`; + const memoryItemId = createItem(mutation, numberedContent); + activeOrder.splice(target.activeIndex, 1, memoryItemId); + removedIds.add(target.item.memoryItemId); }; for (const mutation of batch.mutations) { @@ -801,8 +814,7 @@ export class EvolutionMemoryWriter implements EvolutionTargetWriter { remove(mutation.payload.memoryItemId, mutation.payload.expectedContentDigest); } if (mutation.payload.operation === "REPLACE") { - remove(mutation.payload.removal.memoryItemId, mutation.payload.removal.expectedContentDigest); - add(mutation, { operation: "ADD", ...mutation.payload.addition }); + replace(mutation, mutation.payload); } } const parsedCandidate = parseEvolutionMemory(memory); diff --git a/packages/evolution/src/ports.ts b/packages/evolution/src/ports.ts index 5913507b8..1b08a81c4 100644 --- a/packages/evolution/src/ports.ts +++ b/packages/evolution/src/ports.ts @@ -98,6 +98,7 @@ export interface CuratorModelRequest { readonly input: JsonObject; readonly outputSchema: JsonObject; readonly tools: readonly EvolutionModelTool[]; + readonly validateOutput?: (output: JsonObject) => void; readonly signal?: AbortSignal; } diff --git a/packages/evolution/src/proposal-ranking.ts b/packages/evolution/src/proposal-ranking.ts index 2c59bb3d4..920ce9d89 100644 --- a/packages/evolution/src/proposal-ranking.ts +++ b/packages/evolution/src/proposal-ranking.ts @@ -314,12 +314,12 @@ function removedMemoryItemIds(delta: EvolutionMemoryDelta): readonly string[] { } function additions(delta: EvolutionMemoryDelta): readonly EvolutionMemoryAddDelta[] { - if (delta.operation === "REMOVE") return []; - return [delta.operation === "ADD" ? delta : { operation: "ADD", ...delta.addition }]; + return delta.operation === "ADD" ? [delta] : []; } function addContents(delta: EvolutionMemoryDelta): readonly string[] { - return additions(delta).map((addition) => addition.content); + if (delta.operation === "REMOVE") return []; + return [delta.operation === "ADD" ? delta.content : delta.addition.content]; } function additionDestinationIssues( diff --git a/packages/evolution/test/curator.test.ts b/packages/evolution/test/curator.test.ts index 6be9e86c4..92b80e264 100644 --- a/packages/evolution/test/curator.test.ts +++ b/packages/evolution/test/curator.test.ts @@ -1,8 +1,11 @@ import type { MaterializedEvidence, MaterializedInvocation } from "@earendil-works/diffex-harness-trace"; +import type { TSchema } from "typebox"; +import { Check } from "typebox/value"; import { describe, expect, it } from "vitest"; import { type CuratorInput, computeEvolutionContentDigest, + createCuratorOutputSchema, createEvolutionArtifactMetadata, type EvidenceCorpusManifest, type EvolutionArtifactProducer, @@ -314,6 +317,8 @@ describe("structured delta curator", () => { await curator.curate( curatorInput((request) => { expect(findUnsupportedCodexSchemaKeywords(request.outputSchema)).toEqual([]); + expect(request.validateOutput).toBeTypeOf("function"); + expect(() => request.validateOutput?.({ decisions: [] })).not.toThrow(); expect(request.outputSchema).toMatchObject({ $defs: { destination: { properties: { anchor: { enum: [null, "M1"] } } } }, }); @@ -322,6 +327,66 @@ describe("structured delta curator", () => { ); }); + it("keeps proposal schema variants conformant with parser routes", async () => { + const schema = createCuratorOutputSchema({ + instructionRefs: ["I1", "I2"], + lessonRefs: ["L1"], + memoryRefs: ["M1"], + }) as TSchema; + const addDelta = { + operation: "ADD", + destination: { section: null, anchor: null }, + content: "Keep changes focused.", + intendedScope: ["Repository code changes"], + }; + const removeDelta = { operation: "REMOVE", memoryRef: "M1", reason: "Remove stale guidance" }; + const replaceDelta = { + operation: "REPLACE", + removal: { memoryRef: "M1", reason: "Narrow stale guidance" }, + addition: { content: "Keep changes precisely focused.", intendedScope: ["Repository code changes"] }, + }; + const validDecisions = [ + decision("valid-add", proposal("ADD", { kind: "evolution-memory" }, addDelta)), + decision("valid-remove", proposal("REMOVE", { kind: "evolution-memory" }, removeDelta)), + decision("valid-replace", proposal("REPLACE", { kind: "evolution-memory" }, replaceDelta)), + decision( + "valid-skill-defer", + proposal("DEFER", { kind: "skill", skillName: "review" }, null, { finding: "Review needs precision" }), + ), + decision( + "valid-source-defer", + proposal("DEFER", { kind: "orchestration-source", sourcePath: "src/runner.ts" }, null, { + finding: "Runner retries need isolation", + }), + ), + ]; + expect(Check(schema, { decisions: validDecisions })).toBe(true); + const curator = new StructuredDeltaCurator({ version: "1.2.3" }, { maxResponseAttempts: 1 }); + await expect(curator.curate(curatorInput(() => Promise.resolve(response(validDecisions))))).resolves.toHaveLength( + 5, + ); + + const invalidDecisions = [ + decision("skill-add", proposal("ADD", { kind: "skill", skillName: "review" }, addDelta)), + decision( + "defer-with-delta", + proposal("DEFER", { kind: "skill", skillName: "review" }, addDelta, { finding: "Needs review" }), + ), + decision("remove-with-add-delta", proposal("REMOVE", { kind: "evolution-memory" }, addDelta)), + decision( + "add-with-finding", + proposal("ADD", { kind: "evolution-memory" }, addDelta, { finding: "Unexpected finding" }), + ), + decision("defer-without-finding", proposal("DEFER", { kind: "skill", skillName: "review" }, null)), + ]; + for (const invalidDecision of invalidDecisions) { + expect(Check(schema, { decisions: [invalidDecision] })).toBe(false); + await expect( + curator.curate(curatorInput(() => Promise.resolve(response([invalidDecision])))), + ).rejects.toMatchObject({ reasonCode: "curation-invalid-response" }); + } + }); + it("emits a provenance-rich ADD from final lessons and ordered snapshots", async () => { const capturedInputs: JsonObject[] = []; const toolOutputs: JsonObject[] = []; @@ -485,7 +550,6 @@ describe("structured delta curator", () => { reason: "Narrow the existing rule", }, addition: { - destination: { section: "Evolution memory", anchor: null }, content: "Keep repository code changes focused.", intendedScope: ["Repository code changes"], }, @@ -507,14 +571,13 @@ describe("structured delta curator", () => { operation: "REPLACE", removal: { memoryItemId: "memory-1", expectedContentDigest: MEMORY_ITEM_DIGEST }, addition: { - destination: { section: "Evolution memory", anchor: null }, content: "Keep repository code changes focused.", }, }, }); }); - it("rejects replacement additions anchored to the item being removed", async () => { + it("rejects placement fields on in-place replacements", async () => { const curator = new StructuredDeltaCurator({ version: "1.2.3" }, { maxResponseAttempts: 1 }); const input = curatorInput(() => Promise.resolve( @@ -542,11 +605,41 @@ describe("structured delta curator", () => { await expect(curator.curate(input)).rejects.toMatchObject({ reasonCode: "curation-invalid-response", cause: expect.objectContaining({ - message: "REPLACE addition cannot anchor to the memory item being removed", + message: expect.stringContaining('REPLACE addition does not permit field "destination"'), }), }); }); + it("preserves null destinations as explicit global append", async () => { + const curator = new StructuredDeltaCurator({ version: "1.2.3" }); + const proposals = await curator.curate( + curatorInput(() => + Promise.resolve( + response([ + decision( + "global-append", + proposal( + "ADD", + { kind: "evolution-memory" }, + { + operation: "ADD", + destination: { section: null, anchor: null }, + content: "Append flat memory without inventing a section.", + intendedScope: ["Repository code changes"], + }, + ), + ), + ]), + ), + ), + ); + + expect(proposals[0]).toMatchObject({ + action: "ADD", + exactDelta: { destination: { section: null, anchor: null } }, + }); + }); + it("preserves skill and source findings as non-executable deferred proposals", async () => { const curator = new StructuredDeltaCurator({ version: "1.2.3" }); const proposals = await curator.curate( @@ -721,6 +814,55 @@ describe("structured delta curator", () => { expect(JSON.stringify(inputs[1]?.control)).toContain("L1"); }); + it("aggregates semantic errors with decision keys and field paths", async () => { + let validationError: unknown; + const curator = new StructuredDeltaCurator({ version: "1.2.3" }); + const proposals = await curator.curate( + curatorInput((request) => { + const invalidOutput = response([ + decision( + "wrong-section", + proposal( + "ADD", + { kind: "evolution-memory" }, + { + operation: "ADD", + destination: { section: "Wrong section", anchor: "M1" }, + content: "Keep changes focused.", + intendedScope: ["Repository code changes"], + }, + ), + ), + decision("duplicate-without-reference", null, { + classification: "duplicate", + instructionRefs: [], + memoryRefs: [], + strengthening: null, + }), + ]).output; + try { + request.validateOutput?.(invalidOutput); + } catch (error) { + validationError = error; + } + return Promise.resolve(response([])); + }), + ); + + expect(proposals).toEqual([]); + expect(validationError).toMatchObject({ + name: "CuratorOutputValidationError", + message: expect.stringContaining( + 'Decision "wrong-section" at decisions[0].proposal.exactDelta: ADD anchor "M1" is outside section "Wrong section"', + ), + }); + expect(validationError).toMatchObject({ + message: expect.stringContaining( + 'Decision "duplicate-without-reference" at decisions[1].relationship: A duplicate relationship must cite an existing entry', + ), + }); + }); + it("rejects sentinel addition anchors and reports permitted memory references", async () => { let calls = 0; const curator = new StructuredDeltaCurator({ version: "1.2.3" }, { maxResponseAttempts: 1 }); @@ -939,7 +1081,9 @@ describe("structured delta curator", () => { await expect(curator.curate(input)).rejects.toMatchObject({ reasonCode: "curation-invalid-response", cause: expect.objectContaining({ - message: "Observer documentation and telemetry-format facts are not actionable evolution memory", + message: expect.stringContaining( + "Observer documentation and telemetry-format facts are not actionable evolution memory", + ), }), }); }); diff --git a/packages/evolution/test/evolution-memory-writer.test.ts b/packages/evolution/test/evolution-memory-writer.test.ts index 2ddc79743..58e759c92 100644 --- a/packages/evolution/test/evolution-memory-writer.test.ts +++ b/packages/evolution/test/evolution-memory-writer.test.ts @@ -105,7 +105,7 @@ describe("evolution memory writer", () => { target: { kind: "evolution-memory", expectedHarnessVersion: snapshot.harnessVersion }, payload: { operation: "ADD", - destination: { section: "verification", anchor: null }, + destination: { section: null, anchor: null }, content: "Lesson 99: Verify the runtime. Capture the result. Fix visible regressions.", intendedScope: ["verification"], }, @@ -135,7 +135,7 @@ describe("evolution memory writer", () => { target: { kind: "evolution-memory", expectedHarnessVersion: numberedSnapshot.harnessVersion }, payload: { operation: "ADD", - destination: { section: "verification", anchor: null }, + destination: { section: null, anchor: null }, content: "First sentence. Second sentence. Third sentence. Fourth sentence.", intendedScope: ["verification"], }, @@ -162,7 +162,7 @@ describe("evolution memory writer", () => { target: { kind: "evolution-memory", expectedHarnessVersion: snapshot.harnessVersion }, payload: { operation: "ADD", - destination: { section: "verification", anchor: null }, + destination: { section: null, anchor: null }, content: `Keep ${label} lesson content ${String(index + 1)}.`, intendedScope: ["verification"], }, @@ -250,7 +250,7 @@ describe("evolution memory writer", () => { target: { kind: "evolution-memory", expectedHarnessVersion: snapshot.harnessVersion }, payload: { operation: "ADD", - destination: { section: "verification", anchor: null }, + destination: { section: null, anchor: null }, content: "- Verify UI and scene-flow changes in a real runtime.", intendedScope: ["verification"], }, @@ -261,7 +261,7 @@ describe("evolution memory writer", () => { target: { kind: "evolution-memory", expectedHarnessVersion: snapshot.harnessVersion }, payload: { operation: "ADD", - destination: { section: "capability-honesty", anchor: null }, + destination: { section: null, anchor: null }, content: "- Confirm persistence before promising durable recall.", intendedScope: ["capability-honesty"], }, @@ -303,6 +303,194 @@ describe("evolution memory writer", () => { ]); }); + it("rejects named sections instead of discarding them during blank-memory bootstrap", async () => { + const { harnessState, snapshot, writer } = await emptyFixture(); + const selectedBatch = batch(snapshot, [ + { + mutationId: "mutation-global", + proposalId: "proposal-global", + target: { kind: "evolution-memory", expectedHarnessVersion: snapshot.harnessVersion }, + payload: { + operation: "ADD", + destination: { section: null, anchor: null }, + content: "Keep this batch atomic.", + intendedScope: ["repository"], + }, + }, + { + mutationId: "mutation-missing-section", + proposalId: "proposal-missing-section", + target: { kind: "evolution-memory", expectedHarnessVersion: snapshot.harnessVersion }, + payload: { + operation: "ADD", + destination: { section: "verification", anchor: null }, + content: "Do not discard this requested section.", + intendedScope: ["verification"], + }, + }, + ]); + + await expect( + writer.write(selectedBatch, { idempotencyKey: "write-missing-section", lease: writerLease }), + ).rejects.toMatchObject({ + name: "EvolutionWriterValidationError", + message: 'EVOLVE.md section "verification" must exist exactly once', + }); + expect((await harnessState.loadCurrent()).state.memory).toBe(""); + }); + + it.each([ + ["first", 0], + ["middle", 1], + ["final", 2], + ] as const)("replaces the %s flat memory item in place", async (label, targetIndex) => { + const { harnessState, snapshot, writer } = await emptyFixture(); + const initialBatch = batch( + snapshot, + ["First", "Middle", "Final"].map( + (name): EvolutionMutation => ({ + mutationId: `mutation-add-${name.toLowerCase()}`, + proposalId: `proposal-add-${name.toLowerCase()}`, + target: { kind: "evolution-memory", expectedHarnessVersion: snapshot.harnessVersion }, + payload: { + operation: "ADD", + destination: { section: null, anchor: null }, + content: `${name} lesson.`, + intendedScope: ["repository"], + }, + }), + ), + { batchId: `batch-seed-${label}`, activationEventId: `activation-seed-${label}` }, + ); + const initialWrite = await writer.write(initialBatch, { + idempotencyKey: `write-seed-${label}`, + lease: writerLease, + }); + await writer.activate(initialBatch, initialWrite, { + idempotencyKey: `activate-seed-${label}`, + lease: writerLease, + }); + const seededSnapshot = await writer.captureSnapshot(); + const target = seededSnapshot.activeMemoryItems[targetIndex]; + if (target === undefined) throw new Error(`Expected the ${label} memory item`); + const replacementBatch = batch( + seededSnapshot, + [ + { + mutationId: `mutation-replace-${label}`, + proposalId: `proposal-replace-${label}`, + target: { kind: "evolution-memory", expectedHarnessVersion: seededSnapshot.harnessVersion }, + payload: { + operation: "REPLACE", + removal: { + memoryItemId: target.memoryItemId, + expectedContentDigest: target.contentDigest, + reason: `Replace the ${label} lesson`, + }, + addition: { content: `${label} replacement.`, intendedScope: ["repository"] }, + }, + }, + ], + { batchId: `batch-replace-${label}`, activationEventId: `activation-replace-${label}` }, + ); + const replacementWrite = await writer.write(replacementBatch, { + idempotencyKey: `write-replace-${label}`, + lease: writerLease, + }); + await writer.activate(replacementBatch, replacementWrite, { + idempotencyKey: `activate-replace-${label}`, + lease: writerLease, + }); + + const expected = ["First lesson.", "Middle lesson.", "Final lesson."]; + expected[targetIndex] = `${label} replacement.`; + expect((await harnessState.loadCurrent()).state.memory).toBe( + `${expected.map((content, index) => `Lesson ${String(index + 1)}: ${content}`).join("\n\n")}\n`, + ); + const replacedSnapshot = await writer.captureSnapshot(); + expect(replacedSnapshot.activeMemoryItems).toHaveLength(3); + expect(replacedSnapshot.activeMemoryItems[targetIndex]?.memoryItemId).not.toBe(target.memoryItemId); + expect((await harnessState.loadCurrent()).manifest.metadata.evolutionMemory.nextLessonNumber).toBe(4); + }); + + it("keeps a section-contained replacement in its exact location", async () => { + const root = await mkdtemp(join(tmpdir(), "diffex-evolution-section-replace-")); + temporaryDirectories.push(root); + const harnessState = new HarnessStateStore(join(root, "agent")); + const empty = await harnessState.initialize(); + const initial = await harnessState.advance( + { memory: "# Alpha\n\nAlpha baseline.\n\n# Beta\n\nBeta baseline.\n" }, + empty.manifest.version, + ); + if (initial.status !== "advanced") throw new Error("Expected section fixture harness advance"); + const writer = new EvolutionMemoryWriter({ + version: "1.0.0", + harnessState, + statePath: join(root, "writer-state"), + }); + const snapshot = await writer.captureSnapshot(); + const additionBatch = batch( + snapshot, + [ + { + mutationId: "mutation-add-alpha", + proposalId: "proposal-add-alpha", + target: { kind: "evolution-memory", expectedHarnessVersion: snapshot.harnessVersion }, + payload: { + operation: "ADD", + destination: { section: "Alpha", anchor: null }, + content: "Alpha managed lesson.", + intendedScope: ["alpha"], + }, + }, + ], + { batchId: "batch-add-alpha", activationEventId: "activation-add-alpha" }, + ); + const additionWrite = await writer.write(additionBatch, { + idempotencyKey: "write-add-alpha", + lease: writerLease, + }); + await writer.activate(additionBatch, additionWrite, { + idempotencyKey: "activate-add-alpha", + lease: writerLease, + }); + const addedSnapshot = await writer.captureSnapshot(); + const target = addedSnapshot.activeMemoryItems.find((item) => item.ledger.origin === "evolution"); + if (target === undefined) throw new Error("Expected an Alpha evolution memory item"); + const replacementBatch = batch( + addedSnapshot, + [ + { + mutationId: "mutation-replace-alpha", + proposalId: "proposal-replace-alpha", + target: { kind: "evolution-memory", expectedHarnessVersion: addedSnapshot.harnessVersion }, + payload: { + operation: "REPLACE", + removal: { + memoryItemId: target.memoryItemId, + expectedContentDigest: target.contentDigest, + reason: "Narrow the Alpha lesson", + }, + addition: { content: "Alpha replacement.", intendedScope: ["alpha"] }, + }, + }, + ], + { batchId: "batch-replace-alpha", activationEventId: "activation-replace-alpha" }, + ); + const replacementWrite = await writer.write(replacementBatch, { + idempotencyKey: "write-replace-alpha", + lease: writerLease, + }); + await writer.activate(replacementBatch, replacementWrite, { + idempotencyKey: "activate-replace-alpha", + lease: writerLease, + }); + + expect((await harnessState.loadCurrent()).state.memory).toBe( + "# Alpha\n\nAlpha baseline.\n\nLesson 1: Alpha replacement.\n\n# Beta\n\nBeta baseline.\n", + ); + }); + it("stages selected deltas and atomically activates one harness revision", async () => { const { harnessState, initial, initialMemory, snapshot, writer } = await fixture(); const mutations: readonly EvolutionMutation[] = [