From 837482e68cb4c78cbdf109b8458b11846c8825bb Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Mon, 26 Oct 2026 07:38:37 -0400 Subject: [PATCH 1/2] feat(coding-agent): retain lessons and evolved skills longer --- packages/coding-agent/docs/evolution.md | 4 ++-- .../regressions/4-lesson-grooming.test.ts | 18 +++++++++--------- packages/evolution/src/lesson-score-store.ts | 2 +- packages/evolution/src/skill-score-store.ts | 2 +- .../evolution/test/lesson-grooming.test.ts | 16 ++++++++-------- .../evolution/test/lesson-score-store.test.ts | 2 +- packages/evolution/test/skill-grooming.test.ts | 16 ++++++++-------- .../evolution/test/skill-score-store.test.ts | 2 +- 8 files changed, 31 insertions(+), 31 deletions(-) diff --git a/packages/coding-agent/docs/evolution.md b/packages/coding-agent/docs/evolution.md index 44026c610..41cbe1503 100644 --- a/packages/coding-agent/docs/evolution.md +++ b/packages/coding-agent/docs/evolution.md @@ -51,7 +51,7 @@ After each completed assistant output with active evolution memory, a private co Scores start at 0, persist per workspace, and apply atomically once per session/output. REPLACE creates a new lesson identity at 0; removed or superseded identities stop accruing points. `DEFAULT_LESSON_POLICY` and `LessonPolicy` in `packages/evolution/src/lesson-score-store.ts` expose the shared typed defaults, not per-session settings. Scores never appear in EVOLVE.md or model prompts. -Before an eligible automatic or manual run freezes its input, Diffex grooms the selected revision's lessons, skipping grooming when its memory is empty. Lessons with scores at or below -5 are removed first; if more than 12 remain, the lowest-scoring lessons are removed until 12 remain. Equal scores remove earlier lessons first. This maintenance includes manual lessons, verifies their content digests, and compacts lesson numbering through the memory writer. Below these limits, the selected revision is unchanged. +Before an eligible automatic or manual run freezes its input, Diffex grooms the selected revision's lessons, skipping grooming when its memory is empty. Lessons with scores at or below -7 are removed first; if more than 12 remain, the lowest-scoring lessons are removed until 12 remain. Equal scores remove earlier lessons first. This maintenance includes manual lessons, verifies their content digests, and compacts lesson numbering through the memory writer. Below these limits, the selected revision is unchanged. The job snapshots the post-groom revision, including its evolved skills, rather than the session's pinned revision. Active sessions remain pinned until successful compaction or a new session. The `/evolve` Browse view shows **Reorganizing lessons** during grooming. @@ -63,7 +63,7 @@ Skill reports never enter conversation history, extension events, exports, compa Only output-bound package identities still present in the selected workspace revision accrue points. Updated packages start at 0, retired packages stop accruing, and installed skills are never scored. Scores never modify skill artifacts. `DEFAULT_SKILL_POLICY` and `SkillPolicy` in `packages/evolution/src/skill-score-store.ts` expose the shared typed scoring, grooming, and reset defaults, not per-session settings. Reporting and grooming are skipped without workspace-matching evolved skills in the active harness. -Before an eligible automatic or manual run freezes its input, skill grooming removes workspace-matching evolved skills at or below -8 points, then removes the lowest-scoring survivors until at most 10 remain. Equal scores are ordered by skill ID. Removals verify the frozen package version and content digest and use the skill writer's staging and activation path without candidate evaluation. Installed skills and other workspaces are untouched; historical revisions, scores, and lifecycle observations remain available for audit. Below these limits grooming does not advance the selection. The job freezes the post-groom revision and skill catalog; active sessions adopt that selection only after successful compaction or in a new session. +Before an eligible automatic or manual run freezes its input, skill grooming removes workspace-matching evolved skills at or below -11 points, then removes the lowest-scoring survivors until at most 10 remain. Equal scores are ordered by skill ID. Removals verify the frozen package version and content digest and use the skill writer's staging and activation path without candidate evaluation. Installed skills and other workspaces are untouched; historical revisions, scores, and lifecycle observations remain available for audit. Below these limits grooming does not advance the selection. The job freezes the post-groom revision and skill catalog; active sessions adopt that selection only after successful compaction or in a new session. ## Validation and security diff --git a/packages/coding-agent/test/suite/regressions/4-lesson-grooming.test.ts b/packages/coding-agent/test/suite/regressions/4-lesson-grooming.test.ts index d91504334..69e2a3b20 100644 --- a/packages/coding-agent/test/suite/regressions/4-lesson-grooming.test.ts +++ b/packages/coding-agent/test/suite/regressions/4-lesson-grooming.test.ts @@ -108,7 +108,7 @@ const emptyReflection = () => describe("Item 4: grooming before evolution snapshots", () => { it.each(["manual", "automatic"] as const)( - "grooms five omissions before the %s job and next prompt", + "grooms seven omissions before the %s job and next prompt", async (trigger) => { const env = await setup(); if (trigger === "automatic") { @@ -119,7 +119,7 @@ describe("Item 4: grooming before evolution snapshots", () => { void (async () => { for await (const event of await stream(model, context, options)) { if (event.type === "done") { - event.message.usage = { ...event.message.usage, input: 80_000, output: 0, totalTokens: 80_000 }; + event.message.usage = { ...event.message.usage, input: 60_000, output: 0, totalTokens: 60_000 }; } output.push(event); } @@ -130,12 +130,12 @@ describe("Item 4: grooming before evolution snapshots", () => { await env.coordinator.replace(env.runtime); cleanups.push(() => env.coordinator.stop()); } - for (let index = 0; index < 5; index += 1) { + for (let index = 0; index < 7; index += 1) { const main = fauxAssistantMessage("Verified the requested result."); env.harness.setResponses([ main, assessment(), - ...(index === 4 && trigger === "automatic" ? Array.from({ length: 10 }, emptyReflection) : []), + ...(index === 6 && trigger === "automatic" ? Array.from({ length: 20 }, emptyReflection) : []), ]); await env.session.prompt(`Verify result ${index + 1}.`); } @@ -143,10 +143,10 @@ describe("Item 4: grooming before evolution snapshots", () => { (await env.scores.loadLessonScores(env.scope, env.bundle.manifest.version)).scores.map( (item) => item.points, ), - ).toEqual([-5, 5]); + ).toEqual([-7, 7]); if (trigger === "manual") { expect((await env.store.loadCurrent()).manifest.version).toBe(env.bundle.manifest.version); - env.harness.setResponses(Array.from({ length: 10 }, emptyReflection)); + env.harness.setResponses(Array.from({ length: 20 }, emptyReflection)); await env.runtime.start(); const started = await env.runtime.control.startManual(); expect(started.status).toBe("created"); @@ -208,9 +208,9 @@ describe("Item 4: grooming before evolution snapshots", () => { }, ); - it.each([0, 12])("leaves %i lessons unchanged below grooming thresholds", async (count) => { + it.each([0, 12])("leaves %i lessons unchanged just above the removal threshold", async (count) => { const env = await setup(count); - for (let index = 0; index < 4; index += 1) { + for (let index = 0; index < 6; index += 1) { env.harness.setResponses([fauxAssistantMessage("Verified."), ...(count === 0 ? [] : [assessment()])]); await env.session.prompt(`Verify result ${index + 1}.`); } @@ -219,7 +219,7 @@ describe("Item 4: grooming before evolution snapshots", () => { (await env.scores.loadLessonScores(env.scope, env.bundle.manifest.version)).scores.map( (item) => item.points, ), - ).toEqual(Array.from({ length: count }, (_, index) => (index === 1 ? 4 : -4))); + ).toEqual(Array.from({ length: count }, (_, index) => (index === 1 ? 6 : -6))); } const before = await env.writer.captureSnapshot(); const groom = vi.spyOn(EvolutionMemoryWriter.prototype, "groom"); diff --git a/packages/evolution/src/lesson-score-store.ts b/packages/evolution/src/lesson-score-store.ts index f82506550..573f59a0d 100644 --- a/packages/evolution/src/lesson-score-store.ts +++ b/packages/evolution/src/lesson-score-store.ts @@ -16,7 +16,7 @@ export interface LessonPolicy { export const DEFAULT_LESSON_POLICY: LessonPolicy = Object.freeze({ initialPoints: 0, weights: Object.freeze({ helpful: 1, harmful: -1.5, neutral: -1 }), - removalThreshold: -5, + removalThreshold: -7, lessonCap: 12, resetOnModify: true, }); diff --git a/packages/evolution/src/skill-score-store.ts b/packages/evolution/src/skill-score-store.ts index 8efeed00e..52cdd547b 100644 --- a/packages/evolution/src/skill-score-store.ts +++ b/packages/evolution/src/skill-score-store.ts @@ -16,7 +16,7 @@ export interface SkillPolicy { export const DEFAULT_SKILL_POLICY: SkillPolicy = Object.freeze({ initialPoints: 0, weights: Object.freeze({ helpful: 1, harmful: -1.5, neutral: -1 }), - removalThreshold: -8, + removalThreshold: -11, skillCap: 10, resetOnModify: true, }); diff --git a/packages/evolution/test/lesson-grooming.test.ts b/packages/evolution/test/lesson-grooming.test.ts index b1a5c23ab..2d49e406b 100644 --- a/packages/evolution/test/lesson-grooming.test.ts +++ b/packages/evolution/test/lesson-grooming.test.ts @@ -54,18 +54,18 @@ async function setup(points: readonly number[]) { describe("lesson grooming", () => { it.each([ - { points: [-5, -6, 0], remaining: [3] }, + { points: [-6.5, -7, -8, 0], remaining: [1, 4] }, { points: [...Array(12).fill(0), -1], remaining: Array.from({ length: 12 }, (_, index) => index + 1) }, { - points: [-5, -6, -2, ...Array(12).fill(0)], + points: [-7, -8, -2, ...Array(12).fill(0)], remaining: Array.from({ length: 12 }, (_, index) => index + 4), }, { - points: [-5, -6, -7, ...Array(11).fill(0)], + points: [-7, -8, -9, ...Array(11).fill(0)], remaining: Array.from({ length: 11 }, (_, index) => index + 4), }, { points: Array(13).fill(0), remaining: Array.from({ length: 12 }, (_, index) => index + 2) }, - { points: [-5], remaining: [] }, + { points: [-7], remaining: [] }, ])("removes exactly the threshold/cap victims: $points", async ({ points, remaining }) => { const { state, writer, scores, snapshot } = await setup(points); expect(snapshot.activeMemoryItems.every((item) => item.ledger.origin === "manual")).toBe(true); @@ -88,8 +88,8 @@ describe("lesson grooming", () => { expect((await state.loadCurrent()).manifest.version).toBe(groomed.harnessVersion); }); - it.each([[], [-4.5, 0], Array(12).fill(0)].map((points) => ({ points })))( - "does not advance below thresholds: $points", + it.each([[], [-6.5, -6, -5, 0], Array(12).fill(0)].map((points) => ({ points })))( + "does not advance above the removal threshold and within capacity: $points", async ({ points }) => { const { state, writer, scores, snapshot } = await setup(points); expect(await writer.groom(scores, scope, context)).toEqual(snapshot); @@ -98,7 +98,7 @@ describe("lesson grooming", () => { ); it("keeps manual removal unavailable to ordinary evolution batches", async () => { - const { writer, snapshot } = await setup([-5]); + const { writer, snapshot } = await setup([-7]); const item = snapshot.activeMemoryItems[0]!; const content: Omit = { batchId: "ordinary", @@ -136,7 +136,7 @@ describe("lesson grooming", () => { }); it("still rejects mismatched removal digests", async () => { - const { state, writer, scores, snapshot } = await setup([-5]); + const { state, writer, scores, snapshot } = await setup([-7]); const points = await scores.loadLessonScores(scope, snapshot.harnessVersion); vi.spyOn(scores, "loadLessonScores").mockResolvedValue({ ...points, diff --git a/packages/evolution/test/lesson-score-store.test.ts b/packages/evolution/test/lesson-score-store.test.ts index a82d76255..5cd22cf43 100644 --- a/packages/evolution/test/lesson-score-store.test.ts +++ b/packages/evolution/test/lesson-score-store.test.ts @@ -119,7 +119,7 @@ describe("lesson score store", () => { expect(DEFAULT_LESSON_POLICY).toEqual({ initialPoints: 0, weights: { helpful: 1, harmful: -1.5, neutral: -1 }, - removalThreshold: -5, + removalThreshold: -7, lessonCap: 12, resetOnModify: true, }); diff --git a/packages/evolution/test/skill-grooming.test.ts b/packages/evolution/test/skill-grooming.test.ts index c42019cf1..269f7c05f 100644 --- a/packages/evolution/test/skill-grooming.test.ts +++ b/packages/evolution/test/skill-grooming.test.ts @@ -79,11 +79,11 @@ function packageContent({ filePath: _filePath, baseDir: _baseDir, files, ...skil describe("skill grooming", () => { it.each([ - { points: [-8, -9, 0], removed: [0, 1] }, + { points: [-10.5, -11, -12, 0], removed: [1, 2] }, { points: [...Array(10).fill(0), -1], removed: [10] }, - { points: [-8, -9, -2, ...Array(10).fill(0)], removed: [0, 1, 2] }, - { points: [-8, -9, -10, ...Array(9).fill(0)], removed: [0, 1, 2] }, - { points: [-8], removed: [0] }, + { points: [-11, -12, -2, ...Array(10).fill(0)], removed: [0, 1, 2] }, + { points: [-11, -12, -13, ...Array(9).fill(0)], removed: [0, 1, 2] }, + { points: [-11], removed: [0] }, ])("removes exactly the threshold and capacity victims: $points", async ({ points, removed }) => { const env = await setup(points); const registry = vi.spyOn(env.registry, "list"); @@ -138,8 +138,8 @@ describe("skill grooming", () => { expect(groomed.state.skills).toHaveLength(11); // Ten local skills plus the foreign skill. }); - it.each([[], [-7.5, 0], Array(10).fill(0)].map((points) => ({ points })))( - "does not advance below thresholds: $points", + it.each([[], [-10.5, -10, -8, 0], Array(10).fill(0)].map((points) => ({ points })))( + "does not advance above the removal threshold and within capacity: $points", async ({ points }) => { const env = await setup(points); const installed = join(env.root, "installed.md"); @@ -154,7 +154,7 @@ describe("skill grooming", () => { it.each(["contentDigest", "packageVersion", "workspace"] as const)( "rejects mismatched %s without advancing", async (field) => { - const env = await setup([-8]); + const env = await setup([-11]); const snapshot = await env.scores.loadSkillScores(scope, workspace, env.bundle.manifest.version); vi.spyOn(env.scores, "loadSkillScores").mockResolvedValue({ ...snapshot, @@ -169,7 +169,7 @@ describe("skill grooming", () => { ); it("does not rebase a scored removal onto a concurrently selected revision", async () => { - const env = await setup([-8]); + const env = await setup([-11]); const next = await env.state.saveVersion({ memory: "Changed selection.\n" }, env.bundle.manifest.version); const select = env.state.selectVersionIfCurrent.bind(env.state); vi.spyOn(env.state, "selectVersionIfCurrent").mockImplementationOnce(async (...args) => { diff --git a/packages/evolution/test/skill-score-store.test.ts b/packages/evolution/test/skill-score-store.test.ts index 9b6a9a5f7..48ce088e7 100644 --- a/packages/evolution/test/skill-score-store.test.ts +++ b/packages/evolution/test/skill-score-store.test.ts @@ -78,7 +78,7 @@ describe("skill score store", () => { expect(DEFAULT_SKILL_POLICY).toEqual({ initialPoints: 0, weights: { helpful: 1, harmful: -1.5, neutral: -1 }, - removalThreshold: -8, + removalThreshold: -11, skillCap: 10, resetOnModify: true, }); From f8d9fa7b9398263c6ee6a0d7207baa7fc5fc24f3 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Mon, 26 Oct 2026 07:42:07 -0400 Subject: [PATCH 2/2] fix(coding-agent): align skill grooming regression with retention policy --- .../suite/regressions/4-skill-grooming.test.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/coding-agent/test/suite/regressions/4-skill-grooming.test.ts b/packages/coding-agent/test/suite/regressions/4-skill-grooming.test.ts index eb7ea06ec..b9638fc31 100644 --- a/packages/coding-agent/test/suite/regressions/4-skill-grooming.test.ts +++ b/packages/coding-agent/test/suite/regressions/4-skill-grooming.test.ts @@ -104,7 +104,7 @@ async function setup(count = 2, memory = "") { describe("Item 4: skill grooming before evolution", () => { it.each(["manual", "automatic"] as const)( - "grooms eight omissions before the %s snapshot, curation, and next prompt", + "grooms eleven omissions before the %s snapshot, curation, and next prompt", async (trigger) => { const env = await setup(); const curate = vi.spyOn(StructuredSkillCurator.prototype, "curate"); @@ -152,7 +152,7 @@ describe("Item 4: skill grooming before evolution", () => { void (async () => { for await (const event of await stream(model, context, options)) { if (event.type === "done") - event.message.usage = { ...event.message.usage, input: 50_000, output: 0, totalTokens: 50_000 }; + event.message.usage = { ...event.message.usage, input: 37_000, output: 0, totalTokens: 37_000 }; output.push(event); } output.end(); @@ -162,12 +162,12 @@ describe("Item 4: skill grooming before evolution", () => { await env.coordinator.replace(env.runtime); cleanups.push(() => env.coordinator.stop()); } - for (let index = 0; index < 8; index += 1) { + for (let index = 0; index < 11; index += 1) { env.harness.setResponses([ fauxAssistantMessage("Verified the requested result."), report([{ skillName: "skill-1", signal: "helpful" }]), ]); - if (index === 7) { + if (index === 10) { env.harness.appendResponses( Array.from({ length: 30 }, () => (context) => { const parameters = context.tools?.find( @@ -221,12 +221,12 @@ describe("Item 4: skill grooming before evolution", () => { scores.scores.find( (item) => item.skillId === computeHarnessSkillId(env.workspace, "skills/skill-0/SKILL.md"), )?.points, - ).toBe(-8); + ).toBe(-11); expect( scores.scores.find( (item) => item.skillId === computeHarnessSkillId(env.workspace, "skills/skill-1/SKILL.md"), )?.points, - ).toBe(8); + ).toBe(11); if (trigger === "manual") { expect((await env.store.loadCurrent()).manifest.version).toBe(env.bundle.manifest.version); await env.runtime.start(); @@ -290,7 +290,7 @@ describe("Item 4: skill grooming before evolution", () => { it("freezes one consistent revision after both lesson and skill grooming", async () => { const env = await setup(2, "Lesson 1: Old lesson.\n"); - for (let index = 0; index < 8; index += 1) { + for (let index = 0; index < 11; index += 1) { env.harness.setResponses([ fauxAssistantMessage("Verified."), fauxAssistantMessage(fauxToolCall(LESSON_REPORT_TOOL.name, { lessons: [] }), { stopReason: "toolUse" }),