From dacdc0e1fcef04457dcff43c7bafbf69ab4bbe9c Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 21:39:54 +0000 Subject: [PATCH 01/26] GEN-2907 define run retention eligibility policy --- packages/api/src/postgres/run-retention.ts | 30 +++++++++++++++++ packages/api/test/run-retention.test.ts | 38 ++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 packages/api/src/postgres/run-retention.ts create mode 100644 packages/api/test/run-retention.test.ts diff --git a/packages/api/src/postgres/run-retention.ts b/packages/api/src/postgres/run-retention.ts new file mode 100644 index 0000000..bbf2366 --- /dev/null +++ b/packages/api/src/postgres/run-retention.ts @@ -0,0 +1,30 @@ +/** Global run history retention policy. Keep in sync with the maintenance runbook. */ +export const RUN_HISTORY_RETENTION_MS = 14 * 24 * 60 * 60 * 1_000; + +export type RetainedRunState = "queued" | "reserved" | "starting" | "running" | "blocked" | "stopping" | "succeeded" | "failed" | "stopped"; + +/** + * A run is eligible only after its immutable creation time is older than the + * fixed cutoff, it has reached a terminal state, and normal finalization and + * lifecycle automation handoff have completed. A completion date never starts + * a fresh retention window. + */ +export function isRunEligibleForRetention(input: { + createdAt: Date | string; + state: RetainedRunState; + cleanupComplete: boolean; + automationHandedOff: boolean; +}, now: Date): boolean { + const createdAt = input.createdAt instanceof Date ? input.createdAt.getTime() : Date.parse(input.createdAt); + const currentTime = now.getTime(); + if (!Number.isFinite(createdAt) || !Number.isFinite(currentTime)) throw new TypeError("Retention timestamps must be valid dates"); + if (!input.cleanupComplete || !input.automationHandedOff) return false; + if (input.state !== "succeeded" && input.state !== "failed" && input.state !== "stopped") return false; + return createdAt < currentTime - RUN_HISTORY_RETENTION_MS; +} + +/** Return the one fixed cutoff shared by a bounded sweep. */ +export function runRetentionCutoff(now: Date): Date { + if (!Number.isFinite(now.getTime())) throw new TypeError("Retention timestamp must be a valid date"); + return new Date(now.getTime() - RUN_HISTORY_RETENTION_MS); +} diff --git a/packages/api/test/run-retention.test.ts b/packages/api/test/run-retention.test.ts new file mode 100644 index 0000000..0d22837 --- /dev/null +++ b/packages/api/test/run-retention.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; +import { isRunEligibleForRetention, runRetentionCutoff, RUN_HISTORY_RETENTION_MS } from "../src/postgres/run-retention"; + +describe("run history retention eligibility", () => { + const now = new Date("2026-11-11T12:00:00.000Z"); + const eligible = { + createdAt: new Date(now.getTime() - RUN_HISTORY_RETENTION_MS - 1), + state: "succeeded" as const, + cleanupComplete: true, + automationHandedOff: true, + }; + + it("uses the immutable creation time and a strict 14-day boundary", () => { + const boundary = new Date(now.getTime() - RUN_HISTORY_RETENTION_MS); + expect(isRunEligibleForRetention({ ...eligible, createdAt: boundary }, now)).toBe(false); + expect(isRunEligibleForRetention({ ...eligible, createdAt: new Date(boundary.getTime() - 1) }, now)).toBe(true); + }); + + it("protects every nonterminal state, including old queued and blocked runs", () => { + for (const state of ["queued", "reserved", "starting", "running", "blocked", "stopping"] as const) { + expect(isRunEligibleForRetention({ ...eligible, state }, now), state).toBe(false); + } + }); + + it("requires resource cleanup and automation handoff to finish", () => { + expect(isRunEligibleForRetention({ ...eligible, cleanupComplete: false }, now)).toBe(false); + expect(isRunEligibleForRetention({ ...eligible, automationHandedOff: false }, now)).toBe(false); + }); + + it("uses one fixed cutoff per sweep", () => { + expect(runRetentionCutoff(now)).toEqual(new Date("2026-10-28T12:00:00.000Z")); + }); + + it("rejects invalid timestamps", () => { + expect(() => isRunEligibleForRetention({ ...eligible, createdAt: "invalid" }, now)).toThrow(TypeError); + expect(() => runRetentionCutoff(new Date(Number.NaN))).toThrow(TypeError); + }); +}); From f5ffb8ff0127590a8c1e5a36fde959053f48c335 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 21:55:09 +0000 Subject: [PATCH 02/26] GEN-2907 add transactional retention deletion foundation --- .../migrations/0037_run_retention_cleanup.sql | 28 +++++++ .../src/postgres/run-retention-repository.ts | 82 +++++++++++++++++++ packages/api/src/postgres/schema.ts | 9 +- .../api/test/run-retention-repository.test.ts | 36 ++++++++ 4 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 packages/api/migrations/0037_run_retention_cleanup.sql create mode 100644 packages/api/src/postgres/run-retention-repository.ts create mode 100644 packages/api/test/run-retention-repository.test.ts diff --git a/packages/api/migrations/0037_run_retention_cleanup.sql b/packages/api/migrations/0037_run_retention_cleanup.sql new file mode 100644 index 0000000..d9813f9 --- /dev/null +++ b/packages/api/migrations/0037_run_retention_cleanup.sql @@ -0,0 +1,28 @@ +-- Durable ownership for external objects outlives the run aggregate it cleans. +CREATE TABLE app.run_retention_cleanup ( + id uuid PRIMARY KEY, + tenant_id uuid NOT NULL, + run_id uuid NOT NULL, + object_keys jsonb NOT NULL DEFAULT '[]'::jsonb, + execution_backend_kind text, + execution_handle jsonb, + attempts integer NOT NULL DEFAULT 0, + next_attempt_at timestamptz NOT NULL DEFAULT now(), + completed_at timestamptz, + last_error text, + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX run_retention_cleanup_due ON app.run_retention_cleanup(next_attempt_at, id) WHERE completed_at IS NULL; + +-- Candidate scans use the immutable creation timestamp and terminal-state filter. +CREATE INDEX runs_retention_candidates ON app.runs(created_at, tenant_id, invocation_id) + WHERE state IN ('succeeded','failed','stopped'); + +-- Cascading deletion and cleanup ownership lookup should not scan these child tables. +CREATE INDEX run_activity_by_run ON app.run_activity(tenant_id, run_id); +CREATE INDEX lifecycle_events_by_source_run ON app.lifecycle_events(tenant_id, source_run_id); +CREATE INDEX lifecycle_deliveries_by_source_run ON app.lifecycle_deliveries(tenant_id, source_run_id); +CREATE INDEX credential_run_enrollments_by_run ON app.credential_run_enrollments(tenant_id, run_id); +CREATE INDEX webhook_delivery_events_by_delivery ON app.webhook_delivery_events(tenant_id, delivery_id); + +INSERT INTO app.schema_migrations(version) VALUES ('0037_run_retention_cleanup') ON CONFLICT DO NOTHING; diff --git a/packages/api/src/postgres/run-retention-repository.ts b/packages/api/src/postgres/run-retention-repository.ts new file mode 100644 index 0000000..b1090bd --- /dev/null +++ b/packages/api/src/postgres/run-retention-repository.ts @@ -0,0 +1,82 @@ +import { Database } from "./database"; + +export type RetentionSweepResult = { selected: number; deleted: number; cleanupQueued: number }; + +/** + * Bounded retention deletion. The fixed cutoff is supplied by the caller so a + * complete sweep cannot move its eligibility boundary while it is running. + * Each invocation aggregate and its independent object cleanup receipt commit + * together; SKIP LOCKED lets overlapping maintenance workers make progress. + */ +export class RunRetentionRepository { + constructor(private readonly database: Database) {} + + async sweep(cutoff: Date, limit = 50): Promise { + if (!Number.isFinite(cutoff.getTime())) throw new TypeError("Retention cutoff must be a valid date"); + if (!Number.isInteger(limit) || limit < 1 || limit > 500) throw new RangeError("Retention batch size must be between 1 and 500"); + + const selected = await this.database.pool.query<{ tenant_id: string; invocation_id: string }>(` + SELECT i.tenant_id,i.id invocation_id + FROM app.runs r + JOIN app.job_runs jr ON jr.tenant_id=r.tenant_id AND jr.invocation_id=r.invocation_id + JOIN app.invocations i ON i.tenant_id=r.tenant_id AND i.id=r.invocation_id + WHERE r.created_at < $1 + AND r.state IN ('succeeded','failed','stopped') AND jr.state=r.state + AND r.vm_cleanup_complete AND r.artifact_state NOT IN ('pending','collecting') + AND NOT EXISTS ( + SELECT 1 FROM app.lifecycle_events e + JOIN app.triggers t ON t.tenant_id=e.tenant_id AND t.id=e.trigger_id + JOIN app.jobs j ON j.tenant_id=t.tenant_id AND j.id=t.job_id + WHERE e.tenant_id=r.tenant_id AND e.source_run_id=r.id + AND t.kind='jobLifecycle' AND t.enabled AND t.removed_at IS NULL AND j.enabled + AND NOT EXISTS (SELECT 1 FROM app.lifecycle_deliveries d + WHERE d.tenant_id=e.tenant_id AND d.trigger_id=e.trigger_id + AND d.source_run_id=e.source_run_id AND d.terminal_state=e.terminal_state) + ) + ORDER BY r.created_at,r.tenant_id,r.invocation_id + LIMIT $2`, [cutoff, limit]); + + let deleted = 0; + let cleanupQueued = 0; + for (const candidate of selected.rows) { + const result = await this.database.transaction(async client => { + // Lock in invocation -> job_run -> run order, the aggregate's ownership + // direction. Recheck every eligibility condition after acquiring locks. + const locked = await client.query<{ run_id: string }>(` + SELECT r.id run_id + FROM app.invocations i + JOIN app.job_runs jr ON jr.tenant_id=i.tenant_id AND jr.invocation_id=i.id + JOIN app.runs r ON r.tenant_id=i.tenant_id AND r.invocation_id=i.id + WHERE i.tenant_id=$1 AND i.id=$2 AND r.created_at < $3 + AND r.state IN ('succeeded','failed','stopped') AND jr.state=r.state + AND r.vm_cleanup_complete AND r.artifact_state NOT IN ('pending','collecting') + AND NOT EXISTS ( + SELECT 1 FROM app.lifecycle_events e + JOIN app.triggers t ON t.tenant_id=e.tenant_id AND t.id=e.trigger_id + JOIN app.jobs j ON j.tenant_id=t.tenant_id AND j.id=t.job_id + WHERE e.tenant_id=r.tenant_id AND e.source_run_id=r.id + AND t.kind='jobLifecycle' AND t.enabled AND t.removed_at IS NULL AND j.enabled + AND NOT EXISTS (SELECT 1 FROM app.lifecycle_deliveries d + WHERE d.tenant_id=e.tenant_id AND d.trigger_id=e.trigger_id + AND d.source_run_id=e.source_run_id AND d.terminal_state=e.terminal_state) + ) + FOR UPDATE OF i,jr,r SKIP LOCKED`, [candidate.tenant_id, candidate.invocation_id, cutoff]); + if (!locked.rows.length) return { deleted: false, queued: false }; + + const keys = await client.query<{ object_key: string }>( + "SELECT object_key FROM app.run_artifacts WHERE tenant_id=$1 AND run_id=$2 ORDER BY object_key", + [candidate.tenant_id, locked.rows[0]!.run_id], + ); + if (keys.rows.length) { + await client.query(`INSERT INTO app.run_retention_cleanup(id,tenant_id,run_id,object_keys) + VALUES ($1,$2,$3,$4::jsonb)`, [crypto.randomUUID(), candidate.tenant_id, locked.rows[0]!.run_id, JSON.stringify(keys.rows.map(row => row.object_key))]); + } + const removed = await client.query("DELETE FROM app.invocations WHERE tenant_id=$1 AND id=$2", [candidate.tenant_id, candidate.invocation_id]); + return { deleted: removed.rowCount === 1, queued: keys.rows.length > 0 }; + }); + if (result.deleted) deleted++; + if (result.queued) cleanupQueued++; + } + return { selected: selected.rowCount ?? selected.rows.length, deleted, cleanupQueued }; + } +} diff --git a/packages/api/src/postgres/schema.ts b/packages/api/src/postgres/schema.ts index 5bb398a..bd1dd72 100644 --- a/packages/api/src/postgres/schema.ts +++ b/packages/api/src/postgres/schema.ts @@ -96,6 +96,13 @@ export const runs = app.table("runs", { export const runArtifacts = app.table("run_artifacts", { tenantId: uuid("tenant_id").notNull(), runId: uuid("run_id").notNull(), id: uuid("id").notNull(), kind: text("kind").$type().notNull(), objectKey: text("object_key").notNull(), provider: text("provider").notNull(), format: text("format").notNull(), formatVersion: text("format_version"), cliVersion: text("cli_version"), nativeSessionId: text("native_session_id"), sourceGeneration: text("source_generation"), sourcePath: text("source_path"), mediaType: text("media_type"), harnessVersion: text("harness_version"), byteSize: bigint("byte_size", { mode: "number" }).notNull(), sha256: text("sha256").notNull(), state: text("state").$type().notNull(), createdAt: createdAt(), }, t => [primaryKey({ columns: [t.tenantId, t.id] }), unique().on(t.tenantId, t.objectKey), foreignKey({ columns: [t.tenantId, t.runId], foreignColumns: [runs.tenantId, runs.id] }).onDelete("cascade"), index("run_artifacts_by_run").on(t.tenantId, t.runId, t.createdAt, t.id)]); +/** External cleanup ownership intentionally has no FK to a run, job, or tenant. */ +export const runRetentionCleanup = app.table("run_retention_cleanup", { + id: uuid("id").primaryKey(), tenantId: uuid("tenant_id").notNull(), runId: uuid("run_id").notNull(), + objectKeys: jsonb("object_keys").$type().notNull().default([]), executionBackendKind: text("execution_backend_kind"), executionHandle: jsonb("execution_handle").$type>(), + attempts: integer("attempts").notNull().default(0), nextAttemptAt: timestamp("next_attempt_at", { withTimezone: true }).notNull().defaultNow(), + completedAt: timestamp("completed_at", { withTimezone: true }), lastError: text("last_error"), createdAt: createdAt(), +}, t => [index("run_retention_cleanup_due").on(t.nextAttemptAt, t.id).where(sql`${t.completedAt} IS NULL`)]); export const runTraceEvents = app.table("run_trace_events", { tenantId: uuid("tenant_id").notNull(), runId: uuid("run_id").notNull(), sequence: bigint("sequence", { mode: "number" }).notNull(), id: text("id").notNull(), parentId: text("parent_id"), eventType: text("event_type").notNull(), role: text("role"), title: text("title").notNull(), previewText: text("preview_text").notNull(), displayData: jsonb("display_data").$type>().notNull().default({}), occurredAt: timestamp("occurred_at", { withTimezone: true }), createdAt: createdAt(), searchVector: tsvector("search_vector").generatedAlwaysAs(sql`to_tsvector('english', title || ' ' || preview_text)`), }, t => [primaryKey({ columns: [t.tenantId, t.runId, t.sequence] }), foreignKey({ columns: [t.tenantId, t.runId], foreignColumns: [runs.tenantId, runs.id] }).onDelete("cascade"), index("run_trace_events_search").using("gin", t.searchVector), index("run_trace_events_by_type").on(t.tenantId, t.runId, t.eventType, t.sequence)]); @@ -155,7 +162,7 @@ export const runRelations = relations(runs, ({ one, many }) => ({ export const databaseSchema = { schemaMigrations, tenants, oauthConnections, oauthConsentPreviews, authUsers, authAccounts, authResetTokens, authSessions, oauthDeviceAuthorizations, members, accessTokens, - connections, integrationCollections, integrationRecords, integrationChanges, credentialProxyStates, credentialRunGrants, credentialRunEnrollments, linearWorkspaces, githubInstallations, githubSetupStates, projects, jobs, vmImageVerifications, skills, jobSkills, triggers, webhookConditionsCutover, invocations, manualInvocationPermissions, jobRuns, runCredentials, runs, runArtifacts, + connections, integrationCollections, integrationRecords, integrationChanges, credentialProxyStates, credentialRunGrants, credentialRunEnrollments, linearWorkspaces, githubInstallations, githubSetupStates, projects, jobs, vmImageVerifications, skills, jobSkills, triggers, webhookConditionsCutover, invocations, manualInvocationPermissions, jobRuns, runCredentials, runs, runArtifacts, runRetentionCleanup, runTraceEvents, runActivity, activeClaims, scheduleState, automaticWakes, lifecycleDeliveries, lifecycleEvents, jobEditDeliveries, jobEvents, webhookDeliveries, webhookDeliveryEvents, pendingVerifications, tailFingerprints, wakeHints, runTraceCursors, runTraceChunks, runTraceProjections, traceReplayLimits, traceReplayOperations, diff --git a/packages/api/test/run-retention-repository.test.ts b/packages/api/test/run-retention-repository.test.ts new file mode 100644 index 0000000..b81952f --- /dev/null +++ b/packages/api/test/run-retention-repository.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it, vi } from "vitest"; +import { RunRetentionRepository } from "../src/postgres/run-retention-repository"; + +describe("RunRetentionRepository", () => { + it("bounds selection and rechecks eligibility under aggregate locks", async () => { + const cutoff = new Date("2026-10-28T12:00:00.000Z"); + const statements: string[] = []; + const poolQuery = vi.fn(async (sql: string) => { + statements.push(sql); + return { rows: [{ tenant_id: "tenant", invocation_id: "invocation" }], rowCount: 1 }; + }); + const txQuery = vi.fn(async (sql: string) => { + statements.push(sql); + if (sql.includes("SELECT r.id run_id")) return { rows: [{ run_id: "run" }], rowCount: 1 }; + if (sql.includes("SELECT object_key")) return { rows: [{ object_key: "tenant/run/trace.jsonl" }], rowCount: 1 }; + return { rows: [], rowCount: 1 }; + }); + const database = { pool: { query: poolQuery }, transaction: (work: (client: any) => unknown) => work({ query: txQuery }) } as any; + + await expect(new RunRetentionRepository(database).sweep(cutoff, 8)).resolves.toEqual({ selected: 1, deleted: 1, cleanupQueued: 1 }); + expect(poolQuery).toHaveBeenCalledWith(expect.stringContaining("LIMIT $2"), [cutoff, 8]); + expect(poolQuery.mock.calls[0]![0]).toContain("r.created_at < $1"); + expect(poolQuery.mock.calls[0]![0]).toContain("jr.state=r.state"); + expect(poolQuery.mock.calls[0]![0]).toContain("NOT EXISTS"); + expect(statements.findIndex(sql => sql.includes("INSERT INTO app.run_retention_cleanup"))).toBeLessThan(statements.findIndex(sql => sql.includes("DELETE FROM app.invocations"))); + expect(txQuery.mock.calls.find(([sql]) => sql.includes("SELECT r.id run_id"))![0]).toContain("FOR UPDATE OF i,jr,r SKIP LOCKED"); + }); + + it("rejects invalid cutoffs and unbounded batch sizes before database access", async () => { + const query = vi.fn(); + const repository = new RunRetentionRepository({ pool: { query }, transaction: vi.fn() } as any); + await expect(repository.sweep(new Date(Number.NaN))).rejects.toThrow(TypeError); + await expect(repository.sweep(new Date(), 501)).rejects.toThrow(RangeError); + expect(query).not.toHaveBeenCalled(); + }); +}); From 636d5caed69fe779ebd5ff3e6f14c2b79b44bf93 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 22:10:42 +0000 Subject: [PATCH 03/26] GEN-2907 retry retained artifact cleanup --- packages/api/src/alarm-coordinator.ts | 8 +++- .../src/postgres/run-retention-repository.ts | 46 +++++++++++++++++++ packages/api/src/postgres/run-scheduler.ts | 5 ++ .../api/test/run-retention-repository.test.ts | 21 +++++++++ 4 files changed, 78 insertions(+), 2 deletions(-) diff --git a/packages/api/src/alarm-coordinator.ts b/packages/api/src/alarm-coordinator.ts index 6fd22fe..46d7ea9 100644 --- a/packages/api/src/alarm-coordinator.ts +++ b/packages/api/src/alarm-coordinator.ts @@ -6,6 +6,7 @@ import type { Env } from "./types"; /** The sole remaining Durable Object responsibility: reliable wakeups. */ export class AlarmCoordinator extends DurableObject { + private static readonly MAINTENANCE_INTERVAL_MS = 15 * 60_000; async fetch(request: Request): Promise { if (request.method !== "POST" || new URL(request.url).pathname !== "/wake") return new Response("Not found", { status: 404 }); await this.schedule(); return Response.json({ scheduled: true }); @@ -13,8 +14,11 @@ export class AlarmCoordinator extends DurableObject { async alarm(): Promise { try { await new RunScheduler(databaseFor(this.env), this.env).process(); } finally { await this.schedule(); } } private async schedule() { const next = await new RunRepository(databaseFor(this.env)).nextWakeAt(), current = await this.ctx.storage.getAlarm(); - if (!next) return; - const desired = Math.max(Date.now() + 100, next.getTime()); + // Keep a maintenance heartbeat even with no queued runs or schedules. + // Scheduler wakes still take precedence when they occur sooner. + const now = Date.now(); + const maintenance = now + AlarmCoordinator.MAINTENANCE_INTERVAL_MS; + const desired = Math.max(now + 100, Math.min(next?.getTime() ?? maintenance, maintenance)); if (current === null || current > desired) await this.ctx.storage.setAlarm(desired); } } diff --git a/packages/api/src/postgres/run-retention-repository.ts b/packages/api/src/postgres/run-retention-repository.ts index b1090bd..cca0a57 100644 --- a/packages/api/src/postgres/run-retention-repository.ts +++ b/packages/api/src/postgres/run-retention-repository.ts @@ -1,6 +1,12 @@ import { Database } from "./database"; +import { safeDiagnosticText } from "../harness-diagnostics"; export type RetentionSweepResult = { selected: number; deleted: number; cleanupQueued: number }; +export type ArtifactCleanupResult = { attempted: number; completed: number; retried: number }; + +const CLEANUP_BATCH_SIZE = 20; +const CLEANUP_LEASE_MS = 10 * 60_000; +const CLEANUP_MAX_BACKOFF_MS = 6 * 60 * 60_000; /** * Bounded retention deletion. The fixed cutoff is supplied by the caller so a @@ -11,6 +17,46 @@ export type RetentionSweepResult = { selected: number; deleted: number; cleanupQ export class RunRetentionRepository { constructor(private readonly database: Database) {} + /** + * Delete retained R2 objects outside the run-deletion transaction. A short + * lease prevents concurrent alarms from doing duplicate work; if an alarm + * stops after claiming an item, it becomes eligible again when the lease + * expires. Successful receipts are removed so the queue stays bounded. + */ + async processArtifactCleanup(bucket: Pick | undefined, now = new Date()): Promise { + if (!bucket) return { attempted: 0, completed: 0, retried: 0 }; + let attempted = 0; + let completed = 0; + let retried = 0; + for (let count = 0; count < CLEANUP_BATCH_SIZE; count++) { + const item = await this.database.transaction(async client => { + const claimed = await client.query<{ id: string; object_keys: string[]; attempts: number }>(` + SELECT id,object_keys,attempts FROM app.run_retention_cleanup + WHERE completed_at IS NULL AND next_attempt_at <= $1 + ORDER BY next_attempt_at,id LIMIT 1 FOR UPDATE SKIP LOCKED`, [now]); + const row = claimed.rows[0]; + if (!row) return null; + await client.query(`UPDATE app.run_retention_cleanup + SET attempts=attempts+1,next_attempt_at=$2 WHERE id=$1`, [row.id, new Date(now.getTime() + CLEANUP_LEASE_MS)]); + return { ...row, attempts: row.attempts + 1 }; + }); + if (!item) break; + attempted++; + try { + const keys = Array.isArray(item.object_keys) ? item.object_keys : []; + if (keys.length) await bucket.delete(keys); + await this.database.pool.query("DELETE FROM app.run_retention_cleanup WHERE id=$1", [item.id]); + completed++; + } catch (error) { + const delay = Math.min(CLEANUP_MAX_BACKOFF_MS, 60_000 * 2 ** Math.min(item.attempts - 1, 8)); + await this.database.pool.query(`UPDATE app.run_retention_cleanup + SET next_attempt_at=$2,last_error=$3 WHERE id=$1`, [item.id, new Date(now.getTime() + delay), safeDiagnosticText(error instanceof Error ? error.message : "R2 cleanup failed")]); + retried++; + } + } + return { attempted, completed, retried }; + } + async sweep(cutoff: Date, limit = 50): Promise { if (!Number.isFinite(cutoff.getTime())) throw new TypeError("Retention cutoff must be a valid date"); if (!Number.isInteger(limit) || limit < 1 || limit > 500) throw new RangeError("Retention batch size must be between 1 and 500"); diff --git a/packages/api/src/postgres/run-scheduler.ts b/packages/api/src/postgres/run-scheduler.ts index 7414b55..c15ed45 100644 --- a/packages/api/src/postgres/run-scheduler.ts +++ b/packages/api/src/postgres/run-scheduler.ts @@ -12,6 +12,8 @@ import type { Env } from "../types"; import { ConnectionRepository } from "./connection-repository"; import type { Database } from "./database"; import { RunRepository, type PersistedRun } from "./run-repository"; +import { RunRetentionRepository } from "./run-retention-repository"; +import { runRetentionCutoff } from "./run-retention"; import { AutomationScheduler } from "./automation-scheduler"; import { agentDriver, driverTraceSources, type AgentKind } from "../agent-driver"; import { traceMetric } from "../trace-observability"; @@ -30,6 +32,9 @@ export class RunScheduler { await new AutomationScheduler(this.database, this.env).process(); for (const run of await this.runs.dueForPoll()) await this.poll(run).catch(error => this.fail(run, error, "poll")); for (let count = 0; count < 20; count++) { const run = await this.runs.claimNext(); if (!run) break; await this.launch(run).catch(error => this.fail(run, error, "launch")); } + const retention = new RunRetentionRepository(this.database); + await retention.sweep(runRetentionCutoff(new Date()), 50); + await retention.processArtifactCleanup(this.env.RUN_ARTIFACTS); } private connections(run: PersistedRun) { return new ConnectionRepository(this.database, run.tenantId, this.env.CREDENTIAL_ENCRYPTION_KEY); } diff --git a/packages/api/test/run-retention-repository.test.ts b/packages/api/test/run-retention-repository.test.ts index b81952f..29c8f6a 100644 --- a/packages/api/test/run-retention-repository.test.ts +++ b/packages/api/test/run-retention-repository.test.ts @@ -33,4 +33,25 @@ describe("RunRetentionRepository", () => { await expect(repository.sweep(new Date(), 501)).rejects.toThrow(RangeError); expect(query).not.toHaveBeenCalled(); }); + + it("removes R2 cleanup receipts after deletion and backs failures off", async () => { + const now = new Date("2026-11-11T12:00:00.000Z"); + const bucketDelete = vi.fn().mockRejectedValueOnce(new Error("provider unavailable")).mockResolvedValueOnce(undefined); + let claim: { id: string; object_keys: string[]; attempts: number } | undefined = { id: "cleanup", object_keys: ["tenant/run/object"], attempts: 0 }; + const txQuery = vi.fn(async (sql: string) => { + if (sql.includes("SELECT id,object_keys")) { const row = claim; claim = undefined; return { rows: row ? [row] : [], rowCount: row ? 1 : 0 }; } + return { rows: [], rowCount: 1 }; + }); + const poolQuery = vi.fn(async () => ({ rows: [], rowCount: 1 })); + const database = { pool: { query: poolQuery }, transaction: (work: (client: any) => unknown) => work({ query: txQuery }) } as any; + const repository = new RunRetentionRepository(database); + + await expect(repository.processArtifactCleanup({ delete: bucketDelete } as any, now)).resolves.toEqual({ attempted: 1, completed: 0, retried: 1 }); + expect(poolQuery).toHaveBeenCalledWith(expect.stringContaining("last_error=$3"), ["cleanup", new Date(now.getTime() + 60_000), "provider unavailable"]); + + claim = { id: "cleanup", object_keys: ["tenant/run/object"], attempts: 1 }; + await expect(repository.processArtifactCleanup({ delete: bucketDelete } as any, now)).resolves.toEqual({ attempted: 1, completed: 1, retried: 0 }); + expect(bucketDelete).toHaveBeenCalledWith(["tenant/run/object"]); + expect(poolQuery).toHaveBeenLastCalledWith("DELETE FROM app.run_retention_cleanup WHERE id=$1", ["cleanup"]); + }); }); From 90568468c78dfe9735a63f81a43405078014fecf Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 22:28:22 +0000 Subject: [PATCH 04/26] GEN-2907 reconcile historical provider cleanup --- docs/run-history-retention.md | 21 +++++++++++ .../0038_provider_cleanup_reconciliation.sql | 11 ++++++ packages/api/src/amp-backend.ts | 2 ++ packages/api/src/postgres/run-repository.ts | 35 +++++++++++++++++++ packages/api/src/postgres/run-scheduler.ts | 25 +++++++++++++ packages/api/src/postgres/schema.ts | 2 +- .../run-provider-cleanup-repository.test.ts | 26 ++++++++++++++ 7 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 docs/run-history-retention.md create mode 100644 packages/api/migrations/0038_provider_cleanup_reconciliation.sql create mode 100644 packages/api/test/run-provider-cleanup-repository.test.ts diff --git a/docs/run-history-retention.md b/docs/run-history-retention.md new file mode 100644 index 0000000..e0af929 --- /dev/null +++ b/docs/run-history-retention.md @@ -0,0 +1,21 @@ +# Run history retention + +Factorize uses a global 14-day history window measured from immutable run creation time. A run is eligible only after it is terminal, its execution-provider cleanup is confirmed, artifact finalization is no longer pending, and required lifecycle automation has been handed off. Nonterminal runs remain protected regardless of age. Finishing an overdue run does not give it a new retention window. + +## Cleanup processing + +The scheduler alarm runs bounded retention and artifact cleanup work in addition to ordinary run scheduling. The alarm coordinator keeps a maintenance heartbeat no more than 15 minutes away, even when there are no queued runs or schedules. Each retention batch uses one cutoff, limits candidates, and rechecks state, age, cleanup ownership and pending lifecycle deliveries under aggregate locks before deleting an invocation. + +Artifact object keys are copied into `app.run_retention_cleanup` in the same database transaction that removes the invocation aggregate. The queue has no foreign key to the run, job or tenant. Workers lease one receipt at a time, delete a bounded batch from R2, and remove each receipt after success. Failures retain the receipt with capped exponential backoff; an expired lease makes interrupted work eligible again. + +Terminal runs whose execution-provider cleanup is incomplete use a separate bounded reconciliation lane. Workers claim rows with `FOR UPDATE SKIP LOCKED`, store a ten-minute lease, and ask the recorded backend to stop the recorded handle. A confirmed stop or provider-confirmed absence completes cleanup; failures retain the run and retry with capped exponential backoff. A terminal row with no handle and no provider start timestamp is explicitly treated as never launched. A started row with no handle cannot prove provider absence, so it retains an error and remains ineligible for retention until reconciled. Since retention requires provider cleanup completion, unresolved handles cannot be purged with their authoritative records. + +Successful cleanup removes its retry state from the run. Failed work retains only a bounded diagnostic string and retry timestamp; operator metrics and backlog dashboards remain follow-up work for this rollout. + +## Rollout and recovery + +The retention sweep is intentionally not enabled by a schema migration and no data purge runs during migration. Before enabling or increasing destructive work, refresh the read-only inventory and dry-run eligible, active/exempt, cleanup-blocked, pending-automation and orphan-artifact cohorts. Deploy additive schema and cleanup code first. Use bounded batches and an operational pause switch; retain the pause until provider reconciliation, upload/trace race tests, independent-history cleanup and backlog observability are ready. + +Resume interrupted cleanup by allowing the next alarm to reclaim expired leases. Do not clear a provider handle or cleanup retry record after an error. A provider-confirmed absent resource is idempotently complete. Recheck ownership before deleting any artifact classified as orphaned, and use a grace period so in-flight uploads can commit receipts. Report before/after counts, oldest backlog age, retries, provider results, and pending external deletions. Database deletion is atomic per run, but PostgreSQL space reuse and external object deletion are asynchronous. + +Disabling retention stops future deletion; it does not restore deleted history or objects. Archived branches and backups require separate lifecycle decisions. Never use this workflow to delete the archived validation branch or run a production purge as part of an implementation checkpoint. diff --git a/packages/api/migrations/0038_provider_cleanup_reconciliation.sql b/packages/api/migrations/0038_provider_cleanup_reconciliation.sql new file mode 100644 index 0000000..301c86b --- /dev/null +++ b/packages/api/migrations/0038_provider_cleanup_reconciliation.sql @@ -0,0 +1,11 @@ +-- Historical terminal cleanup is independent from the ordinary run poller. +ALTER TABLE app.runs + ADD COLUMN provider_cleanup_attempt integer NOT NULL DEFAULT 0, + ADD COLUMN provider_cleanup_next_at timestamptz, + ADD COLUMN provider_cleanup_error text; + +CREATE INDEX runs_provider_cleanup_due + ON app.runs(provider_cleanup_next_at, created_at, tenant_id, id) + WHERE NOT vm_cleanup_complete AND state IN ('succeeded','failed','stopped'); + +INSERT INTO app.schema_migrations(version) VALUES ('0038_provider_cleanup_reconciliation') ON CONFLICT DO NOTHING; diff --git a/packages/api/src/amp-backend.ts b/packages/api/src/amp-backend.ts index 3470b42..70d9392 100644 --- a/packages/api/src/amp-backend.ts +++ b/packages/api/src/amp-backend.ts @@ -45,6 +45,8 @@ export class AmpBackend implements ExecutionBackend { } async stop(handle: RunHandle): Promise { const result = await this.call(`/runs/${encodeURIComponent(handle.id)}/stop`, { method: "POST", body: "{}" }); + // Reconciliation is retryable and provider-owned. A missing thread is already cleaned up. + if (result.command.status === 404) return { state: "stopped", detail: "Amp execution is already absent", command: result.command }; return { state: result.command.ok ? mapAmpState(result.data.status ?? "stopped") : "running", detail: result.data.detail ?? result.command.body, command: result.command }; } } diff --git a/packages/api/src/postgres/run-repository.ts b/packages/api/src/postgres/run-repository.ts index e881a0f..7762cce 100644 --- a/packages/api/src/postgres/run-repository.ts +++ b/packages/api/src/postgres/run-repository.ts @@ -55,6 +55,41 @@ export class RunRepository { return result.rows.map(mapped); } + /** Claim a bounded batch of terminal records whose provider cleanup is incomplete. */ + async claimProviderCleanup(limit = 20): Promise; executionHandle: RunHandle | null; backendKind: string; startedAt: Date | null; attempt: number }>> { + return this.database.transaction(async client => { + const result = await client.query<{ tenant_id: string; id: string; execution_config: Record; execution_handle: RunHandle | null; execution_backend_kind: string; provider_cleanup_attempt: number; started_at: Date | null }>(` + WITH candidates AS ( + SELECT x.tenant_id,x.id FROM app.runs x + JOIN app.job_runs jr ON jr.tenant_id=x.tenant_id AND jr.id=x.id + WHERE x.state IN ('succeeded','failed','stopped') AND jr.state=x.state + AND NOT x.vm_cleanup_complete + AND (x.provider_cleanup_next_at IS NULL OR x.provider_cleanup_next_at <= now()) + ORDER BY x.created_at,x.tenant_id,x.id + LIMIT $1 FOR UPDATE OF x SKIP LOCKED + ) + UPDATE app.runs x + SET provider_cleanup_attempt=x.provider_cleanup_attempt+1, + provider_cleanup_next_at=now()+interval '10 minutes',updated_at=now() + FROM candidates c WHERE x.tenant_id=c.tenant_id AND x.id=c.id + RETURNING x.tenant_id,x.id,x.execution_config,x.execution_handle,x.execution_backend_kind,x.provider_cleanup_attempt, + (SELECT jr.started_at FROM app.job_runs jr WHERE jr.tenant_id=x.tenant_id AND jr.id=x.id) started_at`, [limit]); + return result.rows.map(row => ({ tenantId: row.tenant_id, id: row.id, executionConfig: row.execution_config, executionHandle: row.execution_handle, backendKind: row.execution_backend_kind, startedAt: row.started_at, attempt: row.provider_cleanup_attempt })); + }); + } + + async completeProviderCleanup(tenantId: string, runId: string): Promise { + await this.database.pool.query(`UPDATE app.runs SET vm_cleanup_complete=true,provider_cleanup_next_at=NULL,provider_cleanup_error=NULL,updated_at=now() + WHERE tenant_id=$1 AND id=$2 AND state IN ('succeeded','failed','stopped')`, [tenantId, runId]); + } + + async retryProviderCleanup(tenantId: string, runId: string, attempt: number, error: unknown): Promise { + const seconds = Math.min(6 * 60 * 60, 60 * 2 ** Math.min(Math.max(attempt - 1, 0), 8)); + await this.database.pool.query(`UPDATE app.runs SET provider_cleanup_next_at=now()+make_interval(secs=>$3),provider_cleanup_error=$4,updated_at=now() + WHERE tenant_id=$1 AND id=$2 AND state IN ('succeeded','failed','stopped') AND NOT vm_cleanup_complete`, + [tenantId, runId, seconds, safeDiagnosticText(error instanceof Error ? error.message : "Execution provider cleanup failed")]); + } + /** Persist cleanup ownership before provisioning any external resources. */ async recordLaunchHandle(run: PersistedRun, handle: RunHandle): Promise { const result = await this.database.pool.query(`UPDATE app.runs x SET execution_handle=$3,updated_at=now() diff --git a/packages/api/src/postgres/run-scheduler.ts b/packages/api/src/postgres/run-scheduler.ts index c15ed45..d89143f 100644 --- a/packages/api/src/postgres/run-scheduler.ts +++ b/packages/api/src/postgres/run-scheduler.ts @@ -33,10 +33,35 @@ export class RunScheduler { for (const run of await this.runs.dueForPoll()) await this.poll(run).catch(error => this.fail(run, error, "poll")); for (let count = 0; count < 20; count++) { const run = await this.runs.claimNext(); if (!run) break; await this.launch(run).catch(error => this.fail(run, error, "launch")); } const retention = new RunRetentionRepository(this.database); + await this.reconcileProviderCleanup(); await retention.sweep(runRetentionCutoff(new Date()), 50); await retention.processArtifactCleanup(this.env.RUN_ARTIFACTS); } + /** Retry terminal provider cleanup independently from run polling, including old stuck records. */ + private async reconcileProviderCleanup(): Promise { + for (const item of await this.runs.claimProviderCleanup()) { + if (!item.executionHandle) { + if (item.startedAt) { + // A started run without its provider handle cannot be proven absent; retain it for reconciliation. + await this.runs.retryProviderCleanup(item.tenantId, item.id, item.attempt, new Error("Started run has no persisted execution handle; provider absence cannot be verified")); + } else { + // A terminal run with no start timestamp never reached provider launch. + await this.runs.completeProviderCleanup(item.tenantId, item.id); + } + continue; + } + try { + const run = { tenantId: item.tenantId, id: item.id, executionConfig: item.executionConfig, executionHandle: item.executionHandle, backendKind: item.backendKind } as PersistedRun; + const result = await (await this.backend(run)).stop(item.executionHandle); + if (result.state !== "stopped") throw new Error(result.detail ?? "Execution provider did not confirm cleanup"); + await this.runs.completeProviderCleanup(item.tenantId, item.id); + } catch (error) { + await this.runs.retryProviderCleanup(item.tenantId, item.id, item.attempt, error); + } + } + } + private connections(run: PersistedRun) { return new ConnectionRepository(this.database, run.tenantId, this.env.CREDENTIAL_ENCRYPTION_KEY); } private hasCapability(backend: ExecutionBackend, method: keyof T): backend is ExecutionBackend & T { diff --git a/packages/api/src/postgres/schema.ts b/packages/api/src/postgres/schema.ts index bd1dd72..85fcf5b 100644 --- a/packages/api/src/postgres/schema.ts +++ b/packages/api/src/postgres/schema.ts @@ -91,7 +91,7 @@ export const jobRuns = app.table("job_runs", { export const runs = app.table("runs", { executionConfig: jsonb("execution_config").$type().notNull(), - tenantId: uuid("tenant_id").notNull(), id: uuid("id").notNull(), jobId: uuid("job_id").notNull(), invocationId: uuid("invocation_id").notNull(), provider: text("provider").notNull(), issueId: text("issue_id").notNull(), issueUrl: text("issue_url"), issueTitle: text("issue_title").notNull().default(""), runName: text("run_name").notNull().default(""), agentName: text("agent_name").notNull(), workspaceName: text("workspace_name").notNull().default(""), agentKind: text("agent_kind").notNull().default(""), state: text("state").notNull(), executionBackendKind: text("execution_backend_kind").notNull().default(""), executionHandle: jsonb("execution_handle").$type>(), executionCapabilities: jsonb("execution_capabilities").$type().notNull().default([]), destinationUrl: text("destination_url"), artifactState: text("artifact_state").$type().notNull().default("pending"), artifactError: text("artifact_error"), failureCode: text("failure_code"), failureReason: text("failure_reason"), resultProtocolVersion: integer("result_protocol_version").notNull().default(0), jobResult: jsonb("job_result").$type(), failureSource: text("failure_source").$type<"system" | "agent">(), agentErrorReason: text("agent_error_reason"), agentErrorReportedAt: timestamp("agent_error_reported_at", { withTimezone: true }), agentErrorReporterUserId: uuid("agent_error_reporter_user_id"), agentErrorReporterKind: text("agent_error_reporter_kind"), agentErrorReporterId: text("agent_error_reporter_id"), traceSources: jsonb("trace_sources").$type(), executionDiagnostics: jsonb("execution_diagnostics").$type(), claimReleased: boolean("claim_released").notNull().default(false), vmCleanupAttempt: integer("vm_cleanup_attempt").notNull().default(0), cleanupNextAt: timestamp("cleanup_next_at", { withTimezone: true }), vmCleanupComplete: boolean("vm_cleanup_complete").notNull().default(false), createdAt: createdAt(), updatedAt: updatedAt(), + tenantId: uuid("tenant_id").notNull(), id: uuid("id").notNull(), jobId: uuid("job_id").notNull(), invocationId: uuid("invocation_id").notNull(), provider: text("provider").notNull(), issueId: text("issue_id").notNull(), issueUrl: text("issue_url"), issueTitle: text("issue_title").notNull().default(""), runName: text("run_name").notNull().default(""), agentName: text("agent_name").notNull(), workspaceName: text("workspace_name").notNull().default(""), agentKind: text("agent_kind").notNull().default(""), state: text("state").notNull(), executionBackendKind: text("execution_backend_kind").notNull().default(""), executionHandle: jsonb("execution_handle").$type>(), executionCapabilities: jsonb("execution_capabilities").$type().notNull().default([]), destinationUrl: text("destination_url"), artifactState: text("artifact_state").$type().notNull().default("pending"), artifactError: text("artifact_error"), failureCode: text("failure_code"), failureReason: text("failure_reason"), resultProtocolVersion: integer("result_protocol_version").notNull().default(0), jobResult: jsonb("job_result").$type(), failureSource: text("failure_source").$type<"system" | "agent">(), agentErrorReason: text("agent_error_reason"), agentErrorReportedAt: timestamp("agent_error_reported_at", { withTimezone: true }), agentErrorReporterUserId: uuid("agent_error_reporter_user_id"), agentErrorReporterKind: text("agent_error_reporter_kind"), agentErrorReporterId: text("agent_error_reporter_id"), traceSources: jsonb("trace_sources").$type(), executionDiagnostics: jsonb("execution_diagnostics").$type(), claimReleased: boolean("claim_released").notNull().default(false), vmCleanupAttempt: integer("vm_cleanup_attempt").notNull().default(0), cleanupNextAt: timestamp("cleanup_next_at", { withTimezone: true }), vmCleanupComplete: boolean("vm_cleanup_complete").notNull().default(false), providerCleanupAttempt: integer("provider_cleanup_attempt").notNull().default(0), providerCleanupNextAt: timestamp("provider_cleanup_next_at", { withTimezone: true }), providerCleanupError: text("provider_cleanup_error"), createdAt: createdAt(), updatedAt: updatedAt(), }, t => [primaryKey({ columns: [t.tenantId, t.id] }), unique().on(t.tenantId, t.invocationId), foreignKey({ columns: [t.tenantId, t.jobId], foreignColumns: [jobs.tenantId, jobs.id] }).onDelete("cascade"), foreignKey({ columns: [t.tenantId, t.invocationId], foreignColumns: [invocations.tenantId, invocations.id] }).onDelete("cascade"), index("runs_order").on(t.tenantId, t.createdAt.desc(), t.id.desc()), index("runs_state_order").on(t.tenantId, t.state, t.createdAt.desc(), t.id.desc()), index("runs_priority_order").on(t.tenantId, sql`CASE WHEN ${t.state} IN ('queued','reserved','starting') THEN 0 WHEN ${t.state}='running' THEN 1 ELSE 2 END`, t.createdAt.desc(), t.id.desc()), index("runs_priority_job_order").on(t.tenantId, t.jobId, sql`CASE WHEN ${t.state} IN ('queued','reserved','starting') THEN 0 WHEN ${t.state}='running' THEN 1 ELSE 2 END`, t.createdAt.desc(), t.id.desc()), index("runs_cleanup_due").on(t.cleanupNextAt).where(sql`not ${t.vmCleanupComplete}`), index("runs_issue_trgm").using("gin", sql`${t.issueId} gin_trgm_ops`), index("runs_name_trgm").using("gin", sql`${t.runName} gin_trgm_ops`)]); export const runArtifacts = app.table("run_artifacts", { tenantId: uuid("tenant_id").notNull(), runId: uuid("run_id").notNull(), id: uuid("id").notNull(), kind: text("kind").$type().notNull(), objectKey: text("object_key").notNull(), provider: text("provider").notNull(), format: text("format").notNull(), formatVersion: text("format_version"), cliVersion: text("cli_version"), nativeSessionId: text("native_session_id"), sourceGeneration: text("source_generation"), sourcePath: text("source_path"), mediaType: text("media_type"), harnessVersion: text("harness_version"), byteSize: bigint("byte_size", { mode: "number" }).notNull(), sha256: text("sha256").notNull(), state: text("state").$type().notNull(), createdAt: createdAt(), diff --git a/packages/api/test/run-provider-cleanup-repository.test.ts b/packages/api/test/run-provider-cleanup-repository.test.ts new file mode 100644 index 0000000..df9da8e --- /dev/null +++ b/packages/api/test/run-provider-cleanup-repository.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it, vi } from "vitest"; +import { RunRepository } from "../src/postgres/run-repository"; + +describe("terminal provider cleanup reconciliation", () => { + it("claims only terminal incomplete cleanup with a bounded lease", async () => { + const query = vi.fn(async (sql: string) => { + expect(sql).toContain("x.state IN ('succeeded','failed','stopped')"); + expect(sql).toContain("NOT x.vm_cleanup_complete"); + expect(sql).toContain("FOR UPDATE OF x SKIP LOCKED"); + expect(sql).toContain("provider_cleanup_next_at=now()+interval '10 minutes'"); + return { rows: [{ tenant_id: "tenant", id: "run", execution_config: { connectionId: "connection" }, execution_handle: { backendKind: "exe-vm", id: "factorize-run" }, execution_backend_kind: "exe-vm", provider_cleanup_attempt: 2, started_at: new Date("2026-10-01T00:00:00Z") }], rowCount: 1 }; + }); + const database = { transaction: (work: (client: any) => unknown) => work({ query }) } as any; + await expect(new RunRepository(database).claimProviderCleanup()).resolves.toEqual([{ tenantId: "tenant", id: "run", executionConfig: { connectionId: "connection" }, executionHandle: { backendKind: "exe-vm", id: "factorize-run" }, backendKind: "exe-vm", startedAt: new Date("2026-10-01T00:00:00Z"), attempt: 2 }]); + expect(query).toHaveBeenCalledWith(expect.any(String), [20]); + }); + + it("records cleanup success and retries failures with capped backoff", async () => { + const query = vi.fn().mockResolvedValue({ rows: [], rowCount: 1 }); + const repository = new RunRepository({ pool: { query } } as any); + await repository.completeProviderCleanup("tenant", "run"); + expect(query).toHaveBeenNthCalledWith(1, expect.stringContaining("vm_cleanup_complete=true"), ["tenant", "run"]); + await repository.retryProviderCleanup("tenant", "run", 9, new Error("provider unavailable")); + expect(query).toHaveBeenNthCalledWith(2, expect.stringContaining("provider_cleanup_next_at=now()+make_interval(secs=>$3)"), ["tenant", "run", 15_360, "provider unavailable"]); + }); +}); From 1c47ed65d5dd5094f31e711a0dff220ffa2f15af Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 22:43:08 +0000 Subject: [PATCH 05/26] GEN-2907 preserve provider cleanup on job deletion --- docs/run-history-retention.md | 2 ++ .../0039_deleted_job_provider_cleanup.sql | 19 +++++++++++++ packages/api/src/postgres/job-repository.ts | 15 ++++++++++- packages/api/src/postgres/run-repository.ts | 27 +++++++++++++++++++ packages/api/src/postgres/run-scheduler.ts | 18 +++++++++++++ .../run-provider-cleanup-repository.test.ts | 18 +++++++++++++ 6 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 packages/api/migrations/0039_deleted_job_provider_cleanup.sql diff --git a/docs/run-history-retention.md b/docs/run-history-retention.md index e0af929..01cea07 100644 --- a/docs/run-history-retention.md +++ b/docs/run-history-retention.md @@ -10,6 +10,8 @@ Artifact object keys are copied into `app.run_retention_cleanup` in the same dat Terminal runs whose execution-provider cleanup is incomplete use a separate bounded reconciliation lane. Workers claim rows with `FOR UPDATE SKIP LOCKED`, store a ten-minute lease, and ask the recorded backend to stop the recorded handle. A confirmed stop or provider-confirmed absence completes cleanup; failures retain the run and retry with capped exponential backoff. A terminal row with no handle and no provider start timestamp is explicitly treated as never launched. A started row with no handle cannot prove provider absence, so it retains an error and remains ineligible for retention until reconciled. Since retention requires provider cleanup completion, unresolved handles cannot be purged with their authoritative records. +Explicit job deletion also snapshots every incomplete run's provider configuration, handle, and start evidence into `app.deleted_job_provider_cleanup` in the same transaction as the job cascade. This independent queue lets the maintenance lane stop or reconcile provider resources after the source job and run rows are gone. A started run without a saved handle stays queued for retry and manual reconciliation; a never-started run can be removed from the queue safely. R2 artifacts for explicit job deletion continue to be deleted before the database cascade so a storage failure leaves the authoritative keys available for retry. + Successful cleanup removes its retry state from the run. Failed work retains only a bounded diagnostic string and retry timestamp; operator metrics and backlog dashboards remain follow-up work for this rollout. ## Rollout and recovery diff --git a/packages/api/migrations/0039_deleted_job_provider_cleanup.sql b/packages/api/migrations/0039_deleted_job_provider_cleanup.sql new file mode 100644 index 0000000..8145c21 --- /dev/null +++ b/packages/api/migrations/0039_deleted_job_provider_cleanup.sql @@ -0,0 +1,19 @@ +-- Provider ownership must survive the job/run cascades in explicit job deletion. +CREATE TABLE app.deleted_job_provider_cleanup ( + tenant_id uuid NOT NULL, + run_id uuid NOT NULL, + execution_config jsonb NOT NULL, + execution_backend_kind text NOT NULL, + execution_handle jsonb, + started_at timestamptz, + attempts integer NOT NULL DEFAULT 0, + next_attempt_at timestamptz NOT NULL DEFAULT now(), + lease_expires_at timestamptz, + last_error text, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (tenant_id, run_id) +); +CREATE INDEX deleted_job_provider_cleanup_due + ON app.deleted_job_provider_cleanup(next_attempt_at, tenant_id, run_id); + +INSERT INTO app.schema_migrations(version) VALUES ('0039_deleted_job_provider_cleanup') ON CONFLICT DO NOTHING; diff --git a/packages/api/src/postgres/job-repository.ts b/packages/api/src/postgres/job-repository.ts index e725afa..8365a32 100644 --- a/packages/api/src/postgres/job-repository.ts +++ b/packages/api/src/postgres/job-repository.ts @@ -151,7 +151,20 @@ export class PostgresJobRepository implements JobRepository { for (const [position, skill] of selected.entries()) await tx.execute(sql`INSERT INTO app.job_skills(tenant_id,job_id,skill_id,position) VALUES(${this.tenantId}::uuid,${job.id}::uuid,${skill.id}::uuid,${position})`); } - async delete(id: string): Promise { return Boolean((await this.database.orm.delete(jobs).where(and(eq(jobs.tenantId, this.tenantId), eq(jobs.id, id))).returning({ id: jobs.id })).length); } + async delete(id: string): Promise { + return this.database.transaction(async client => { + // Preserve provider cleanup ownership before job deletion cascades through + // runs. This durable queue has no FK to the job or tenant by design. + await client.query(`INSERT INTO app.deleted_job_provider_cleanup + (tenant_id,run_id,execution_config,execution_backend_kind,execution_handle,started_at) + SELECT x.tenant_id,x.id,x.execution_config,x.execution_backend_kind,x.execution_handle,jr.started_at + FROM app.runs x JOIN app.job_runs jr ON jr.tenant_id=x.tenant_id AND jr.id=x.id + WHERE x.tenant_id=$1 AND x.job_id=$2 AND NOT x.vm_cleanup_complete + ON CONFLICT (tenant_id,run_id) DO NOTHING`, [this.tenantId, id]); + const result = await client.query("DELETE FROM app.jobs WHERE tenant_id=$1 AND id=$2 RETURNING id", [this.tenantId, id]); + return result.rowCount === 1; + }); + } async findInvocation(jobId: string, claimKey: string) { const result = await this.database.pool.query(`SELECT i.id invocation_id,i.job_id,i.source,i.claim_key,i.trigger_id,i.context,i.occurrence,i.source_job_id,i.source_run_id,i.created_at invocation_created_at,r.id run_id,r.state run_state,r.encrypted_prompt,r.encrypted_skill_snapshot,r.run_name,r.created_at run_created_at,r.updated_at run_updated_at,r.started_at FROM app.invocations i JOIN app.job_runs r ON r.tenant_id=i.tenant_id AND r.invocation_id=i.id WHERE i.tenant_id=$1 AND i.job_id=$2 AND i.claim_key=$3`, [this.tenantId, jobId, claimKey]); diff --git a/packages/api/src/postgres/run-repository.ts b/packages/api/src/postgres/run-repository.ts index 7762cce..3e23cd7 100644 --- a/packages/api/src/postgres/run-repository.ts +++ b/packages/api/src/postgres/run-repository.ts @@ -90,6 +90,33 @@ export class RunRepository { [tenantId, runId, seconds, safeDiagnosticText(error instanceof Error ? error.message : "Execution provider cleanup failed")]); } + /** Claim provider cleanup whose source job/run has already been deleted. */ + async claimDeletedJobProviderCleanup(limit = 20): Promise; executionHandle: RunHandle | null; backendKind: string; startedAt: Date | null; attempt: number }>> { + return this.database.transaction(async client => { + const result = await client.query<{ tenant_id: string; run_id: string; execution_config: Record; execution_handle: RunHandle | null; execution_backend_kind: string; started_at: Date | null; attempts: number }>(` + WITH candidates AS ( + SELECT tenant_id,run_id FROM app.deleted_job_provider_cleanup + WHERE next_attempt_at <= now() AND (lease_expires_at IS NULL OR lease_expires_at <= now()) + ORDER BY next_attempt_at,tenant_id,run_id LIMIT $1 FOR UPDATE SKIP LOCKED + ) + UPDATE app.deleted_job_provider_cleanup q + SET attempts=q.attempts+1,lease_expires_at=now()+interval '10 minutes',next_attempt_at=now()+interval '10 minutes' + FROM candidates c WHERE q.tenant_id=c.tenant_id AND q.run_id=c.run_id + RETURNING q.tenant_id,q.run_id,q.execution_config,q.execution_handle,q.execution_backend_kind,q.started_at,q.attempts`, [limit]); + return result.rows.map(row => ({ tenantId: row.tenant_id, id: row.run_id, executionConfig: row.execution_config, executionHandle: row.execution_handle, backendKind: row.execution_backend_kind, startedAt: row.started_at, attempt: row.attempts })); + }); + } + + async completeDeletedJobProviderCleanup(tenantId: string, runId: string): Promise { + await this.database.pool.query("DELETE FROM app.deleted_job_provider_cleanup WHERE tenant_id=$1 AND run_id=$2", [tenantId, runId]); + } + + async retryDeletedJobProviderCleanup(tenantId: string, runId: string, attempt: number, error: unknown): Promise { + const seconds = Math.min(6 * 60 * 60, 60 * 2 ** Math.min(Math.max(attempt - 1, 0), 8)); + await this.database.pool.query(`UPDATE app.deleted_job_provider_cleanup SET next_attempt_at=now()+make_interval(secs=>$3),lease_expires_at=NULL,last_error=$4 + WHERE tenant_id=$1 AND run_id=$2`, [tenantId, runId, seconds, safeDiagnosticText(error instanceof Error ? error.message : "Execution provider cleanup failed")]); + } + /** Persist cleanup ownership before provisioning any external resources. */ async recordLaunchHandle(run: PersistedRun, handle: RunHandle): Promise { const result = await this.database.pool.query(`UPDATE app.runs x SET execution_handle=$3,updated_at=now() diff --git a/packages/api/src/postgres/run-scheduler.ts b/packages/api/src/postgres/run-scheduler.ts index d89143f..4147699 100644 --- a/packages/api/src/postgres/run-scheduler.ts +++ b/packages/api/src/postgres/run-scheduler.ts @@ -60,6 +60,24 @@ export class RunScheduler { await this.runs.retryProviderCleanup(item.tenantId, item.id, item.attempt, error); } } + for (const item of await this.runs.claimDeletedJobProviderCleanup()) { + if (!item.executionHandle) { + if (item.startedAt) { + await this.runs.retryDeletedJobProviderCleanup(item.tenantId, item.id, item.attempt, new Error("Deleted job run started without a persisted provider handle; provider absence cannot be verified")); + } else { + await this.runs.completeDeletedJobProviderCleanup(item.tenantId, item.id); + } + continue; + } + try { + const run = { tenantId: item.tenantId, id: item.id, executionConfig: item.executionConfig, executionHandle: item.executionHandle, backendKind: item.backendKind } as PersistedRun; + const result = await (await this.backend(run)).stop(item.executionHandle); + if (result.state !== "stopped") throw new Error(result.detail ?? "Execution provider did not confirm cleanup"); + await this.runs.completeDeletedJobProviderCleanup(item.tenantId, item.id); + } catch (error) { + await this.runs.retryDeletedJobProviderCleanup(item.tenantId, item.id, item.attempt, error); + } + } } private connections(run: PersistedRun) { return new ConnectionRepository(this.database, run.tenantId, this.env.CREDENTIAL_ENCRYPTION_KEY); } diff --git a/packages/api/test/run-provider-cleanup-repository.test.ts b/packages/api/test/run-provider-cleanup-repository.test.ts index df9da8e..4f47a78 100644 --- a/packages/api/test/run-provider-cleanup-repository.test.ts +++ b/packages/api/test/run-provider-cleanup-repository.test.ts @@ -23,4 +23,22 @@ describe("terminal provider cleanup reconciliation", () => { await repository.retryProviderCleanup("tenant", "run", 9, new Error("provider unavailable")); expect(query).toHaveBeenNthCalledWith(2, expect.stringContaining("provider_cleanup_next_at=now()+make_interval(secs=>$3)"), ["tenant", "run", 15_360, "provider unavailable"]); }); + + it("leases independent ownership after job deletion and keeps failed provider work retryable", async () => { + const query = vi.fn(async (sql: string) => { + expect(sql).toContain("app.deleted_job_provider_cleanup"); + expect(sql).toContain("FOR UPDATE SKIP LOCKED"); + return { rows: [{ tenant_id: "tenant", run_id: "deleted-run", execution_config: { connectionId: "connection" }, execution_handle: { backendKind: "exe-vm", id: "provider-run" }, execution_backend_kind: "exe-vm", started_at: new Date("2026-10-01T00:00:00Z"), attempts: 3 }], rowCount: 1 }; + }); + const database = { transaction: (work: (client: any) => unknown) => work({ query }) } as any; + const repository = new RunRepository(database); + await expect(repository.claimDeletedJobProviderCleanup()).resolves.toEqual([{ tenantId: "tenant", id: "deleted-run", executionConfig: { connectionId: "connection" }, executionHandle: { backendKind: "exe-vm", id: "provider-run" }, backendKind: "exe-vm", startedAt: new Date("2026-10-01T00:00:00Z"), attempt: 3 }]); + + const writes = vi.fn().mockResolvedValue({ rowCount: 1 }); + const writer = new RunRepository({ pool: { query: writes } } as any); + await writer.retryDeletedJobProviderCleanup("tenant", "deleted-run", 3, new Error("provider unavailable")); + expect(writes).toHaveBeenCalledWith(expect.stringContaining("lease_expires_at=NULL,last_error=$4"), ["tenant", "deleted-run", 240, "provider unavailable"]); + await writer.completeDeletedJobProviderCleanup("tenant", "deleted-run"); + expect(writes).toHaveBeenLastCalledWith(expect.stringContaining("DELETE FROM app.deleted_job_provider_cleanup"), ["tenant", "deleted-run"]); + }); }); From 4b676b980b9ccd1a0395ba06051e31954dd5cdbc Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 23:22:00 +0000 Subject: [PATCH 06/26] GEN-2907 test provider deletion ownership and report backlog --- docs/run-history-retention.md | 2 +- ...nup.sql => 0038_run_retention_cleanup.sql} | 2 +- ... 0039_provider_cleanup_reconciliation.sql} | 2 +- ... => 0040_deleted_job_provider_cleanup.sql} | 2 +- .../src/postgres/run-retention-repository.ts | 25 ++++++++++++++ packages/api/src/postgres/run-scheduler.ts | 13 +++++-- .../api/test/job-lifecycle.postgres.test.ts | 34 +++++++++++++++++++ .../api/test/run-retention-repository.test.ts | 18 ++++++++++ 8 files changed, 92 insertions(+), 6 deletions(-) rename packages/api/migrations/{0037_run_retention_cleanup.sql => 0038_run_retention_cleanup.sql} (95%) rename packages/api/migrations/{0038_provider_cleanup_reconciliation.sql => 0039_provider_cleanup_reconciliation.sql} (88%) rename packages/api/migrations/{0039_deleted_job_provider_cleanup.sql => 0040_deleted_job_provider_cleanup.sql} (91%) diff --git a/docs/run-history-retention.md b/docs/run-history-retention.md index 01cea07..1107904 100644 --- a/docs/run-history-retention.md +++ b/docs/run-history-retention.md @@ -12,7 +12,7 @@ Terminal runs whose execution-provider cleanup is incomplete use a separate boun Explicit job deletion also snapshots every incomplete run's provider configuration, handle, and start evidence into `app.deleted_job_provider_cleanup` in the same transaction as the job cascade. This independent queue lets the maintenance lane stop or reconcile provider resources after the source job and run rows are gone. A started run without a saved handle stays queued for retry and manual reconciliation; a never-started run can be removed from the queue safely. R2 artifacts for explicit job deletion continue to be deleted before the database cascade so a storage failure leaves the authoritative keys available for retry. -Successful cleanup removes its retry state from the run. Failed work retains only a bounded diagnostic string and retry timestamp; operator metrics and backlog dashboards remain follow-up work for this rollout. +Successful cleanup removes its retry state from the run. Failed work retains only a bounded diagnostic string and retry timestamp. Each maintenance alarm emits a structured `run_retention_maintenance` log with sweep and artifact-cleanup progress plus pending counts and oldest age for terminal-provider cleanup, deleted-job provider cleanup, artifact objects, and overdue nonterminal runs. The snapshot contains no tenant, run, or object identifiers. A failed snapshot is logged separately and does not fail maintenance. These logs provide rollout evidence; a retained time-series dashboard and independent-history cleanup remain follow-up work. ## Rollout and recovery diff --git a/packages/api/migrations/0037_run_retention_cleanup.sql b/packages/api/migrations/0038_run_retention_cleanup.sql similarity index 95% rename from packages/api/migrations/0037_run_retention_cleanup.sql rename to packages/api/migrations/0038_run_retention_cleanup.sql index d9813f9..a2dcaa3 100644 --- a/packages/api/migrations/0037_run_retention_cleanup.sql +++ b/packages/api/migrations/0038_run_retention_cleanup.sql @@ -25,4 +25,4 @@ CREATE INDEX lifecycle_deliveries_by_source_run ON app.lifecycle_deliveries(tena CREATE INDEX credential_run_enrollments_by_run ON app.credential_run_enrollments(tenant_id, run_id); CREATE INDEX webhook_delivery_events_by_delivery ON app.webhook_delivery_events(tenant_id, delivery_id); -INSERT INTO app.schema_migrations(version) VALUES ('0037_run_retention_cleanup') ON CONFLICT DO NOTHING; +INSERT INTO app.schema_migrations(version) VALUES ('0038_run_retention_cleanup') ON CONFLICT DO NOTHING; diff --git a/packages/api/migrations/0038_provider_cleanup_reconciliation.sql b/packages/api/migrations/0039_provider_cleanup_reconciliation.sql similarity index 88% rename from packages/api/migrations/0038_provider_cleanup_reconciliation.sql rename to packages/api/migrations/0039_provider_cleanup_reconciliation.sql index 301c86b..402dd2e 100644 --- a/packages/api/migrations/0038_provider_cleanup_reconciliation.sql +++ b/packages/api/migrations/0039_provider_cleanup_reconciliation.sql @@ -8,4 +8,4 @@ CREATE INDEX runs_provider_cleanup_due ON app.runs(provider_cleanup_next_at, created_at, tenant_id, id) WHERE NOT vm_cleanup_complete AND state IN ('succeeded','failed','stopped'); -INSERT INTO app.schema_migrations(version) VALUES ('0038_provider_cleanup_reconciliation') ON CONFLICT DO NOTHING; +INSERT INTO app.schema_migrations(version) VALUES ('0039_provider_cleanup_reconciliation') ON CONFLICT DO NOTHING; diff --git a/packages/api/migrations/0039_deleted_job_provider_cleanup.sql b/packages/api/migrations/0040_deleted_job_provider_cleanup.sql similarity index 91% rename from packages/api/migrations/0039_deleted_job_provider_cleanup.sql rename to packages/api/migrations/0040_deleted_job_provider_cleanup.sql index 8145c21..3ec0025 100644 --- a/packages/api/migrations/0039_deleted_job_provider_cleanup.sql +++ b/packages/api/migrations/0040_deleted_job_provider_cleanup.sql @@ -16,4 +16,4 @@ CREATE TABLE app.deleted_job_provider_cleanup ( CREATE INDEX deleted_job_provider_cleanup_due ON app.deleted_job_provider_cleanup(next_attempt_at, tenant_id, run_id); -INSERT INTO app.schema_migrations(version) VALUES ('0039_deleted_job_provider_cleanup') ON CONFLICT DO NOTHING; +INSERT INTO app.schema_migrations(version) VALUES ('0040_deleted_job_provider_cleanup') ON CONFLICT DO NOTHING; diff --git a/packages/api/src/postgres/run-retention-repository.ts b/packages/api/src/postgres/run-retention-repository.ts index cca0a57..c73e1bf 100644 --- a/packages/api/src/postgres/run-retention-repository.ts +++ b/packages/api/src/postgres/run-retention-repository.ts @@ -3,6 +3,7 @@ import { safeDiagnosticText } from "../harness-diagnostics"; export type RetentionSweepResult = { selected: number; deleted: number; cleanupQueued: number }; export type ArtifactCleanupResult = { attempted: number; completed: number; retried: number }; +export type RetentionBacklog = { queue: string; pending: number; oldestAgeSeconds: number | null }; const CLEANUP_BATCH_SIZE = 20; const CLEANUP_LEASE_MS = 10 * 60_000; @@ -17,6 +18,30 @@ const CLEANUP_MAX_BACKOFF_MS = 6 * 60 * 60_000; export class RunRetentionRepository { constructor(private readonly database: Database) {} + /** Aggregate cleanup pressure without emitting tenant, run, or object identifiers. */ + async backlogSnapshot(): Promise { + const result = await this.database.pool.query<{ queue: string; pending: number; oldest_age_seconds: number | null }>(` + SELECT queue,pending,oldest_age_seconds FROM ( + SELECT 'terminal_provider'::text queue,count(*)::int pending, + floor(extract(epoch FROM now()-min(created_at)))::int oldest_age_seconds + FROM app.runs WHERE state IN ('succeeded','failed','stopped') AND NOT vm_cleanup_complete + UNION ALL + SELECT 'deleted_job_provider'::text,count(*)::int, + floor(extract(epoch FROM now()-min(created_at)))::int + FROM app.deleted_job_provider_cleanup + UNION ALL + SELECT 'artifact_objects'::text,count(*)::int, + floor(extract(epoch FROM now()-min(created_at)))::int + FROM app.run_retention_cleanup WHERE completed_at IS NULL + UNION ALL + SELECT 'overdue_nonterminal'::text,count(*)::int, + floor(extract(epoch FROM now()-min(created_at)))::int + FROM app.runs WHERE state IN ('queued','reserved','starting','running','blocked','stopping') + AND created_at < now()-interval '14 days' + ) backlog ORDER BY queue`); + return result.rows.map(row => ({ queue: row.queue, pending: row.pending, oldestAgeSeconds: row.oldest_age_seconds })); + } + /** * Delete retained R2 objects outside the run-deletion transaction. A short * lease prevents concurrent alarms from doing duplicate work; if an alarm diff --git a/packages/api/src/postgres/run-scheduler.ts b/packages/api/src/postgres/run-scheduler.ts index 4147699..b8843a5 100644 --- a/packages/api/src/postgres/run-scheduler.ts +++ b/packages/api/src/postgres/run-scheduler.ts @@ -34,8 +34,17 @@ export class RunScheduler { for (let count = 0; count < 20; count++) { const run = await this.runs.claimNext(); if (!run) break; await this.launch(run).catch(error => this.fail(run, error, "launch")); } const retention = new RunRetentionRepository(this.database); await this.reconcileProviderCleanup(); - await retention.sweep(runRetentionCutoff(new Date()), 50); - await retention.processArtifactCleanup(this.env.RUN_ARTIFACTS); + const cutoff = runRetentionCutoff(new Date()); + const sweep = await retention.sweep(cutoff, 50); + const artifacts = await retention.processArtifactCleanup(this.env.RUN_ARTIFACTS); + // Aggregate observability is best effort: a metrics query failure must not + // turn a successful maintenance pass into a failed scheduler alarm. + try { + const backlog = await retention.backlogSnapshot(); + console.info(JSON.stringify({ event: "run_retention_maintenance", cutoff: cutoff.toISOString(), sweep, artifacts, backlog })); + } catch (error) { + console.warn(JSON.stringify({ event: "run_retention_backlog_snapshot_failed", message: error instanceof Error ? error.message : "unknown error" })); + } } /** Retry terminal provider cleanup independently from run polling, including old stuck records. */ diff --git a/packages/api/test/job-lifecycle.postgres.test.ts b/packages/api/test/job-lifecycle.postgres.test.ts index 6141c50..83f7feb 100644 --- a/packages/api/test/job-lifecycle.postgres.test.ts +++ b/packages/api/test/job-lifecycle.postgres.test.ts @@ -65,6 +65,9 @@ describe.skipIf(!process.env.AUTH_TEST_DATABASE_URL)("job lifecycle delivery (Po }); afterEach(async () => { vi.restoreAllMocks(); + // Provider cleanup ownership deliberately has no tenant FK, so this test + // database must remove its retained queue rows explicitly. + await database.pool.query("DELETE FROM app.deleted_job_provider_cleanup WHERE tenant_id=ANY($1::uuid[])", [tenants]); await database.pool.query("DELETE FROM app.tenants WHERE id=ANY($1::uuid[])", [tenants]); await runs.consumeWakeHint(); tenants.length = 0; @@ -183,6 +186,37 @@ describe.skipIf(!process.env.AUTH_TEST_DATABASE_URL)("job lifecycle delivery (Po expect(edges.rowCount).toBe(0); }); + it("commits provider cleanup ownership with job deletion and rolls both back on failure", async () => { + const a = await tenant(), source = await job(a); + const run = await start(a, source); + const handle = { backendKind: "exe-vm", id: `factorize-${run.id}` }; + await runs.recordLaunchHandle(run, handle); + await runs.markLaunched(run, handle, "https://vm.example", ["stop"]); + + const suffix = crypto.randomUUID().replaceAll("-", ""); + const functionName = `reject_job_delete_${suffix}`; + const triggerName = `reject_job_delete_${suffix}`; + await database.pool.query(`CREATE FUNCTION app.${functionName}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected job delete failure'; END $$`); + await database.pool.query(`CREATE TRIGGER ${triggerName} BEFORE DELETE ON app.jobs FOR EACH ROW EXECUTE FUNCTION app.${functionName}()`); + try { + await expect(repository(a).delete(source.id)).rejects.toThrow("injected job delete failure"); + expect((await database.pool.query("SELECT id FROM app.jobs WHERE tenant_id=$1 AND id=$2", [a, source.id])).rowCount).toBe(1); + expect((await database.pool.query("SELECT run_id FROM app.deleted_job_provider_cleanup WHERE tenant_id=$1 AND run_id=$2", [a, run.id])).rowCount).toBe(0); + } finally { + await database.pool.query(`DROP TRIGGER ${triggerName} ON app.jobs`); + await database.pool.query(`DROP FUNCTION app.${functionName}()`); + } + + expect(await repository(a).delete(source.id)).toBe(true); + expect((await database.pool.query("SELECT execution_config,execution_backend_kind,execution_handle,started_at FROM app.deleted_job_provider_cleanup WHERE tenant_id=$1 AND run_id=$2", [a, run.id])).rows[0]).toMatchObject({ + execution_config: expect.objectContaining({ connectionId: "test" }), + execution_backend_kind: "exe-vm", + execution_handle: handle, + started_at: expect.any(Date), + }); + expect((await database.pool.query("SELECT 1 FROM app.runs WHERE tenant_id=$1 AND id=$2", [a, run.id])).rowCount).toBe(0); + }); + it("stores and revokes an explicit self-invocation permission", async () => { const a = await tenant(), self = await job(a); await database.pool.query("INSERT INTO app.manual_invocation_permissions(tenant_id,source_job_id,destination_job_id) VALUES ($1,$2,$2)", [a, self.id]); diff --git a/packages/api/test/run-retention-repository.test.ts b/packages/api/test/run-retention-repository.test.ts index 29c8f6a..0e0447c 100644 --- a/packages/api/test/run-retention-repository.test.ts +++ b/packages/api/test/run-retention-repository.test.ts @@ -54,4 +54,22 @@ describe("RunRetentionRepository", () => { expect(bucketDelete).toHaveBeenCalledWith(["tenant/run/object"]); expect(poolQuery).toHaveBeenLastCalledWith("DELETE FROM app.run_retention_cleanup WHERE id=$1", ["cleanup"]); }); + + it("reports bounded backlog counts and oldest age without exposing row identifiers", async () => { + const query = vi.fn().mockResolvedValue({ rows: [ + { queue: "artifact_objects", pending: 2, oldest_age_seconds: 120 }, + { queue: "deleted_job_provider", pending: 0, oldest_age_seconds: null }, + { queue: "overdue_nonterminal", pending: 1, oldest_age_seconds: 90_000 }, + { queue: "terminal_provider", pending: 3, oldest_age_seconds: 600 }, + ] }); + const repository = new RunRetentionRepository({ pool: { query } } as any); + await expect(repository.backlogSnapshot()).resolves.toEqual([ + { queue: "artifact_objects", pending: 2, oldestAgeSeconds: 120 }, + { queue: "deleted_job_provider", pending: 0, oldestAgeSeconds: null }, + { queue: "overdue_nonterminal", pending: 1, oldestAgeSeconds: 90_000 }, + { queue: "terminal_provider", pending: 3, oldestAgeSeconds: 600 }, + ]); + expect(query.mock.calls[0]![0]).toContain("'overdue_nonterminal'"); + expect(query.mock.calls[0]![0]).not.toContain("run_id"); + }); }); From ca3e6d9f314fbba61e17b9bc00e277e36d98399e Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Wed, 11 Nov 2026 23:41:55 +0000 Subject: [PATCH 07/26] GEN-2907 queue artifacts rejected by retention races --- packages/api/src/artifact-upload.ts | 21 +++++++++++---- packages/api/test/artifact-upload.test.ts | 24 +++++++++++++++++ .../api/test/run-retention-repository.test.ts | 26 +++++++++++++++++++ 3 files changed, 66 insertions(+), 5 deletions(-) diff --git a/packages/api/src/artifact-upload.ts b/packages/api/src/artifact-upload.ts index aa4fdac..e7ea37c 100644 --- a/packages/api/src/artifact-upload.ts +++ b/packages/api/src/artifact-upload.ts @@ -86,22 +86,33 @@ export async function artifactUpload(request: Request, env: Env, token: string): const stored = await env.RUN_ARTIFACTS.get(key); if (!stored) return new Response("Artifact disappeared after upload", { status: 502 }); if (stored.size !== size) { await stored.body.cancel(); return new Response("Retained artifact size does not match upload receipt", { status: 422 }); } + const objectWasUnreceipted = Boolean(object && !existing.some(item => item.object_key === key)); const database = databaseFor(env); const conflict = await database.transaction(async client => { + // R2 writes cannot participate in the database transaction. If the run + // disappears or changes generation while the upload is in flight, retain + // independent cleanup ownership for the newly written object before + // returning the conflict. Retention may already have removed the run and + // its artifact receipt, so this queue intentionally has no run FK. + const queueUnrecordedObject = async () => { + if (!objectWasUnreceipted) return; + await client.query(`INSERT INTO app.run_retention_cleanup(id,tenant_id,run_id,object_keys,last_error) + VALUES ($1,$2,$3,$4::jsonb,'Artifact upload lost a race with run retention')`, [crypto.randomUUID(), grant.tenantId, grant.runId, JSON.stringify([key])]); + }; // Shares the lock with incremental ingestion: terminal projection wins atomically. const run = (await client.query("SELECT id,trace_sources FROM app.runs WHERE tenant_id=$1 AND id=$2 FOR UPDATE", [grant.tenantId, grant.runId])).rows[0]; - if (!run) return "Run not found"; + if (!run) { await queueUnrecordedObject(); return "Run not found"; } if (grant.artifactType !== "job_file" && (grant.sourceKind === "execution_stream" || grant.primary === true)) { const primary = run.trace_sources?.primary; - if (primary && (primary.kind !== (grant.sourceKind ?? "native_session") || (primary.provider && primary.provider !== grant.provider) || (primary.path && primary.path !== grant.sourcePath))) return "Terminal trace source mismatch"; + if (primary && (primary.kind !== (grant.sourceKind ?? "native_session") || (primary.provider && primary.provider !== grant.provider) || (primary.path && primary.path !== grant.sourcePath))) { await queueUnrecordedObject(); return "Terminal trace source mismatch"; } const cursor = (await client.query("SELECT generation FROM app.run_trace_cursors WHERE tenant_id=$1 AND run_id=$2", [grant.tenantId, grant.runId])).rows[0]; - if ((cursor?.generation ?? null) !== grant.traceGeneration) return "Stale terminal trace generation"; + if ((cursor?.generation ?? null) !== grant.traceGeneration) { await queueUnrecordedObject(); return "Stale terminal trace generation"; } } const artifacts = await repository.list(grant.runId, client); const terminal = artifacts.find(item => item.kind === "execution_stream" && item.state === "stored"); - if (grant.sourceKind === "execution_stream" && terminal && terminal.sha256 !== sha256) return "Terminal trace stream is immutable"; + if (grant.sourceKind === "execution_stream" && terminal && terminal.sha256 !== sha256) { await queueUnrecordedObject(); return "Terminal trace stream is immutable"; } const native = artifacts.find(item => item.kind === "native_session" && item.state === "stored"); - if (grant.artifactType !== "job_file" && grant.sourceKind !== "execution_stream" && native && native.sha256 !== sha256) return "Terminal native session is immutable"; + if (grant.artifactType !== "job_file" && grant.sourceKind !== "execution_stream" && native && native.sha256 !== sha256) { await queueUnrecordedObject(); return "Terminal native session is immutable"; } const primary = grant.artifactType !== "job_file" && (grant.sourceKind === "execution_stream" || (grant.primary !== false && !terminal && run?.trace_sources?.primary?.kind !== "execution_stream")); const projection = (await client.query("SELECT source_kind,artifact_sha256,parser_version FROM app.run_trace_projections WHERE tenant_id=$1 AND run_id=$2", [grant.tenantId, grant.runId])).rows[0]; await record(client); diff --git a/packages/api/test/artifact-upload.test.ts b/packages/api/test/artifact-upload.test.ts index d936c06..d4663db 100644 --- a/packages/api/test/artifact-upload.test.ts +++ b/packages/api/test/artifact-upload.test.ts @@ -26,6 +26,30 @@ describe("artifact upload grants", () => { expect(put).toHaveBeenCalledWith(`tenants/tenant-1/runs/run-1/native/${"a".repeat(64)}.jsonl`, expect.any(ReadableStream), expect.objectContaining({ customMetadata: expect.objectContaining({ tenantId: "tenant-1", runId: "run-1" }) })); }); + it("queues a newly written object when run retention wins the upload race", async () => { + const key = `tenants/tenant-1/runs/run-1/native/${"a".repeat(64)}.jsonl`; + const queries: Array<{ sql: string; values: unknown[] }> = []; + const query = vi.fn(async (sql: string, values: unknown[] = []) => { + queries.push({ sql, values }); + // Receipt lookup sees no row and retention has deleted the run before + // the upload acquires its aggregate lock. + return { rows: [], rowCount: 0 }; + }); + const put = vi.fn(async () => ({ httpEtag: "etag" })); + const get = vi.fn(async () => ({ size: 7, httpEtag: "etag", body: new Response("session").body })); + const token = await issueArtifactUploadGrant(grant, secret); + const response = await artifactUpload(new Request("https://app/upload", { method: "PUT", headers: { "Content-Type": grant.contentType, "Content-Length": "7", "X-Artifact-SHA256": "a".repeat(64) }, body: "session" }), { + SESSION_SIGNING_SECRET: secret, + RUN_ARTIFACTS: { put, get }, + DATABASE: { pool: { query }, transaction: (work: any) => work({ query }) }, + } as any, token); + + expect(response.status).toBe(409); + const queued = queries.find(({ sql }) => sql.includes("INSERT INTO app.run_retention_cleanup")); + expect(queued?.values).toEqual([expect.any(String), grant.tenantId, grant.runId, JSON.stringify([key])]); + expect(queued?.sql).toContain("Artifact upload lost a race with run retention"); + }); + it("rejects mismatched content types before storage", async () => { const token = await issueArtifactUploadGrant(grant, secret), put = vi.fn(); const response = await artifactUpload(new Request("https://app/upload", { method: "PUT", headers: { "Content-Type": "text/plain" }, body: "session" }), { SESSION_SIGNING_SECRET: secret, RUN_ARTIFACTS: { put } } as any, token); diff --git a/packages/api/test/run-retention-repository.test.ts b/packages/api/test/run-retention-repository.test.ts index 0e0447c..7fbb52b 100644 --- a/packages/api/test/run-retention-repository.test.ts +++ b/packages/api/test/run-retention-repository.test.ts @@ -55,6 +55,32 @@ describe("RunRetentionRepository", () => { expect(poolQuery).toHaveBeenLastCalledWith("DELETE FROM app.run_retention_cleanup WHERE id=$1", ["cleanup"]); }); + it("retains R2 ownership when external deletion succeeds but acknowledgment fails", async () => { + const now = new Date("2026-11-11T12:00:00.000Z"); + const bucketDelete = vi.fn().mockResolvedValue(undefined); + let claimAvailable = true; + const txQuery = vi.fn(async (sql: string) => { + if (!sql.includes("SELECT id,object_keys") || !claimAvailable) return { rows: [], rowCount: 0 }; + claimAvailable = false; + return { rows: [{ id: "cleanup", object_keys: ["tenant/run/object"], attempts: 0 }], rowCount: 1 }; + }); + const acknowledge = vi.fn() + .mockRejectedValueOnce(new Error("database disconnected after R2 deletion")) + .mockResolvedValueOnce({ rows: [], rowCount: 1 }); + const poolQuery = vi.fn(async (sql: string) => { + if (sql.startsWith("DELETE FROM app.run_retention_cleanup")) return acknowledge(sql); + return { rows: [], rowCount: 1 }; + }); + const repository = new RunRetentionRepository({ pool: { query: poolQuery }, transaction: (work: (client: any) => unknown) => work({ query: txQuery }) } as any); + + await expect(repository.processArtifactCleanup({ delete: bucketDelete } as any, now)).resolves.toEqual({ attempted: 1, completed: 0, retried: 1 }); + expect(poolQuery).toHaveBeenCalledWith(expect.stringContaining("last_error=$3"), ["cleanup", new Date(now.getTime() + 60_000), "database disconnected after R2 deletion"]); + claimAvailable = true; + await expect(repository.processArtifactCleanup({ delete: bucketDelete } as any, now)).resolves.toEqual({ attempted: 1, completed: 1, retried: 0 }); + expect(bucketDelete).toHaveBeenCalledTimes(2); + expect(acknowledge).toHaveBeenCalledTimes(2); + }); + it("reports bounded backlog counts and oldest age without exposing row identifiers", async () => { const query = vi.fn().mockResolvedValue({ rows: [ { queue: "artifact_objects", pending: 2, oldest_age_seconds: 120 }, From 6a6d3870cc219de055b63a656fe2d2b3c252b7a1 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 00:01:55 +0000 Subject: [PATCH 08/26] GEN-2907 cover retention cross-resource failures --- packages/api/src/postgres/webhook-service.ts | 8 ++++-- .../api/test/job-lifecycle.postgres.test.ts | 21 ++++++++++++++ .../api/test/native-auth.postgres.test.ts | 16 +++++++++++ .../run-provider-cleanup-repository.test.ts | 28 +++++++++++++++++++ 4 files changed, 70 insertions(+), 3 deletions(-) diff --git a/packages/api/src/postgres/webhook-service.ts b/packages/api/src/postgres/webhook-service.ts index 3340610..5ed3c5f 100644 --- a/packages/api/src/postgres/webhook-service.ts +++ b/packages/api/src/postgres/webhook-service.ts @@ -26,9 +26,11 @@ export class WebhookService { return result.rows[0]?.id ?? (await this.database.pool.query("SELECT id FROM app.webhook_deliveries WHERE tenant_id=$1 AND provider=$2 AND delivery_id=$3", [this.tenantId, provider, deliveryId])).rows[0].id; } private async event(deliveryPk: string, jobId: string, provider: string, deliveryId: string, outcome: string, detail: string) { - await this.database.pool.query("INSERT INTO app.job_events(tenant_id,id,job_id,provider,delivery_id,outcome,detail) VALUES ($1,$2,$3,$4,$5,$6,$7)", [this.tenantId, crypto.randomUUID(), jobId, provider, deliveryId, outcome, detail]); - await this.database.pool.query("INSERT INTO app.webhook_delivery_events(tenant_id,id,delivery_id,job_id,outcome,detail) VALUES ($1,$2,$3,$4,$5,$6)", [this.tenantId, crypto.randomUUID(), deliveryPk, jobId, outcome, detail]); - await this.database.pool.query("UPDATE app.webhook_deliveries SET outcome=$4,detail=$5 WHERE tenant_id=$1 AND id=$2 AND delivery_id=$3", [this.tenantId, deliveryPk, deliveryId, outcome, detail]); + await this.database.transaction(async client => { + await client.query("INSERT INTO app.job_events(tenant_id,id,job_id,provider,delivery_id,outcome,detail) VALUES ($1,$2,$3,$4,$5,$6,$7)", [this.tenantId, crypto.randomUUID(), jobId, provider, deliveryId, outcome, detail]); + await client.query("INSERT INTO app.webhook_delivery_events(tenant_id,id,delivery_id,job_id,outcome,detail) VALUES ($1,$2,$3,$4,$5,$6)", [this.tenantId, crypto.randomUUID(), deliveryPk, jobId, outcome, detail]); + await client.query("UPDATE app.webhook_deliveries SET outcome=$4,detail=$5 WHERE tenant_id=$1 AND id=$2 AND delivery_id=$3", [this.tenantId, deliveryPk, deliveryId, outcome, detail]); + }); } private async invoke(provider: WebhookProvider, deliveryId: string, source: Record | ((config: WebhookTriggerConfig) => Record), eventName?: string) { const base = typeof source === "function" ? {} : source; diff --git a/packages/api/test/job-lifecycle.postgres.test.ts b/packages/api/test/job-lifecycle.postgres.test.ts index 83f7feb..37db382 100644 --- a/packages/api/test/job-lifecycle.postgres.test.ts +++ b/packages/api/test/job-lifecycle.postgres.test.ts @@ -4,6 +4,7 @@ import { Database } from "../src/postgres/database"; import { PostgresJobRepository } from "../src/postgres/job-repository"; import { AutomationScheduler } from "../src/postgres/automation-scheduler"; import { RunRepository } from "../src/postgres/run-repository"; +import { RunRetentionRepository } from "../src/postgres/run-retention-repository"; import { RunQueryRepository } from "../src/postgres/run-query-repository"; import { InvocationService, type Job, type Trigger } from "../src/job-domain"; import { decrypt, encrypt } from "../src/crypto"; @@ -87,6 +88,26 @@ describe.skipIf(!process.env.AUTH_TEST_DATABASE_URL)("job lifecycle delivery (Po expect(await deliveries()).toHaveLength(1); }); + it("retains an expired source run until its enabled lifecycle delivery is acknowledged", async () => { + const a = await tenant(), source = await job(a); + const target = await job(a, { sourceJobIds: [source.id], states: ["succeeded"] }); + const sourceRunId = await complete(a, source); + const cutoff = new Date(Date.now() - 14 * 24 * 60 * 60_000); + await database.pool.query("UPDATE app.runs SET created_at=$3 WHERE tenant_id=$1 AND id=$2", [a, sourceRunId, new Date(cutoff.getTime() - 60_000)]); + + const retention = new RunRetentionRepository(database); + await expect(retention.sweep(cutoff, 10)).resolves.toMatchObject({ deleted: 0 }); + expect((await database.pool.query("SELECT 1 FROM app.runs WHERE tenant_id=$1 AND id=$2", [a, sourceRunId])).rowCount).toBe(1); + expect(await deliveries()).toHaveLength(0); + + await deliver(); + expect(await invocations(a, target.id)).toMatchObject([{ context: { "trigger-1": { sourceRunId, state: "succeeded" } } }]); + expect(await deliveries()).toHaveLength(1); + await expect(retention.sweep(cutoff, 10)).resolves.toMatchObject({ deleted: 1 }); + expect((await database.pool.query("SELECT 1 FROM app.runs WHERE tenant_id=$1 AND id=$2", [a, sourceRunId])).rowCount).toBe(0); + expect(await invocations(a, target.id)).toHaveLength(1); + }); + it("does not replay terminal history when a trigger is created disabled then enabled", async () => { const a = await tenant(), source = await job(a); await complete(a, source, "failed"); diff --git a/packages/api/test/native-auth.postgres.test.ts b/packages/api/test/native-auth.postgres.test.ts index 03f9bf9..cdbc2ff 100644 --- a/packages/api/test/native-auth.postgres.test.ts +++ b/packages/api/test/native-auth.postgres.test.ts @@ -8,6 +8,7 @@ import { ApiService } from "../src/flow-service"; import { protectedApiFetch } from "../src/protected-api"; import { tokenDigest } from "../src/auth"; import { hmac } from "../src/crypto"; +import { WebhookService } from "../src/postgres/webhook-service"; import { signSetupState } from "../src/github"; import app, { readSession, signSession } from "../src/index"; import type { Env } from "../src/types"; @@ -232,6 +233,21 @@ describe.skipIf(!process.env.AUTH_TEST_DATABASE_URL)("native authentication with const webhook = await app.request("/webhooks/linear", { method: "POST", body: raw, headers: { "linear-delivery": "delivery", "linear-signature": Buffer.from(await hmac(raw, env.LINEAR_WEBHOOK_SIGNING_SECRET), "base64").toString("hex") } }, env); expect(webhook.status).toBe(200); expect((await db.pool.query("SELECT tenant_id FROM app.webhook_deliveries WHERE delivery_id='linear:delivery'")).rows[0].tenant_id).toBe(session.tenantId); + const suffix = crypto.randomUUID().replaceAll("-", ""), functionName = `reject_webhook_event_${suffix}`, triggerName = `reject_webhook_event_${suffix}`; + const deliveryPk = (await db.pool.query("SELECT id FROM app.webhook_deliveries WHERE tenant_id=$1 AND delivery_id='linear:delivery'", [session.tenantId])).rows[0].id; + const projectId = (await db.pool.query("INSERT INTO app.projects(tenant_id,id,name,description) VALUES ($1::uuid,md5($1::uuid::text || ':default')::uuid,'Default','Webhook test') ON CONFLICT (tenant_id,name) DO UPDATE SET updated_at=app.projects.updated_at RETURNING id", [session.tenantId])).rows[0].id; + const eventJobId = crypto.randomUUID(); + await db.pool.query("INSERT INTO app.jobs(tenant_id,id,project_id,name,slug,encrypted_prompt_template,execution_config,concurrency_limit) VALUES ($1,$2,$3,'Webhook test job',$4,'encrypted','{}',1)", [session.tenantId, eventJobId, projectId, `webhook-test-${eventJobId}`]); + await db.pool.query(`CREATE FUNCTION app.${functionName}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected webhook timeline failure'; END $$`); + await db.pool.query(`CREATE TRIGGER ${triggerName} BEFORE INSERT ON app.webhook_delivery_events FOR EACH ROW EXECUTE FUNCTION app.${functionName}()`); + try { + await expect((new WebhookService(db, env, session.tenantId) as any).event(deliveryPk, eventJobId, "linear", "linear:delivery", "accepted", "queued")).rejects.toThrow("injected webhook timeline failure"); + expect((await db.pool.query("SELECT 1 FROM app.job_events WHERE tenant_id=$1 AND delivery_id='linear:delivery'", [session.tenantId])).rowCount).toBe(0); + expect((await db.pool.query("SELECT outcome FROM app.webhook_deliveries WHERE tenant_id=$1 AND delivery_id='linear:delivery'", [session.tenantId])).rows[0].outcome).toBe("ignored"); + } finally { + await db.pool.query(`DROP TRIGGER ${triggerName} ON app.webhook_delivery_events`); + await db.pool.query(`DROP FUNCTION app.${functionName}()`); + } const other = await verified(), otherSession = await sessionFor(other.email); expect((await app.request(`/auth/linear/callback?code=code&state=${encodeURIComponent(state)}`, { headers: { Cookie: `${otherSession.cookie}; linear_oauth_state=${encodeURIComponent(state)}` } }, env)).status).toBe(400); expect(await new ConnectionRepository(db, otherSession.tenantId, env.CREDENTIAL_ENCRYPTION_KEY).putLinear(organizationId, {})).toBe(false); diff --git a/packages/api/test/run-provider-cleanup-repository.test.ts b/packages/api/test/run-provider-cleanup-repository.test.ts index 4f47a78..75112fa 100644 --- a/packages/api/test/run-provider-cleanup-repository.test.ts +++ b/packages/api/test/run-provider-cleanup-repository.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from "vitest"; import { RunRepository } from "../src/postgres/run-repository"; +import { RunScheduler } from "../src/postgres/run-scheduler"; describe("terminal provider cleanup reconciliation", () => { it("claims only terminal incomplete cleanup with a bounded lease", async () => { @@ -24,6 +25,33 @@ describe("terminal provider cleanup reconciliation", () => { expect(query).toHaveBeenNthCalledWith(2, expect.stringContaining("provider_cleanup_next_at=now()+make_interval(secs=>$3)"), ["tenant", "run", 15_360, "provider unavailable"]); }); + it("retains provider cleanup ownership when stop succeeds but database acknowledgment fails", async () => { + const writes: Array<{ sql: string; values: unknown[] }> = []; + const poolQuery = vi.fn(async (sql: string, values: unknown[] = []) => { + writes.push({ sql, values }); + if (sql.includes("vm_cleanup_complete=true")) throw new Error("database acknowledgment unavailable"); + return { rows: [], rowCount: 1 }; + }); + const txQuery = vi.fn(async (sql: string) => { + if (sql.includes("app.deleted_job_provider_cleanup")) return { rows: [], rowCount: 0 }; + if (sql.includes("WITH candidates")) return { rows: [{ tenant_id: "tenant", id: "run", execution_config: {}, execution_handle: { backendKind: "exe-vm", id: "provider-run" }, execution_backend_kind: "exe-vm", provider_cleanup_attempt: 1, started_at: new Date("2026-10-01T00:00:00Z") }], rowCount: 1 }; + return { rows: [], rowCount: 0 }; + }); + const database = { pool: { query: poolQuery }, transaction: (work: (client: any) => unknown) => work({ query: txQuery }) } as any; + const scheduler = new RunScheduler(database, {} as any); + const stop = vi.fn().mockResolvedValue({ state: "stopped" }); + vi.spyOn(scheduler as any, "backend").mockResolvedValue({ stop }); + + await (scheduler as any).reconcileProviderCleanup(); + + expect(stop).toHaveBeenCalledWith({ backendKind: "exe-vm", id: "provider-run" }); + expect(writes.map(write => write.sql)).toEqual(expect.arrayContaining([ + expect.stringContaining("vm_cleanup_complete=true"), + expect.stringContaining("provider_cleanup_next_at=now()+make_interval(secs=>$3)"), + ])); + expect(writes.at(-1)).toMatchObject({ values: ["tenant", "run", 60, "database acknowledgment unavailable"] }); + }); + it("leases independent ownership after job deletion and keeps failed provider work retryable", async () => { const query = vi.fn(async (sql: string) => { expect(sql).toContain("app.deleted_job_provider_cleanup"); From 2b127ae5fab4c039f99912d03c294669a7998a5f Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 00:22:41 +0000 Subject: [PATCH 09/26] GEN-2907 bound independent history cleanup --- docs/run-history-retention.md | 6 +- .../0041_independent_history_retention.sql | 11 +++ .../src/postgres/run-retention-repository.ts | 91 +++++++++++++++++++ packages/api/src/postgres/run-scheduler.ts | 3 +- packages/api/src/postgres/schema.ts | 16 ++-- packages/api/src/postgres/webhook-service.ts | 4 +- .../api/test/run-retention-repository.test.ts | 14 +++ scripts/hosting-runtime.test.mjs | 2 +- 8 files changed, 133 insertions(+), 14 deletions(-) create mode 100644 packages/api/migrations/0041_independent_history_retention.sql diff --git a/docs/run-history-retention.md b/docs/run-history-retention.md index 1107904..bdfc845 100644 --- a/docs/run-history-retention.md +++ b/docs/run-history-retention.md @@ -10,13 +10,15 @@ Artifact object keys are copied into `app.run_retention_cleanup` in the same dat Terminal runs whose execution-provider cleanup is incomplete use a separate bounded reconciliation lane. Workers claim rows with `FOR UPDATE SKIP LOCKED`, store a ten-minute lease, and ask the recorded backend to stop the recorded handle. A confirmed stop or provider-confirmed absence completes cleanup; failures retain the run and retry with capped exponential backoff. A terminal row with no handle and no provider start timestamp is explicitly treated as never launched. A started row with no handle cannot prove provider absence, so it retains an error and remains ineligible for retention until reconciled. Since retention requires provider cleanup completion, unresolved handles cannot be purged with their authoritative records. +Each alarm also removes up to 500 rows per independent-history lane: job events, completed webhook deliveries and their cascading timelines, and completed VM image verifications. Webhook rows still marked `received` and any delivery referenced by a pending GitHub verification are protected. Image verifications in `cleanup_failed` and other nonterminal states remain as cleanup ownership. Expired or used GitHub setup states, consent previews, device authorizations, OAuth continuations, password-reset tokens, revoked or expired sessions, and revoked or expired access tokens are removed in bounded batches; valid long-lived access tokens survive regardless of age. Expiring a webhook delivery removes its deduplication key, so a provider retry after the retention window may be processed as a new delivery. A retry racing deletion uses an atomic upsert to reuse or recreate its delivery record, and event/timeline writes remain transactional. + Explicit job deletion also snapshots every incomplete run's provider configuration, handle, and start evidence into `app.deleted_job_provider_cleanup` in the same transaction as the job cascade. This independent queue lets the maintenance lane stop or reconcile provider resources after the source job and run rows are gone. A started run without a saved handle stays queued for retry and manual reconciliation; a never-started run can be removed from the queue safely. R2 artifacts for explicit job deletion continue to be deleted before the database cascade so a storage failure leaves the authoritative keys available for retry. -Successful cleanup removes its retry state from the run. Failed work retains only a bounded diagnostic string and retry timestamp. Each maintenance alarm emits a structured `run_retention_maintenance` log with sweep and artifact-cleanup progress plus pending counts and oldest age for terminal-provider cleanup, deleted-job provider cleanup, artifact objects, and overdue nonterminal runs. The snapshot contains no tenant, run, or object identifiers. A failed snapshot is logged separately and does not fail maintenance. These logs provide rollout evidence; a retained time-series dashboard and independent-history cleanup remain follow-up work. +Successful cleanup removes its retry state from the run. Failed work retains only a bounded diagnostic string and retry timestamp. Each maintenance alarm emits a structured `run_retention_maintenance` log with run sweep, artifact and independent-history deletion progress plus pending counts and oldest age for terminal-provider cleanup, deleted-job provider cleanup, artifact objects, overdue nonterminal runs, independent histories, and expired temporary records. The snapshot contains no tenant, run, or object identifiers. A failed snapshot is logged separately and does not fail maintenance. These logs provide rollout evidence; a retained time-series dashboard remains follow-up work. ## Rollout and recovery -The retention sweep is intentionally not enabled by a schema migration and no data purge runs during migration. Before enabling or increasing destructive work, refresh the read-only inventory and dry-run eligible, active/exempt, cleanup-blocked, pending-automation and orphan-artifact cohorts. Deploy additive schema and cleanup code first. Use bounded batches and an operational pause switch; retain the pause until provider reconciliation, upload/trace race tests, independent-history cleanup and backlog observability are ready. +The retention sweep is intentionally not enabled by a schema migration and no data purge runs during migration. Before enabling or increasing destructive work, refresh the read-only inventory and dry-run eligible, active/exempt, cleanup-blocked, pending-automation and orphan-artifact cohorts. Deploy additive schema and cleanup code first. Use bounded batches and an operational pause switch; retain the pause until provider reconciliation, upload/trace race tests and the remaining rollout gates are ready. Maintenance runs from a heartbeat no more than 15 minutes away; cleanup lag is bounded by that cadence, each lane's batch cap, and any older backlog ahead of new work. Resume interrupted cleanup by allowing the next alarm to reclaim expired leases. Do not clear a provider handle or cleanup retry record after an error. A provider-confirmed absent resource is idempotently complete. Recheck ownership before deleting any artifact classified as orphaned, and use a grace period so in-flight uploads can commit receipts. Report before/after counts, oldest backlog age, retries, provider results, and pending external deletions. Database deletion is atomic per run, but PostgreSQL space reuse and external object deletion are asynchronous. diff --git a/packages/api/migrations/0041_independent_history_retention.sql b/packages/api/migrations/0041_independent_history_retention.sql new file mode 100644 index 0000000..ae11c35 --- /dev/null +++ b/packages/api/migrations/0041_independent_history_retention.sql @@ -0,0 +1,11 @@ +-- Bounded 14-day cleanup scans for histories and temporary authentication records. +CREATE INDEX job_events_retention ON app.job_events(received_at,tenant_id,id); +CREATE INDEX webhook_deliveries_retention ON app.webhook_deliveries(received_at,tenant_id,id) WHERE outcome <> 'received'; +CREATE INDEX pending_verifications_by_delivery ON app.pending_verifications(tenant_id,delivery_id); +CREATE INDEX vm_image_verifications_retention ON app.vm_image_verifications(created_at,tenant_id,id) WHERE status IN ('succeeded','failed'); +CREATE INDEX github_setup_states_expiration ON app.github_setup_states(expires_at,tenant_id,nonce); +CREATE INDEX auth_reset_tokens_expiration ON app.auth_reset_tokens(expires_at,digest); +CREATE INDEX auth_sessions_expiration ON app.auth_sessions(expires_at,id); +CREATE INDEX access_tokens_expiration ON app.access_tokens(expires_at,tenant_id,id); + +INSERT INTO app.schema_migrations(version) VALUES ('0041_independent_history_retention') ON CONFLICT DO NOTHING; diff --git a/packages/api/src/postgres/run-retention-repository.ts b/packages/api/src/postgres/run-retention-repository.ts index c73e1bf..c77e6c0 100644 --- a/packages/api/src/postgres/run-retention-repository.ts +++ b/packages/api/src/postgres/run-retention-repository.ts @@ -3,11 +3,13 @@ import { safeDiagnosticText } from "../harness-diagnostics"; export type RetentionSweepResult = { selected: number; deleted: number; cleanupQueued: number }; export type ArtifactCleanupResult = { attempted: number; completed: number; retried: number }; +export type IndependentHistoryCleanupResult = { queue: string; deleted: number }[]; export type RetentionBacklog = { queue: string; pending: number; oldestAgeSeconds: number | null }; const CLEANUP_BATCH_SIZE = 20; const CLEANUP_LEASE_MS = 10 * 60_000; const CLEANUP_MAX_BACKOFF_MS = 6 * 60 * 60_000; +const HISTORY_BATCH_SIZE = 500; /** * Bounded retention deletion. The fixed cutoff is supplied by the caller so a @@ -38,10 +40,99 @@ export class RunRetentionRepository { floor(extract(epoch FROM now()-min(created_at)))::int FROM app.runs WHERE state IN ('queued','reserved','starting','running','blocked','stopping') AND created_at < now()-interval '14 days' + UNION ALL + SELECT 'job_events'::text,count(*)::int,floor(extract(epoch FROM now()-min(received_at)))::int + FROM app.job_events WHERE received_at < now()-interval '14 days' + UNION ALL + SELECT 'webhook_deliveries'::text,count(*)::int,floor(extract(epoch FROM now()-min(d.received_at)))::int + FROM app.webhook_deliveries d + WHERE d.received_at < now()-interval '14 days' AND d.outcome <> 'received' + AND NOT EXISTS (SELECT 1 FROM app.pending_verifications p WHERE p.tenant_id=d.tenant_id AND p.delivery_id=d.delivery_id) + UNION ALL + SELECT 'vm_image_verifications'::text,count(*)::int,floor(extract(epoch FROM now()-min(created_at)))::int + FROM app.vm_image_verifications WHERE status IN ('succeeded','failed') AND created_at < now()-interval '14 days' + UNION ALL + SELECT 'expired_temporary_records'::text, + (SELECT count(*)::int FROM app.github_setup_states WHERE expires_at < now() OR used_at IS NOT NULL) + +(SELECT count(*)::int FROM app.oauth_consent_previews WHERE expires_at < now() OR used_at IS NOT NULL) + +(SELECT count(*)::int FROM app.oauth_device_authorizations WHERE expires_at < now()) + +(SELECT count(*)::int FROM app.oauth_connections WHERE expires_at < now() OR used_at IS NOT NULL) + +(SELECT count(*)::int FROM app.auth_reset_tokens WHERE expires_at < now() OR used_at IS NOT NULL) + +(SELECT count(*)::int FROM app.auth_sessions WHERE expires_at < now() OR revoked_at IS NOT NULL) + +(SELECT count(*)::int FROM app.access_tokens WHERE expires_at < now() OR revoked_at IS NOT NULL), + GREATEST( + COALESCE(floor(extract(epoch FROM now()-(SELECT min(expires_at) FROM app.github_setup_states WHERE expires_at < now() OR used_at IS NOT NULL))),0), + COALESCE(floor(extract(epoch FROM now()-(SELECT min(expires_at) FROM app.oauth_consent_previews WHERE expires_at < now() OR used_at IS NOT NULL))),0), + COALESCE(floor(extract(epoch FROM now()-(SELECT min(expires_at) FROM app.oauth_device_authorizations WHERE expires_at < now()))),0), + COALESCE(floor(extract(epoch FROM now()-(SELECT min(expires_at) FROM app.oauth_connections WHERE expires_at < now() OR used_at IS NOT NULL))),0), + COALESCE(floor(extract(epoch FROM now()-(SELECT min(expires_at) FROM app.auth_reset_tokens WHERE expires_at < now() OR used_at IS NOT NULL))),0), + COALESCE(floor(extract(epoch FROM now()-(SELECT min(expires_at) FROM app.auth_sessions WHERE expires_at < now() OR revoked_at IS NOT NULL))),0), + COALESCE(floor(extract(epoch FROM now()-(SELECT min(expires_at) FROM app.access_tokens WHERE expires_at < now() OR revoked_at IS NOT NULL))),0) + )::int ) backlog ORDER BY queue`); return result.rows.map(row => ({ queue: row.queue, pending: row.pending, oldestAgeSeconds: row.oldest_age_seconds })); } + /** + * Bound independent history and expired one-time record growth. Every lane + * selects and deletes under row locks, so concurrent maintenance workers + * divide the work instead of blocking each other. Pending webhook delivery + * and resource cleanup ownership are deliberately retained. + */ + async cleanupIndependentHistory(): Promise { + const lanes: Array<[string, string]> = [ + ["job_events", `WITH expired AS ( + SELECT tenant_id,id FROM app.job_events WHERE received_at < now()-interval '14 days' + ORDER BY received_at,tenant_id,id LIMIT $1 FOR UPDATE SKIP LOCKED + ) DELETE FROM app.job_events e USING expired x WHERE e.tenant_id=x.tenant_id AND e.id=x.id`], + ["webhook_deliveries", `WITH expired AS ( + SELECT d.tenant_id,d.id FROM app.webhook_deliveries d + WHERE d.received_at < now()-interval '14 days' AND d.outcome <> 'received' + AND NOT EXISTS (SELECT 1 FROM app.pending_verifications p WHERE p.tenant_id=d.tenant_id AND p.delivery_id=d.delivery_id) + ORDER BY d.received_at,d.tenant_id,d.id LIMIT $1 FOR UPDATE OF d SKIP LOCKED + ) DELETE FROM app.webhook_deliveries d USING expired x WHERE d.tenant_id=x.tenant_id AND d.id=x.id`], + ["vm_image_verifications", `WITH expired AS ( + SELECT tenant_id,id FROM app.vm_image_verifications WHERE status IN ('succeeded','failed') + AND created_at < now()-interval '14 days' + ORDER BY created_at,tenant_id,id LIMIT $1 FOR UPDATE SKIP LOCKED + ) DELETE FROM app.vm_image_verifications v USING expired x WHERE v.tenant_id=x.tenant_id AND v.id=x.id`], + ["github_setup_states", `WITH expired AS ( + SELECT tenant_id,nonce FROM app.github_setup_states WHERE expires_at < now() OR used_at IS NOT NULL + ORDER BY expires_at,tenant_id,nonce LIMIT $1 FOR UPDATE SKIP LOCKED + ) DELETE FROM app.github_setup_states s USING expired x WHERE s.tenant_id=x.tenant_id AND s.nonce=x.nonce`], + ["oauth_consent_previews", `WITH expired AS ( + SELECT id FROM app.oauth_consent_previews WHERE expires_at < now() OR used_at IS NOT NULL + ORDER BY expires_at,id LIMIT $1 FOR UPDATE SKIP LOCKED + ) DELETE FROM app.oauth_consent_previews p USING expired x WHERE p.id=x.id`], + ["oauth_device_authorizations", `WITH expired AS ( + SELECT device_code FROM app.oauth_device_authorizations WHERE expires_at < now() + ORDER BY expires_at,device_code LIMIT $1 FOR UPDATE SKIP LOCKED + ) DELETE FROM app.oauth_device_authorizations a USING expired x WHERE a.device_code=x.device_code`], + ["oauth_connections", `WITH expired AS ( + SELECT id FROM app.oauth_connections WHERE expires_at < now() OR used_at IS NOT NULL + ORDER BY expires_at,id LIMIT $1 FOR UPDATE SKIP LOCKED + ) DELETE FROM app.oauth_connections c USING expired x WHERE c.id=x.id`], + ["auth_reset_tokens", `WITH expired AS ( + SELECT digest FROM app.auth_reset_tokens WHERE expires_at < now() OR used_at IS NOT NULL + ORDER BY expires_at,digest LIMIT $1 FOR UPDATE SKIP LOCKED + ) DELETE FROM app.auth_reset_tokens t USING expired x WHERE t.digest=x.digest`], + ["auth_sessions", `WITH expired AS ( + SELECT id FROM app.auth_sessions WHERE expires_at < now() OR revoked_at IS NOT NULL + ORDER BY expires_at,id LIMIT $1 FOR UPDATE SKIP LOCKED + ) DELETE FROM app.auth_sessions s USING expired x WHERE s.id=x.id`], + ["access_tokens", `WITH expired AS ( + SELECT tenant_id,id FROM app.access_tokens WHERE expires_at < now() OR revoked_at IS NOT NULL + ORDER BY expires_at,tenant_id,id LIMIT $1 FOR UPDATE SKIP LOCKED + ) DELETE FROM app.access_tokens t USING expired x WHERE t.tenant_id=x.tenant_id AND t.id=x.id`], + ]; + const result: IndependentHistoryCleanupResult = []; + for (const [queue, statement] of lanes) { + const deleted = await this.database.pool.query(statement, [HISTORY_BATCH_SIZE]); + result.push({ queue, deleted: deleted.rowCount ?? 0 }); + } + return result; + } + /** * Delete retained R2 objects outside the run-deletion transaction. A short * lease prevents concurrent alarms from doing duplicate work; if an alarm diff --git a/packages/api/src/postgres/run-scheduler.ts b/packages/api/src/postgres/run-scheduler.ts index b8843a5..8ec5875 100644 --- a/packages/api/src/postgres/run-scheduler.ts +++ b/packages/api/src/postgres/run-scheduler.ts @@ -37,11 +37,12 @@ export class RunScheduler { const cutoff = runRetentionCutoff(new Date()); const sweep = await retention.sweep(cutoff, 50); const artifacts = await retention.processArtifactCleanup(this.env.RUN_ARTIFACTS); + const independentHistory = await retention.cleanupIndependentHistory(); // Aggregate observability is best effort: a metrics query failure must not // turn a successful maintenance pass into a failed scheduler alarm. try { const backlog = await retention.backlogSnapshot(); - console.info(JSON.stringify({ event: "run_retention_maintenance", cutoff: cutoff.toISOString(), sweep, artifacts, backlog })); + console.info(JSON.stringify({ event: "run_retention_maintenance", cutoff: cutoff.toISOString(), sweep, artifacts, independentHistory, backlog })); } catch (error) { console.warn(JSON.stringify({ event: "run_retention_backlog_snapshot_failed", message: error instanceof Error ? error.message : "unknown error" })); } diff --git a/packages/api/src/postgres/schema.ts b/packages/api/src/postgres/schema.ts index 85fcf5b..8cc970d 100644 --- a/packages/api/src/postgres/schema.ts +++ b/packages/api/src/postgres/schema.ts @@ -28,8 +28,8 @@ export const oauthConnections = app.table("oauth_connections", { userId: uuid("user_id").references(() => authUsers.id, { onDelete: "cascade" }), readyVersion: integer("ready_version"), ready: boolean("ready").notNull().default(false), usedAt: timestamp("used_at", { withTimezone: true }), }, t => [index("oauth_connections_expires").on(t.expiresAt), check("oauth_connections_kind_check", sql`${t.kind} IN ('callback','device')`)]); export const oauthConsentPreviews = app.table("oauth_consent_previews", { id: uuid("id").primaryKey(), expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), usedAt: timestamp("used_at", { withTimezone: true }) }, t => [index("oauth_consent_previews_expires").on(t.expiresAt)]); -export const authResetTokens = app.table("auth_reset_tokens", { connectionId: uuid("connection_id").references(() => oauthConnections.id, { onDelete: "set null" }), purpose: text("purpose").$type<"verify" | "reset">().notNull().default("reset"), digest: text("digest").primaryKey(), userId: uuid("user_id").notNull().references(() => authUsers.id, { onDelete: "cascade" }), expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), usedAt: timestamp("used_at", { withTimezone: true }) }, t => [check("auth_reset_tokens_purpose_check", sql`${t.purpose} IN ('verify','reset')`)]); -export const authSessions = app.table("auth_sessions", { id: uuid("id").primaryKey(), userId: uuid("user_id").notNull().references(() => authUsers.id, { onDelete: "cascade" }), expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), revokedAt: timestamp("revoked_at", { withTimezone: true }) }); +export const authResetTokens = app.table("auth_reset_tokens", { connectionId: uuid("connection_id").references(() => oauthConnections.id, { onDelete: "set null" }), purpose: text("purpose").$type<"verify" | "reset">().notNull().default("reset"), digest: text("digest").primaryKey(), userId: uuid("user_id").notNull().references(() => authUsers.id, { onDelete: "cascade" }), expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), usedAt: timestamp("used_at", { withTimezone: true }) }, t => [check("auth_reset_tokens_purpose_check", sql`${t.purpose} IN ('verify','reset')`), index("auth_reset_tokens_expiration").on(t.expiresAt,t.digest)]); +export const authSessions = app.table("auth_sessions", { id: uuid("id").primaryKey(), userId: uuid("user_id").notNull().references(() => authUsers.id, { onDelete: "cascade" }), expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), revokedAt: timestamp("revoked_at", { withTimezone: true }) }, t => [index("auth_sessions_expiration").on(t.expiresAt,t.id)]); export const oauthDeviceAuthorizations = app.table("oauth_device_authorizations", { deviceCode: text("device_code").primaryKey(), userCode: text("user_code").notNull().unique(), record: jsonb("record").$type>().notNull(), expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), createdAt: createdAt(), updatedAt: updatedAt(), }, t => [index("oauth_device_authorizations_expires").on(t.expiresAt)]); @@ -38,7 +38,7 @@ export const members = app.table("members", { }, t => [primaryKey({ columns: [t.tenantId, t.userId] }), check("members_role_check", sql`${t.role} in ('owner','member')`), check("members_session_version_check", sql`${t.sessionVersion} > 0`)]); export const accessTokens = app.table("access_tokens", { tenantId: uuid("tenant_id").notNull().references(() => tenants.id, { onDelete: "cascade" }), id: uuid("id").notNull(), name: text("name").notNull(), digest: text("digest").notNull().unique(), userId: uuid("user_id").notNull(), sessionVersion: integer("session_version").notNull(), scopes: jsonb("scopes").$type().notNull().default([]), createdAt: createdAt(), expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), lastUsedAt: timestamp("last_used_at", { withTimezone: true }), revokedAt: timestamp("revoked_at", { withTimezone: true }), -}, t => [primaryKey({ columns: [t.tenantId, t.id] }), foreignKey({ columns: [t.tenantId, t.userId], foreignColumns: [members.tenantId, members.userId] }).onDelete("cascade")]); +}, t => [primaryKey({ columns: [t.tenantId, t.id] }), foreignKey({ columns: [t.tenantId, t.userId], foreignColumns: [members.tenantId, members.userId] }).onDelete("cascade"), index("access_tokens_expiration").on(t.expiresAt,t.tenantId,t.id)]); export const connections = app.table("connections", { tenantId: uuid("tenant_id").notNull().references(() => tenants.id, { onDelete: "cascade" }), kind: text("kind").notNull(), encryptedValue: text("encrypted_value").notNull(), updatedAt: updatedAt() }, t => [primaryKey({ columns: [t.tenantId, t.kind] })]); export const integrationCollections = app.table("integration_collections", { @@ -61,7 +61,7 @@ export const credentialRunEnrollments = app.table("credential_run_enrollments", }, t => [primaryKey({ columns: [t.tenantId, t.id] }), unique().on(t.tokenDigest), foreignKey({ columns: [t.tenantId, t.runId], foreignColumns: [credentialRunGrants.tenantId, credentialRunGrants.runId] }).onDelete("cascade"), index("credential_run_enrollments_expiry").on(t.expiresAt).where(sql`${t.consumedAt} is null`), check("credential_run_enrollments_revision_check", sql`${t.grantRevision} > 0`)]); export const linearWorkspaces = app.table("linear_workspaces", { organizationId: uuid("organization_id").primaryKey(), tenantId: uuid("tenant_id").notNull().unique().references(() => tenants.id, { onDelete: "cascade" }) }); export const githubInstallations = app.table("github_installations", { installationId: bigint("installation_id", { mode: "number" }).primaryKey(), tenantId: uuid("tenant_id").notNull().references(() => tenants.id, { onDelete: "cascade" }), accountLogin: text("account_login").notNull(), accountType: text("account_type").notNull(), state: text("state").$type<"active" | "suspended">().notNull(), updatedAt: updatedAt() }, t => [unique().on(t.tenantId, t.installationId), check("github_installations_state_check", sql`${t.state} in ('active','suspended')`)]); -export const githubSetupStates = app.table("github_setup_states", { tenantId: uuid("tenant_id").notNull().references(() => tenants.id, { onDelete: "cascade" }), nonce: text("nonce").notNull(), expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), usedAt: timestamp("used_at", { withTimezone: true }) }, t => [primaryKey({ columns: [t.tenantId, t.nonce] })]); +export const githubSetupStates = app.table("github_setup_states", { tenantId: uuid("tenant_id").notNull().references(() => tenants.id, { onDelete: "cascade" }), nonce: text("nonce").notNull(), expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), usedAt: timestamp("used_at", { withTimezone: true }) }, t => [primaryKey({ columns: [t.tenantId, t.nonce] }), index("github_setup_states_expiration").on(t.expiresAt,t.tenantId,t.nonce)]); export const projects = app.table("projects", { tenantId: uuid("tenant_id").notNull().references(() => tenants.id, { onDelete: "cascade" }), id: uuid("id").notNull(), name: text("name").notNull(), description: text("description"), createdAt: createdAt(), updatedAt: updatedAt() }, t => [primaryKey({ columns: [t.tenantId, t.id] }), unique().on(t.tenantId, t.name), index("projects_tenant_updated").on(t.tenantId, t.updatedAt)]); export const jobs = app.table("jobs", { @@ -69,7 +69,7 @@ export const jobs = app.table("jobs", { }, t => [primaryKey({ columns: [t.tenantId, t.id] }), unique().on(t.tenantId, t.slug), foreignKey({ columns: [t.tenantId, t.projectId], foreignColumns: [projects.tenantId, projects.id] }).onDelete("restrict"), index("jobs_tenant_project_updated").on(t.tenantId, t.projectId, t.updatedAt), check("jobs_concurrency_limit_check", sql`${t.concurrencyLimit} >= 1`)]); export const vmImageVerifications = app.table("vm_image_verifications", { tenantId: uuid("tenant_id").notNull().references(() => tenants.id, { onDelete: "cascade" }), id: uuid("id").notNull(), jobId: uuid("job_id"), status: text("status").notNull(), phase: text("phase").notNull(), config: jsonb("config").$type>().notNull(), errorKind: text("error_kind"), error: text("error"), vmName: text("vm_name"), createdAt: createdAt(), updatedAt: updatedAt(), -}, t => [primaryKey({ columns: [t.tenantId, t.id] }), check("vm_image_verifications_status_check", sql`${t.status} IN ('queued','creating','checking_ssh','checking_agent','cleaning_up','succeeded','failed','cleanup_failed')`), index("vm_image_verifications_status_idx").on(t.status, t.updatedAt)]); +}, t => [primaryKey({ columns: [t.tenantId, t.id] }), check("vm_image_verifications_status_check", sql`${t.status} IN ('queued','creating','checking_ssh','checking_agent','cleaning_up','succeeded','failed','cleanup_failed')`), index("vm_image_verifications_status_idx").on(t.status, t.updatedAt), index("vm_image_verifications_retention").on(t.createdAt,t.tenantId,t.id).where(sql`${t.status} IN ('succeeded','failed')`)]); export const skills = app.table("skills", { tenantId: uuid("tenant_id").notNull().references(() => tenants.id, { onDelete: "cascade" }), id: uuid("id").notNull(), key: text("key").notNull(), encryptedBody: text("encrypted_body").notNull(), revision: integer("revision").notNull().default(1), createdAt: createdAt(), updatedAt: updatedAt(), }, t => [primaryKey({ columns: [t.tenantId, t.id] }), unique().on(t.tenantId, t.key)]); @@ -117,9 +117,9 @@ export const jobEditDeliveries = app.table("job_edit_deliveries", { }, t => [primaryKey({ columns: [t.tenantId, t.eventId, t.triggerId] }), foreignKey({ columns: [t.tenantId, t.triggerId], foreignColumns: [triggers.tenantId, triggers.id] }).onDelete("cascade"), foreignKey({ columns: [t.tenantId, t.sourceJobId], foreignColumns: [jobs.tenantId, jobs.id] }).onDelete("cascade"), index("job_edit_deliveries_pending").on(t.editedAt, t.eventId)]); export const lifecycleDeliveries = app.table("lifecycle_deliveries", { tenantId: uuid("tenant_id").notNull(), triggerId: uuid("trigger_id").notNull(), sourceRunId: uuid("source_run_id").notNull(), terminalState: text("terminal_state").notNull(), createdAt: createdAt() }, t => [primaryKey({ columns: [t.tenantId, t.triggerId, t.sourceRunId, t.terminalState] }), foreignKey({ columns: [t.tenantId, t.triggerId], foreignColumns: [triggers.tenantId, triggers.id] }).onDelete("cascade"), foreignKey({ columns: [t.tenantId, t.sourceRunId], foreignColumns: [runs.tenantId, runs.id] }).onDelete("cascade")]); export const lifecycleEvents = app.table("lifecycle_events", { eventId: bigint("event_id", { mode: "number" }).primaryKey(), tenantId: uuid("tenant_id").notNull(), triggerId: uuid("trigger_id").notNull(), sourceRunId: uuid("source_run_id").notNull(), terminalState: text("terminal_state").notNull(), occurredAt: timestamp("occurred_at", { withTimezone: true, mode: "date" }).notNull().defaultNow() }, t => [unique().on(t.tenantId, t.triggerId, t.sourceRunId, t.terminalState), foreignKey({ columns: [t.tenantId, t.triggerId], foreignColumns: [triggers.tenantId, triggers.id] }).onDelete("cascade"), foreignKey({ columns: [t.tenantId, t.sourceRunId], foreignColumns: [runs.tenantId, runs.id] }).onDelete("cascade"), index("lifecycle_events_pending").on(t.eventId)]); -export const jobEvents = app.table("job_events", { tenantId: uuid("tenant_id").notNull(), id: uuid("id").notNull(), jobId: uuid("job_id").notNull(), provider: text("provider").notNull(), deliveryId: text("delivery_id").notNull(), outcome: text("outcome").notNull(), detail: text("detail").notNull(), receivedAt: timestamp("received_at", { withTimezone: true }).notNull().defaultNow() }, t => [primaryKey({ columns: [t.tenantId, t.id] }), foreignKey({ columns: [t.tenantId, t.jobId], foreignColumns: [jobs.tenantId, jobs.id] }).onDelete("cascade"), index("job_events_by_job").on(t.tenantId, t.jobId, t.receivedAt)]); -export const webhookDeliveries = app.table("webhook_deliveries", { tenantId: uuid("tenant_id").notNull().references(() => tenants.id, { onDelete: "cascade" }), id: uuid("id").notNull(), provider: text("provider").notNull(), deliveryId: text("delivery_id").notNull(), eventType: text("event_type").notNull().default("unknown"), eventAction: text("event_action").notNull().default("unknown"), outcome: text("outcome").notNull(), detail: text("detail").notNull(), receivedAt: timestamp("received_at", { withTimezone: true }).notNull().defaultNow() }, t => [primaryKey({ columns: [t.tenantId, t.id] }), unique().on(t.tenantId, t.provider, t.deliveryId)]); -export const webhookDeliveryEvents = app.table("webhook_delivery_events", { tenantId: uuid("tenant_id").notNull(), id: uuid("id").notNull(), deliveryId: uuid("delivery_id").notNull(), jobId: uuid("job_id"), outcome: text("outcome").notNull(), detail: text("detail").notNull(), createdAt: createdAt() }, t => [primaryKey({ columns: [t.tenantId, t.id] }), foreignKey({ columns: [t.tenantId, t.deliveryId], foreignColumns: [webhookDeliveries.tenantId, webhookDeliveries.id] }).onDelete("cascade"), foreignKey({ columns: [t.tenantId, t.jobId], foreignColumns: [jobs.tenantId, jobs.id] }).onDelete("cascade")]); +export const jobEvents = app.table("job_events", { tenantId: uuid("tenant_id").notNull(), id: uuid("id").notNull(), jobId: uuid("job_id").notNull(), provider: text("provider").notNull(), deliveryId: text("delivery_id").notNull(), outcome: text("outcome").notNull(), detail: text("detail").notNull(), receivedAt: timestamp("received_at", { withTimezone: true }).notNull().defaultNow() }, t => [primaryKey({ columns: [t.tenantId, t.id] }), foreignKey({ columns: [t.tenantId, t.jobId], foreignColumns: [jobs.tenantId, jobs.id] }).onDelete("cascade"), index("job_events_by_job").on(t.tenantId, t.jobId, t.receivedAt), index("job_events_retention").on(t.receivedAt,t.tenantId,t.id)]); +export const webhookDeliveries = app.table("webhook_deliveries", { tenantId: uuid("tenant_id").notNull().references(() => tenants.id, { onDelete: "cascade" }), id: uuid("id").notNull(), provider: text("provider").notNull(), deliveryId: text("delivery_id").notNull(), eventType: text("event_type").notNull().default("unknown"), eventAction: text("event_action").notNull().default("unknown"), outcome: text("outcome").notNull(), detail: text("detail").notNull(), receivedAt: timestamp("received_at", { withTimezone: true }).notNull().defaultNow() }, t => [primaryKey({ columns: [t.tenantId, t.id] }), unique().on(t.tenantId, t.provider, t.deliveryId), index("webhook_deliveries_retention").on(t.receivedAt,t.tenantId,t.id).where(sql`${t.outcome} <> 'received'`)]); +export const webhookDeliveryEvents = app.table("webhook_delivery_events", { tenantId: uuid("tenant_id").notNull(), id: uuid("id").notNull(), deliveryId: uuid("delivery_id").notNull(), jobId: uuid("job_id"), outcome: text("outcome").notNull(), detail: text("detail").notNull(), createdAt: createdAt() }, t => [primaryKey({ columns: [t.tenantId, t.id] }), foreignKey({ columns: [t.tenantId, t.deliveryId], foreignColumns: [webhookDeliveries.tenantId, webhookDeliveries.id] }).onDelete("cascade"), foreignKey({ columns: [t.tenantId, t.jobId], foreignColumns: [jobs.tenantId, jobs.id] }).onDelete("cascade"), index("webhook_delivery_events_by_delivery").on(t.tenantId,t.deliveryId)]); export const pendingVerifications = app.table("pending_verifications", { tenantId: uuid("tenant_id").notNull().references(() => tenants.id, { onDelete: "cascade" }), id: uuid("id").notNull(), jobId: uuid("job_id").notNull(), triggerId: uuid("trigger_id").notNull(), deliveryId: text("delivery_id").notNull(), installationId: bigint("installation_id", { mode: "number" }).notNull(), repositoryId: bigint("repository_id", { mode: "number" }).notNull(), repositoryOwner: text("repository_owner").notNull(), repositoryName: text("repository_name").notNull(), pullNumber: integer("pull_number").notNull(), attempt: integer("attempt").notNull().default(0), nextAttemptAt: timestamp("next_attempt_at", { withTimezone: true }).notNull(), payload: jsonb("payload").$type>().notNull() }, t => [primaryKey({ columns: [t.tenantId, t.id] }), foreignKey({ columns: [t.tenantId, t.jobId], foreignColumns: [jobs.tenantId, jobs.id] }).onDelete("cascade"), foreignKey({ columns: [t.tenantId, t.triggerId], foreignColumns: [triggers.tenantId, triggers.id] }).onDelete("cascade"), index("pending_verifications_due").on(t.nextAttemptAt)]); export const tailFingerprints = app.table("tail_fingerprints", { tenantId: uuid("tenant_id").notNull().references(() => tenants.id, { onDelete: "cascade" }), jobId: uuid("job_id").notNull(), fingerprint: text("fingerprint").notNull(), windowStartedAt: timestamp("window_started_at", { withTimezone: true }).notNull(), count: integer("count").notNull() }, t => [primaryKey({ columns: [t.tenantId, t.jobId, t.fingerprint] }), foreignKey({ columns: [t.tenantId, t.jobId], foreignColumns: [jobs.tenantId, jobs.id] }).onDelete("cascade")]); export const wakeHints = app.table("wake_hints", { key: text("key").primaryKey().default("global"), notBefore: timestamp("not_before", { withTimezone: true }).notNull(), updatedAt: updatedAt() }); diff --git a/packages/api/src/postgres/webhook-service.ts b/packages/api/src/postgres/webhook-service.ts index 5ed3c5f..f603362 100644 --- a/packages/api/src/postgres/webhook-service.ts +++ b/packages/api/src/postgres/webhook-service.ts @@ -22,8 +22,8 @@ export class WebhookService { } private async ensure(provider: string, deliveryId: string, type: unknown, action: unknown) { const id = crypto.randomUUID(); - const result = await this.database.pool.query(`INSERT INTO app.webhook_deliveries(tenant_id,id,provider,delivery_id,event_type,event_action,outcome,detail) VALUES ($1,$2,$3,$4,$5,$6,'received','Delivery received; processing is in progress.') ON CONFLICT (tenant_id,provider,delivery_id) DO NOTHING RETURNING id`, [this.tenantId, id, provider, deliveryId, String(type || "unknown"), String(action || "unknown")]); - return result.rows[0]?.id ?? (await this.database.pool.query("SELECT id FROM app.webhook_deliveries WHERE tenant_id=$1 AND provider=$2 AND delivery_id=$3", [this.tenantId, provider, deliveryId])).rows[0].id; + const result = await this.database.pool.query(`INSERT INTO app.webhook_deliveries(tenant_id,id,provider,delivery_id,event_type,event_action,outcome,detail) VALUES ($1,$2,$3,$4,$5,$6,'received','Delivery received; processing is in progress.') ON CONFLICT (tenant_id,provider,delivery_id) DO UPDATE SET delivery_id=EXCLUDED.delivery_id RETURNING id`, [this.tenantId, id, provider, deliveryId, String(type || "unknown"), String(action || "unknown")]); + return result.rows[0].id; } private async event(deliveryPk: string, jobId: string, provider: string, deliveryId: string, outcome: string, detail: string) { await this.database.transaction(async client => { diff --git a/packages/api/test/run-retention-repository.test.ts b/packages/api/test/run-retention-repository.test.ts index 7fbb52b..f39d5b3 100644 --- a/packages/api/test/run-retention-repository.test.ts +++ b/packages/api/test/run-retention-repository.test.ts @@ -98,4 +98,18 @@ describe("RunRetentionRepository", () => { expect(query.mock.calls[0]![0]).toContain("'overdue_nonterminal'"); expect(query.mock.calls[0]![0]).not.toContain("run_id"); }); + + it("cleans independent histories in bounded lanes and protects pending work", async () => { + const statements: string[] = []; + const query = vi.fn(async (sql: string) => { statements.push(sql); return { rows: [], rowCount: 7 }; }); + const result = await new RunRetentionRepository({ pool: { query } } as any).cleanupIndependentHistory(); + expect(result).toHaveLength(10); + expect(result.every(lane => lane.deleted === 7)).toBe(true); + expect(query).toHaveBeenCalledTimes(10); + expect(query).toHaveBeenCalledWith(expect.stringContaining("FOR UPDATE SKIP LOCKED"), [500]); + expect(statements.find(sql => sql.includes("webhook_deliveries d"))).toContain("d.outcome <> 'received'"); + expect(statements.find(sql => sql.includes("webhook_deliveries d"))).toContain("app.pending_verifications"); + expect(statements.find(sql => sql.includes("vm_image_verifications"))).toContain("status IN ('succeeded','failed')"); + expect(statements.find(sql => sql.includes("access_tokens"))).toContain("expires_at < now() OR revoked_at IS NOT NULL"); + }); }); diff --git a/scripts/hosting-runtime.test.mjs b/scripts/hosting-runtime.test.mjs index cd024ef..0dbd670 100644 --- a/scripts/hosting-runtime.test.mjs +++ b/scripts/hosting-runtime.test.mjs @@ -31,7 +31,7 @@ test("actual Cloudflare local runtime enforces static/API/protocol ownership, he } const database = databaseUrl.href; assert.ok(["localhost", "127.0.0.1"].includes(new URL(database).hostname), "Hosting runtime requires a local database URL"); - const child = spawn("npm", ["exec", "--workspace=factorize", "--", "wrangler", "dev", "--local", "--local-protocol", "https", "--port", String(port), "--var", `APP_ORIGIN:https://localhost:${port}`], { env: { ...process.env, CLOUDFLARE_HYPERDRIVE_LOCAL_CONNECTION_STRING_HYPERDRIVE: database }, detached: true, stdio: ["ignore", "pipe", "pipe"] }); + const child = spawn("npm", ["exec", "-w", "factorize", "--", "wrangler", "dev", "--local", "--local-protocol", "https", "--port", String(port), "--var", `APP_ORIGIN:https://localhost:${port}`], { env: { ...process.env, CLOUDFLARE_HYPERDRIVE_LOCAL_CONNECTION_STRING_HYPERDRIVE: database }, detached: true, stdio: ["ignore", "pipe", "pipe"] }); let output = ""; const ready = new Promise((resolve, reject) => { const deadline = setTimeout(() => reject(new Error("Local Worker did not start: " + output)), 30_000); From 824559f8964e6c29977239941c1dbc819c498ad7 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 00:25:32 +0000 Subject: [PATCH 10/26] GEN-2907 fix runtime validation command --- scripts/hosting-runtime.test.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/hosting-runtime.test.mjs b/scripts/hosting-runtime.test.mjs index 0dbd670..edfaeaa 100644 --- a/scripts/hosting-runtime.test.mjs +++ b/scripts/hosting-runtime.test.mjs @@ -31,7 +31,7 @@ test("actual Cloudflare local runtime enforces static/API/protocol ownership, he } const database = databaseUrl.href; assert.ok(["localhost", "127.0.0.1"].includes(new URL(database).hostname), "Hosting runtime requires a local database URL"); - const child = spawn("npm", ["exec", "-w", "factorize", "--", "wrangler", "dev", "--local", "--local-protocol", "https", "--port", String(port), "--var", `APP_ORIGIN:https://localhost:${port}`], { env: { ...process.env, CLOUDFLARE_HYPERDRIVE_LOCAL_CONNECTION_STRING_HYPERDRIVE: database }, detached: true, stdio: ["ignore", "pipe", "pipe"] }); + const child = spawn("npm", ["exec", "--", "wrangler", "dev", "--local", "--local-protocol", "https", "--port", String(port), "--var", `APP_ORIGIN:https://localhost:${port}`], { env: { ...process.env, CLOUDFLARE_HYPERDRIVE_LOCAL_CONNECTION_STRING_HYPERDRIVE: database }, detached: true, stdio: ["ignore", "pipe", "pipe"] }); let output = ""; const ready = new Promise((resolve, reject) => { const deadline = setTimeout(() => reject(new Error("Local Worker did not start: " + output)), 30_000); From 5aed3d58d8c9a464ea2a3a7e1f5fc89e6851093b Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 00:26:43 +0000 Subject: [PATCH 11/26] GEN-2907 launch runtime check with local Wrangler CLI --- scripts/hosting-runtime.test.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/hosting-runtime.test.mjs b/scripts/hosting-runtime.test.mjs index edfaeaa..77f8ce0 100644 --- a/scripts/hosting-runtime.test.mjs +++ b/scripts/hosting-runtime.test.mjs @@ -6,6 +6,7 @@ import https from "node:https"; import pg from "pg"; import { randomUUID } from "node:crypto"; import { readdir, readFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; function read(port, path, method = "GET", headers = {}) { return new Promise((resolve, reject) => { @@ -31,7 +32,8 @@ test("actual Cloudflare local runtime enforces static/API/protocol ownership, he } const database = databaseUrl.href; assert.ok(["localhost", "127.0.0.1"].includes(new URL(database).hostname), "Hosting runtime requires a local database URL"); - const child = spawn("npm", ["exec", "--", "wrangler", "dev", "--local", "--local-protocol", "https", "--port", String(port), "--var", `APP_ORIGIN:https://localhost:${port}`], { env: { ...process.env, CLOUDFLARE_HYPERDRIVE_LOCAL_CONNECTION_STRING_HYPERDRIVE: database }, detached: true, stdio: ["ignore", "pipe", "pipe"] }); + const wranglerCli = fileURLToPath(new URL("../node_modules/wrangler/bin/wrangler.js", import.meta.url)); + const child = spawn(process.execPath, [wranglerCli, "dev", "--local", "--local-protocol", "https", "--port", String(port), "--var", `APP_ORIGIN:https://localhost:${port}`], { cwd: fileURLToPath(new URL("../packages/api/", import.meta.url)), env: { ...process.env, CLOUDFLARE_HYPERDRIVE_LOCAL_CONNECTION_STRING_HYPERDRIVE: database }, detached: true, stdio: ["ignore", "pipe", "pipe"] }); let output = ""; const ready = new Promise((resolve, reject) => { const deadline = setTimeout(() => reject(new Error("Local Worker did not start: " + output)), 30_000); From 250f13c257ed85ca42b890196dc1d5ec81b9ca13 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 00:29:52 +0000 Subject: [PATCH 12/26] GEN-2907 launch OAuth runtime with local Wrangler CLI --- scripts/oauth-runtime.test.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/oauth-runtime.test.mjs b/scripts/oauth-runtime.test.mjs index c93bd02..6e6f31a 100644 --- a/scripts/oauth-runtime.test.mjs +++ b/scripts/oauth-runtime.test.mjs @@ -6,6 +6,7 @@ import https from 'node:https'; import { randomUUID, randomBytes, createHmac, createHash } from 'node:crypto'; import { mkdtemp, rm, readdir, readFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; +import { fileURLToPath } from 'node:url'; import pg from 'pg'; // Real public wire operations only; no imports from the backend or mocked OAuth helpers. @@ -30,7 +31,8 @@ test('local OAuth consent and device grants authorize REST/MCP and respect revoc const origin = `https://localhost:${port}`, secret = randomBytes(32).toString('hex'); const body = Buffer.from(JSON.stringify({ tenantId, userId, email, exp: Math.floor(Date.now()/1000)+600, sessionVersion: 1 })).toString('base64url'); const cookie = 'factorize_session=' + encodeURIComponent(body + '.' + createHmac('sha256', secret).update(body).digest('base64')); - child = spawn('npm', ['exec', '--workspace=factorize', '--', 'wrangler', 'dev', '--local', '--local-protocol', 'https', '--port', String(port), '--persist-to', directory, '--var', `APP_ORIGIN:${origin}`, '--var', `SESSION_SIGNING_SECRET:${secret}`, '--var', `CREDENTIAL_ENCRYPTION_KEY:${randomBytes(32).toString('base64')}`], { detached: true, stdio: ['ignore','pipe','pipe'], env: { ...process.env, CLOUDFLARE_HYPERDRIVE_LOCAL_CONNECTION_STRING_HYPERDRIVE: base.href } }); + const wranglerCli = fileURLToPath(new URL('../node_modules/wrangler/bin/wrangler.js', import.meta.url)); + child = spawn(process.execPath, [wranglerCli, 'dev', '--local', '--local-protocol', 'https', '--port', String(port), '--persist-to', directory, '--var', `APP_ORIGIN:${origin}`, '--var', `SESSION_SIGNING_SECRET:${secret}`, '--var', `CREDENTIAL_ENCRYPTION_KEY:${randomBytes(32).toString('base64')}`], { cwd: fileURLToPath(new URL('../packages/api/', import.meta.url)), detached: true, stdio: ['ignore','pipe','pipe'], env: { ...process.env, CLOUDFLARE_HYPERDRIVE_LOCAL_CONNECTION_STRING_HYPERDRIVE: base.href } }); await new Promise((resolve, reject) => { let readyOutput = ''; const deadline = setTimeout(() => reject(new Error('Local protocol Worker did not start')), 30000); const chunk = value => { readyOutput += value; if (readyOutput.includes('Ready on')) { clearTimeout(deadline); resolve(); } }; From 925c537556ae43156a886c7ef8fbd2f5b01d5145 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 00:51:17 +0000 Subject: [PATCH 13/26] GEN-2907 audit retention inventory and lock race --- docs/run-history-retention.md | 16 ++++++++++++++++ .../api/test/run-retention-repository.test.ts | 17 +++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/docs/run-history-retention.md b/docs/run-history-retention.md index bdfc845..af88ba7 100644 --- a/docs/run-history-retention.md +++ b/docs/run-history-retention.md @@ -12,6 +12,22 @@ Terminal runs whose execution-provider cleanup is incomplete use a separate boun Each alarm also removes up to 500 rows per independent-history lane: job events, completed webhook deliveries and their cascading timelines, and completed VM image verifications. Webhook rows still marked `received` and any delivery referenced by a pending GitHub verification are protected. Image verifications in `cleanup_failed` and other nonterminal states remain as cleanup ownership. Expired or used GitHub setup states, consent previews, device authorizations, OAuth continuations, password-reset tokens, revoked or expired sessions, and revoked or expired access tokens are removed in bounded batches; valid long-lived access tokens survive regardless of age. Expiring a webhook delivery removes its deduplication key, so a provider retry after the retention window may be processed as a new delivery. A retry racing deletion uses an atomic upsert to reuse or recreate its delivery record, and event/timeline writes remain transactional. +## Retention inventory audit + +The application schema and migrations classify retained rows by ownership rather than applying age deletion to every table: + +| Class | Tables | Policy | +| --- | --- | --- | +| Durable identity and configuration | `tenants`, `auth_users`, `auth_accounts`, `members`, `connections`, `linear_workspaces`, `github_installations`, `projects`, `jobs`, `triggers`, `skills`, `job_skills`, `manual_invocation_permissions`, `integration_collections`, `integration_records`, `webhook_conditions_cutover`, `fly_connections`, `fly_execution_configurations`, `execution_targets` | Preserve. User, tenant, and current configuration are outside run-history retention. | +| Run aggregate | `invocations`, `job_runs`, `runs`, `run_artifacts`, `run_trace_events`, `run_trace_cursors`, `run_trace_chunks`, `run_trace_projections`, `run_activity`, `active_claims`, `lifecycle_events`, `lifecycle_deliveries`, `run_credentials`, `credential_proxy_states`, `credential_run_grants`, `credential_run_enrollments`, `trace_replay_limits`, `trace_replay_operations` | Remove through the invocation aggregate after terminal-state, immutable-age, cleanup, finalization, and automation checks. Foreign-key cascades remove run-owned history and temporary run access together. | +| Independent operational history | `job_events`, `webhook_deliveries`, `webhook_delivery_events`, `vm_image_verifications` | Bounded 14-day cleanup of completed history; retain pending delivery, verification, or resource-cleanup ownership. | +| One-time or expiring authentication records | `github_setup_states`, `oauth_consent_previews`, `oauth_device_authorizations`, `oauth_connections`, `auth_reset_tokens`, `auth_sessions`, `access_tokens` | Bounded cleanup after use, revocation, or actual expiry. Creation age alone never removes a still-valid credential. | +| Retryable cleanup ownership | `run_retention_cleanup`, `deleted_job_provider_cleanup` | Keep until external object/provider work is acknowledged; use leases and retry. These queues do not cascade with deleted run/job rows. | +| Scheduler and migration state | `schema_migrations`, `schedule_state`, `automatic_wakes`, `wake_hints`, `tail_fingerprints`, `pending_verifications`, `job_edit_deliveries`, `unlock_key_verifications`, `gen_2416_migration_checkpoint`, `gen_2416_migration_inventory` | Preserve while active or required for correctness; expiry/acknowledgment is governed by the owning workflow, not run age. Review one-time migration bookkeeping separately after its recovery window. | +| Integration synchronization history | `integration_changes`, `integration_generation_journal`, `integration_generation_records`, `integration_generation_changes`, `integration_generation_previous_records`, `integration_generation_previous_changes` | Still requires bounded retention and a stale-consumer full-resynchronization contract. The existing `cleanupGenerations` helper is not wired to recurring maintenance; do not imply these tables are bounded yet. Current encrypted integration records and tombstones remain durable configuration. | + +The retention indexes cover candidate age/order and the audited foreign-key lookups for run activity, lifecycle events/deliveries, credential enrollments, webhook timelines, and pending verifications by delivery. The migrations create 66 tables, including migration-only bookkeeping tables; the 2026-11-11 production inventory observed 63 tables. Adding a table requires assigning it to an inventory class and documenting whether it cascades, expires independently, or represents durable state. No global age rule is safe for configuration, active work, or synchronization state. + Explicit job deletion also snapshots every incomplete run's provider configuration, handle, and start evidence into `app.deleted_job_provider_cleanup` in the same transaction as the job cascade. This independent queue lets the maintenance lane stop or reconcile provider resources after the source job and run rows are gone. A started run without a saved handle stays queued for retry and manual reconciliation; a never-started run can be removed from the queue safely. R2 artifacts for explicit job deletion continue to be deleted before the database cascade so a storage failure leaves the authoritative keys available for retry. Successful cleanup removes its retry state from the run. Failed work retains only a bounded diagnostic string and retry timestamp. Each maintenance alarm emits a structured `run_retention_maintenance` log with run sweep, artifact and independent-history deletion progress plus pending counts and oldest age for terminal-provider cleanup, deleted-job provider cleanup, artifact objects, overdue nonterminal runs, independent histories, and expired temporary records. The snapshot contains no tenant, run, or object identifiers. A failed snapshot is logged separately and does not fail maintenance. These logs provide rollout evidence; a retained time-series dashboard remains follow-up work. diff --git a/packages/api/test/run-retention-repository.test.ts b/packages/api/test/run-retention-repository.test.ts index f39d5b3..f277bb0 100644 --- a/packages/api/test/run-retention-repository.test.ts +++ b/packages/api/test/run-retention-repository.test.ts @@ -26,6 +26,23 @@ describe("RunRetentionRepository", () => { expect(txQuery.mock.calls.find(([sql]) => sql.includes("SELECT r.id run_id"))![0]).toContain("FOR UPDATE OF i,jr,r SKIP LOCKED"); }); + it("skips a candidate that loses eligibility before its aggregate lock is acquired", async () => { + const cutoff = new Date("2026-10-28T12:00:00.000Z"); + const poolQuery = vi.fn(async () => ({ rows: [{ tenant_id: "tenant", invocation_id: "invocation" }], rowCount: 1 })); + const txQuery = vi.fn(async (sql: string) => { + // A finalizer or another retention worker changed the aggregate after + // the bounded candidate scan. The locked eligibility recheck sees it + // as unavailable, so no artifacts or aggregate rows may be removed. + if (sql.includes("SELECT r.id run_id")) return { rows: [], rowCount: 0 }; + return { rows: [], rowCount: 0 }; + }); + const database = { pool: { query: poolQuery }, transaction: (work: (client: any) => unknown) => work({ query: txQuery }) } as any; + + await expect(new RunRetentionRepository(database).sweep(cutoff, 8)).resolves.toEqual({ selected: 1, deleted: 0, cleanupQueued: 0 }); + expect(txQuery).toHaveBeenCalledTimes(1); + expect(txQuery.mock.calls[0]![0]).toContain("FOR UPDATE OF i,jr,r SKIP LOCKED"); + }); + it("rejects invalid cutoffs and unbounded batch sizes before database access", async () => { const query = vi.fn(); const repository = new RunRetentionRepository({ pool: { query }, transaction: vi.fn() } as any); From 5ff8fdbb662269d3f736597446662d063edef367 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 01:07:10 +0000 Subject: [PATCH 14/26] GEN-2907 bound integration sync history --- docs/run-history-retention.md | 4 ++- .../0042_integration_history_retention.sql | 5 ++++ packages/api/src/flow-service.ts | 2 +- .../encrypted-integration-repository.ts | 8 ++++-- .../src/postgres/run-retention-repository.ts | 25 +++++++++++++++++++ packages/api/src/postgres/schema.ts | 2 +- ...ed-integration-repository.postgres.test.ts | 14 +++++++++++ .../api/test/run-retention-repository.test.ts | 4 +-- 8 files changed, 57 insertions(+), 7 deletions(-) create mode 100644 packages/api/migrations/0042_integration_history_retention.sql diff --git a/docs/run-history-retention.md b/docs/run-history-retention.md index af88ba7..46723d0 100644 --- a/docs/run-history-retention.md +++ b/docs/run-history-retention.md @@ -12,6 +12,8 @@ Terminal runs whose execution-provider cleanup is incomplete use a separate boun Each alarm also removes up to 500 rows per independent-history lane: job events, completed webhook deliveries and their cascading timelines, and completed VM image verifications. Webhook rows still marked `received` and any delivery referenced by a pending GitHub verification are protected. Image verifications in `cleanup_failed` and other nonterminal states remain as cleanup ownership. Expired or used GitHub setup states, consent previews, device authorizations, OAuth continuations, password-reset tokens, revoked or expired sessions, and revoked or expired access tokens are removed in bounded batches; valid long-lived access tokens survive regardless of age. Expiring a webhook delivery removes its deduplication key, so a provider retry after the retention window may be processed as a new delivery. A retry racing deletion uses an atomic upsert to reuse or recreate its delivery record, and event/timeline writes remain transactional. +Integration synchronization history is also bounded on each alarm. At most 500 change rows older than 14 days are removed per pass, and committed generation rollback archives are retained for seven days; abandoned staged, rolled-back, and expired generations are removed after one day. Current encrypted integration records and tombstones are preserved. `changes_pruned_through` advances atomically with each deleted change batch. A consumer whose cursor is below that floor receives a stale-revision response marked `resyncRequired`; it must fetch the current full collection before resuming incremental changes. This prevents a pruned gap from being mistaken for a complete change page. + ## Retention inventory audit The application schema and migrations classify retained rows by ownership rather than applying age deletion to every table: @@ -24,7 +26,7 @@ The application schema and migrations classify retained rows by ownership rather | One-time or expiring authentication records | `github_setup_states`, `oauth_consent_previews`, `oauth_device_authorizations`, `oauth_connections`, `auth_reset_tokens`, `auth_sessions`, `access_tokens` | Bounded cleanup after use, revocation, or actual expiry. Creation age alone never removes a still-valid credential. | | Retryable cleanup ownership | `run_retention_cleanup`, `deleted_job_provider_cleanup` | Keep until external object/provider work is acknowledged; use leases and retry. These queues do not cascade with deleted run/job rows. | | Scheduler and migration state | `schema_migrations`, `schedule_state`, `automatic_wakes`, `wake_hints`, `tail_fingerprints`, `pending_verifications`, `job_edit_deliveries`, `unlock_key_verifications`, `gen_2416_migration_checkpoint`, `gen_2416_migration_inventory` | Preserve while active or required for correctness; expiry/acknowledgment is governed by the owning workflow, not run age. Review one-time migration bookkeeping separately after its recovery window. | -| Integration synchronization history | `integration_changes`, `integration_generation_journal`, `integration_generation_records`, `integration_generation_changes`, `integration_generation_previous_records`, `integration_generation_previous_changes` | Still requires bounded retention and a stale-consumer full-resynchronization contract. The existing `cleanupGenerations` helper is not wired to recurring maintenance; do not imply these tables are bounded yet. Current encrypted integration records and tombstones remain durable configuration. | +| Integration synchronization history | `integration_changes`, `integration_generation_journal`, `integration_generation_records`, `integration_generation_changes`, `integration_generation_previous_records`, `integration_generation_previous_changes` | Change log: bounded 14-day retention with a revision floor and full-resynchronization response for stale consumers. Committed generation rollback archives: seven days; abandoned staging and completed journal records: one day. Cascading journal deletion removes generation snapshots and archives. Current encrypted integration records and tombstones remain durable configuration. | The retention indexes cover candidate age/order and the audited foreign-key lookups for run activity, lifecycle events/deliveries, credential enrollments, webhook timelines, and pending verifications by delivery. The migrations create 66 tables, including migration-only bookkeeping tables; the 2026-11-11 production inventory observed 63 tables. Adding a table requires assigning it to an inventory class and documenting whether it cascades, expires independently, or represents durable state. No global age rule is safe for configuration, active work, or synchronization state. diff --git a/packages/api/migrations/0042_integration_history_retention.sql b/packages/api/migrations/0042_integration_history_retention.sql new file mode 100644 index 0000000..0d94542 --- /dev/null +++ b/packages/api/migrations/0042_integration_history_retention.sql @@ -0,0 +1,5 @@ +ALTER TABLE app.integration_collections + ADD COLUMN changes_pruned_through bigint NOT NULL DEFAULT 0 CHECK (changes_pruned_through >= 0); + +CREATE INDEX integration_changes_retention + ON app.integration_changes(created_at,tenant_id,revision); diff --git a/packages/api/src/flow-service.ts b/packages/api/src/flow-service.ts index 0b1ede5..93a390f 100644 --- a/packages/api/src/flow-service.ts +++ b/packages/api/src/flow-service.ts @@ -523,7 +523,7 @@ export class ApiService { private async bearerWrite(input: { id: string; expectedRevision: number; idempotencyKey: string; name: string; origin: string; description: string | null; encryptedPayload?: string; operation: "upsert" | "delete" }) { try { return await this.encryptedIntegrations().write({ ...input, kind: "bearer" }); } catch (error) { - if (error instanceof StaleIntegrationRevisionError) throw new ServiceError(409, "stale_revision", `The integration collection changed. Current revision is ${error.currentRevision}.`); + if (error instanceof StaleIntegrationRevisionError) throw new ServiceError(409, "stale_revision", `${error.resyncRequired ? "Integration change history expired; fetch the current collection to resynchronize. " : "The integration collection changed. "}Current revision is ${error.currentRevision}.`); if (error instanceof IntegrationIdempotencyConflictError) throw new ServiceError(409, "conflict", "The idempotency key was already used for a different integration change."); if (error instanceof IntegrationValidationError) throw new ServiceError(400, "invalid_request", error.message); throw error; diff --git a/packages/api/src/postgres/encrypted-integration-repository.ts b/packages/api/src/postgres/encrypted-integration-repository.ts index 095f6d0..9e02d3f 100644 --- a/packages/api/src/postgres/encrypted-integration-repository.ts +++ b/packages/api/src/postgres/encrypted-integration-repository.ts @@ -20,7 +20,7 @@ export type GenerationRecord = IntegrationMetadata & { revision: number; encrypt export type GenerationChange = IntegrationMetadata & { revision: number; idempotencyKey: string; operation: "upsert" | "delete"; encryptedPayload: string | null }; export class StaleIntegrationRevisionError extends Error { - constructor(readonly currentRevision: number) { super("Integration collection revision is stale"); this.name = "StaleIntegrationRevisionError"; } + constructor(readonly currentRevision: number, readonly resyncRequired = false) { super(resyncRequired ? "Integration change history expired; fetch the current collection before continuing" : "Integration collection revision is stale"); this.name = "StaleIntegrationRevisionError"; } } export class IntegrationIdempotencyConflictError extends Error { constructor() { super("Integration idempotency key was already used for a different change"); this.name = "IntegrationIdempotencyConflictError"; } @@ -129,7 +129,11 @@ export class EncryptedIntegrationRepository { async changesSince(revision: number, limit = 100): Promise { if (!Number.isSafeInteger(revision) || revision < 0) throw new IntegrationValidationError("revision must be a non-negative safe integer"); if (!Number.isSafeInteger(limit) || limit < 1 || limit > 500) throw new IntegrationValidationError("limit must be between 1 and 500"); - const { rows } = await this.database.pool.query("SELECT revision,operation,record_id,kind,name,origin,description,encrypted_payload FROM app.integration_changes WHERE tenant_id=$1 AND revision>$2 ORDER BY revision LIMIT $3", [this.tenantId, revision, limit]); + const rows = await this.database.transaction(async client => { + const collection = (await client.query<{ revision: string | number; changes_pruned_through: string | number }>("SELECT revision,changes_pruned_through FROM app.integration_collections WHERE tenant_id=$1 FOR SHARE", [this.tenantId])).rows[0]; + if (collection && revision < safeRevision(collection.changes_pruned_through)) throw new StaleIntegrationRevisionError(safeRevision(collection.revision), true); + return (await client.query("SELECT revision,operation,record_id,kind,name,origin,description,encrypted_payload FROM app.integration_changes WHERE tenant_id=$1 AND revision>$2 ORDER BY revision LIMIT $3", [this.tenantId, revision, limit])).rows; + }); return Promise.all(rows.map(async row => ({ id: row.record_id, kind: row.kind, name: row.name, origin: row.origin, description: row.description, revision: safeRevision(row.revision), operation: row.operation, encryptedPayload: row.encrypted_payload === null ? null : await decrypt(row.encrypted_payload, this.serverEncryptionKey) }))); } diff --git a/packages/api/src/postgres/run-retention-repository.ts b/packages/api/src/postgres/run-retention-repository.ts index c77e6c0..37643f4 100644 --- a/packages/api/src/postgres/run-retention-repository.ts +++ b/packages/api/src/postgres/run-retention-repository.ts @@ -52,6 +52,14 @@ export class RunRetentionRepository { SELECT 'vm_image_verifications'::text,count(*)::int,floor(extract(epoch FROM now()-min(created_at)))::int FROM app.vm_image_verifications WHERE status IN ('succeeded','failed') AND created_at < now()-interval '14 days' UNION ALL + SELECT 'integration_changes'::text,count(*)::int,floor(extract(epoch FROM now()-min(created_at)))::int + FROM app.integration_changes WHERE created_at < now()-interval '14 days' + UNION ALL + SELECT 'integration_generations'::text,count(*)::int,floor(extract(epoch FROM now()-min(created_at)))::int + FROM app.integration_generation_journal WHERE (state='staged' AND created_at < now()-interval '1 day') + OR (state='committed' AND committed_at < now()-interval '7 days') + OR (state IN ('rolled_back','expired') AND created_at < now()-interval '1 day') + UNION ALL SELECT 'expired_temporary_records'::text, (SELECT count(*)::int FROM app.github_setup_states WHERE expires_at < now() OR used_at IS NOT NULL) +(SELECT count(*)::int FROM app.oauth_consent_previews WHERE expires_at < now() OR used_at IS NOT NULL) @@ -124,6 +132,23 @@ export class RunRetentionRepository { SELECT tenant_id,id FROM app.access_tokens WHERE expires_at < now() OR revoked_at IS NOT NULL ORDER BY expires_at,tenant_id,id LIMIT $1 FOR UPDATE SKIP LOCKED ) DELETE FROM app.access_tokens t USING expired x WHERE t.tenant_id=x.tenant_id AND t.id=x.id`], + // Keep the incremental cursor honest: any consumer older than this + // floor must fetch the current collection instead of applying a gap. + ["integration_changes", `WITH expired AS ( + SELECT tenant_id,revision FROM app.integration_changes WHERE created_at < now()-interval '14 days' + ORDER BY created_at,tenant_id,revision LIMIT $1 + ), floors AS (SELECT tenant_id,max(revision) AS revision FROM expired GROUP BY tenant_id), + advanced AS (UPDATE app.integration_collections c SET changes_pruned_through=GREATEST(c.changes_pruned_through,f.revision) + FROM floors f WHERE c.tenant_id=f.tenant_id RETURNING c.tenant_id) + DELETE FROM app.integration_changes c USING expired x WHERE c.tenant_id=x.tenant_id AND c.revision=x.revision + AND EXISTS (SELECT 1 FROM advanced) RETURNING c.revision`], + ["integration_generations", `WITH expired AS ( + SELECT tenant_id,generation_id FROM app.integration_generation_journal + WHERE (state='staged' AND created_at < now()-interval '1 day') + OR (state='committed' AND committed_at < now()-interval '7 days') + OR (state IN ('rolled_back','expired') AND created_at < now()-interval '1 day') + ORDER BY created_at,tenant_id,generation_id LIMIT $1 FOR UPDATE SKIP LOCKED + ) DELETE FROM app.integration_generation_journal j USING expired x WHERE j.tenant_id=x.tenant_id AND j.generation_id=x.generation_id`], ]; const result: IndependentHistoryCleanupResult = []; for (const [queue, statement] of lanes) { diff --git a/packages/api/src/postgres/schema.ts b/packages/api/src/postgres/schema.ts index 8cc970d..554df98 100644 --- a/packages/api/src/postgres/schema.ts +++ b/packages/api/src/postgres/schema.ts @@ -42,7 +42,7 @@ export const accessTokens = app.table("access_tokens", { export const connections = app.table("connections", { tenantId: uuid("tenant_id").notNull().references(() => tenants.id, { onDelete: "cascade" }), kind: text("kind").notNull(), encryptedValue: text("encrypted_value").notNull(), updatedAt: updatedAt() }, t => [primaryKey({ columns: [t.tenantId, t.kind] })]); export const integrationCollections = app.table("integration_collections", { - tenantId: uuid("tenant_id").primaryKey().references(() => tenants.id, { onDelete: "cascade" }), revision: bigint("revision", { mode: "number" }).notNull().default(0), createdAt: createdAt(), updatedAt: updatedAt(), + tenantId: uuid("tenant_id").primaryKey().references(() => tenants.id, { onDelete: "cascade" }), revision: bigint("revision", { mode: "number" }).notNull().default(0), changesPrunedThrough: bigint("changes_pruned_through", { mode: "number" }).notNull().default(0), createdAt: createdAt(), updatedAt: updatedAt(), }); export const integrationRecords = app.table("integration_records", { tenantId: uuid("tenant_id").notNull().references(() => integrationCollections.tenantId, { onDelete: "cascade" }), id: uuid("id").notNull(), kind: text("kind").$type<"bearer" | "model">().notNull(), name: text("name").notNull(), origin: text("origin"), description: text("description"), revision: bigint("revision", { mode: "number" }).notNull(), encryptedPayload: text("encrypted_payload"), deletedAt: timestamp("deleted_at", { withTimezone: true, mode: "date" }), createdAt: createdAt(), updatedAt: updatedAt(), diff --git a/packages/api/test/encrypted-integration-repository.postgres.test.ts b/packages/api/test/encrypted-integration-repository.postgres.test.ts index ee0531c..d4911b8 100644 --- a/packages/api/test/encrypted-integration-repository.postgres.test.ts +++ b/packages/api/test/encrypted-integration-repository.postgres.test.ts @@ -3,6 +3,7 @@ import { readdir, readFile } from "node:fs/promises"; import pg from "pg"; import { decrypt } from "../src/crypto"; import { Database } from "../src/postgres/database"; +import { RunRetentionRepository } from "../src/postgres/run-retention-repository"; import { EncryptedIntegrationRepository, IntegrationIdempotencyConflictError, StaleIntegrationGenerationError, StaleIntegrationRevisionError } from "../src/postgres/encrypted-integration-repository"; const url = process.env.AUTH_TEST_DATABASE_URL; @@ -107,6 +108,19 @@ describe.skipIf(!url)("encrypted integration persistence (PostgreSQL)", () => { await expect(repository.write({ ...first, name: "Different operation with same key" })).rejects.toBeInstanceOf(IntegrationIdempotencyConflictError); }); + it("prunes bounded old change history and tells stale consumers to reload the full collection", async () => { + const { tenantId, repository } = await freshRepository(); + const encryptedPayload = await userEncrypt("retained synthetic integration"); + await repository.write({ id: crypto.randomUUID(), idempotencyKey: crypto.randomUUID(), expectedRevision: 0, operation: "upsert", kind: "model", name: "Retained", origin: null, description: null, encryptedPayload }); + await db.pool.query("UPDATE app.integration_changes SET created_at=now()-interval '15 days' WHERE tenant_id=$1", [tenantId]); + + const result = await new RunRetentionRepository(db).cleanupIndependentHistory(); + expect(result.find(row => row.queue === "integration_changes")?.deleted).toBe(1); + await expect(repository.changesSince(0)).rejects.toMatchObject({ name: "StaleIntegrationRevisionError", currentRevision: 1, resyncRequired: true }); + expect(await repository.changesSince(1)).toEqual([]); + expect((await repository.records()).map(record => record.name)).toEqual(["Retained"]); + }); + it("stages ciphertext only, atomically replaces the whole generation, and makes a lost commit reply retryable", async () => { const { repository, tenantId } = await freshRepository(); const id = crypto.randomUUID(), original = await userEncrypt("old credential fixture"), rotated = await userEncrypt("rotated credential fixture"); diff --git a/packages/api/test/run-retention-repository.test.ts b/packages/api/test/run-retention-repository.test.ts index f277bb0..8d7da07 100644 --- a/packages/api/test/run-retention-repository.test.ts +++ b/packages/api/test/run-retention-repository.test.ts @@ -120,9 +120,9 @@ describe("RunRetentionRepository", () => { const statements: string[] = []; const query = vi.fn(async (sql: string) => { statements.push(sql); return { rows: [], rowCount: 7 }; }); const result = await new RunRetentionRepository({ pool: { query } } as any).cleanupIndependentHistory(); - expect(result).toHaveLength(10); + expect(result).toHaveLength(12); expect(result.every(lane => lane.deleted === 7)).toBe(true); - expect(query).toHaveBeenCalledTimes(10); + expect(query).toHaveBeenCalledTimes(12); expect(query).toHaveBeenCalledWith(expect.stringContaining("FOR UPDATE SKIP LOCKED"), [500]); expect(statements.find(sql => sql.includes("webhook_deliveries d"))).toContain("d.outcome <> 'received'"); expect(statements.find(sql => sql.includes("webhook_deliveries d"))).toContain("app.pending_verifications"); From 5d85955688a21c4bd1dec66b7331ecc5cd27c787 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 01:32:37 +0000 Subject: [PATCH 15/26] GEN-2907 stop polling expired run views --- .../src/features/runs/continuous-trace.tsx | 13 +++++----- packages/app/src/features/runs/detail.tsx | 6 ++++- packages/app/src/features/runs/queries.ts | 24 ++++++++++++------- packages/app/test/runs.browser.test.mjs | 15 ++++++++++++ 4 files changed, 42 insertions(+), 16 deletions(-) diff --git a/packages/app/src/features/runs/continuous-trace.tsx b/packages/app/src/features/runs/continuous-trace.tsx index ce00948..d0e585b 100644 --- a/packages/app/src/features/runs/continuous-trace.tsx +++ b/packages/app/src/features/runs/continuous-trace.tsx @@ -4,6 +4,7 @@ import { useEffect, useLayoutEffect, useMemo, useRef, useState, type TouchEvent import { useInfiniteQuery, useQuery, type InfiniteData } from "@tanstack/react-query"; import { api } from "factorize-api-client"; import { messageOf } from "../auth/session"; +import { RunNotFoundError } from "./queries"; import type { operations } from "factorize-api-client"; import { defaultRangeExtractor, useVirtualizer } from "@tanstack/react-virtual"; @@ -44,12 +45,12 @@ export function ContinuousTrace({ runId, revision, active, onReset, preview = fa initialPageParam: { mode: "recent" } as PageParam, queryFn: async ({ signal, pageParam }) => { const query = pageParam.mode === "recent" ? { recent: true, revision: pageParam.revision, limit: preview ? 20 : 100 } : { before: pageParam.cursor, revision: pageParam.revision, limit: 100 }; - const { data, error } = await api.GET("/api/v1/runs/{runId}/trace-pages", { signal, params: { path: { runId }, query } }); - if (!data || error) throw new Error(messageOf(error)); + const { data, error, response } = await api.GET("/api/v1/runs/{runId}/trace-pages", { signal, params: { path: { runId }, query } }); + if (!data || error) throw response.status === 404 ? new RunNotFoundError() : new Error(messageOf(error)); return data; }, enabled: visible && !!revision, getNextPageParam: page => page.previousCursor == null ? undefined : { mode: "before", cursor: page.previousCursor, revision: page.revision } as PageParam, - staleTime: Infinity, retry: 1, + staleTime: Infinity, retry: (failureCount, error) => !(error instanceof RunNotFoundError) && failureCount < 1, }); const pages = trace.data?.pages ?? []; const items = useMemo(() => { @@ -72,10 +73,10 @@ export function ContinuousTrace({ runId, revision, active, onReset, preview = fa queryKey: ["runs", runId, "conversation-tail", revision], enabled: !preview && active && visible && !!trace.data, queryFn: async ({ signal }) => { - const { data, error } = await api.GET("/api/v1/runs/{runId}/trace-pages", { signal, params: { path: { runId }, query: { after: latestSequence, revision, limit: 100 } } }); - if (!data || error) throw new Error(messageOf(error)); return data; + const { data, error, response } = await api.GET("/api/v1/runs/{runId}/trace-pages", { signal, params: { path: { runId }, query: { after: latestSequence, revision, limit: 100 } } }); + if (!data || error) throw response.status === 404 ? new RunNotFoundError() : new Error(messageOf(error)); return data; }, - refetchIntervalInBackground: false, refetchInterval: query => query.state.error ? 10_000 : 2_000, retry: 1, + refetchIntervalInBackground: false, refetchInterval: query => query.state.error instanceof RunNotFoundError ? false : query.state.error ? 10_000 : 2_000, retry: (failureCount, error) => !(error instanceof RunNotFoundError) && failureCount < 1, }); useEffect(() => { const changed = live.data?.reset ? live.data : pages.find(page => page.reset || page.revision !== revision); if (changed) onReset(changed.revision); }, [live.data, pages, revision, onReset]); useEffect(() => { if (live.data && !live.data.reset && live.data.revision === revision && live.data.items.length) setLiveItems(previous => [...previous, ...live.data!.items]); }, [live.data, revision]); diff --git a/packages/app/src/features/runs/detail.tsx b/packages/app/src/features/runs/detail.tsx index 836abef..372e0ee 100644 --- a/packages/app/src/features/runs/detail.tsx +++ b/packages/app/src/features/runs/detail.tsx @@ -6,7 +6,7 @@ import { useEffect, useState, useRef } from "react"; import { Link, useLocation, useParams, useNavigate } from "@tanstack/react-router"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { api } from "factorize-api-client"; -import { runPageQuery, runStatusQuery } from "./queries"; +import { runPageQuery, runStatusQuery, RunNotFoundError } from "./queries"; import { jobQuery } from "../jobs/queries"; import { JobNav } from "../jobs/job-nav"; import { ResourceSwitcher } from "../../shared/resource-switcher"; @@ -34,6 +34,10 @@ export function RunDetail() { const [renaming, setRenaming] = useState(false), [nameDraft, setNameDraft] = useState(""), [nameExpected, setNameExpected] = useState(""), [nameError, setNameError] = useState(""); const active = !status.data || !["succeeded", "failed", "stopped"].includes(status.data.state) || status.data.finalizing; const finalizedSnapshot = useRef(""); + useEffect(() => { + if (!(status.error instanceof RunNotFoundError)) return; + void cache.removeQueries({ queryKey: ["runs", runId], predicate: query => query.queryKey[2] !== "status" }); + }, [status.error, cache, runId]); useEffect(() => { if (!status.data) return; const next = `${status.data.state}:${status.data.finalizing}:${status.data.artifact_state}`; diff --git a/packages/app/src/features/runs/queries.ts b/packages/app/src/features/runs/queries.ts index 23642e5..57f470a 100644 --- a/packages/app/src/features/runs/queries.ts +++ b/packages/app/src/features/runs/queries.ts @@ -2,23 +2,29 @@ import { api } from "factorize-api-client"; import { keepPreviousData, queryOptions } from "@tanstack/react-query"; import { messageOf } from "../auth/session"; import type { RunStatus } from "../../shared/ui"; +export class RunNotFoundError extends Error { + constructor() { super("This run is no longer available. It may have expired from retained history."); this.name = "RunNotFoundError"; } +} +function queryError(error: unknown, status?: number): Error { + return status === 404 ? new RunNotFoundError() : new Error(messageOf(error)); +} export const runStatusQuery = (runId: string) => queryOptions({ queryKey: ["runs", runId, "status"], queryFn: async ({ signal }) => { - const { data, error } = await api.GET("/api/v1/runs/{runId}/status", { signal, params: { path: { runId } } }); - if (!data || error) throw new Error(messageOf(error)); return data; - }, retry: 1, refetchIntervalInBackground: false, - refetchInterval: query => query.state.data && ["succeeded", "failed", "stopped"].includes(query.state.data.state) && !query.state.data.finalizing ? false : query.state.error ? 10_000 : 2_000, + const { data, error, response } = await api.GET("/api/v1/runs/{runId}/status", { signal, params: { path: { runId } } }); + if (!data || error) throw queryError(error, response.status); return data; + }, retry: (failureCount, error) => !(error instanceof RunNotFoundError) && failureCount < 1, refetchIntervalInBackground: false, + refetchInterval: query => query.state.error instanceof RunNotFoundError ? false : query.state.data && ["succeeded", "failed", "stopped"].includes(query.state.data.state) && !query.state.data.finalizing ? false : query.state.error ? 10_000 : 2_000, }); export const runTraceQuery = (runId: string, revision: string, after: number, polling: boolean) => queryOptions({ queryKey: ["runs", runId, "trace", revision, after], queryFn: async ({ signal }) => { - const { data, error } = await api.GET("/api/v1/runs/{runId}/trace-pages", { signal, params: { path: { runId }, query: { revision, after, limit: 100 } } }); - if (!data || error) throw new Error(messageOf(error)); return data; - }, retry: 1, enabled: !!revision, refetchIntervalInBackground: false, refetchInterval: polling ? 2_000 : false, + const { data, error, response } = await api.GET("/api/v1/runs/{runId}/trace-pages", { signal, params: { path: { runId }, query: { revision, after, limit: 100 } } }); + if (!data || error) throw queryError(error, response.status); return data; + }, retry: (failureCount, error) => !(error instanceof RunNotFoundError) && failureCount < 1, enabled: !!revision, refetchIntervalInBackground: false, refetchInterval: query => query.state.error instanceof RunNotFoundError ? false : polling ? 2_000 : false, }); -export const runDetailQuery = (runId: string) => queryOptions({ queryKey: ["runs", runId, "detail"], queryFn: async ({ signal }) => { const { data, error } = await api.GET("/api/v1/runs/{runId}", { signal, params: { path: { runId } } }); if (!data || error) throw new Error(messageOf(error)); return data; }, staleTime: 60_000 }); -export const diagnosticsQuery = (runId: string) => queryOptions({ queryKey: ["runs", runId, "diagnostics"], queryFn: async ({ signal }) => { const { data, error } = await api.GET("/api/v1/runs/{runId}/diagnostics", { signal, params: { path: { runId } } }); if (!data || error) throw new Error(messageOf(error)); return data; }, staleTime: 30_000 }); +export const runDetailQuery = (runId: string) => queryOptions({ queryKey: ["runs", runId, "detail"], queryFn: async ({ signal }) => { const { data, error, response } = await api.GET("/api/v1/runs/{runId}", { signal, params: { path: { runId } } }); if (!data || error) throw queryError(error, response.status); return data; }, retry: (failureCount, error) => !(error instanceof RunNotFoundError) && failureCount < 1, staleTime: 60_000 }); +export const diagnosticsQuery = (runId: string) => queryOptions({ queryKey: ["runs", runId, "diagnostics"], queryFn: async ({ signal }) => { const { data, error, response } = await api.GET("/api/v1/runs/{runId}/diagnostics", { signal, params: { path: { runId } } }); if (!data || error) throw queryError(error, response.status); return data; }, retry: (failureCount, error) => !(error instanceof RunNotFoundError) && failureCount < 1, staleTime: 30_000 }); export const runPageQuery = ({ jobId, projectIds, jobIds, cursor, limit, sort, direction, states }: { jobId?: string; projectIds?: string[]; jobIds?: string[]; cursor?: string; limit: number; sort?: "job" | "run" | "status" | "created" | "agent"; direction?: "asc" | "desc"; states?: RunStatus[] }) => queryOptions({ queryKey: ["runs", "page", jobId, projectIds, jobIds, cursor, limit, sort, direction, states], queryFn: async ({ signal }) => { const { data, error } = await api.GET("/api/v1/runs", { signal, params: { query: { jobId: jobIds ?? jobId, projectId: projectIds, cursor, limit, sort, direction, state: states } } }); if (!data || error) throw new Error(messageOf(error)); return data; }, diff --git a/packages/app/test/runs.browser.test.mjs b/packages/app/test/runs.browser.test.mjs index 29ef3ca..fb02f91 100644 --- a/packages/app/test/runs.browser.test.mjs +++ b/packages/app/test/runs.browser.test.mjs @@ -31,6 +31,21 @@ test("individual run keeps its parent project and job navigation context", async assert.equal(await page.getByRole("navigation", { name: "Breadcrumb" }).count(), 0); await context.close(); }); +test("an expired run shows a missing state, stops polling, and never loads retained details", async () => { + const { context, page } = await setup(); let statusReads = 0, detailReads = 0, traceReads = 0; + await page.clock.install({ time: new Date(time) }); + await page.route(`**/api/v1/runs/${runId}/status`, route => { statusReads++; return route.fulfill({ status: 404, json: { error: { code: "not_found", message: "Not found" } } }); }); + await page.route(`**/api/v1/runs/${runId}`, route => { detailReads++; return route.fulfill({ status: 404, json: { error: { code: "not_found", message: "Not found" } } }); }); + await page.route(`**/api/v1/runs/${runId}/trace-pages?**`, route => { traceReads++; return route.fulfill({ status: 404, json: { error: { code: "not_found", message: "Not found" } } }); }); + await page.goto(origin + `/job-runs/${runId}`); + await page.getByRole("alert").filter({ hasText: "This run is no longer available" }).waitFor(); + const readsAtMissingState = statusReads; + await page.clock.runFor(30_000); + assert.equal(statusReads, readsAtMissingState, "a definitive 404 must not poll again"); + assert.equal(detailReads, 0, "an expired run must not load prompt or provenance data"); + assert.equal(traceReads, 0, "an expired run must not load trace data"); + await context.close(); +}); test("run page polling backs off when idle, discovers runs, and stays fast while active", async () => { const idle = await setup(); let idleCount = 0; await idle.page.clock.install({ time: new Date(time) }); From 3d6f5ce9c29dda0cbe809b5eace1451d0ff0a58a Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 01:49:11 +0000 Subject: [PATCH 16/26] GEN-2907 cover expired REST and MCP reads --- .../run-retention-missing-boundary.test.ts | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 packages/api/test/run-retention-missing-boundary.test.ts diff --git a/packages/api/test/run-retention-missing-boundary.test.ts b/packages/api/test/run-retention-missing-boundary.test.ts new file mode 100644 index 0000000..fd9910b --- /dev/null +++ b/packages/api/test/run-retention-missing-boundary.test.ts @@ -0,0 +1,48 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { ApiService, ServiceError } from "../src/flow-service"; +import { protectedApiFetch } from "../src/protected-api"; + +vi.mock("cloudflare:workers", () => ({ WorkerEntrypoint: class {} })); +afterEach(() => vi.restoreAllMocks()); + +const runId = "00000000-0000-4000-8000-000000000001"; +const auth = { tenantId: "tenant-a", userId: "owner", authMethod: "bearer", scopes: ["runs:read"] } as any; +const env = { APP_ORIGIN: "https://example.com" } as any; + +function request(transport: "REST" | "MCP", pathOrTool: string, args: Record = {}) { + const mcp = transport === "MCP"; + return protectedApiFetch(new Request(`https://example.com${mcp ? "/mcp" : pathOrTool}`, { + method: mcp ? "POST" : "GET", + headers: { Host: "example.com", "Content-Type": "application/json", Accept: "application/json, text/event-stream" }, + ...(mcp ? { body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name: pathOrTool, arguments: { runId, ...args } } }) } : {}), + }), env, auth, {} as any); +} + +describe("retained-history missing-run API boundaries", () => { + it("returns authoritative not_found responses for expired REST reads", async () => { + vi.spyOn(ApiService.prototype, "getRunStatus").mockRejectedValue(new ServiceError(404, "not_found", "Run not found")); + vi.spyOn(ApiService.prototype, "getRun").mockRejectedValue(new ServiceError(404, "not_found", "Run not found")); + vi.spyOn(ApiService.prototype, "getRunDiagnostics").mockRejectedValue(new ServiceError(404, "not_found", "Run not found")); + vi.spyOn(ApiService.prototype, "getRunTrace").mockRejectedValue(new ServiceError(404, "not_found", "Run not found")); + + const reads = [ + ["/api/v1/runs/" + runId + "/status", "getRunStatus"], + ["/api/v1/runs/" + runId, "getRun"], + ["/api/v1/runs/" + runId + "/diagnostics", "getRunDiagnostics"], + ["/api/v1/runs/" + runId + "/trace?after=0&limit=10", "getRunTrace"], + ] as const; + for (const [path] of reads) { + const response = await request("REST", path); + expect(response.status).toBe(404); + expect(await response.json()).toEqual({ error: { code: "not_found", message: "Run not found" } }); + } + }); + + it.each(["get_run", "get_run_diagnostics"] as const)("preserves missing history as not_found over MCP tool %s", async tool => { + const method = tool === "get_run" ? "getRun" : "getRunDiagnostics"; + vi.spyOn(ApiService.prototype, method).mockRejectedValue(new ServiceError(404, "not_found", "Run not found")); + const response = await request("MCP", tool); + expect(response.status).toBe(200); + expect(await response.text()).toContain("not_found: Run not found"); + }); +}); From 7fcdff41529ab7ad43db2cc1000dfd3c7eda1c07 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 02:00:52 +0000 Subject: [PATCH 17/26] GEN-2907 add retention pause switch --- docs/run-history-retention.md | 2 +- packages/api/src/postgres/run-scheduler.ts | 16 ++++++++++ packages/api/src/types.ts | 2 ++ packages/api/test/run-retention-pause.test.ts | 29 +++++++++++++++++++ 4 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 packages/api/test/run-retention-pause.test.ts diff --git a/docs/run-history-retention.md b/docs/run-history-retention.md index 46723d0..09ac5ab 100644 --- a/docs/run-history-retention.md +++ b/docs/run-history-retention.md @@ -36,7 +36,7 @@ Successful cleanup removes its retry state from the run. Failed work retains onl ## Rollout and recovery -The retention sweep is intentionally not enabled by a schema migration and no data purge runs during migration. Before enabling or increasing destructive work, refresh the read-only inventory and dry-run eligible, active/exempt, cleanup-blocked, pending-automation and orphan-artifact cohorts. Deploy additive schema and cleanup code first. Use bounded batches and an operational pause switch; retain the pause until provider reconciliation, upload/trace race tests and the remaining rollout gates are ready. Maintenance runs from a heartbeat no more than 15 minutes away; cleanup lag is bounded by that cadence, each lane's batch cap, and any older backlog ahead of new work. +The retention sweep is intentionally not enabled by a schema migration and no data purge runs during migration. Before enabling or increasing destructive work, refresh the read-only inventory and dry-run eligible, active/exempt, cleanup-blocked, pending-automation and orphan-artifact cohorts. Deploy additive schema and cleanup code first. Set the Worker variable `RUN_HISTORY_RETENTION_PAUSED=true` to pause run deletion, R2 artifact deletion, and independent-history expiry. The scheduler continues its read-only backlog snapshot and terminal provider resource reconciliation; the latter completes normal finalization and is required before an overdue run can expire. Remove the variable or set it to `false` to resume bounded retention work. Keep retention paused until provider reconciliation, upload/trace race tests and the remaining rollout gates are ready. Maintenance runs from a heartbeat no more than 15 minutes away; cleanup lag is bounded by that cadence, each lane's batch cap, and any older backlog ahead of new work. Resume interrupted cleanup by allowing the next alarm to reclaim expired leases. Do not clear a provider handle or cleanup retry record after an error. A provider-confirmed absent resource is idempotently complete. Recheck ownership before deleting any artifact classified as orphaned, and use a grace period so in-flight uploads can commit receipts. Report before/after counts, oldest backlog age, retries, provider results, and pending external deletions. Database deletion is atomic per run, but PostgreSQL space reuse and external object deletion are asynchronous. diff --git a/packages/api/src/postgres/run-scheduler.ts b/packages/api/src/postgres/run-scheduler.ts index 8ec5875..077ca64 100644 --- a/packages/api/src/postgres/run-scheduler.ts +++ b/packages/api/src/postgres/run-scheduler.ts @@ -34,7 +34,23 @@ export class RunScheduler { for (let count = 0; count < 20; count++) { const run = await this.runs.claimNext(); if (!run) break; await this.launch(run).catch(error => this.fail(run, error, "launch")); } const retention = new RunRetentionRepository(this.database); await this.reconcileProviderCleanup(); + await this.runRetentionMaintenance(retention); + } + + private async runRetentionMaintenance(retention = new RunRetentionRepository(this.database)): Promise { const cutoff = runRetentionCutoff(new Date()); + if (this.env.RUN_HISTORY_RETENTION_PAUSED?.trim().toLowerCase() === "true") { + // Keep the read-only backlog signal while pausing all automatic history + // and object deletion. Provider cleanup above remains active because it + // is required resource finalization, not history expiration. + try { + const backlog = await retention.backlogSnapshot(); + console.info(JSON.stringify({ event: "run_retention_maintenance", cutoff: cutoff.toISOString(), paused: true, backlog })); + } catch (error) { + console.warn(JSON.stringify({ event: "run_retention_backlog_snapshot_failed", message: error instanceof Error ? error.message : "unknown error", paused: true })); + } + return; + } const sweep = await retention.sweep(cutoff, 50); const artifacts = await retention.processArtifactCleanup(this.env.RUN_ARTIFACTS); const independentHistory = await retention.cleanupIndependentHistory(); diff --git a/packages/api/src/types.ts b/packages/api/src/types.ts index 3aa3b95..1fc1e39 100644 --- a/packages/api/src/types.ts +++ b/packages/api/src/types.ts @@ -27,6 +27,8 @@ export interface Env { POSTMARK_FROM_EMAIL?: string; POSTMARK_MESSAGE_STREAM?: string; RUN_ARTIFACTS?: R2Bucket; + /** Pause automatic retention and artifact-history deletion while retaining maintenance visibility. */ + RUN_HISTORY_RETENTION_PAUSED?: string; TRACE_PRIMARY_MODE?: "execution_stream" | "native_session"; TRACE_STREAM_TENANTS?: string; /** OpenAI-compatible Codex endpoint configured for the execution environment. */ diff --git a/packages/api/test/run-retention-pause.test.ts b/packages/api/test/run-retention-pause.test.ts new file mode 100644 index 0000000..170245e --- /dev/null +++ b/packages/api/test/run-retention-pause.test.ts @@ -0,0 +1,29 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { RunRetentionRepository } from "../src/postgres/run-retention-repository"; +import { RunScheduler } from "../src/postgres/run-scheduler"; + +afterEach(() => vi.restoreAllMocks()); + +describe("run history retention pause switch", () => { + it("keeps backlog visibility but skips history and artifact deletion while paused", async () => { + const backlog = [{ queue: "artifact_objects", pending: 4, oldestAgeSeconds: 900 }]; + const snapshot = vi.spyOn(RunRetentionRepository.prototype, "backlogSnapshot").mockResolvedValue(backlog); + const sweep = vi.spyOn(RunRetentionRepository.prototype, "sweep"); + const artifacts = vi.spyOn(RunRetentionRepository.prototype, "processArtifactCleanup"); + const history = vi.spyOn(RunRetentionRepository.prototype, "cleanupIndependentHistory"); + const info = vi.spyOn(console, "info").mockImplementation(() => {}); + const scheduler: any = new RunScheduler({} as any, { RUN_HISTORY_RETENTION_PAUSED: " TRUE " } as any); + + await scheduler.runRetentionMaintenance(new RunRetentionRepository({} as any)); + + expect(snapshot).toHaveBeenCalledOnce(); + expect(sweep).not.toHaveBeenCalled(); + expect(artifacts).not.toHaveBeenCalled(); + expect(history).not.toHaveBeenCalled(); + expect(JSON.parse(info.mock.calls[0]![0] as string)).toMatchObject({ + event: "run_retention_maintenance", + paused: true, + backlog, + }); + }); +}); From 41aa7a983d100debff3c06104744537c3c96d0d2 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 02:29:40 +0000 Subject: [PATCH 18/26] GEN-2907 cover retention upload race --- .../api/test/trace-source.postgres.test.ts | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/packages/api/test/trace-source.postgres.test.ts b/packages/api/test/trace-source.postgres.test.ts index ef37692..b880944 100644 --- a/packages/api/test/trace-source.postgres.test.ts +++ b/packages/api/test/trace-source.postgres.test.ts @@ -14,6 +14,7 @@ import { ExeVmBackend } from "../src/exe-vm-backend"; import { agentDriver } from "../src/agent-driver"; import { IdentityRepository } from "../src/postgres/identity-repository"; import { authenticateRunCredential, issueRunCredential } from "../src/run-credentials"; +import { RunRetentionRepository } from "../src/postgres/run-retention-repository"; const url = process.env.TRACE_TEST_DATABASE_URL ?? process.env.AUTH_TEST_DATABASE_URL; const zero = "0".repeat(64), secret = "test-signing-secret"; @@ -186,6 +187,42 @@ describe.skipIf(!url)("trace sources with PostgreSQL", () => { expect((await new TraceProjectionRepository(database, f.tenantId).diagnostics(f.runId)).projection.reconciliation).toMatchObject({ state: "no_live_cursor", codexCompletion: "missing_turn_completed" }); }); + it("keeps durable cleanup ownership when retention deletes a run during artifact upload", async () => { + const f = await fixture(); + const { job_id: jobId, invocation_id: invocationId } = (await database.pool.query("SELECT job_id,invocation_id FROM app.runs WHERE tenant_id=$1 AND id=$2", [f.tenantId, f.runId])).rows[0]; + await database.pool.query("UPDATE app.runs SET state='succeeded',vm_cleanup_complete=true,artifact_state='stored',created_at=now()-interval '15 days' WHERE tenant_id=$1 AND id=$2", [f.tenantId, f.runId]); + await database.pool.query("INSERT INTO app.job_runs(tenant_id,id,job_id,invocation_id,state,encrypted_prompt) VALUES ($1,$2,$3,$4,'succeeded','test')", [f.tenantId, f.runId, jobId, invocationId]); + + let resumeUpload!: () => void; + let signalPut!: () => void; + const putStarted = new Promise(resolve => { signalPut = resolve; }); + const uploadResumed = new Promise(resolve => { resumeUpload = resolve; }); + const originalPut = f.env.RUN_ARTIFACTS.put; + f.env.RUN_ARTIFACTS.put = async (...args: Parameters) => { + signalPut(); + await uploadResumed; + return originalPut(...args); + }; + + const pendingUpload = f.upload(record("racing-upload")); + await putStarted; + const cutoff = new Date(Date.now() - 14 * 24 * 60 * 60 * 1000); + await expect(new RunRetentionRepository(database).sweep(cutoff, 10)).resolves.toMatchObject({ deleted: 1, cleanupQueued: 0 }); + resumeUpload(); + + expect((await pendingUpload).status).toBe(409); + const key = `tenants/${f.tenantId}/runs/${f.runId}/trace/${await hash(new TextEncoder().encode(record("racing-upload")))}.jsonl`; + expect(f.blobs.has(key)).toBe(true); + const cleanup = await database.pool.query("SELECT object_keys FROM app.run_retention_cleanup WHERE tenant_id=$1 AND run_id=$2", [f.tenantId, f.runId]); + expect(cleanup.rows).toEqual([{ object_keys: [key] }]); + + const deleted: string[] = []; + await expect(new RunRetentionRepository(database).processArtifactCleanup({ delete: async (keys: string[]) => { deleted.push(...keys); for (const item of keys) f.blobs.delete(item); } } as any)).resolves.toMatchObject({ attempted: 1, completed: 1, retried: 0 }); + expect(deleted).toEqual([key]); + expect(f.blobs.has(key)).toBe(false); + expect((await database.pool.query("SELECT id FROM app.run_retention_cleanup WHERE tenant_id=$1 AND run_id=$2", [f.tenantId, f.runId])).rows).toHaveLength(0); + }); + it.each(["codex", "claude"] as const)("reconciles %s durable bytes, preserves native audit data, and rejects late chunks", async provider => { const f = await fixture(provider), text = record("event-1") + '{"partial":', bytes = new TextEncoder().encode(text); const chunk = { generation: "file-1", expectedGeneration: null, startOffset: 0, chunkSha256: await hash(bytes), previousHash: zero, bytes }; From 5551cc02cf2ab52c1f9bff73c22052d692eeba0c Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 02:45:14 +0000 Subject: [PATCH 19/26] GEN-2907 cover trace retention race --- .../api/test/trace-source.postgres.test.ts | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/packages/api/test/trace-source.postgres.test.ts b/packages/api/test/trace-source.postgres.test.ts index b880944..7f5e839 100644 --- a/packages/api/test/trace-source.postgres.test.ts +++ b/packages/api/test/trace-source.postgres.test.ts @@ -223,6 +223,31 @@ describe.skipIf(!url)("trace sources with PostgreSQL", () => { expect((await database.pool.query("SELECT id FROM app.run_retention_cleanup WHERE tenant_id=$1 AND run_id=$2", [f.tenantId, f.runId])).rows).toHaveLength(0); }); + it("serializes incremental trace ingestion with retention aggregate deletion", async () => { + const f = await fixture(); + const { job_id: jobId, invocation_id: invocationId } = (await database.pool.query("SELECT job_id,invocation_id FROM app.runs WHERE tenant_id=$1 AND id=$2", [f.tenantId, f.runId])).rows[0]; + await database.pool.query("UPDATE app.runs SET state='succeeded',vm_cleanup_complete=true,artifact_state='stored',created_at=now()-interval '15 days' WHERE tenant_id=$1 AND id=$2", [f.tenantId, f.runId]); + await database.pool.query("INSERT INTO app.job_runs(tenant_id,id,job_id,invocation_id,state,encrypted_prompt) VALUES ($1,$2,$3,$4,'succeeded','test')", [f.tenantId, f.runId, jobId, invocationId]); + + const bytes = new TextEncoder().encode(record("retention-race")); + const append = f.live.append(f.runId, "codex", { + generation: "retention-race", + expectedGeneration: null, + startOffset: 0, + chunkSha256: await hash(bytes), + previousHash: zero, + bytes, + }, "execution_stream"); + const sweep = new RunRetentionRepository(database).sweep(new Date(Date.now() - 14 * 24 * 60 * 60 * 1000), 10); + const [appendResult, sweepResult] = await Promise.allSettled([append, sweep]); + + expect(sweepResult).toMatchObject({ status: "fulfilled", value: { deleted: 1 } }); + if (appendResult.status === "rejected") expect(String(appendResult.reason)).toContain("Trace run not found"); + else expect(appendResult.value).toMatchObject({ duplicate: false, events: 1 }); + expect((await database.pool.query("SELECT id FROM app.runs WHERE tenant_id=$1 AND id=$2", [f.tenantId, f.runId])).rows).toHaveLength(0); + expect((await database.pool.query("SELECT run_id FROM app.run_trace_events WHERE tenant_id=$1 AND run_id=$2 UNION ALL SELECT run_id FROM app.run_trace_chunks WHERE tenant_id=$1 AND run_id=$2 UNION ALL SELECT run_id FROM app.run_trace_cursors WHERE tenant_id=$1 AND run_id=$2", [f.tenantId, f.runId])).rows).toHaveLength(0); + }); + it.each(["codex", "claude"] as const)("reconciles %s durable bytes, preserves native audit data, and rejects late chunks", async provider => { const f = await fixture(provider), text = record("event-1") + '{"partial":', bytes = new TextEncoder().encode(text); const chunk = { generation: "file-1", expectedGeneration: null, startOffset: 0, chunkSha256: await hash(bytes), previousHash: zero, bytes }; From 4c09fde21421774e165d7662ffb6e93aa030de14 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 02:59:34 +0000 Subject: [PATCH 20/26] GEN-2907 cover webhook retention safety --- .../api/test/native-auth.postgres.test.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/packages/api/test/native-auth.postgres.test.ts b/packages/api/test/native-auth.postgres.test.ts index cdbc2ff..c06e195 100644 --- a/packages/api/test/native-auth.postgres.test.ts +++ b/packages/api/test/native-auth.postgres.test.ts @@ -9,6 +9,7 @@ import { protectedApiFetch } from "../src/protected-api"; import { tokenDigest } from "../src/auth"; import { hmac } from "../src/crypto"; import { WebhookService } from "../src/postgres/webhook-service"; +import { RunRetentionRepository } from "../src/postgres/run-retention-repository"; import { signSetupState } from "../src/github"; import app, { readSession, signSession } from "../src/index"; import type { Env } from "../src/types"; @@ -256,6 +257,24 @@ describe.skipIf(!process.env.AUTH_TEST_DATABASE_URL)("native authentication with expect((await app.request(`/auth/linear/callback?code=code&state=${expired}`, { headers: { Cookie: `${session.cookie}; linear_oauth_state=${expired}` } }, env)).status).toBe(400); expect(await readSession(session.cookie.slice("factorize_session=".length), env.SESSION_SIGNING_SECRET)).toMatchObject({ tenantId: session.tenantId }); }); + it("retains in-progress webhook deliveries and expires completed delivery history with its events", async () => { + const user = await verified(); + const session = await sessionFor(user.email); + const receivedId = crypto.randomUUID(); + const completedId = crypto.randomUUID(); + await db.pool.query(`INSERT INTO app.webhook_deliveries(tenant_id,id,provider,delivery_id,outcome,detail,received_at) + VALUES ($1,$2,'linear','retention:received','received','processing',now()-interval '15 days'), + ($1,$3,'linear','retention:completed','accepted','complete',now()-interval '15 days')`, [session.tenantId, receivedId, completedId]); + await db.pool.query(`INSERT INTO app.webhook_delivery_events(tenant_id,id,delivery_id,outcome,detail,created_at) + VALUES ($1,$2,$3,'accepted','complete',now()-interval '15 days')`, [session.tenantId, crypto.randomUUID(), completedId]); + + await new RunRetentionRepository(db).cleanupIndependentHistory(); + + expect((await db.pool.query("SELECT outcome FROM app.webhook_deliveries WHERE tenant_id=$1 AND id=$2", [session.tenantId, receivedId])).rows).toEqual([{ outcome: "received" }]); + expect((await db.pool.query("SELECT 1 FROM app.webhook_deliveries WHERE tenant_id=$1 AND id=$2", [session.tenantId, completedId])).rowCount).toBe(0); + expect((await db.pool.query("SELECT 1 FROM app.webhook_delivery_events WHERE tenant_id=$1 AND delivery_id=$2", [session.tenantId, completedId])).rowCount).toBe(0); + }); + it("fails closed on missing Postmark configuration and allows retry after delivery failure", async () => { const broken = new AuthRepository(db, { ...env, POSTMARK_SERVER_TOKEN: undefined }); await expect(broken.signup({ email: "config@example.test", password })).rejects.toThrow("Authentication email requires"); From 36fae74fdc04eab52e1554676d51220d94f32009 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 03:15:02 +0000 Subject: [PATCH 21/26] GEN-2907 verify retention rollback and dry-run ops --- docs/run-history-retention.md | 44 ++++++++++++++++++- .../api/test/job-lifecycle.postgres.test.ts | 35 +++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/docs/run-history-retention.md b/docs/run-history-retention.md index 09ac5ab..b49ecaa 100644 --- a/docs/run-history-retention.md +++ b/docs/run-history-retention.md @@ -32,7 +32,49 @@ The retention indexes cover candidate age/order and the audited foreign-key look Explicit job deletion also snapshots every incomplete run's provider configuration, handle, and start evidence into `app.deleted_job_provider_cleanup` in the same transaction as the job cascade. This independent queue lets the maintenance lane stop or reconcile provider resources after the source job and run rows are gone. A started run without a saved handle stays queued for retry and manual reconciliation; a never-started run can be removed from the queue safely. R2 artifacts for explicit job deletion continue to be deleted before the database cascade so a storage failure leaves the authoritative keys available for retry. -Successful cleanup removes its retry state from the run. Failed work retains only a bounded diagnostic string and retry timestamp. Each maintenance alarm emits a structured `run_retention_maintenance` log with run sweep, artifact and independent-history deletion progress plus pending counts and oldest age for terminal-provider cleanup, deleted-job provider cleanup, artifact objects, overdue nonterminal runs, independent histories, and expired temporary records. The snapshot contains no tenant, run, or object identifiers. A failed snapshot is logged separately and does not fail maintenance. These logs provide rollout evidence; a retained time-series dashboard remains follow-up work. +Successful cleanup removes its retry state from the run. Failed work retains only a bounded diagnostic string and retry timestamp. Each maintenance alarm emits a structured `run_retention_maintenance` log with run sweep, artifact and independent-history deletion progress plus pending counts and oldest age for terminal-provider cleanup, deleted-job provider cleanup, artifact objects, overdue nonterminal runs, independent histories, and expired temporary records. The snapshot contains no tenant, run, or object identifiers. A failed snapshot is logged separately and does not fail maintenance. + +### Read-only dry run and rollout + +Before enabling deletion, run the following against a read-only database snapshot with a single captured `:cutoff` value (UTC now minus 14 days). It reports database cohorts without locking or changing rows; compare R2 listing results separately because orphan classification spans object storage and PostgreSQL. + +```sql +WITH old AS ( + SELECT r.*, jr.state AS job_run_state + FROM app.runs r + JOIN app.invocations i ON i.tenant_id=r.tenant_id AND i.id=r.invocation_id + JOIN app.job_runs jr ON jr.tenant_id=r.tenant_id AND jr.invocation_id=r.invocation_id + WHERE r.created_at < :cutoff +), pending_automation AS ( + SELECT DISTINCT e.tenant_id,e.source_run_id + FROM app.lifecycle_events e + JOIN app.triggers t ON t.tenant_id=e.tenant_id AND t.id=e.trigger_id + JOIN app.jobs j ON j.tenant_id=t.tenant_id AND j.id=t.job_id + WHERE t.kind='jobLifecycle' AND t.enabled AND t.removed_at IS NULL AND j.enabled + AND NOT EXISTS (SELECT 1 FROM app.lifecycle_deliveries d + WHERE d.tenant_id=e.tenant_id AND d.trigger_id=e.trigger_id + AND d.source_run_id=e.source_run_id AND d.terminal_state=e.terminal_state) +) +SELECT cohort,count(*) AS runs FROM ( + SELECT CASE + WHEN r.state IN ('queued','reserved','starting','running','blocked','stopping') THEN 'active_or_exempt' + WHEN r.state IN ('succeeded','failed','stopped') AND NOT r.vm_cleanup_complete THEN 'provider_cleanup_blocked' + WHEN r.state IN ('succeeded','failed','stopped') AND r.artifact_state IN ('pending','collecting') THEN 'finalization_blocked' + WHEN r.state IN ('succeeded','failed','stopped') AND r.job_run_state <> r.state THEN 'state_mismatch_review' + WHEN r.state IN ('succeeded','failed','stopped') AND EXISTS + (SELECT 1 FROM pending_automation p WHERE p.tenant_id=r.tenant_id AND p.source_run_id=r.id) THEN 'automation_pending' + WHEN r.state IN ('succeeded','failed','stopped') THEN 'eligible_terminal' + ELSE 'other_state_review' + END AS cohort + FROM old r +) cohorts GROUP BY cohort ORDER BY cohort; +``` + +Roll out in stages: (1) apply additive migrations while retention is paused; (2) deploy cleanup code with `RUN_HISTORY_RETENTION_PAUSED=true`; (3) refresh the read-only cohort and R2 inventory, resolve old provider cleanup (including no-handle cases), and verify upload/trace/finalization/lifecycle fault tests; (4) enable the default bounded batches and review the first maintenance logs before increasing throughput. The pause switch stops run deletion, R2 deletion, and independent-history expiry. It does not disable read-only backlog metrics or terminal provider cleanup. Do not execute a production purge as part of code validation. + +### Dashboard and alert signals + +Build a retained time-series dashboard from `run_retention_maintenance` structured logs. Plot `backlog[].pending` and `backlog[].oldestAgeSeconds` per queue, `sweep.selected/deleted/cleanupQueued`, `artifacts.attempted/completed/retried`, and each independent-history lane's deleted count. Keep `paused` visible on every panel. Alert on a nonzero or rising `overdue_nonterminal` cohort for operator review, any sustained rise in provider or artifact oldest age, repeated artifact retries, or missing maintenance events for more than 30 minutes. A paused deployment should show backlog growth without deletion progress. Never put tenant, run, provider handle, or object key dimensions on these panels. ## Rollout and recovery diff --git a/packages/api/test/job-lifecycle.postgres.test.ts b/packages/api/test/job-lifecycle.postgres.test.ts index 37db382..17bc7c2 100644 --- a/packages/api/test/job-lifecycle.postgres.test.ts +++ b/packages/api/test/job-lifecycle.postgres.test.ts @@ -108,6 +108,41 @@ describe.skipIf(!process.env.AUTH_TEST_DATABASE_URL)("job lifecycle delivery (Po expect(await invocations(a, target.id)).toHaveLength(1); }); + it("rolls back aggregate deletion with its artifact receipt, then expires replay deduplication with the run", async () => { + const a = await tenant(), source = await job(a); + const runId = await complete(a, source); + const cutoff = new Date(Date.now() - 14 * 24 * 60 * 60_000); + await database.pool.query("UPDATE app.runs SET created_at=$3 WHERE tenant_id=$1 AND id=$2", [a, runId, new Date(cutoff.getTime() - 60_000)]); + const artifactId = crypto.randomUUID(), requestId = crypto.randomUUID(); + await database.pool.query(`INSERT INTO app.run_artifacts + (tenant_id,id,run_id,kind,object_key,provider,format,byte_size,sha256,state) + VALUES ($1,$2,$3,'execution_stream',$4,'codex','jsonl',1,$5,'stored')`, + [a, artifactId, runId, `${a}/${runId}/trace.jsonl`, "a".repeat(64)]); + await database.pool.query(`INSERT INTO app.trace_replay_operations + (tenant_id,request_id,run_id,source,actor_id,result) + VALUES ($1,$2,$3,'primary','owner','{"status":"projected"}')`, [a, requestId, runId]); + + const suffix = crypto.randomUUID().replaceAll("-", ""); + const functionName = `reject_retention_delete_${suffix}`, triggerName = `reject_retention_delete_${suffix}`; + await database.pool.query(`CREATE FUNCTION app.${functionName}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected retention delete failure'; END $$`); + await database.pool.query(`CREATE TRIGGER ${triggerName} BEFORE DELETE ON app.invocations FOR EACH ROW EXECUTE FUNCTION app.${functionName}()`); + const retention = new RunRetentionRepository(database); + try { + await expect(retention.sweep(cutoff, 10)).rejects.toThrow("injected retention delete failure"); + } finally { + await database.pool.query(`DROP TRIGGER ${triggerName} ON app.invocations`); + await database.pool.query(`DROP FUNCTION app.${functionName}()`); + } + expect((await database.pool.query("SELECT 1 FROM app.runs WHERE tenant_id=$1 AND id=$2", [a, runId])).rowCount).toBe(1); + expect((await database.pool.query("SELECT 1 FROM app.run_artifacts WHERE tenant_id=$1 AND id=$2", [a, artifactId])).rowCount).toBe(1); + expect((await database.pool.query("SELECT 1 FROM app.run_retention_cleanup WHERE tenant_id=$1 AND run_id=$2", [a, runId])).rowCount).toBe(0); + + await expect(retention.sweep(cutoff, 10)).resolves.toMatchObject({ deleted: 1, cleanupQueued: 1 }); + expect((await database.pool.query("SELECT 1 FROM app.runs WHERE tenant_id=$1 AND id=$2", [a, runId])).rowCount).toBe(0); + expect((await database.pool.query("SELECT 1 FROM app.trace_replay_operations WHERE tenant_id=$1 AND request_id=$2", [a, requestId])).rowCount).toBe(0); + expect((await database.pool.query("SELECT object_keys FROM app.run_retention_cleanup WHERE tenant_id=$1 AND run_id=$2", [a, runId])).rows[0]).toEqual({ object_keys: [`${a}/${runId}/trace.jsonl`] }); + }); + it("does not replay terminal history when a trigger is created disabled then enabled", async () => { const a = await tenant(), source = await job(a); await complete(a, source, "failed"); From a0686814b6c8977a27c5974cd41ad91d9f45ec56 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 03:29:09 +0000 Subject: [PATCH 22/26] GEN-2907 retain provider cleanup after stop failure --- .../run-provider-cleanup-repository.test.ts | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/packages/api/test/run-provider-cleanup-repository.test.ts b/packages/api/test/run-provider-cleanup-repository.test.ts index 75112fa..001d615 100644 --- a/packages/api/test/run-provider-cleanup-repository.test.ts +++ b/packages/api/test/run-provider-cleanup-repository.test.ts @@ -52,6 +52,34 @@ describe("terminal provider cleanup reconciliation", () => { expect(writes.at(-1)).toMatchObject({ values: ["tenant", "run", 60, "database acknowledgment unavailable"] }); }); + it("retains terminal cleanup ownership and schedules a retry when the provider cannot confirm absence", async () => { + const writes: Array<{ sql: string; values: unknown[] }> = []; + const poolQuery = vi.fn(async (sql: string, values: unknown[] = []) => { + writes.push({ sql, values }); + return { rows: [], rowCount: 1 }; + }); + const txQuery = vi.fn(async (sql: string) => { + if (sql.includes("app.deleted_job_provider_cleanup")) return { rows: [], rowCount: 0 }; + if (sql.includes("WITH candidates")) return { rows: [{ + tenant_id: "tenant", id: "run", execution_config: { connectionId: "connection" }, + execution_handle: { backendKind: "exe-vm", id: "provider-run" }, execution_backend_kind: "exe-vm", + provider_cleanup_attempt: 2, started_at: new Date("2026-10-01T00:00:00Z"), + }], rowCount: 1 }; + return { rows: [], rowCount: 0 }; + }); + const scheduler = new RunScheduler({ pool: { query: poolQuery }, transaction: (work: (client: any) => unknown) => work({ query: txQuery }) } as any, {} as any); + const stop = vi.fn().mockRejectedValue(new Error("provider temporarily unavailable")); + vi.spyOn(scheduler as any, "backend").mockResolvedValue({ stop }); + + await (scheduler as any).reconcileProviderCleanup(); + + expect(stop).toHaveBeenCalledWith({ backendKind: "exe-vm", id: "provider-run" }); + expect(writes).toEqual([{ + sql: expect.stringContaining("provider_cleanup_next_at=now()+make_interval(secs=>$3)"), + values: ["tenant", "run", 120, "provider temporarily unavailable"], + }]); + }); + it("leases independent ownership after job deletion and keeps failed provider work retryable", async () => { const query = vi.fn(async (sql: string) => { expect(sql).toContain("app.deleted_job_provider_cleanup"); From d00173ce52c1057d22a1caaf5dff7ffe8ccd420e Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 03:45:35 +0000 Subject: [PATCH 23/26] GEN-2907 verify cleanup boundaries and dashboard signal --- packages/api/src/postgres/run-scheduler.ts | 2 +- .../run-provider-cleanup-repository.test.ts | 38 +++++++++++++++- packages/api/test/run-retention-pause.test.ts | 43 +++++++++++++++++++ 3 files changed, 81 insertions(+), 2 deletions(-) diff --git a/packages/api/src/postgres/run-scheduler.ts b/packages/api/src/postgres/run-scheduler.ts index 077ca64..b2df5c4 100644 --- a/packages/api/src/postgres/run-scheduler.ts +++ b/packages/api/src/postgres/run-scheduler.ts @@ -58,7 +58,7 @@ export class RunScheduler { // turn a successful maintenance pass into a failed scheduler alarm. try { const backlog = await retention.backlogSnapshot(); - console.info(JSON.stringify({ event: "run_retention_maintenance", cutoff: cutoff.toISOString(), sweep, artifacts, independentHistory, backlog })); + console.info(JSON.stringify({ event: "run_retention_maintenance", cutoff: cutoff.toISOString(), paused: false, sweep, artifacts, independentHistory, backlog })); } catch (error) { console.warn(JSON.stringify({ event: "run_retention_backlog_snapshot_failed", message: error instanceof Error ? error.message : "unknown error" })); } diff --git a/packages/api/test/run-provider-cleanup-repository.test.ts b/packages/api/test/run-provider-cleanup-repository.test.ts index 001d615..7304364 100644 --- a/packages/api/test/run-provider-cleanup-repository.test.ts +++ b/packages/api/test/run-provider-cleanup-repository.test.ts @@ -1,8 +1,44 @@ -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { RunRepository } from "../src/postgres/run-repository"; import { RunScheduler } from "../src/postgres/run-scheduler"; +afterEach(() => vi.restoreAllMocks()); + describe("terminal provider cleanup reconciliation", () => { + it("keeps a started run without a handle in retryable cleanup ownership", async () => { + const database = { pool: { query: vi.fn() } } as any; + const scheduler = new RunScheduler(database, {} as any); + vi.spyOn(RunRepository.prototype, "claimProviderCleanup").mockResolvedValue([{ + tenantId: "tenant", id: "started-without-handle", executionConfig: {}, executionHandle: null, + backendKind: "exe-vm", startedAt: new Date("2026-10-01T00:00:00Z"), attempt: 3, + }]); + vi.spyOn(RunRepository.prototype, "claimDeletedJobProviderCleanup").mockResolvedValue([]); + const retry = vi.spyOn(RunRepository.prototype, "retryProviderCleanup").mockResolvedValue(undefined); + const complete = vi.spyOn(RunRepository.prototype, "completeProviderCleanup").mockResolvedValue(undefined); + + await (scheduler as any).reconcileProviderCleanup(); + + expect(retry).toHaveBeenCalledWith("tenant", "started-without-handle", 3, + expect.objectContaining({ message: expect.stringContaining("provider absence cannot be verified") })); + expect(complete).not.toHaveBeenCalled(); + }); + + it("completes provider cleanup for a terminal run that was never launched", async () => { + const scheduler = new RunScheduler({ pool: { query: vi.fn() } } as any, {} as any); + vi.spyOn(RunRepository.prototype, "claimProviderCleanup").mockResolvedValue([{ + tenantId: "tenant", id: "never-launched", executionConfig: {}, executionHandle: null, + backendKind: "exe-vm", startedAt: null, attempt: 1, + }]); + vi.spyOn(RunRepository.prototype, "claimDeletedJobProviderCleanup").mockResolvedValue([]); + const complete = vi.spyOn(RunRepository.prototype, "completeProviderCleanup").mockResolvedValue(undefined); + const retry = vi.spyOn(RunRepository.prototype, "retryProviderCleanup").mockResolvedValue(undefined); + + await (scheduler as any).reconcileProviderCleanup(); + + expect(complete).toHaveBeenCalledWith("tenant", "never-launched"); + expect(retry).not.toHaveBeenCalled(); + }); + it("claims only terminal incomplete cleanup with a bounded lease", async () => { const query = vi.fn(async (sql: string) => { expect(sql).toContain("x.state IN ('succeeded','failed','stopped')"); diff --git a/packages/api/test/run-retention-pause.test.ts b/packages/api/test/run-retention-pause.test.ts index 170245e..d2ea5f3 100644 --- a/packages/api/test/run-retention-pause.test.ts +++ b/packages/api/test/run-retention-pause.test.ts @@ -26,4 +26,47 @@ describe("run history retention pause switch", () => { backlog, }); }); + + it("emits the aggregate fields needed by retention dashboard panels without identifiers", async () => { + vi.spyOn(RunRetentionRepository.prototype, "sweep").mockResolvedValue({ selected: 8, deleted: 6, cleanupQueued: 2 }); + vi.spyOn(RunRetentionRepository.prototype, "processArtifactCleanup").mockResolvedValue({ attempted: 3, completed: 1, retried: 2 }); + vi.spyOn(RunRetentionRepository.prototype, "cleanupIndependentHistory").mockResolvedValue([ + { queue: "job_events", deleted: 12 }, { queue: "webhook_deliveries", deleted: 4 }, + ]); + vi.spyOn(RunRetentionRepository.prototype, "backlogSnapshot").mockResolvedValue([ + { queue: "terminal_provider", pending: 2, oldestAgeSeconds: 7200 }, + { queue: "artifact_objects", pending: 1, oldestAgeSeconds: 90 }, + ]); + const info = vi.spyOn(console, "info").mockImplementation(() => {}); + const scheduler: any = new RunScheduler({} as any, {} as any); + + await scheduler.runRetentionMaintenance(new RunRetentionRepository({} as any)); + + const event = JSON.parse(info.mock.calls[0]![0] as string); + expect(event).toMatchObject({ + event: "run_retention_maintenance", + paused: false, + sweep: { selected: 8, deleted: 6, cleanupQueued: 2 }, + artifacts: { attempted: 3, completed: 1, retried: 2 }, + independentHistory: [{ queue: "job_events", deleted: 12 }, { queue: "webhook_deliveries", deleted: 4 }], + backlog: [{ queue: "terminal_provider", pending: 2, oldestAgeSeconds: 7200 }, { queue: "artifact_objects", pending: 1, oldestAgeSeconds: 90 }], + }); + expect(JSON.stringify(event)).not.toMatch(/tenant|runId|objectKey|providerHandle/i); + }); + + it("keeps maintenance successful when the post-sweep dashboard snapshot fails", async () => { + vi.spyOn(RunRetentionRepository.prototype, "sweep").mockResolvedValue({ selected: 1, deleted: 1, cleanupQueued: 0 }); + vi.spyOn(RunRetentionRepository.prototype, "processArtifactCleanup").mockResolvedValue({ attempted: 0, completed: 0, retried: 0 }); + vi.spyOn(RunRetentionRepository.prototype, "cleanupIndependentHistory").mockResolvedValue([]); + vi.spyOn(RunRetentionRepository.prototype, "backlogSnapshot").mockRejectedValue(new Error("snapshot unavailable")); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const scheduler: any = new RunScheduler({} as any, {} as any); + + await expect(scheduler.runRetentionMaintenance(new RunRetentionRepository({} as any))).resolves.toBeUndefined(); + + expect(JSON.parse(warn.mock.calls[0]![0] as string)).toMatchObject({ + event: "run_retention_backlog_snapshot_failed", + message: "snapshot unavailable", + }); + }); }); From c5ce839cf01cdd117d59f23e601fa3dcda64c153 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 03:58:48 +0000 Subject: [PATCH 24/26] GEN-2907 retain provider cleanup until stopped --- .../run-provider-cleanup-repository.test.ts | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/packages/api/test/run-provider-cleanup-repository.test.ts b/packages/api/test/run-provider-cleanup-repository.test.ts index 7304364..eb48705 100644 --- a/packages/api/test/run-provider-cleanup-repository.test.ts +++ b/packages/api/test/run-provider-cleanup-repository.test.ts @@ -116,6 +116,35 @@ describe("terminal provider cleanup reconciliation", () => { }]); }); + it("does not acknowledge cleanup when a provider stop response leaves the run active", async () => { + const writes: Array<{ sql: string; values: unknown[] }> = []; + const poolQuery = vi.fn(async (sql: string, values: unknown[] = []) => { + writes.push({ sql, values }); + return { rows: [], rowCount: 1 }; + }); + const txQuery = vi.fn(async (sql: string) => { + if (sql.includes("app.deleted_job_provider_cleanup")) return { rows: [], rowCount: 0 }; + if (sql.includes("WITH candidates")) return { rows: [{ + tenant_id: "tenant", id: "run", execution_config: {}, + execution_handle: { backendKind: "exe-vm", id: "provider-run" }, execution_backend_kind: "exe-vm", + provider_cleanup_attempt: 1, started_at: new Date("2026-10-01T00:00:00Z"), + }], rowCount: 1 }; + return { rows: [], rowCount: 0 }; + }); + const scheduler = new RunScheduler({ pool: { query: poolQuery }, transaction: (work: (client: any) => unknown) => work({ query: txQuery }) } as any, {} as any); + const stop = vi.fn().mockResolvedValue({ state: "running", detail: "Cancellation is still in progress" }); + vi.spyOn(scheduler as any, "backend").mockResolvedValue({ stop }); + + await (scheduler as any).reconcileProviderCleanup(); + + expect(stop).toHaveBeenCalledWith({ backendKind: "exe-vm", id: "provider-run" }); + expect(writes).toEqual([{ + sql: expect.stringContaining("provider_cleanup_next_at=now()+make_interval(secs=>$3)"), + values: ["tenant", "run", 60, "Cancellation is still in progress"], + }]); + expect(writes.every(({ sql }) => !sql.includes("vm_cleanup_complete=true"))).toBe(true); + }); + it("leases independent ownership after job deletion and keeps failed provider work retryable", async () => { const query = vi.fn(async (sql: string) => { expect(sql).toContain("app.deleted_job_provider_cleanup"); From f8eeae8d00a158cbc204c7d0497cbe2dee0e8eb8 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 04:21:56 +0000 Subject: [PATCH 25/26] GEN-2907 expose retention backlog stages --- docs/run-history-retention.md | 2 +- .../src/postgres/run-retention-repository.ts | 24 +++++++++++++++++++ .../api/test/run-retention-repository.test.ts | 18 ++++++++++++-- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/docs/run-history-retention.md b/docs/run-history-retention.md index b49ecaa..f53c02a 100644 --- a/docs/run-history-retention.md +++ b/docs/run-history-retention.md @@ -74,7 +74,7 @@ Roll out in stages: (1) apply additive migrations while retention is paused; (2) ### Dashboard and alert signals -Build a retained time-series dashboard from `run_retention_maintenance` structured logs. Plot `backlog[].pending` and `backlog[].oldestAgeSeconds` per queue, `sweep.selected/deleted/cleanupQueued`, `artifacts.attempted/completed/retried`, and each independent-history lane's deleted count. Keep `paused` visible on every panel. Alert on a nonzero or rising `overdue_nonterminal` cohort for operator review, any sustained rise in provider or artifact oldest age, repeated artifact retries, or missing maintenance events for more than 30 minutes. A paused deployment should show backlog growth without deletion progress. Never put tenant, run, provider handle, or object key dimensions on these panels. +Build a retained time-series dashboard from `run_retention_maintenance` structured logs. Plot `backlog[].pending` and `backlog[].oldestAgeSeconds` per queue, `sweep.selected/deleted/cleanupQueued`, `artifacts.attempted/completed/retried`, and each independent-history lane's deleted count. Include `finalization_pending`, `lifecycle_handoff`, `webhook_processing`, and `webhook_verification` alongside the provider, artifact, and overdue nonterminal queues so stalls before aggregate deletion are visible. Keep `paused` visible on every panel. Alert on a nonzero or rising `overdue_nonterminal` cohort for operator review, any sustained rise in provider, finalization, lifecycle, webhook, or artifact oldest age, repeated artifact retries, or missing maintenance events for more than 30 minutes. A paused deployment should show backlog growth without deletion progress. Never put tenant, run, provider handle, or object key dimensions on these panels. ## Rollout and recovery diff --git a/packages/api/src/postgres/run-retention-repository.ts b/packages/api/src/postgres/run-retention-repository.ts index 37643f4..f0c086f 100644 --- a/packages/api/src/postgres/run-retention-repository.ts +++ b/packages/api/src/postgres/run-retention-repository.ts @@ -41,6 +41,30 @@ export class RunRetentionRepository { FROM app.runs WHERE state IN ('queued','reserved','starting','running','blocked','stopping') AND created_at < now()-interval '14 days' UNION ALL + SELECT 'finalization_pending'::text,count(*)::int, + floor(extract(epoch FROM now()-min(r.created_at)))::int + FROM app.runs r JOIN app.job_runs jr ON jr.tenant_id=r.tenant_id AND jr.id=r.id + WHERE r.state IN ('succeeded','failed','stopped') + AND (r.artifact_state IN ('pending','collecting') OR jr.state<>r.state) + UNION ALL + SELECT 'lifecycle_handoff'::text,count(DISTINCT (e.tenant_id,e.source_run_id))::int, + floor(extract(epoch FROM now()-min(e.occurred_at)))::int + FROM app.lifecycle_events e + JOIN app.triggers t ON t.tenant_id=e.tenant_id AND t.id=e.trigger_id + JOIN app.jobs j ON j.tenant_id=t.tenant_id AND j.id=t.job_id + WHERE t.kind='jobLifecycle' AND t.enabled AND t.removed_at IS NULL AND j.enabled + AND NOT EXISTS (SELECT 1 FROM app.lifecycle_deliveries d + WHERE d.tenant_id=e.tenant_id AND d.trigger_id=e.trigger_id + AND d.source_run_id=e.source_run_id AND d.terminal_state=e.terminal_state) + UNION ALL + SELECT 'webhook_processing'::text,count(*)::int, + floor(extract(epoch FROM now()-min(received_at)))::int + FROM app.webhook_deliveries WHERE outcome='received' + UNION ALL + SELECT 'webhook_verification'::text,count(*)::int, + GREATEST(0,floor(extract(epoch FROM now()-min(next_attempt_at)))::int) + FROM app.pending_verifications + UNION ALL SELECT 'job_events'::text,count(*)::int,floor(extract(epoch FROM now()-min(received_at)))::int FROM app.job_events WHERE received_at < now()-interval '14 days' UNION ALL diff --git a/packages/api/test/run-retention-repository.test.ts b/packages/api/test/run-retention-repository.test.ts index 8d7da07..d4de676 100644 --- a/packages/api/test/run-retention-repository.test.ts +++ b/packages/api/test/run-retention-repository.test.ts @@ -102,18 +102,32 @@ describe("RunRetentionRepository", () => { const query = vi.fn().mockResolvedValue({ rows: [ { queue: "artifact_objects", pending: 2, oldest_age_seconds: 120 }, { queue: "deleted_job_provider", pending: 0, oldest_age_seconds: null }, + { queue: "finalization_pending", pending: 1, oldest_age_seconds: 240 }, + { queue: "lifecycle_handoff", pending: 2, oldest_age_seconds: 300 }, { queue: "overdue_nonterminal", pending: 1, oldest_age_seconds: 90_000 }, { queue: "terminal_provider", pending: 3, oldest_age_seconds: 600 }, + { queue: "webhook_processing", pending: 1, oldest_age_seconds: 45 }, + { queue: "webhook_verification", pending: 1, oldest_age_seconds: 0 }, ] }); const repository = new RunRetentionRepository({ pool: { query } } as any); - await expect(repository.backlogSnapshot()).resolves.toEqual([ + const snapshot = await repository.backlogSnapshot(); + expect(snapshot).toEqual([ { queue: "artifact_objects", pending: 2, oldestAgeSeconds: 120 }, { queue: "deleted_job_provider", pending: 0, oldestAgeSeconds: null }, + { queue: "finalization_pending", pending: 1, oldestAgeSeconds: 240 }, + { queue: "lifecycle_handoff", pending: 2, oldestAgeSeconds: 300 }, { queue: "overdue_nonterminal", pending: 1, oldestAgeSeconds: 90_000 }, { queue: "terminal_provider", pending: 3, oldestAgeSeconds: 600 }, + { queue: "webhook_processing", pending: 1, oldestAgeSeconds: 45 }, + { queue: "webhook_verification", pending: 1, oldestAgeSeconds: 0 }, ]); expect(query.mock.calls[0]![0]).toContain("'overdue_nonterminal'"); - expect(query.mock.calls[0]![0]).not.toContain("run_id"); + expect(query.mock.calls[0]![0]).toContain("'finalization_pending'"); + expect(query.mock.calls[0]![0]).toContain("'lifecycle_handoff'"); + expect(query.mock.calls[0]![0]).toContain("'webhook_processing'"); + expect(query.mock.calls[0]![0]).toContain("'webhook_verification'"); + expect(query.mock.calls[0]![0]).toContain("SELECT queue,pending,oldest_age_seconds"); + expect(snapshot.every(item => !("runId" in item) && !("tenantId" in item))).toBe(true); }); it("cleans independent histories in bounded lanes and protects pending work", async () => { From a2bf34f6a3934a846e6e67a99a4898fc60a6414e Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 12 Nov 2026 05:16:12 +0000 Subject: [PATCH 26/26] GEN-2907 cover replay retention race --- .../api/test/trace-source.postgres.test.ts | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/packages/api/test/trace-source.postgres.test.ts b/packages/api/test/trace-source.postgres.test.ts index 7f5e839..42b16b5 100644 --- a/packages/api/test/trace-source.postgres.test.ts +++ b/packages/api/test/trace-source.postgres.test.ts @@ -241,13 +241,41 @@ describe.skipIf(!url)("trace sources with PostgreSQL", () => { const sweep = new RunRetentionRepository(database).sweep(new Date(Date.now() - 14 * 24 * 60 * 60 * 1000), 10); const [appendResult, sweepResult] = await Promise.allSettled([append, sweep]); - expect(sweepResult).toMatchObject({ status: "fulfilled", value: { deleted: 1 } }); + expect(sweepResult.status).toBe("fulfilled"); if (appendResult.status === "rejected") expect(String(appendResult.reason)).toContain("Trace run not found"); else expect(appendResult.value).toMatchObject({ duplicate: false, events: 1 }); + // Retention intentionally skips a candidate locked by ingestion so one + // busy run cannot stall the batch. A later alarm must finish that cleanup. + const remaining = await database.pool.query("SELECT id FROM app.runs WHERE tenant_id=$1 AND id=$2", [f.tenantId, f.runId]); + if (remaining.rowCount) await expect(new RunRetentionRepository(database).sweep(new Date(Date.now() - 14 * 24 * 60 * 60 * 1000), 10)).resolves.toMatchObject({ deleted: 1 }); expect((await database.pool.query("SELECT id FROM app.runs WHERE tenant_id=$1 AND id=$2", [f.tenantId, f.runId])).rows).toHaveLength(0); expect((await database.pool.query("SELECT run_id FROM app.run_trace_events WHERE tenant_id=$1 AND run_id=$2 UNION ALL SELECT run_id FROM app.run_trace_chunks WHERE tenant_id=$1 AND run_id=$2 UNION ALL SELECT run_id FROM app.run_trace_cursors WHERE tenant_id=$1 AND run_id=$2", [f.tenantId, f.runId])).rows).toHaveLength(0); }); + it("makes a replay racing retention observe the expired run without recreating replay history", async () => { + const f = await fixture("claude"); + const { job_id: jobId, invocation_id: invocationId } = (await database.pool.query("SELECT job_id,invocation_id FROM app.runs WHERE tenant_id=$1 AND id=$2", [f.tenantId, f.runId])).rows[0]; + await database.pool.query("UPDATE app.runs SET state='succeeded',vm_cleanup_complete=true,artifact_state='stored',created_at=now()-interval '15 days' WHERE tenant_id=$1 AND id=$2", [f.tenantId, f.runId]); + await database.pool.query("INSERT INTO app.job_runs(tenant_id,id,job_id,invocation_id,state,encrypted_prompt) VALUES ($1,$2,$3,$4,'succeeded','test')", [f.tenantId, f.runId, jobId, invocationId]); + + const owner = await database.pool.connect(); + await owner.query("BEGIN"); + try { + await owner.query("SELECT id FROM app.runs WHERE tenant_id=$1 AND id=$2 FOR UPDATE", [f.tenantId, f.runId]); + const replay = replayTrace(f.env, f.tenantId, "owner", f.runId, { requestId: crypto.randomUUID(), source: "primary" }); + await new Promise(resolve => setTimeout(resolve, 25)); + await owner.query("DELETE FROM app.invocations WHERE tenant_id=$1 AND id=$2", [f.tenantId, invocationId]); + await owner.query("COMMIT"); + + await expect(replay).rejects.toMatchObject({ status: 404, code: "not_found" }); + expect((await database.pool.query("SELECT 1 FROM app.trace_replay_operations WHERE tenant_id=$1 AND run_id=$2", [f.tenantId, f.runId])).rowCount).toBe(0); + expect((await database.pool.query("SELECT 1 FROM app.run_trace_events WHERE tenant_id=$1 AND run_id=$2", [f.tenantId, f.runId])).rowCount).toBe(0); + } finally { + await owner.query("ROLLBACK").catch(() => undefined); + owner.release(); + } + }); + it.each(["codex", "claude"] as const)("reconciles %s durable bytes, preserves native audit data, and rejects late chunks", async provider => { const f = await fixture(provider), text = record("event-1") + '{"partial":', bytes = new TextEncoder().encode(text); const chunk = { generation: "file-1", expectedGeneration: null, startOffset: 0, chunkSha256: await hash(bytes), previousHash: zero, bytes };