From e08dd50e16fe5cca23ac72207b8775a3497f1496 Mon Sep 17 00:00:00 2001 From: flint Date: Sat, 3 Oct 2026 18:02:48 +0000 Subject: [PATCH 01/17] fix(openclaw-tps-mail): surface and reconcile a failed cur/ record write after a terminal transition (cli#492) Route every ack/nack stamp through one locked, existing-only, atomic cur/ writer that reuses the CLI's own mailbox lock. A failed write is logged by message id, record path and error code instead of being swallowed, and the terminal obligation is re-stamped onto the record on the next scan. --- .../fixed-492-cur-record-stamp-reconcile.md | 1 + packages/cli/package.json | 1 + plugins/openclaw-tps-mail/src/index.ts | 162 ++++++++++-- .../test/cur-record-write.test.ts | 245 ++++++++++++++++++ 4 files changed, 394 insertions(+), 15 deletions(-) create mode 100644 .changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md create mode 100644 plugins/openclaw-tps-mail/test/cur-record-write.test.ts diff --git a/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md b/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md new file mode 100644 index 00000000..7c81712e --- /dev/null +++ b/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md @@ -0,0 +1 @@ +- **A failed cur/ record write after a terminal transition is logged by id, path and code and reconciled on the next scan (Closes #492)**. Every ack/nack stamp now goes through one locked, existing-only, atomic writer that reuses the CLI's mailbox lock, so a failed write no longer leaves the on-disk record and the plugin's view diverged in silence. diff --git a/packages/cli/package.json b/packages/cli/package.json index 7c050421..81bea909 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -16,6 +16,7 @@ "./utils/mail-verify": "./dist/src/utils/mail-verify.js", "./utils/mail-routing": "./dist/src/utils/mail-routing.js", "./utils/mail-producer": "./dist/src/utils/mail-producer.js", + "./utils/mail-lock": "./dist/src/utils/mail-lock.js", "./utils/relay": "./dist/src/utils/relay.js" }, "optionalDependencies": { diff --git a/plugins/openclaw-tps-mail/src/index.ts b/plugins/openclaw-tps-mail/src/index.ts index d073fb8b..25ebfb46 100644 --- a/plugins/openclaw-tps-mail/src/index.ts +++ b/plugins/openclaw-tps-mail/src/index.ts @@ -51,10 +51,10 @@ * (hook always landed in `main` session, accumulating noise). */ -import { randomUUID } from "node:crypto"; -import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, writeFileSync, watch as fsWatch, type FSWatcher } from "node:fs"; +import { randomBytes, randomUUID } from "node:crypto"; +import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync, watch as fsWatch, type FSWatcher } from "node:fs"; import { homedir } from "node:os"; -import { basename, resolve } from "node:path"; +import { basename, dirname, resolve } from "node:path"; import type { Envelope, ChainEntry } from "@tpsdev-ai/agent"; import { signEnvelope, verifyEnvelope, verifiedMailTier } from "@tpsdev-ai/agent"; import { readAgentPrivateKey } from "@tpsdev-ai/cli/utils/agent-keys"; @@ -62,6 +62,7 @@ import { signForDelivery } from "@tpsdev-ai/cli/utils/mail-producer"; import { isValidEnvelopeId, mailRootForRecordPath, promote, recoverPromoted, verifyRecordForMailbox, sweepStrandedPromoteScratch } from "@tpsdev-ai/cli/utils/mail"; import { createMailVerifyClient } from "@tpsdev-ai/cli/utils/mail-verify"; import { resolveMailRoute, type MailRoute } from "@tpsdev-ai/cli/utils/mail-routing"; +import { mailLockPath, tryAcquireMailLock } from "@tpsdev-ai/cli/utils/mail-lock"; import { deliverToRemoteBranch, deliverToSandbox, resolveAgentMailRoot } from "@tpsdev-ai/cli/utils/relay"; import { TERMINAL_STATES, @@ -619,18 +620,129 @@ function routeFor(mailDir: string, cfg: any, accountId: string, to: string): Mai } /** - * Patch a mail record in place. Used to ack/nack a message that the shared - * promote() enforcement point has already moved new/ → cur/. The promotion - * itself is atomic inside promote() (new/ → tmp/ → cur/); this only enriches - * the cur/ record after dispatch, so a crash here cannot replay a message. + * ONE locked, existing-only writer for a cur/ record — cli#469's rule, applied + * where it reaches the plugin. Every ack/nack stamp written AFTER a durable + * terminal transition goes through this, and only this: + * + * - it REUSES the CLI's own mailbox lock (`tryAcquireMailLock` from + * `@tpsdev-ai/cli/utils/mail-lock`), so the plugin's stamp coordinates with + * the CLI's `promote()` — both lock the same `//.mail-lock`; + * - it is EXISTING-ONLY: a missing or unreadable record is a NAMED refusal, + * never a create; + * - it re-reads under the lock and REPLACES atomically (dot-temp + rename), + * so a crash can never leave a torn record. + * + * It never throws. The caller gets a structured outcome it must ACT on, so a + * failed write is surfaced (logged by message id, record path and error code) + * and reconciled on the next scan — never silently ignored (cli#492). */ -function patchMailFile(path: string, patch: Partial): void { +type CurRecordUpdate = + | { ok: true } + | { + ok: false; + reason: "record-missing" | "record-unreadable" | "lock-busy" | "lock-unverified" | "write-failed"; + path: string; + code: string; + }; + +/** Read a cur/ record (existing-only), distinguishing absent from unreadable. */ +function readCurRecord(path: string): + | { status: "ok"; record: TpsMailBody } + | { status: "missing" } + | { status: "unreadable"; code: string } { try { - const current = readMailFile(path); - if (!current) return; - writeFileSync(path, JSON.stringify({ ...current, ...patch }, null, 2), "utf-8"); - } catch { - // best effort — don't crash the watcher on state-transition errors + const record = JSON.parse(readFileSync(path, "utf-8")) as TpsMailBody; + if (!record || typeof record !== "object" || typeof record.id !== "string") { + return { status: "unreadable", code: "INVALID_RECORD" }; + } + return { status: "ok", record }; + } catch (err) { + const code = (err as NodeJS.ErrnoException).code ?? "INVALID_JSON"; + return code === "ENOENT" ? { status: "missing" } : { status: "unreadable", code }; + } +} + +function updateExistingCurRecord(path: string, patch: Partial): CurRecordUpdate { + // The mailbox root is the record's grandparent: //cur/. + const root = dirname(dirname(path)); + // EXISTING-ONLY: refuse by name before taking the lock when there is nothing + // to update. A missing record is benign (there is no stamp to diverge from); + // an UNREADABLE one is refused, never replaced. + let current = readCurRecord(path); + if (current.status !== "ok") { + return { + ok: false, + reason: current.status === "missing" ? "record-missing" : "record-unreadable", + path, + code: current.status === "missing" ? "ENOENT" : current.code, + }; + } + let lock: ReturnType; + try { + lock = tryAcquireMailLock(mailLockPath(root)); + } catch (err: any) { + // An unverifiable lock owner is NOT "no lock": fail closed, never write + // outside the lock (cli#469's rule and this plugin's unknown-evidence rule). + return { ok: false, reason: "lock-unverified", path, code: err?.name ?? "LOCK_ERROR" }; + } + if (!lock) return { ok: false, reason: "lock-busy", path, code: "EEXIST" }; + try { + // Re-read under the lock: lock, re-read, mutate, atomically replace. + current = readCurRecord(path); + if (current.status !== "ok") { + return { + ok: false, + reason: current.status === "missing" ? "record-missing" : "record-unreadable", + path, + code: current.status === "missing" ? "ENOENT" : current.code, + }; + } + const tmp = resolve(dirname(path), `.${basename(path)}.${process.pid}.${randomBytes(6).toString("hex")}.tmp`); + try { + writeFileSync(tmp, JSON.stringify({ ...current.record, ...patch }, null, 2), "utf-8"); + renameSync(tmp, path); + } catch (err: any) { + try { + rmSync(tmp, { force: true }); + } catch { + /* the fault persists — the caller's diagnostic names it */ + } + return { ok: false, reason: "write-failed", path, code: err?.code ?? "WRITE_FAILED" }; + } + return { ok: true }; + } finally { + lock.release(); + } +} + +/** + * Re-stamp an `acked` or `failed` obligation whose cur/ record never received + * its stamp (the write failed after the durable transition — cli#492). The + * obligation record is the durable truth, so on a scan the record and the + * maildir are made to agree: an `acked` obligation re-stamps `ackedAt`/`read`, + * a `failed` one `nackedAt`/`nackReason`. Idempotent; a failure here is logged + * by name so the next scan tries again. + */ +function reconcileTerminalCurStamps(mailDir: string, agent: string, log: any): void { + for (const rec of listObligations(mailDir, agent)) { + if (rec.state !== "acked" && rec.state !== "failed") continue; + const curPath = findCurPath(mailDir, agent, rec.inboundId); + if (!curPath) continue; + const cur = readMailFile(curPath); + if (!cur) continue; // absent/unreadable now; the next scan retries + if (rec.state === "acked" && !cur.ackedAt) { + const r = updateExistingCurRecord(curPath, { ackedAt: new Date().toISOString(), read: true }); + if (r.ok) log?.info?.(`tps-mail: reconciled the acked stamp for ${rec.inboundId} at ${curPath}`); + else if (r.reason !== "record-missing") { + log?.warn?.(`tps-mail: ack-stamp-reconcile-failed: ${rec.inboundId} at ${r.path} (${r.code})`); + } + } else if (rec.state === "failed" && !cur.nackedAt) { + const r = updateExistingCurRecord(curPath, { nackedAt: new Date().toISOString(), nackReason: rec.failure ?? "failed" }); + if (r.ok) log?.info?.(`tps-mail: reconciled the nacked stamp for ${rec.inboundId} at ${curPath}`); + else if (r.reason !== "record-missing") { + log?.warn?.(`tps-mail: nack-stamp-reconcile-failed: ${rec.inboundId} at ${r.path} (${r.code})`); + } + } } } @@ -897,7 +1009,13 @@ function ackObligation(ctx: YieldContext, obligationId: string, why: string): vo ); return; } - patchMailFile(ctx.curPath, { ackedAt: new Date().toISOString(), read: true }); + const stamped = updateExistingCurRecord(ctx.curPath, { ackedAt: new Date().toISOString(), read: true }); + if (!stamped.ok && stamped.reason !== "record-missing") { + ctx.log?.warn?.( + `tps-mail: ack-stamp-failed: could not stamp ackedAt on ${ctx.inboundId} at ${stamped.path} (${stamped.code}); ` + + `the obligation is acked and the stamp is reconciled on the next scan`, + ); + } ctx.log?.info?.(`tps-mail: acked ${ctx.inboundId} — ${why}`); releaseObligationState(obligationId); } @@ -1068,7 +1186,13 @@ async function settleObligation( // handed off (cli#403). releaseObligationState(obligationId); if (!s.alreadyStamped) { - patchMailFile(ctx.curPath, { nackedAt: new Date().toISOString(), nackReason: failedOn }); + const stamped = updateExistingCurRecord(ctx.curPath, { nackedAt: new Date().toISOString(), nackReason: failedOn }); + if (!stamped.ok && stamped.reason !== "record-missing") { + ctx.log?.warn?.( + `tps-mail: nack-stamp-failed: could not stamp nackedAt on ${ctx.inboundId} at ${stamped.path} (${stamped.code}); ` + + `the obligation is failed and the stamp is reconciled on the next scan`, + ); + } } // cli#389 round 10, item 1: AWAIT the one send, and record its outcome on the // record. The mail is owed until `nackSentAt` says otherwise (at-least-once). @@ -2270,6 +2394,14 @@ const gateway: ChannelGatewayAdapter = { } } catch { /* ignore */ } + // STAMP RECONCILIATION (cli#492): an acked or failed obligation whose + // cur/ record never received its stamp — the write failed AFTER the + // durable transition — is re-stamped here, under the same lock that + // write uses, BEFORE the recovery loop below decides whether to + // re-dispatch the record. Idempotent; a failure here is logged by name + // and retried on the next scan. + reconcileTerminalCurStamps(account.mailDir, agentId, log); + // Crash recovery (at-least-once): re-dispatch cur/ records that were // promoted but never acked/nacked. cur/ is a DESTINATION, so the record // must PROVE it came through promote() (envelopeId + stored signed diff --git a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts new file mode 100644 index 00000000..00cced73 --- /dev/null +++ b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts @@ -0,0 +1,245 @@ +/** + * cur-record-write.test.ts — cli#492. + * + * After a DURABLE terminal transition (acked/failed) the plugin stamps the cur/ + * record (`ackedAt` / `nackedAt`). The old `patchMailFile` swallowed a failed + * write, so the on-disk record and the plugin's view could diverge with no + * diagnostic. + * + * These tests inject a write failure (the cur/ DIRECTORY is made unwritable, so + * the atomic stamp's temp file cannot be created) and assert: + * (a) a diagnostic naming the message id, the record path and the error code; + * (b) a later scan reconciles the record to the terminal state. + * A third test pins the successful path as unchanged. + * + * The first two FAIL on the pre-fix tree: the write failure is silent and no + * scan ever reconciles the record. + */ +import { describe, expect, it, beforeEach, afterEach, mock } from "bun:test"; +import { chmodSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { tmpdir } from "node:os"; +import * as ed from "@noble/ed25519"; +import { createHash } from "node:crypto"; +import { signEnvelope, type ChainEntry } from "@tpsdev-ai/agent"; + +import { hashes } from "@noble/ed25519"; +hashes.sha512 = (m: Uint8Array) => new Uint8Array(createHash("sha512").update(m).digest()); + +const FLINT_SEED = Buffer.alloc(32, 0x01); +const ANVIL_SEED = Buffer.alloc(32, 0x02); +const pubkeyFromSeed = (s: Buffer): Buffer => Buffer.from(ed.getPublicKey(new Uint8Array(s))); + +import pluginModule from "../src/index.js"; + +let capturedPlugin: any; +const mockApi: any = { + registerChannel: ({ plugin }: { plugin: any }) => { capturedPlugin = plugin; }, + registerAgentEventSubscription: () => {}, + logger: { info: () => {}, warn: () => {}, error: () => {} }, +}; +pluginModule.register(mockApi); + +let mailDir: string; +let keysDir: string; +let home: string; +let origHome: string | undefined; +let origKeys: string | undefined; + +beforeEach(() => { + mailDir = mkdtempSync(join(tmpdir(), "tps-492-mail-")); + keysDir = mkdtempSync(join(tmpdir(), "tps-492-keys-")); + home = mkdtempSync(join(tmpdir(), "tps-492-home-")); + writeFileSync(join(keysDir, "anvil.key"), ANVIL_SEED); + writeFileSync(join(keysDir, "flint.key"), FLINT_SEED); + origHome = process.env.HOME; process.env.HOME = home; + origKeys = process.env.TPS_TEST_KEYS_DIR; process.env.TPS_TEST_KEYS_DIR = keysDir; + mock.module("@tpsdev-ai/cli/utils/mail-verify", () => ({ + createMailVerifyClient: async () => ({ + async getAgent(name: string) { + if (name === "flint") return { publicKey: pubkeyFromSeed(FLINT_SEED) }; + if (name === "anvil") return { publicKey: pubkeyFromSeed(ANVIL_SEED) }; + return null; + }, + }), + })); +}); + +afterEach(() => { + if (origHome === undefined) delete process.env.HOME; else process.env.HOME = origHome; + if (origKeys === undefined) delete process.env.TPS_TEST_KEYS_DIR; else process.env.TPS_TEST_KEYS_DIR = origKeys; + for (const d of [mailDir, keysDir, home]) { try { rmSync(d, { recursive: true, force: true }); } catch { /* best effort */ } } +}); + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); +async function pollUntil(pred: () => boolean, ms = 4000): Promise { + const t = Date.now(); + while (Date.now() - t < ms) { if (pred()) return true; await sleep(10); } + return pred(); +} + +function signedBody(from: string, to: string, body: string, seed: Buffer): string { + const chain: ChainEntry[] = [ + { agent: "system", kind: "human", timestamp: new Date().toISOString(), rationale: "originates", signature: null }, + { agent: from, kind: "agent", timestamp: new Date().toISOString(), rationale: `agent ${from}`, signature: null }, + ]; + return JSON.stringify(signEnvelope( + { v: 1, from, to, body, messageId: `env-${Math.random().toString(36).slice(2, 10)}`, timestamp: new Date().toISOString(), delegationChain: chain }, + { [from]: seed }, + )); +} + +/** The anvil cur/ record (promote() names it for the inbound id). */ +function readCur(): { path: string; record: any } | null { + const dir = resolve(mailDir, "anvil", "cur"); + let names: string[]; + try { names = readdirSync(dir); } catch { return null; } + for (const n of names) { + if (!n.endsWith(".json")) continue; + const p = join(dir, n); + try { return { path: p, record: JSON.parse(readFileSync(p, "utf-8")) }; } catch { /* torn */ } + } + return null; +} +function obligation(id: string): any | null { + try { return JSON.parse(readFileSync(resolve(mailDir, "anvil", ".obligations", `${id}.json`), "utf-8")); } catch { return null; } +} + +interface Boot { + inboundId: string; + logs: string[]; + dispatch: () => any; + deliver: (text: string) => Promise; + skip: (reason?: string) => void; + settle: () => void; + stop: () => Promise; +} + +/** Start one account. `withInbound` writes a new inbound so a turn is dispatched. */ +async function boot(withInbound: boolean): Promise { + mkdirSync(resolve(mailDir, "flint", "new"), { recursive: true }); + mkdirSync(resolve(mailDir, "anvil", "new"), { recursive: true }); + const inboundId = `msg-${Math.random().toString(36).slice(2, 10)}`; + if (withInbound) { + writeFileSync(resolve(mailDir, "anvil", "new", `2026-05-26T00-00-00-${inboundId}.json`), JSON.stringify({ + id: inboundId, from: "flint", to: "anvil", body: signedBody("flint", "anvil", "inbound", FLINT_SEED), + timestamp: new Date().toISOString(), headers: { "X-TPS-Trust": "agent", "X-TPS-Surface": "tps-mail" }, deliveryAttempts: 0, + }, null, 2), "utf-8"); + } + let dispatchedArgs: any = null; + let settleFn: (() => void) | null = null; + const controller = new AbortController(); + const logs: string[] = []; + const channelRuntime = { + routing: { buildAgentSessionKey: (p: any) => `agent:${p.agentId}:tps-mail:default:${p.peer.id}` }, + reply: { + finalizeInboundContext: async (c: any) => ({ ...c, CommandAuthorized: false }), + dispatchReplyWithBufferedBlockDispatcher: async (args: any) => { + dispatchedArgs = args; + await new Promise((res) => { settleFn = res; }); + return { failedCounts: 0 }; + }, + }, + }; + const cfg = { bindings: [{ agentId: "anvil", match: { channel: "tps-mail", accountId: "default" } }] }; + const ctx = { + account: { accountId: "default", mailDir, enabled: true }, + cfg, + log: { info: (m: string) => logs.push(String(m)), warn: (m: string) => logs.push(String(m)), error: (m: string) => logs.push(String(m)) }, + channelRuntime, + abortSignal: controller.signal, + }; + const startPromise = capturedPlugin.gateway.startAccount(ctx); + return { + inboundId, logs, + dispatch: () => dispatchedArgs, + deliver: async (text: string) => { await dispatchedArgs.dispatcherOptions.deliver({ text }, { kind: "final" }); }, + skip: (reason = "empty") => dispatchedArgs.dispatcherOptions.onSkip?.({ text: "" }, { kind: "final", reason }), + settle: () => settleFn?.(), + stop: async () => { controller.abort(); try { await startPromise; } catch { /* aborted */ } }, + }; +} + +describe("cli#492 — a failed cur/ stamp write is surfaced and reconciled", () => { + it("ack: a failed ackedAt write logs id/path/code and is reconciled on the next scan", async () => { + const h = await boot(true); + expect(await pollUntil(() => h.dispatch() !== null, 4000), "dispatch started").toBe(true); + const cur = readCur(); + expect(cur, "the inbound was promoted to cur/").not.toBeNull(); + + // The cur/ DIRECTORY and the record are both unwritable: a direct overwrite + // of the record AND the atomic stamp's temp file fail, so the write FAILS + // after the durable `acked` transition. + const curDir = resolve(mailDir, "anvil", "cur"); + chmodSync(cur!.path, 0o444); + chmodSync(curDir, 0o555); + try { + await h.deliver("verdict"); + h.settle(); + expect(await pollUntil(() => obligation(h.inboundId)?.state === "acked", 4000), "the acked transition is durable").toBe(true); + expect(readCur()?.record?.ackedAt, "the stamp did NOT land").toBeUndefined(); + expect( + await pollUntil( + () => h.logs.some((m) => m.includes("ack-stamp-failed") && m.includes(h.inboundId) && m.includes(cur!.path) && m.includes("EACCES")), + 4000, + ), + "the failed write is logged by id, path and code", + ).toBe(true); + } finally { + chmodSync(curDir, 0o755); + chmodSync(cur!.path, 0o644); + } + await h.stop(); + + // NEXT SCAN: a fresh start reconciles the durable terminal state onto the record. + const h2 = await boot(false); + expect(await pollUntil(() => !!readCur()?.record?.ackedAt, 4000), "reconciled on the next scan").toBe(true); + expect(readCur()?.record?.read).toBe(true); + expect(obligation(h.inboundId)?.state).toBe("acked"); + await h2.stop(); + }, 20000); + + it("nack: a failed nackedAt write logs id/path/code and is reconciled on the next scan", async () => { + const h = await boot(true); + expect(await pollUntil(() => h.dispatch() !== null, 4000), "dispatch started").toBe(true); + const cur = readCur(); + expect(cur, "the inbound was promoted to cur/").not.toBeNull(); + + const curDir = resolve(mailDir, "anvil", "cur"); + chmodSync(cur!.path, 0o444); + chmodSync(curDir, 0o555); + try { + h.skip("empty"); // an empty/silent final → the named failure path + h.settle(); + expect(await pollUntil(() => obligation(h.inboundId)?.state === "failed", 4000), "the failed transition is durable").toBe(true); + expect(readCur()?.record?.nackedAt, "the stamp did NOT land").toBeUndefined(); + expect( + await pollUntil( + () => h.logs.some((m) => m.includes("nack-stamp-failed") && m.includes(h.inboundId) && m.includes(cur!.path) && m.includes("EACCES")), + 4000, + ), + "the failed write is logged by id, path and code", + ).toBe(true); + } finally { + chmodSync(curDir, 0o755); + chmodSync(cur!.path, 0o644); + } + await h.stop(); + + const h2 = await boot(false); + expect(await pollUntil(() => !!readCur()?.record?.nackedAt, 4000), "reconciled on the next scan").toBe(true); + expect(obligation(h.inboundId)?.state).toBe("failed"); + await h2.stop(); + }, 20000); + + it("the successful write path is unchanged: a normal ack stamps ackedAt with no diagnostic", async () => { + const h = await boot(true); + expect(await pollUntil(() => h.dispatch() !== null, 4000), "dispatch started").toBe(true); + await h.deliver("verdict"); + h.settle(); + expect(await pollUntil(() => !!readCur()?.record?.ackedAt, 4000)).toBe(true); + expect(readCur()?.record?.read).toBe(true); + expect(h.logs.some((m) => m.includes("ack-stamp-failed"))).toBe(false); + await h.stop(); + }, 15000); +}); From 80e8c7bfead6b37ca309640d71ce2e9e68f6f982 Mon Sep 17 00:00:00 2001 From: flint Date: Sat, 3 Oct 2026 12:32:41 -0700 Subject: [PATCH 02/17] fix(openclaw-tps-mail): a failed terminal stamp is retried in-process (bounded, cancelled on stop); startup reconcile stays the backstop (#492) Co-Authored-By: Claude Opus 5.5 --- .../fixed-492-cur-record-stamp-reconcile.md | 2 +- plugins/openclaw-tps-mail/src/index.ts | 77 +++++--- .../test/cur-record-write.test.ts | 175 +++++++++++------- 3 files changed, 159 insertions(+), 95 deletions(-) diff --git a/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md b/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md index 7c81712e..bca56549 100644 --- a/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md +++ b/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md @@ -1 +1 @@ -- **A failed cur/ record write after a terminal transition is logged by id, path and code and reconciled on the next scan (Closes #492)**. Every ack/nack stamp now goes through one locked, existing-only, atomic writer that reuses the CLI's mailbox lock, so a failed write no longer leaves the on-disk record and the plugin's view diverged in silence. +- **A failed cur/ stamp write after a terminal transition is logged, retried in-process up to 3 times, then re-stamped at the next account start**. Each failure is logged by message id, record path and error code. Every ack/nack stamp goes through one locked, existing-only, atomic writer that reuses the CLI mailbox lock (Closes #492). diff --git a/plugins/openclaw-tps-mail/src/index.ts b/plugins/openclaw-tps-mail/src/index.ts index 25ebfb46..a0dc40f1 100644 --- a/plugins/openclaw-tps-mail/src/index.ts +++ b/plugins/openclaw-tps-mail/src/index.ts @@ -634,7 +634,7 @@ function routeFor(mailDir: string, cfg: any, accountId: string, to: string): Mai * * It never throws. The caller gets a structured outcome it must ACT on, so a * failed write is surfaced (logged by message id, record path and error code) - * and reconciled on the next scan — never silently ignored (cli#492). + * and retried — never silently ignored (cli#492). */ type CurRecordUpdate = | { ok: true } @@ -718,10 +718,10 @@ function updateExistingCurRecord(path: string, patch: Partial): Cur /** * Re-stamp an `acked` or `failed` obligation whose cur/ record never received * its stamp (the write failed after the durable transition — cli#492). The - * obligation record is the durable truth, so on a scan the record and the - * maildir are made to agree: an `acked` obligation re-stamps `ackedAt`/`read`, - * a `failed` one `nackedAt`/`nackReason`. Idempotent; a failure here is logged - * by name so the next scan tries again. + * obligation record is the durable truth, so at account start the record and + * the maildir are made to agree: an `acked` obligation re-stamps + * `ackedAt`/`read`, a `failed` one `nackedAt`/`nackReason`. Idempotent; a + * failure here is logged by name. */ function reconcileTerminalCurStamps(mailDir: string, agent: string, log: any): void { for (const rec of listObligations(mailDir, agent)) { @@ -729,7 +729,7 @@ function reconcileTerminalCurStamps(mailDir: string, agent: string, log: any): v const curPath = findCurPath(mailDir, agent, rec.inboundId); if (!curPath) continue; const cur = readMailFile(curPath); - if (!cur) continue; // absent/unreadable now; the next scan retries + if (!cur) continue; if (rec.state === "acked" && !cur.ackedAt) { const r = updateExistingCurRecord(curPath, { ackedAt: new Date().toISOString(), read: true }); if (r.ok) log?.info?.(`tps-mail: reconciled the acked stamp for ${rec.inboundId} at ${curPath}`); @@ -992,6 +992,52 @@ function readObligationForCleanup(ctx: YieldContext, obligationId: string) { return result; } +/** Delays before each in-process retry of a failed terminal stamp (cli#492). */ +let stampRetryDelaysMs = [1000, 4000, 16000]; + +/** Test-only: shorten the stamp retry delays. Returns the previous delays. */ +export function setStampRetryDelaysForTests(delays: number[]): number[] { + const prev = stampRetryDelaysMs; + stampRetryDelaysMs = delays; + return prev; +} + +/** + * Stamp the cur/ record after a durable terminal transition. A failure is + * logged by id, path and code, then retried under the account's live + * incarnation after each delay in `stampRetryDelaysMs`; the timers are + * cancelled when the account stops. After the last retry the stamp is left to + * `reconcileTerminalCurStamps` at the next account start. + */ +function stampTerminalCur( + ctx: YieldContext, + kind: "ack" | "nack", + patch: Partial, + attempt = 0, +): void { + const key = kind === "ack" ? "ackedAt" : "nackedAt"; + if (attempt > 0) { + const cur = readCurRecord(ctx.curPath); + if (cur.status === "ok" && cur.record[key]) return; + } + const stamped = updateExistingCurRecord(ctx.curPath, patch); + if (stamped.ok) { + if (attempt > 0) ctx.log?.info?.(`tps-mail: ${kind}-stamp-retry-ok: ${ctx.inboundId} at ${ctx.curPath} (retry ${attempt})`); + return; + } + if (stamped.reason === "record-missing") return; + const delay = stampRetryDelaysMs[attempt]; + ctx.log?.warn?.( + `tps-mail: ${kind}-stamp-failed: could not stamp ${key} on ${ctx.inboundId} at ${stamped.path} (${stamped.code}); ` + + (delay === undefined + ? `no retries left, the next account start re-stamps it` + : `retry ${attempt + 1} of ${stampRetryDelaysMs.length} in ${delay}ms`), + ); + if (delay === undefined || !isLiveContext(ctx)) return; + const timer = accountTimer(ctx, () => stampTerminalCur(ctx, kind, patch, attempt + 1), delay); + if (typeof (timer as any).unref === "function") (timer as any).unref(); +} + function ackObligation(ctx: YieldContext, obligationId: string, why: string): void { if (!isLiveContext(ctx)) return; // Only stamp the inbound when the ACK TRANSITION actually landed. A terminal @@ -1009,13 +1055,7 @@ function ackObligation(ctx: YieldContext, obligationId: string, why: string): vo ); return; } - const stamped = updateExistingCurRecord(ctx.curPath, { ackedAt: new Date().toISOString(), read: true }); - if (!stamped.ok && stamped.reason !== "record-missing") { - ctx.log?.warn?.( - `tps-mail: ack-stamp-failed: could not stamp ackedAt on ${ctx.inboundId} at ${stamped.path} (${stamped.code}); ` + - `the obligation is acked and the stamp is reconciled on the next scan`, - ); - } + stampTerminalCur(ctx, "ack", { ackedAt: new Date().toISOString(), read: true }); ctx.log?.info?.(`tps-mail: acked ${ctx.inboundId} — ${why}`); releaseObligationState(obligationId); } @@ -1186,13 +1226,7 @@ async function settleObligation( // handed off (cli#403). releaseObligationState(obligationId); if (!s.alreadyStamped) { - const stamped = updateExistingCurRecord(ctx.curPath, { nackedAt: new Date().toISOString(), nackReason: failedOn }); - if (!stamped.ok && stamped.reason !== "record-missing") { - ctx.log?.warn?.( - `tps-mail: nack-stamp-failed: could not stamp nackedAt on ${ctx.inboundId} at ${stamped.path} (${stamped.code}); ` + - `the obligation is failed and the stamp is reconciled on the next scan`, - ); - } + stampTerminalCur(ctx, "nack", { nackedAt: new Date().toISOString(), nackReason: failedOn }); } // cli#389 round 10, item 1: AWAIT the one send, and record its outcome on the // record. The mail is owed until `nackSentAt` says otherwise (at-least-once). @@ -2398,8 +2432,7 @@ const gateway: ChannelGatewayAdapter = { // cur/ record never received its stamp — the write failed AFTER the // durable transition — is re-stamped here, under the same lock that // write uses, BEFORE the recovery loop below decides whether to - // re-dispatch the record. Idempotent; a failure here is logged by name - // and retried on the next scan. + // re-dispatch the record. Idempotent; a failure here is logged by name. reconcileTerminalCurStamps(account.mailDir, agentId, log); // Crash recovery (at-least-once): re-dispatch cur/ records that were diff --git a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts index 00cced73..1dd7714c 100644 --- a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts +++ b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts @@ -9,11 +9,11 @@ * These tests inject a write failure (the cur/ DIRECTORY is made unwritable, so * the atomic stamp's temp file cannot be created) and assert: * (a) a diagnostic naming the message id, the record path and the error code; - * (b) a later scan reconciles the record to the terminal state. - * A third test pins the successful path as unchanged. - * - * The first two FAIL on the pre-fix tree: the write failure is silent and no - * scan ever reconciles the record. + * (b) a bounded in-process retry fixes a transient failure without a restart; + * (c) a persistent failure stops after the bound and the next account start + * re-stamps the record; + * (d) stopping the account cancels pending retries. + * A further test pins the successful path as unchanged. */ import { describe, expect, it, beforeEach, afterEach, mock } from "bun:test"; import { chmodSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; @@ -30,7 +30,7 @@ const FLINT_SEED = Buffer.alloc(32, 0x01); const ANVIL_SEED = Buffer.alloc(32, 0x02); const pubkeyFromSeed = (s: Buffer): Buffer => Buffer.from(ed.getPublicKey(new Uint8Array(s))); -import pluginModule from "../src/index.js"; +import pluginModule, { setStampRetryDelaysForTests } from "../src/index.js"; let capturedPlugin: any; const mockApi: any = { @@ -45,6 +45,7 @@ let keysDir: string; let home: string; let origHome: string | undefined; let origKeys: string | undefined; +let prevDelays: number[]; beforeEach(() => { mailDir = mkdtempSync(join(tmpdir(), "tps-492-mail-")); @@ -52,6 +53,7 @@ beforeEach(() => { home = mkdtempSync(join(tmpdir(), "tps-492-home-")); writeFileSync(join(keysDir, "anvil.key"), ANVIL_SEED); writeFileSync(join(keysDir, "flint.key"), FLINT_SEED); + prevDelays = setStampRetryDelaysForTests([100, 100, 100]); origHome = process.env.HOME; process.env.HOME = home; origKeys = process.env.TPS_TEST_KEYS_DIR; process.env.TPS_TEST_KEYS_DIR = keysDir; mock.module("@tpsdev-ai/cli/utils/mail-verify", () => ({ @@ -66,6 +68,7 @@ beforeEach(() => { }); afterEach(() => { + setStampRetryDelaysForTests(prevDelays); if (origHome === undefined) delete process.env.HOME; else process.env.HOME = origHome; if (origKeys === undefined) delete process.env.TPS_TEST_KEYS_DIR; else process.env.TPS_TEST_KEYS_DIR = origKeys; for (const d of [mailDir, keysDir, home]) { try { rmSync(d, { recursive: true, force: true }); } catch { /* best effort */ } } @@ -160,77 +163,105 @@ async function boot(withInbound: boolean): Promise { }; } -describe("cli#492 — a failed cur/ stamp write is surfaced and reconciled", () => { - it("ack: a failed ackedAt write logs id/path/code and is reconciled on the next scan", async () => { - const h = await boot(true); - expect(await pollUntil(() => h.dispatch() !== null, 4000), "dispatch started").toBe(true); - const cur = readCur(); - expect(cur, "the inbound was promoted to cur/").not.toBeNull(); - - // The cur/ DIRECTORY and the record are both unwritable: a direct overwrite - // of the record AND the atomic stamp's temp file fail, so the write FAILS - // after the durable `acked` transition. - const curDir = resolve(mailDir, "anvil", "cur"); - chmodSync(cur!.path, 0o444); - chmodSync(curDir, 0o555); - try { - await h.deliver("verdict"); - h.settle(); - expect(await pollUntil(() => obligation(h.inboundId)?.state === "acked", 4000), "the acked transition is durable").toBe(true); - expect(readCur()?.record?.ackedAt, "the stamp did NOT land").toBeUndefined(); - expect( - await pollUntil( - () => h.logs.some((m) => m.includes("ack-stamp-failed") && m.includes(h.inboundId) && m.includes(cur!.path) && m.includes("EACCES")), - 4000, - ), - "the failed write is logged by id, path and code", - ).toBe(true); - } finally { - chmodSync(curDir, 0o755); - chmodSync(cur!.path, 0o644); - } - await h.stop(); - - // NEXT SCAN: a fresh start reconciles the durable terminal state onto the record. - const h2 = await boot(false); - expect(await pollUntil(() => !!readCur()?.record?.ackedAt, 4000), "reconciled on the next scan").toBe(true); - expect(readCur()?.record?.read).toBe(true); - expect(obligation(h.inboundId)?.state).toBe("acked"); - await h2.stop(); - }, 20000); +const failedLogs = (h: Boot, tag: string) => h.logs.filter((m) => m.includes(tag) && m.includes(h.inboundId)); - it("nack: a failed nackedAt write logs id/path/code and is reconciled on the next scan", async () => { - const h = await boot(true); - expect(await pollUntil(() => h.dispatch() !== null, 4000), "dispatch started").toBe(true); - const cur = readCur(); - expect(cur, "the inbound was promoted to cur/").not.toBeNull(); +/** Make the cur/ record and directory unwritable so the stamp write fails with EACCES. */ +function breakCur(path: string): () => void { + const curDir = resolve(mailDir, "anvil", "cur"); + chmodSync(path, 0o444); + chmodSync(curDir, 0o555); + return () => { chmodSync(curDir, 0o755); chmodSync(path, 0o644); }; +} - const curDir = resolve(mailDir, "anvil", "cur"); - chmodSync(cur!.path, 0o444); - chmodSync(curDir, 0o555); - try { +describe("cli#492 — a failed cur/ stamp write is surfaced and retried", () => { + for (const kind of ["ack", "nack"] as const) { + const stampKey = kind === "ack" ? "ackedAt" : "nackedAt"; + const state = kind === "ack" ? "acked" : "failed"; + const finish = (h: Boot) => { + if (kind === "ack") return h.deliver("verdict").then(() => h.settle()); h.skip("empty"); // an empty/silent final → the named failure path h.settle(); - expect(await pollUntil(() => obligation(h.inboundId)?.state === "failed", 4000), "the failed transition is durable").toBe(true); - expect(readCur()?.record?.nackedAt, "the stamp did NOT land").toBeUndefined(); - expect( - await pollUntil( - () => h.logs.some((m) => m.includes("nack-stamp-failed") && m.includes(h.inboundId) && m.includes(cur!.path) && m.includes("EACCES")), - 4000, - ), - "the failed write is logged by id, path and code", - ).toBe(true); - } finally { - chmodSync(curDir, 0o755); - chmodSync(cur!.path, 0o644); - } - await h.stop(); + return Promise.resolve(); + }; + + it(`${kind}: a transient failed ${stampKey} write is logged by id/path/code and fixed by the in-process retry, no restart`, async () => { + const h = await boot(true); + expect(await pollUntil(() => h.dispatch() !== null, 4000), "dispatch started").toBe(true); + const cur = readCur(); + expect(cur, "the inbound was promoted to cur/").not.toBeNull(); + const restore = breakCur(cur!.path); + let restored = false; + try { + await finish(h); + expect(await pollUntil(() => obligation(h.inboundId)?.state === state, 4000), "the terminal transition is durable").toBe(true); + expect(readCur()?.record?.[stampKey], "the stamp did NOT land").toBeUndefined(); + expect( + await pollUntil( + () => failedLogs(h, `${kind}-stamp-failed`).some((m) => m.includes(cur!.path) && m.includes("EACCES")), + 4000, + ), + "the failed write is logged by id, path and code", + ).toBe(true); + } finally { + restore(); + restored = true; + } + expect(restored).toBe(true); + expect(await pollUntil(() => !!readCur()?.record?.[stampKey], 4000), "the in-process retry stamped the record").toBe(true); + expect(obligation(h.inboundId)?.state).toBe(state); + await h.stop(); + }, 20000); - const h2 = await boot(false); - expect(await pollUntil(() => !!readCur()?.record?.nackedAt, 4000), "reconciled on the next scan").toBe(true); - expect(obligation(h.inboundId)?.state).toBe("failed"); - await h2.stop(); - }, 20000); + it(`${kind}: a persistent failure stops after the bound and the next account start re-stamps the record`, async () => { + const h = await boot(true); + expect(await pollUntil(() => h.dispatch() !== null, 4000), "dispatch started").toBe(true); + const cur = readCur(); + expect(cur, "the inbound was promoted to cur/").not.toBeNull(); + const restore = breakCur(cur!.path); + try { + await finish(h); + // initial attempt + 3 retries = 4 logged failures, then no more. + expect(await pollUntil(() => failedLogs(h, `${kind}-stamp-failed`).length >= 4, 4000), "initial attempt + 3 retries").toBe(true); + await sleep(400); + const logged = failedLogs(h, `${kind}-stamp-failed`); + expect(logged.length, "no attempt beyond the bound").toBe(4); + expect(logged[3]).toContain("no retries left"); + expect(readCur()?.record?.[stampKey]).toBeUndefined(); + } finally { + restore(); + } + await sleep(300); + expect(readCur()?.record?.[stampKey], "nothing retries after the bound").toBeUndefined(); + await h.stop(); + + // NEXT ACCOUNT START: the durable terminal state is re-stamped onto the record. + const h2 = await boot(false); + expect(await pollUntil(() => !!readCur()?.record?.[stampKey], 4000), "re-stamped at the next account start").toBe(true); + if (kind === "ack") expect(readCur()?.record?.read).toBe(true); + expect(obligation(h.inboundId)?.state).toBe(state); + await h2.stop(); + }, 20000); + + it(`${kind}: stopping the account cancels the pending stamp retries`, async () => { + setStampRetryDelaysForTests([300, 300, 300]); + const h = await boot(true); + expect(await pollUntil(() => h.dispatch() !== null, 4000), "dispatch started").toBe(true); + const cur = readCur(); + expect(cur, "the inbound was promoted to cur/").not.toBeNull(); + const restore = breakCur(cur!.path); + try { + await finish(h); + expect(await pollUntil(() => failedLogs(h, `${kind}-stamp-failed`).length >= 1, 4000), "first failure logged").toBe(true); + await h.stop(); + } finally { + restore(); + } + const before = failedLogs(h, `${kind}-stamp-failed`).length; + await sleep(1200); + expect(failedLogs(h, `${kind}-stamp-failed`).length, "no retry ran after stop").toBe(before); + expect(readCur()?.record?.[stampKey], "the cancelled retry never stamped").toBeUndefined(); + }, 20000); + } it("the successful write path is unchanged: a normal ack stamps ackedAt with no diagnostic", async () => { const h = await boot(true); From 58f9a56c4e6021202fe59e8cd8bc24beb0b0670c Mon Sep 17 00:00:00 2001 From: flint Date: Sat, 3 Oct 2026 15:48:05 -0700 Subject: [PATCH 03/17] fix(openclaw-tps-mail): drop the unused mail-lock subpath export; body and changelog name the writer actually used Co-Authored-By: Claude Opus 5.5 --- .../fixed-492-cur-record-stamp-reconcile.md | 2 +- packages/cli/package.json | 1 - pr-body.new.md | 189 ------------------ 3 files changed, 1 insertion(+), 191 deletions(-) delete mode 100644 pr-body.new.md diff --git a/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md b/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md index bca56549..43a3000f 100644 --- a/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md +++ b/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md @@ -1 +1 @@ -- **A failed cur/ stamp write after a terminal transition is logged, retried in-process up to 3 times, then re-stamped at the next account start**. Each failure is logged by message id, record path and error code. Every ack/nack stamp goes through one locked, existing-only, atomic writer that reuses the CLI mailbox lock (Closes #492). +- **A failed cur/ stamp write after a terminal transition is logged, retried in-process up to 3 times, then re-stamped at the next account start**. Each failure is logged by message id, record path and error code. Ack/nack stamps use the CLI's locked, existing-only `updateExistingRecord` (#469) (Closes #492). diff --git a/packages/cli/package.json b/packages/cli/package.json index 81bea909..7c050421 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -16,7 +16,6 @@ "./utils/mail-verify": "./dist/src/utils/mail-verify.js", "./utils/mail-routing": "./dist/src/utils/mail-routing.js", "./utils/mail-producer": "./dist/src/utils/mail-producer.js", - "./utils/mail-lock": "./dist/src/utils/mail-lock.js", "./utils/relay": "./dist/src/utils/relay.js" }, "optionalDependencies": { diff --git a/pr-body.new.md b/pr-body.new.md deleted file mode 100644 index a3307ac9..00000000 --- a/pr-body.new.md +++ /dev/null @@ -1,189 +0,0 @@ -Closes #380. -Closes #487. - -For CLI signed-inbox delivery, promote() is the only first-delivery writer of cur/. Updates touch only existing records. MailClient and CLI promotion share the envelope policy and locked consumed-ID replay store; CLI promotion additionally rejects an invalid signed non-bridge trust value, which MailClient instead maps to external. Outbox and internal mail are separate stores. - -## What changed - -- **The deploy bot (both copies) and the channel bridge promote instead of renaming.** Failed verification refuses delivery and attempts dead-lettering. The bridge's parse-failure path no longer moves an unparseable record into `cur/`. -- **Bridge acknowledgement uses the promoted path.** The bridge attempts to persist a sent marker after adapter.send succeeds; recovery skips sent or acknowledged records. Reported send failures leave no marker and are retried on restart. Ack atomically updates existing records. Ack or cleanup failure preserves the sent marker. A crash or marker-write failure after successful send can permit a duplicate. -- **Both retry eligible `dlq/` entries.** The bridge also retries `new/` on its timer and serializes mailbox work. -- **One mailbox policy for `promote()`, `MailClient`, and topic catch-up.** The signature/sender/recipient/id-shape/timestamp decision, envelope parser, and topic-recipient rules live in `packages/agent/src/lib/mailbox-policy.ts`. The CLI and `MailClient` use its consumed-id replay store under the shared mailbox lock; topic catch-up retains its cursor when verification or recipient policy is unavailable. -- **`MailClient` throws when constructed without a verifier.** The parameter is still optional in TypeScript; the refusal is at runtime. `AgentRuntime` constructs a verifier using `config.flair?.url`, `FLAIR_URL`, then `http://127.0.0.1:9926`. -- **Updates use `updateExistingRecord()`.** It locks, re-reads, mutates and atomically replaces an existing record; lease checkout revalidates its verified snapshot. The writer scan allows promotion, the locked existing-only helper, `MailClient`, the container outbox/cur archive (`relay.ts`) and the office internal-mail store (`internal-mail.ts`); unrecognized destinations may be missed. The delegation check covers `checkMessages`, `setBridgeSentAtPath`, `ackMessageAtPath`, `nackMessage` and `patchMailFile`. -- **Follow-ups:** #482 same-filename promotion overwrite; the per-process bridge queue. - -- **Registry verification accepts hex and canonical base64 public keys; the agent provider accepts raw private seeds.** Non-404 registry read failures are retryable. -- **Lock acquisition and stale reclamation share an atomic claim.** A stranded claim requires operator recovery; polling re-reads birth tokens. -- **Unrecoverable consumed history withholds delivery.** Appends preserve a line boundary; initialized ledger loss refuses delivery; CLI cur recovery requires a ledger ID. -- **The scan uses the filesystem destination position before options or callbacks.** The forged deploy-bot fixture includes an ID. -- **The delivery-control test checks its build prerequisites.** Missing agent entry points report `packages/agent/dist missing — run bun run build`; the root test script is unchanged. - -## Evidence - -### Touched tests, measured on b5527966 - -| File | Fixture | Pass | Fail | -|---|---|---|---| -| `test/mail-cur-writers.test.ts` | native | 8 | 0 | -| `packages/cli/test/mail-final-controls.test.ts` | native | 14 | 0 | -| `packages/agent/test/mail-promote-guard.test.ts` | in-process HTTP | 20 | 0 | -| `packages/cli/test/bridge-mail-promote.test.ts` | in-process HTTP and polling watcher | 3 | 0 | - -Native bind/watch integration remains unverified. - -### Measured on 9d86f15f versus origin/main 3df27695 - -Isolated launchers, canonical paths, empty launcher HOME; per-file runs with a 90-second deadline. Plugin dependencies came from the existing offline installation; both trees were built. - -| Suite | 9d86f15f pass/fail | origin/main 3df27695 pass/fail | Timeouts (head/main) | -|---|---|---|---| -| agent | 134/0 | 120/0 | 0/0 | -| cli | 1507/16 | 1496/16 | 0/0 | -| pi-tps-mail | 0/0 | 0/0 | 0/0 | -| root-test | 135/2 | 123/2 | 0/0 | -| plugin | 208/0 | 205/0 | 1/1 | -| github-review | 195/6 | 218/6 | 0/0 | -| Whole socket-free lane, observed cases | 2179/24 | 2162/24 | 1/1 | - -Counts combine completed cases and successful reruns without double-counting. Both trees skip the same three pi-tps-mail cases and have identical failing test names. - -The updated cur-writers test passes 10/0 on 9d86f15f. Applied as a test fixture to origin/main 3df27695 sources (which have no native cur-writers test), it reports 6/4 and detects the original deploy-bot and bridge bypasses. - -Timed-out file on both trees: - -```text -plugins/openclaw-tps-mail/test/locality.test.ts -``` - -Failed test names on both trees: - -```text -4e positive — real nono (Landlock/Seatbelt): the premise > nono denies a path outside every grant while the granted twin is readable -4e positive — the attested launch against the pinned nono > tps agent start is RELEASED: real nono, canaries verified, session bound to the spawned pid -4e — the private dir is removed on every path > createPrivateLaunchDir + removePrivateLaunchDir leave nothing behind -4e+r4f positive — a TTY parent still RELEASES (real nono) > script(1) gives the launch a PTY; real nono keeps it and the child attests anyway -4g — the launch socket path is bounded to sun_path > a 64-char id under a long HOME still yields a within-limit socket path -4g — the launch socket path is bounded to sun_path > a HOME too long for even the shortened label refuses LOUDLY, naming the length -A1 — independent plugin loading > the built entry loads under node -E — the gateway-boundary lane (OpenClaw loader + gateway tool dispatch, in a separate node process) > from the lane's manifest OVERLAY: the probe runs in the gateway process, reads the host-only marker, sees a sandboxed session; a concurrent pair posts ONCE -E — the gateway-boundary lane (OpenClaw loader + gateway tool dispatch, in a separate node process) > the SHIPPED manifest REJECTS the CI probe even with the CI flag set -E — the gateway-boundary lane (OpenClaw loader + gateway tool dispatch, in a separate node process) > the SHIPPED manifest: zero diagnostics, default sandbox policy withholds the verb, the documented allow offers it, and it POSTS with the audit acknowledged -T5 — the pinned-path launch spawns nono and the child argv asserts the flags > agent start --sandbox-required with a fake nono at NONO_BIN: the run argv carries both flags -changelog fragments — two PRs with distinct fragment filenames (cli#449) > B merged into A, and A's original commit merged into B: both clean, both fragments present -get > runs verify and returns live value -latch-admin reconcile — decisions, records, application > the BUILT command runs under node: list, and clear refusing a posted latch -runCommandUnderNono() > warns and falls back when nono not on PATH (non-strict) -runVerify — nonzero exit > false command returns nonzero_exit -tps agent commit > creates a branch and commits only the requested paths -tps agent commit > pushes the branch and opens a PR via gh-as -type coercion > string coercion — rejects empty -``` - -Loopback-bind exclusions (socket-free cases retained where possible; launch-attestation requires Unix sockets). Loopback binds were refused. - -```text -packages/agent/test/flair-context.test.ts -packages/agent/test/mail-promote-guard.test.ts -packages/cli/test/branch-join.test.ts -packages/cli/test/bridge-mail-promote.test.ts -packages/cli/test/codex-presence-444.test.ts -packages/cli/test/flair-sync.test.ts -packages/cli/test/launch-attestation.test.ts -packages/cli/test/mail-bridge.test.ts -packages/cli/test/mail-producers-sign.test.ts -packages/cli/test/mail-promote.test.ts -packages/cli/test/mail-receipt-thread.test.ts -packages/cli/test/mail-remote.test.ts -packages/cli/test/mail-send-routes.test.ts -packages/cli/test/mail-send-stdin-reply.test.ts -packages/cli/test/mail-unresolvable-principal.test.ts -packages/cli/test/mail-watch.test.ts -packages/cli/test/mail.test.ts -packages/cli/test/mock-llm.test.ts -packages/cli/test/noise-ik-transport.test.ts -packages/cli/test/plain-tcp-transport.test.ts -packages/cli/test/runtime-mail-lifecycle.test.ts -packages/cli/test/service-proxy.test.ts -packages/cli/test/wire-mail.test.ts -packages/cli/test/ws-noise-transport.test.ts -packages/pi-tps-mail/test/reply-send.test.ts -test/deploy-bot-promote.test.ts -``` - -### Measured on bbd6d313 versus origin/main 42de3b4b - -Isolated per-file launchers; both trees built with offline plugin dependencies. Counts replace timing-failure runs with matching reruns. - -| Suite | bbd6d313 pass/fail | origin/main 42de3b4b pass/fail | -|---|---|---| -| agent | 135/0 | 121/0 | -| cli | 1621/13 | 1854/16 | -| pi-tps-mail | 20/0 | 20/0 | -| root-test | 135/1 | 123/1 | -| plugin | 212/0 | 209/0 | -| github-review | 221/2 | 221/2 | -| Whole socket-free lane | 2344/16 | 2548/19 | - -Shared files have the same failing test names; `runtime-attested-launch.test.ts` exists only on origin/main and adds three failures there. No final runs timed out. - -Bridge/mail/cur checks present in each tree: bbd6d313 72/3; origin/main 42de3b4b 34/3, with the same failures in the shared tests. `bridge-ack-path.test.ts` reports 11/0 on bbd6d313 and 2/9 on 32104f71; the requested regressions fail on 32104f71. - -### Measured on 056338b4 versus origin/main 42de3b4b - -| Suite | 056338b4 pass/fail | origin/main 42de3b4b pass/fail | -|---|---|---| -| agent | 134/0 | 120/0 | -| cli | 1613/22 | 1838/25 | -| pi-tps-mail | 20/0 | 20/0 | -| root-test | 135/1 | 123/1 | -| plugin | 212/0 | 209/0 | -| github-review | 221/2 | 221/2 | -| Whole socket-free lane | 2335/25 | 2531/28 | -| Bridge/mail/cur target files | 47/0 | 4/0 | - - -### Measured on a8f1fd1a versus origin/main 42de3b4b - -| Suite | a8f1fd1a pass/fail | origin/main 42de3b4b pass/fail | -|---|---|---| -| agent | 135/0 | 121/0 | -| cli | 1629/19 | 1852/23 | -| pi-tps-mail | 20/0 | 20/0 | -| root-test | 135/1 | 123/1 | -| plugin | 197/2 | 194/2 | -| github-review | 216/7 | 216/7 | -| Whole socket-free lane | 2332/29 | 2526/33 | -| Bridge/mail/cur target files | 60/0 | — | - -### Measured on bb055580 versus origin/main 42de3b4b - -| Suite | bb055580 pass/fail | origin/main 42de3b4b pass/fail | -|---|---|---| -| agent | 135/0 | 121/0 | -| cli | 1615/15 | 1822/18 | -| pi-tps-mail | 20/0 | 20/0 | -| root-test | 137/1 | 123/1 | -| plugin | 212/0 | 209/0 | -| github-review | 220/3 | 220/3 | -| Whole socket-free lane | 2339/19 | 2515/22 | -| Bridge/mail/cur | 243/0 | 143/0 | - -Shared files have the same failing test names; origin/main-only `runtime-attested-launch.test.ts` adds failures there. Matching sequential reruns replace watcher timing failures. Final runs had no timeouts. - -Lease-sweep-vs-ack and nack-vs-ack: bb055580 2/0; a8f1fd1a 0/2. Existing ack-vs-ack: bb055580 1/0. - - -## Summary by CodeRabbit - -* **New Features** - * Mail that cannot be verified during a temporary service outage can be retried when verification becomes available. - * Mail processing validates signed envelopes, recipients, and message IDs before delivery. - * Deployment bots and bridges can retry eligible dead-lettered mail. -* **Bug Fixes** - * Invalid, forged, or previously delivered messages are withheld from delivery. - * Mailbox history and locking safeguards help prevent duplicate delivery during concurrent processing or storage issues. - * Deployment bots and bridges verify incoming mail before forwarding it. - - -Merged main’s #485 strict synchronous outbox lock entry point; mailbox callers retain the shared agent lock. From 694177306ffb89bdb4c913c1e1fc74e379a86ee0 Mon Sep 17 00:00:00 2001 From: flint Date: Sat, 3 Oct 2026 17:15:39 -0700 Subject: [PATCH 04/17] fix(openclaw-tps-mail): failed stamps and unreadable obligations are reported; reconcile is idempotent; stop cancels its timer (tested) Co-Authored-By: Claude Opus 5.5 --- .../fixed-492-cur-record-stamp-reconcile.md | 2 +- plugins/openclaw-tps-mail/src/index.ts | 116 ++++++++------- plugins/openclaw-tps-mail/src/obligations.ts | 15 +- .../test/cur-record-write.test.ts | 100 ++++++++++--- .../test/patch-mail-file.test.ts | 134 +++++++++++++++++- 5 files changed, 285 insertions(+), 82 deletions(-) diff --git a/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md b/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md index 43a3000f..af543e34 100644 --- a/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md +++ b/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md @@ -1 +1 @@ -- **A failed cur/ stamp write after a terminal transition is logged, retried in-process up to 3 times, then re-stamped at the next account start**. Each failure is logged by message id, record path and error code. Ack/nack stamps use the CLI's locked, existing-only `updateExistingRecord` (#469) (Closes #492). +- **Ack/nack stamps use locked writes**. Stamps use the CLI's existing-only `updateExistingRecord` (#469) (Closes #492). diff --git a/plugins/openclaw-tps-mail/src/index.ts b/plugins/openclaw-tps-mail/src/index.ts index 67647a42..f6eb2d41 100644 --- a/plugins/openclaw-tps-mail/src/index.ts +++ b/plugins/openclaw-tps-mail/src/index.ts @@ -262,11 +262,12 @@ function findBoundAgents(cfg: any, accountId: string): string[] { // ─── TPS mail envelope helpers ─────────────────────────────────────────────── -function readMailFile(filePath: string): TpsMailBody | null { +function readMailFile(filePath: string, onReadError?: (path: string, code: string) => void): TpsMailBody | null { try { const raw = readFileSync(filePath, "utf-8"); return JSON.parse(raw) as TpsMailBody; - } catch { + } catch (err: any) { + if (err?.code !== "ENOENT") onReadError?.(filePath, err?.code ?? "INVALID_RECORD"); return null; } } @@ -618,54 +619,66 @@ function routeFor(mailDir: string, cfg: any, accountId: string, to: string): Mai return resolveMailRoute({ to, mailDir, localAgents: findBoundAgents(cfg, accountId) }); } -/** - * Update an existing cur/ record through the CLI's `updateExistingRecord` - * (cli#469). Never throws: a missing record is `record-missing`; any other - * failure is `write-failed` with its error code, for the caller to log and - * retry (cli#492). - */ type CurRecordUpdate = | { ok: true } | { ok: false; reason: "record-missing" | "write-failed"; path: string; code: string }; -export function patchMailFile(path: string, patch: Partial): CurRecordUpdate { +export function patchMailFile(path: string, patch: Partial, stampKey?: "ackedAt" | "nackedAt"): CurRecordUpdate { try { - const r = updateExistingRecord(path, (current) => Object.assign(current, patch)); + let alreadyStamped = false; + const r = updateExistingRecord(path, (current) => { + if (stampKey && current[stampKey]) { + alreadyStamped = true; + return null; + } + return Object.assign(current, patch); + }); + if (r.status === "changed" && alreadyStamped) return { ok: true }; if (r.status === "updated") return { ok: true }; if (r.status === "gone") return { ok: false, reason: "record-missing", path, code: "ENOENT" }; return { ok: false, reason: "write-failed", path, code: r.status }; } catch (err: any) { - return { ok: false, reason: "write-failed", path, code: err?.code ?? err?.name ?? "WRITE_FAILED" }; + return { ok: false, reason: "write-failed", path, code: err?.code ?? "WRITE_FAILED" }; } } -/** - * Re-stamp an `acked` or `failed` obligation whose cur/ record never received - * its stamp (the write failed after the durable transition — cli#492). The - * obligation record is the durable truth, so at account start the record and - * the maildir are made to agree: an `acked` obligation re-stamps - * `ackedAt`/`read`, a `failed` one `nackedAt`/`nackReason`. Idempotent; a - * failure here is logged by name. - */ -function reconcileTerminalCurStamps(mailDir: string, agent: string, log: any): void { - for (const rec of listObligations(mailDir, agent)) { +export function reconcileTerminalCurStamps(mailDir: string, agent: string, log: any): void { + const reportReadError = (state: string, id: string) => (path: string, code: string) => log?.warn?.( + `tps-mail: stamp-reconcile-read-failed: ${id} actor=${agent} state=${state} path=${path} code=${code}; restore readable records and restart the account`, + ); + for (const rec of listObligations(mailDir, agent, reportReadError("unknown", "unknown"))) { if (rec.state !== "acked" && rec.state !== "failed") continue; - const curPath = findCurPath(mailDir, agent, rec.inboundId); - if (!curPath) continue; - const cur = readMailFile(curPath); - if (!cur) continue; - if (rec.state === "acked" && !cur.ackedAt) { - const r = patchMailFile(curPath, { ackedAt: new Date().toISOString(), read: true }); - if (r.ok) log?.info?.(`tps-mail: reconciled the acked stamp for ${rec.inboundId} at ${curPath}`); - else if (r.reason !== "record-missing") { - log?.warn?.(`tps-mail: ack-stamp-reconcile-failed: ${rec.inboundId} at ${r.path} (${r.code})`); - } - } else if (rec.state === "failed" && !cur.nackedAt) { - const r = patchMailFile(curPath, { nackedAt: new Date().toISOString(), nackReason: rec.failure ?? "failed" }); - if (r.ok) log?.info?.(`tps-mail: reconciled the nacked stamp for ${rec.inboundId} at ${curPath}`); - else if (r.reason !== "record-missing") { - log?.warn?.(`tps-mail: nack-stamp-reconcile-failed: ${rec.inboundId} at ${r.path} (${r.code})`); - } + let unreadable = false; + const onReadError = (path: string, code: string) => { + unreadable = true; + reportReadError(rec.state, rec.inboundId)(path, code); + }; + const kind = rec.state === "acked" ? "ack" : "nack"; + const key = kind === "ack" ? "ackedAt" : "nackedAt"; + const curPath = findCurPath(mailDir, agent, rec.inboundId, onReadError); + const missing = (path: string) => log?.warn?.( + `tps-mail: ${kind}-stamp-reconcile-failed: ${rec.inboundId} actor=${agent} state=${rec.state} path=${path} code=ENOENT; inspect the missing cur record; obligation retained`, + ); + if (!curPath) { + if (!unreadable) missing(resolve(mailDir, agent, "cur")); + continue; + } + const cur = readMailFile(curPath, onReadError); + if (!cur) { + if (!unreadable) missing(curPath); + continue; + } + if (cur[key]) continue; + const patch = kind === "ack" + ? { ackedAt: new Date().toISOString(), read: true } + : { nackedAt: new Date().toISOString(), nackReason: rec.failure ?? "failed" }; + const r = patchMailFile(curPath, patch, key); + if (r.ok) log?.info?.(`tps-mail: reconciled the ${kind} stamp for ${rec.inboundId} at ${curPath}`); + else { + log?.warn?.( + `tps-mail: ${kind}-stamp-reconcile-failed: ${rec.inboundId} actor=${agent} state=${rec.state} path=${r.path} code=${r.code}; ` + + (r.reason === "record-missing" ? "inspect the missing cur record; obligation retained" : "restore writable records and restart the account; obligation retained"), + ); } } } @@ -926,13 +939,6 @@ export function setStampRetryDelaysForTests(delays: number[]): number[] { return prev; } -/** - * Stamp the cur/ record after a durable terminal transition. A failure is - * logged by id, path and code, then retried under the account's live - * incarnation after each delay in `stampRetryDelaysMs`; the timers are - * cancelled when the account stops. After the last retry the stamp is left to - * `reconcileTerminalCurStamps` at the next account start. - */ function stampTerminalCur( ctx: YieldContext, kind: "ack" | "nack", @@ -940,18 +946,18 @@ function stampTerminalCur( attempt = 0, ): void { const key = kind === "ack" ? "ackedAt" : "nackedAt"; - if (attempt > 0 && readMailFile(ctx.curPath)?.[key]) return; - const stamped = patchMailFile(ctx.curPath, patch); + const stamped = patchMailFile(ctx.curPath, patch, key); if (stamped.ok) { if (attempt > 0) ctx.log?.info?.(`tps-mail: ${kind}-stamp-retry-ok: ${ctx.inboundId} at ${ctx.curPath} (retry ${attempt})`); return; } - if (stamped.reason === "record-missing") return; - const delay = stampRetryDelaysMs[attempt]; + const delay = stamped.reason === "record-missing" ? undefined : stampRetryDelaysMs[attempt]; ctx.log?.warn?.( - `tps-mail: ${kind}-stamp-failed: could not stamp ${key} on ${ctx.inboundId} at ${stamped.path} (${stamped.code}); ` + - (delay === undefined - ? `no retries left, the next account start re-stamps it` + `tps-mail: ${kind}-stamp-failed: ${ctx.inboundId} actor=${ctx.agent} state=${kind === "ack" ? "acked" : "failed"} path=${stamped.path} code=${stamped.code}; ` + + (stamped.reason === "record-missing" + ? "inspect the missing cur record; obligation retained" + : delay === undefined + ? "no retries left; restore writable records and restart the account; obligation retained" : `retry ${attempt + 1} of ${stampRetryDelaysMs.length} in ${delay}ms`), ); if (delay === undefined || !isLiveContext(ctx)) return; @@ -1622,17 +1628,17 @@ function installYieldSubscription(api: any): boolean { } /** The cur/ path for an inbound id (cur filenames are timestamp-id, not the id). */ -function findCurPath(mailDir: string, agent: string, inboundId: string): string | null { +function findCurPath(mailDir: string, agent: string, inboundId: string, onReadError?: (path: string, code: string) => void): string | null { const curDir = resolve(mailDir, agent, "cur"); try { for (const name of readdirSync(curDir)) { if (!name.endsWith(".json")) continue; const p = resolve(curDir, name); - const rec = readMailFile(p); + const rec = readMailFile(p, onReadError); if (rec?.id === inboundId) return p; } - } catch { - // no cur dir yet + } catch (err: any) { + if (err?.code !== "ENOENT") onReadError?.(curDir, err?.code ?? "READ_FAILED"); } return null; } diff --git a/plugins/openclaw-tps-mail/src/obligations.ts b/plugins/openclaw-tps-mail/src/obligations.ts index d5207db2..5b6dafe2 100644 --- a/plugins/openclaw-tps-mail/src/obligations.ts +++ b/plugins/openclaw-tps-mail/src/obligations.ts @@ -200,12 +200,19 @@ export function readObligationResult(mailDir: string, agent: string, inboundId: } } -export function listObligations(mailDir: string, agent: string): ObligationRecord[] { +export function listObligations( + mailDir: string, + agent: string, + onReadError: (path: string, code: string) => void = (path, code) => console.warn( + `tps-mail: obligation-list-read-failed: actor=${agent} state=unknown path=${path} code=${code}; restore readable obligations and restart the account`, + ), +): ObligationRecord[] { const dir = obligationsDir(mailDir, agent); let names: string[]; try { names = readdirSync(dir); - } catch { + } catch (err: any) { + if (err?.code !== "ENOENT") onReadError(dir, err?.code ?? "READ_FAILED"); return []; } const out: ObligationRecord[] = []; @@ -213,8 +220,8 @@ export function listObligations(mailDir: string, agent: string): ObligationRecor if (!name.endsWith(".json") || name.startsWith(".")) continue; try { out.push(JSON.parse(readFileSync(resolve(dir, name), "utf-8")) as ObligationRecord); - } catch { - // A torn record is not readable truth; skip it rather than crash recovery. + } catch (err: any) { + if (err?.code !== "ENOENT") onReadError(resolve(dir, name), err?.code ?? "INVALID_RECORD"); } } return out; diff --git a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts index 1dd7714c..53ba25a7 100644 --- a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts +++ b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts @@ -1,21 +1,5 @@ -/** - * cur-record-write.test.ts — cli#492. - * - * After a DURABLE terminal transition (acked/failed) the plugin stamps the cur/ - * record (`ackedAt` / `nackedAt`). The old `patchMailFile` swallowed a failed - * write, so the on-disk record and the plugin's view could diverge with no - * diagnostic. - * - * These tests inject a write failure (the cur/ DIRECTORY is made unwritable, so - * the atomic stamp's temp file cannot be created) and assert: - * (a) a diagnostic naming the message id, the record path and the error code; - * (b) a bounded in-process retry fixes a transient failure without a restart; - * (c) a persistent failure stops after the bound and the next account start - * re-stamps the record; - * (d) stopping the account cancels pending retries. - * A further test pins the successful path as unchanged. - */ -import { describe, expect, it, beforeEach, afterEach, mock } from "bun:test"; +import { describe, expect, it, beforeEach, afterEach, mock, spyOn } from "bun:test"; +import * as fs from "node:fs"; import { chmodSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { join, resolve } from "node:path"; import { tmpdir } from "node:os"; @@ -26,6 +10,27 @@ import { signEnvelope, type ChainEntry } from "@tpsdev-ai/agent"; import { hashes } from "@noble/ed25519"; hashes.sha512 = (m: Uint8Array) => new Uint8Array(createHash("sha512").update(m).digest()); +const realFs = { ...fs }; +let removeOnTerminal: { path: string; state: string } | undefined; +let stampError: Error | undefined; +mock.module("node:fs", () => ({ + ...realFs, + openSync: (...args: any[]) => { + if (stampError && String(args[0]).includes(".ack-")) throw stampError; + return (realFs.openSync as any)(...args); + }, + writeFileSync: (...args: any[]) => { + const result = (realFs.writeFileSync as any)(...args); + if (removeOnTerminal && String(args[0]).includes(".obligations/") && typeof args[1] === "string") { + if (JSON.parse(args[1]).state === removeOnTerminal.state) { + realFs.unlinkSync(removeOnTerminal.path); + removeOnTerminal = undefined; + } + } + return result; + }, +})); + const FLINT_SEED = Buffer.alloc(32, 0x01); const ANVIL_SEED = Buffer.alloc(32, 0x02); const pubkeyFromSeed = (s: Buffer): Buffer => Buffer.from(ed.getPublicKey(new Uint8Array(s))); @@ -68,6 +73,8 @@ beforeEach(() => { }); afterEach(() => { + removeOnTerminal = undefined; + stampError = undefined; setStampRetryDelaysForTests(prevDelays); if (origHome === undefined) delete process.env.HOME; else process.env.HOME = origHome; if (origKeys === undefined) delete process.env.TPS_TEST_KEYS_DIR; else process.env.TPS_TEST_KEYS_DIR = origKeys; @@ -242,8 +249,55 @@ describe("cli#492 — a failed cur/ stamp write is surfaced and retried", () => await h2.stop(); }, 20000); + it(`${kind}: a missing record is reported after the durable transition`, async () => { + const h = await boot(true); + try { + expect(await pollUntil(() => h.dispatch() !== null)).toBe(true); + const cur = readCur()!; + removeOnTerminal = { path: cur.path, state }; + await finish(h); + expect(await pollUntil(() => failedLogs(h, `${kind}-stamp-failed`).length > 0)).toBe(true); + expect(failedLogs(h, `${kind}-stamp-failed`)[0]).toContain(`actor=anvil state=${state} path=${cur.path} code=ENOENT`); + expect(failedLogs(h, `${kind}-stamp-failed`)[0]).toContain("inspect the missing cur record; obligation retained"); + expect(obligation(h.inboundId)?.state).toBe(state); + await sleep(400); + expect(failedLogs(h, `${kind}-stamp-failed`).length).toBe(1); + expect(realFs.existsSync(cur.path)).toBe(false); + } finally { + await h.stop(); + } + }, 15000); + + it(`${kind}: an exception without a code is reported and remains retryable`, async () => { + const h = await boot(true); + try { + expect(await pollUntil(() => h.dispatch() !== null)).toBe(true); + stampError = new Error("injected stamp failure"); + await finish(h); + expect(await pollUntil(() => failedLogs(h, `${kind}-stamp-failed`).length > 0)).toBe(true); + const diagnostic = failedLogs(h, `${kind}-stamp-failed`)[0]; + expect(diagnostic).toContain(`actor=anvil state=${state}`); + expect(diagnostic).toContain("code=WRITE_FAILED"); + expect(diagnostic).toContain("retry 1"); + stampError = undefined; + expect(await pollUntil(() => !!readCur()?.record?.[stampKey])).toBe(true); + expect(obligation(h.inboundId)?.state).toBe(state); + } finally { + stampError = undefined; + await h.stop(); + } + }, 15000); + it(`${kind}: stopping the account cancels the pending stamp retries`, async () => { - setStampRetryDelaysForTests([300, 300, 300]); + setStampRetryDelaysForTests([731, 731, 731]); + const realSetTimeout = globalThis.setTimeout; + let scheduled = 0; + let fired = 0; + const timerSpy = spyOn(globalThis, "setTimeout").mockImplementation(((fn: any, delay: number, ...args: any[]) => { + if (delay !== 731) return realSetTimeout(fn, delay, ...args); + scheduled++; + return realSetTimeout(() => { fired++; fn(...args); }, delay); + }) as typeof setTimeout); const h = await boot(true); expect(await pollUntil(() => h.dispatch() !== null, 4000), "dispatch started").toBe(true); const cur = readCur(); @@ -252,12 +306,18 @@ describe("cli#492 — a failed cur/ stamp write is surfaced and retried", () => try { await finish(h); expect(await pollUntil(() => failedLogs(h, `${kind}-stamp-failed`).length >= 1, 4000), "first failure logged").toBe(true); + expect(scheduled).toBe(1); await h.stop(); } finally { restore(); } const before = failedLogs(h, `${kind}-stamp-failed`).length; - await sleep(1200); + try { + await sleep(1200); + expect(fired, "the pending timer callback never runs after stop").toBe(0); + } finally { + timerSpy.mockRestore(); + } expect(failedLogs(h, `${kind}-stamp-failed`).length, "no retry ran after stop").toBe(before); expect(readCur()?.record?.[stampKey], "the cancelled retry never stamped").toBeUndefined(); }, 20000); diff --git a/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts b/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts index 34fb901f..c4450e23 100644 --- a/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts +++ b/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts @@ -5,11 +5,33 @@ import { join } from "node:path"; const realFs = { ...fs }; let removeAfterRead: string | undefined; +let readFailure: string | undefined; +let readsUntilFailure = 1; +let listFailure: string | undefined; +let stampOnRead: { path: string; count: number } | undefined; +let readsBeforeRemoval = 1; +let uncodedWriteFailure = false; mock.module("node:fs", () => ({ ...realFs, + readdirSync: (...args: any[]) => { + if (args[0] === listFailure) throw Object.assign(new Error("injected list failure"), { code: "EACCES" }); + return (realFs.readdirSync as any)(...args); + }, + openSync: (...args: any[]) => { + if (uncodedWriteFailure && String(args[0]).includes(".ack-")) throw new Error("injected write failure"); + return (realFs.openSync as any)(...args); + }, readFileSync: (...args: any[]) => { + if (args[0] === readFailure && --readsUntilFailure <= 0) throw Object.assign(new Error("injected read failure"), { code: "EACCES" }); const result = (realFs.readFileSync as any)(...args); - if (args[0] === removeAfterRead) { + if (stampOnRead?.path === args[0] && --stampOnRead.count === 0) { + const record = JSON.parse(String(result)); + record.ackedAt = "concurrent-ack"; + record.nackedAt = "concurrent-nack"; + realFs.writeFileSync(stampOnRead.path, JSON.stringify(record)); + stampOnRead = undefined; + } + if (args[0] === removeAfterRead && --readsBeforeRemoval === 0) { realFs.unlinkSync(removeAfterRead!); removeAfterRead = undefined; } @@ -17,12 +39,17 @@ mock.module("node:fs", () => ({ }, })); -const { patchMailFile } = await import("../src/index.js"); +const { patchMailFile, reconcileTerminalCurStamps } = await import("../src/index.js"); const root = realFs.mkdtempSync(join(tmpdir(), "patch-mail-")); const path = join(root, "record.json"); afterEach(() => { removeAfterRead = undefined; + readsBeforeRemoval = readsUntilFailure = 1; + readFailure = listFailure = undefined; + stampOnRead = undefined; + uncodedWriteFailure = false; + realFs.rmSync(join(root, "anvil"), { recursive: true, force: true }); realFs.rmSync(path, { force: true }); }); @@ -48,3 +75,106 @@ describe("patchMailFile", () => { expect(realFs.existsSync(path)).toBe(false); }); }); + + +function terminalFixture(state: "acked" | "failed") { + const obligationDir = join(root, "anvil", ".obligations"); + const curDir = join(root, "anvil", "cur"); + realFs.mkdirSync(obligationDir, { recursive: true }); + realFs.mkdirSync(curDir, { recursive: true }); + const obligationPath = join(obligationDir, "inbound.json"); + const curPath = join(curDir, "timestamp-inbound.json"); + realFs.writeFileSync(obligationPath, JSON.stringify({ inboundId: "inbound", state, failure: "empty" })); + realFs.writeFileSync(curPath, JSON.stringify({ id: "inbound", body: "hello" })); + const logs: string[] = []; + const reconcile = () => reconcileTerminalCurStamps(root, "anvil", { warn: (message: string) => logs.push(message) }); + return { obligationDir, obligationPath, curDir, curPath, logs, reconcile }; +} + +describe("terminal stamp reconciliation", () => { + for (const state of ["acked", "failed"] as const) { + const kind = state === "acked" ? "ack" : "nack"; + const key = state === "acked" ? "ackedAt" : "nackedAt"; + + test(`${kind}: a record disappearing during the locked write is reported`, () => { + const f = terminalFixture(state); + removeAfterRead = f.curPath; + readsBeforeRemoval = 3; + f.reconcile(); + expect(removeAfterRead).toBeUndefined(); + expect(f.logs).toHaveLength(1); + expect(f.logs[0]).toContain(`${kind}-stamp-reconcile-failed: inbound actor=anvil state=${state} path=${f.curPath} code=ENOENT`); + expect(f.logs[0]).toContain("inspect the missing cur record; obligation retained"); + expect(realFs.existsSync(f.curPath)).toBe(false); + expect(JSON.parse(realFs.readFileSync(f.obligationPath, "utf8")).state).toBe(state); + }); + + test(`${kind}: an uncoded write failure is reported and the next run can stamp`, () => { + const f = terminalFixture(state); + uncodedWriteFailure = true; + f.reconcile(); + expect(f.logs[0]).toContain(`actor=anvil state=${state} path=${f.curPath} code=WRITE_FAILED`); + expect(f.logs[0]).toContain("restore writable records and restart the account; obligation retained"); + uncodedWriteFailure = false; + f.reconcile(); + expect(JSON.parse(realFs.readFileSync(f.curPath, "utf8"))[key]).toBeDefined(); + }); + + test(`${kind}: running reconcile twice leaves bytes and file metadata unchanged`, () => { + const f = terminalFixture(state); + f.reconcile(); + const bytes = realFs.readFileSync(f.curPath, "utf8"); + const stat = realFs.statSync(f.curPath); + f.reconcile(); + expect(realFs.readFileSync(f.curPath, "utf8")).toBe(bytes); + expect(realFs.statSync(f.curPath).mtimeMs).toBe(stat.mtimeMs); + expect(realFs.statSync(f.curPath).ino).toBe(stat.ino); + expect(f.logs).toEqual([]); + }); + + test(`${kind}: a concurrent stamp between the precheck and lock is preserved`, () => { + const f = terminalFixture(state); + stampOnRead = { path: f.curPath, count: 2 }; + f.reconcile(); + expect(stampOnRead).toBeUndefined(); + expect(JSON.parse(realFs.readFileSync(f.curPath, "utf8"))[key]).toBe(`concurrent-${kind}`); + }); + } + + for (const stage of ["obligation-list", "obligation-read", "cur-list", "cur-read", "cur-reread"] as const) { + test(`${stage}: a read failure is diagnosed and remains recoverable`, () => { + const f = terminalFixture("acked"); + const target = stage === "obligation-list" ? f.obligationDir + : stage === "obligation-read" ? f.obligationPath + : stage === "cur-list" ? f.curDir : f.curPath; + if (stage.endsWith("list")) listFailure = target; + else { + readFailure = target; + if (stage === "cur-reread") readsUntilFailure = 2; + } + f.reconcile(); + expect(f.logs[0]).toContain("stamp-reconcile-read-failed"); + expect(f.logs[0]).toContain("actor=anvil state="); + expect(f.logs[0]).toContain(`path=${target} code=EACCES`); + expect(f.logs[0]).toContain("restore readable records and restart the account"); + listFailure = readFailure = undefined; + f.reconcile(); + expect(JSON.parse(realFs.readFileSync(f.curPath, "utf8")).ackedAt).toBeDefined(); + }); + } + + test("an absent obligation directory has no read failure", () => { + const logs: string[] = []; + reconcileTerminalCurStamps(root, "anvil", { warn: (m: string) => logs.push(m) }); + expect(logs).toEqual([]); + }); + + test("a terminal obligation without a cur record is reported", () => { + const f = terminalFixture("acked"); + realFs.unlinkSync(f.curPath); + f.reconcile(); + expect(f.logs[0]).toContain("code=ENOENT"); + expect(f.logs[0]).toContain("obligation retained"); + expect(realFs.existsSync(f.obligationPath)).toBe(true); + }); +}); From e6869216f2c5a699dae2197c4a9fd9b00697f2fa Mon Sep 17 00:00:00 2001 From: flint Date: Sat, 3 Oct 2026 18:01:25 -0700 Subject: [PATCH 05/17] fix(openclaw-tps-mail): an unreadable obligation is never erased or re-dispatched; bad records never stop reconcile Co-Authored-By: Claude Opus 5.5 --- plugins/openclaw-tps-mail/src/index.ts | 84 ++++++++++++------- plugins/openclaw-tps-mail/src/obligations.ts | 30 +++++-- .../test/cur-record-write.test.ts | 55 ++++++++++++ .../test/patch-mail-file.test.ts | 33 ++++++++ 4 files changed, 164 insertions(+), 38 deletions(-) diff --git a/plugins/openclaw-tps-mail/src/index.ts b/plugins/openclaw-tps-mail/src/index.ts index f6eb2d41..ce1842de 100644 --- a/plugins/openclaw-tps-mail/src/index.ts +++ b/plugins/openclaw-tps-mail/src/index.ts @@ -265,7 +265,12 @@ function findBoundAgents(cfg: any, accountId: string): string[] { function readMailFile(filePath: string, onReadError?: (path: string, code: string) => void): TpsMailBody | null { try { const raw = readFileSync(filePath, "utf-8"); - return JSON.parse(raw) as TpsMailBody; + const record: unknown = JSON.parse(raw); + if (!record || typeof record !== "object" || Array.isArray(record) || typeof (record as TpsMailBody).id !== "string") { + onReadError?.(filePath, "INVALID_RECORD"); + return null; + } + return record as TpsMailBody; } catch (err: any) { if (err?.code !== "ENOENT") onReadError?.(filePath, err?.code ?? "INVALID_RECORD"); return null; @@ -642,11 +647,16 @@ export function patchMailFile(path: string, patch: Partial, stampKe } } -export function reconcileTerminalCurStamps(mailDir: string, agent: string, log: any): void { +export function reconcileTerminalCurStamps(mailDir: string, agent: string, log: any): Set { + const unknownInbounds = new Set(); const reportReadError = (state: string, id: string) => (path: string, code: string) => log?.warn?.( `tps-mail: stamp-reconcile-read-failed: ${id} actor=${agent} state=${state} path=${path} code=${code}; restore readable records and restart the account`, ); - for (const rec of listObligations(mailDir, agent, reportReadError("unknown", "unknown"))) { + const onObligationReadError = (path: string, code: string) => { + unknownInbounds.add(path.endsWith(".json") ? basename(path, ".json") : "*"); + reportReadError("unknown", "unknown")(path, code); + }; + for (const rec of listObligations(mailDir, agent, onObligationReadError)) { if (rec.state !== "acked" && rec.state !== "failed") continue; let unreadable = false; const onReadError = (path: string, code: string) => { @@ -681,6 +691,7 @@ export function reconcileTerminalCurStamps(mailDir: string, agent: string, log: ); } } + return unknownInbounds; } /** @@ -1951,25 +1962,31 @@ const gateway: ChannelGatewayAdapter = { // The obligation record is created HERE, keyed on the inbound id — a // replayed inbound finds its record and opens NO second obligation. const obligationId = randomUUID(); - const created = createObligation( - account.mailDir, - recipient, - () => ({ - obligationId, - inboundId: msg.id, - // cli#429: the durable thread id — the inbound's SIGNED envelope id - // (promote()/recoverPromoted() stamp it, and the id rule holds there). - ...(isValidEnvelopeId(msg.envelopeId) ? { inboundEnvelopeId: msg.envelopeId } : {}), - inboundTimestamp: msg.timestamp, - from: msg.from, - to: recipient, - accountId: account.accountId, - state: "pending", - deadlineAt: null, - attempts: 1, - }), - log, - ); + let created: ReturnType; + try { + created = createObligation( + account.mailDir, + recipient, + () => ({ + obligationId, + inboundId: msg.id, + // cli#429: the durable thread id — the inbound's SIGNED envelope id + // (promote()/recoverPromoted() stamp it, and the id rule holds there). + ...(isValidEnvelopeId(msg.envelopeId) ? { inboundEnvelopeId: msg.envelopeId } : {}), + inboundTimestamp: msg.timestamp, + from: msg.from, + to: recipient, + accountId: account.accountId, + state: "pending", + deadlineAt: null, + attempts: 1, + }), + log, + ); + } catch (err: any) { + log?.warn?.(`tps-mail: obligation creation deferred for ${msg.id}: ${err?.message ?? err}`); + return; + } const obId = created.record.obligationId; const yieldCtx = makeYieldCtx( account.mailDir, @@ -2359,8 +2376,8 @@ const gateway: ChannelGatewayAdapter = { // cur/ record never received its stamp — the write failed AFTER the // durable transition — is re-stamped here, under the same lock that // write uses, BEFORE the recovery loop below decides whether to - // re-dispatch the record. Idempotent; a failure here is logged by name. - reconcileTerminalCurStamps(account.mailDir, agentId, log); + // re-dispatch the record. + const unknownInbounds = reconcileTerminalCurStamps(account.mailDir, agentId, log); // Crash recovery (at-least-once): re-dispatch cur/ records that were // promoted but never acked/nacked. cur/ is a DESTINATION, so the record @@ -2379,6 +2396,7 @@ const gateway: ChannelGatewayAdapter = { if (seenFiles.has(curPath)) continue; const record = readMailFile(curPath); if (!record || record.ackedAt || record.nackedAt) continue; + if (unknownInbounds.has("*") || unknownInbounds.has(record.id)) continue; void recoverUnackedCurRecord(agentId, curPath, record); } } @@ -2441,14 +2459,16 @@ const gateway: ChannelGatewayAdapter = { // a route cannot pin a record forever. try { const retentionDays = resolveObligationRetentionDays(pluginConfig, (cfg as any)?.channels?.[CHANNEL_ID]); - sweepTerminalObligations( - account.mailDir, - agentId, - retentionDays, - log, - Date.now(), - resolveObligationNackHoldDays(retentionDays, pluginConfig, (cfg as any)?.channels?.[CHANNEL_ID], log), - ); + if (unknownInbounds.size === 0) { + sweepTerminalObligations( + account.mailDir, + agentId, + retentionDays, + log, + Date.now(), + resolveObligationNackHoldDays(retentionDays, pluginConfig, (cfg as any)?.channels?.[CHANNEL_ID], log), + ); + } } catch (err: any) { log?.warn?.(`tps-mail: obligation retention sweep failed (ignored): ${err?.message ?? String(err)}`); } diff --git a/plugins/openclaw-tps-mail/src/obligations.ts b/plugins/openclaw-tps-mail/src/obligations.ts index 5b6dafe2..f4ea6d86 100644 --- a/plugins/openclaw-tps-mail/src/obligations.ts +++ b/plugins/openclaw-tps-mail/src/obligations.ts @@ -174,6 +174,13 @@ export function obligationPath(mailDir: string, agent: string, inboundId: string return resolve(obligationsDir(mailDir, agent), `${inboundId}.json`); } +function isObligationRecord(record: unknown): record is ObligationRecord { + if (!record || typeof record !== "object" || Array.isArray(record)) return false; + const value = record as Partial; + return typeof value.inboundId === "string" && typeof value.state === "string" && + ALL_STATES.has(value.state); +} + export function readObligation(mailDir: string, agent: string, inboundId: string): ObligationRecord | null { const p = obligationPath(mailDir, agent, inboundId); try { @@ -189,8 +196,8 @@ export function readObligationResult(mailDir: string, agent: string, inboundId: | { status: "unverified"; path: string; code: string } { const path = obligationPath(mailDir, agent, inboundId); try { - const record = JSON.parse(readFileSync(path, "utf-8")) as ObligationRecord; - if (!record || typeof record.state !== "string") { + const record: unknown = JSON.parse(readFileSync(path, "utf-8")); + if (!isObligationRecord(record)) { return { status: "unverified", path, code: "INVALID_RECORD" }; } return { status: "found", record }; @@ -219,9 +226,14 @@ export function listObligations( for (const name of names) { if (!name.endsWith(".json") || name.startsWith(".")) continue; try { - out.push(JSON.parse(readFileSync(resolve(dir, name), "utf-8")) as ObligationRecord); + const record: unknown = JSON.parse(readFileSync(resolve(dir, name), "utf-8")); + if (!isObligationRecord(record)) { + onReadError(resolve(dir, name), "INVALID_RECORD"); + continue; + } + out.push(record); } catch (err: any) { - if (err?.code !== "ENOENT") onReadError(resolve(dir, name), err?.code ?? "INVALID_RECORD"); + onReadError(resolve(dir, name), err?.code ?? "INVALID_RECORD"); } } return out; @@ -249,8 +261,14 @@ export function createObligation( log?: ObligationLog, ): { created: boolean; record: ObligationRecord } { const draft = make(); - const existing = readObligation(mailDir, agent, draft.inboundId); - if (existing) { + const result = readObligationResult(mailDir, agent, draft.inboundId); + if (result.status === "unverified") { + const message = `tps-mail: obligation-create-read-failed: ${draft.inboundId} actor=${agent} state=unknown path=${result.path} code=${result.code}; restore readable records and restart the account; obligation retained`; + log?.warn?.(message); + throw new Error(message); + } + if (result.status === "found") { + const existing = result.record; log?.info?.( `tps-mail: obligation for inbound ${draft.inboundId} already exists (${existing.obligationId}); not creating a second`, ); diff --git a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts index 53ba25a7..c9f62f62 100644 --- a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts +++ b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts @@ -13,8 +13,17 @@ hashes.sha512 = (m: Uint8Array) => new Uint8Array(createHash("sha512").update(m) const realFs = { ...fs }; let removeOnTerminal: { path: string; state: string } | undefined; let stampError: Error | undefined; +let obligationReadFailure: string | undefined; +let obligationReads = 0; +let failObligationReads = Infinity; mock.module("node:fs", () => ({ ...realFs, + readFileSync: (...args: any[]) => { + if (args[0] === obligationReadFailure && ++obligationReads <= failObligationReads) { + throw Object.assign(new Error("injected obligation read failure"), { code: "EACCES" }); + } + return (realFs.readFileSync as any)(...args); + }, openSync: (...args: any[]) => { if (stampError && String(args[0]).includes(".ack-")) throw stampError; return (realFs.openSync as any)(...args); @@ -75,6 +84,9 @@ beforeEach(() => { afterEach(() => { removeOnTerminal = undefined; stampError = undefined; + obligationReadFailure = undefined; + obligationReads = 0; + failObligationReads = Infinity; setStampRetryDelaysForTests(prevDelays); if (origHome === undefined) delete process.env.HOME; else process.env.HOME = origHome; if (origKeys === undefined) delete process.env.TPS_TEST_KEYS_DIR; else process.env.TPS_TEST_KEYS_DIR = origKeys; @@ -334,3 +346,46 @@ describe("cli#492 — a failed cur/ stamp write is surfaced and retried", () => await h.stop(); }, 15000); }); + +for (const failReads of [Infinity, 1]) { + it(`startup retains an unreadable terminal obligation without redispatch (failed reads: ${failReads})`, async () => { + const first = await boot(true); + expect(await pollUntil(() => first.dispatch() !== null)).toBe(true); + await first.deliver("verdict"); + first.settle(); + expect(await pollUntil(() => !!readCur()?.record?.ackedAt)).toBe(true); + await first.stop(); + const cur = readCur()!; + delete cur.record.ackedAt; + realFs.writeFileSync(cur.path, JSON.stringify(cur.record)); + const path = resolve(mailDir, "anvil", ".obligations", `${first.inboundId}.json`); + const terminal = JSON.parse(realFs.readFileSync(path, "utf8")); + terminal.lastTransitionAt = new Date(0).toISOString(); + realFs.writeFileSync(path, JSON.stringify(terminal)); + const bytes = realFs.readFileSync(path, "utf8"); + obligationReadFailure = path; + failObligationReads = failReads; + const second = await boot(false); + try { + expect(await pollUntil(() => second.logs.some((m) => m.includes(path) && m.includes("code=EACCES")))).toBe(true); + await sleep(300); + expect(second.dispatch()).toBeNull(); + expect(second.logs.some((m) => m.includes(`delivering ${first.inboundId} `))).toBe(false); + expect(realFs.readFileSync(path, "utf8")).toBe(bytes); + expect(realFs.existsSync(cur.path)).toBe(true); + expect(JSON.parse(realFs.readFileSync(cur.path, "utf8")).ackedAt).toBeUndefined(); + expect(second.logs.some((m) => m.includes("actor=anvil state=unknown") && m.includes(path) && m.includes("restore readable records and restart the account"))).toBe(true); + } finally { + await second.stop(); + obligationReadFailure = undefined; + } + const third = await boot(false); + try { + expect(await pollUntil(() => !!readCur()?.record?.ackedAt)).toBe(true); + expect(third.dispatch()).toBeNull(); + + } finally { + await third.stop(); + } + }, 15000); +} diff --git a/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts b/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts index c4450e23..4790bb67 100644 --- a/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts +++ b/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts @@ -40,6 +40,7 @@ mock.module("node:fs", () => ({ })); const { patchMailFile, reconcileTerminalCurStamps } = await import("../src/index.js"); +const { createObligation, listObligations } = await import("../src/obligations.js"); const root = realFs.mkdtempSync(join(tmpdir(), "patch-mail-")); const path = join(root, "record.json"); @@ -178,3 +179,35 @@ describe("terminal stamp reconciliation", () => { expect(realFs.existsSync(f.obligationPath)).toBe(true); }); }); + +for (const invalid of [null, 7, "bad", [], { inboundId: "bad" }]) { + test(`invalid obligation ${JSON.stringify(invalid)} is reported and a later terminal record is reconciled`, () => { + const f = terminalFixture("acked"); + const badPath = join(f.obligationDir, "000-bad.json"); + realFs.writeFileSync(badPath, JSON.stringify(invalid)); + const errors: string[] = []; + expect(listObligations(root, "anvil", (path, code) => errors.push(`${path}:${code}`))).toHaveLength(1); + expect(errors).toEqual([`${badPath}:INVALID_RECORD`]); + expect(() => f.reconcile()).not.toThrow(); + expect(f.logs[0]).toContain(`actor=anvil state=unknown path=${badPath} code=INVALID_RECORD`); + expect(f.logs[0]).toContain("restore readable records and restart the account"); + expect(JSON.parse(realFs.readFileSync(f.curPath, "utf8")).ackedAt).toBeDefined(); + expect(realFs.readFileSync(badPath, "utf8")).toBe(JSON.stringify(invalid)); + }); +} + +test("a null cur record is reported as unreadable", () => { + const f = terminalFixture("acked"); + realFs.writeFileSync(f.curPath, "null"); + f.reconcile(); + expect(f.logs[0]).toContain(`actor=anvil state=acked path=${f.curPath} code=INVALID_RECORD`); + expect(f.logs[0]).toContain("restore readable records and restart the account"); +}); + +test("creation refuses an unreadable existing terminal obligation", () => { + const f = terminalFixture("acked"); + const bytes = realFs.readFileSync(f.obligationPath, "utf8"); + readFailure = f.obligationPath; + expect(() => createObligation(root, "anvil", () => ({ inboundId: "inbound", state: "pending" }) as any)).toThrow("state=unknown"); + expect(realFs.readFileSync(f.obligationPath, "utf8")).toBe(bytes); +}); From 8aee8e95bae80e1bb15a8bbee2b293ccf665ea80 Mon Sep 17 00:00:00 2001 From: flint Date: Sat, 3 Oct 2026 18:54:21 -0700 Subject: [PATCH 06/17] fix(openclaw-tps-mail): every startup reconcile failure leaves the inbound unresolved; validation text exact Co-Authored-By: Claude Opus 5.5 --- plugins/openclaw-tps-mail/src/index.ts | 91 ++++++--- plugins/openclaw-tps-mail/src/obligations.ts | 181 +++++++++++++----- .../test/cur-record-write.test.ts | 60 +++++- .../test/obligation-retention.test.ts | 6 + .../test/patch-mail-file.test.ts | 145 +++++++++++++- 5 files changed, 403 insertions(+), 80 deletions(-) diff --git a/plugins/openclaw-tps-mail/src/index.ts b/plugins/openclaw-tps-mail/src/index.ts index ce1842de..43f2071c 100644 --- a/plugins/openclaw-tps-mail/src/index.ts +++ b/plugins/openclaw-tps-mail/src/index.ts @@ -272,7 +272,7 @@ function readMailFile(filePath: string, onReadError?: (path: string, code: strin } return record as TpsMailBody; } catch (err: any) { - if (err?.code !== "ENOENT") onReadError?.(filePath, err?.code ?? "INVALID_RECORD"); + onReadError?.(filePath, err?.code ?? "INVALID_RECORD"); return null; } } @@ -628,10 +628,13 @@ type CurRecordUpdate = | { ok: true } | { ok: false; reason: "record-missing" | "write-failed"; path: string; code: string }; -export function patchMailFile(path: string, patch: Partial, stampKey?: "ackedAt" | "nackedAt"): CurRecordUpdate { +export function patchMailFile(path: string, patch: Partial, stampKey?: "ackedAt" | "nackedAt", inboundId?: string): CurRecordUpdate { try { let alreadyStamped = false; const r = updateExistingRecord(path, (current) => { + if (inboundId !== undefined && current.id !== inboundId) { + throw Object.assign(new Error("cur identity changed"), { code: "ID_MISMATCH" }); + } if (stampKey && current[stampKey]) { alreadyStamped = true; return null; @@ -647,16 +650,19 @@ export function patchMailFile(path: string, patch: Partial, stampKe } } -export function reconcileTerminalCurStamps(mailDir: string, agent: string, log: any): Set { - const unknownInbounds = new Set(); - const reportReadError = (state: string, id: string) => (path: string, code: string) => log?.warn?.( - `tps-mail: stamp-reconcile-read-failed: ${id} actor=${agent} state=${state} path=${path} code=${code}; restore readable records and restart the account`, - ); +export function reconcileTerminalCurStamps(mailDir: string, agent: string, log: any, records?: ObligationRecord[], unknownInbounds = new Set()): Set { + const reportReadError = (state: string, id: string) => (path: string, code: string) => { + if (unknownInbounds.has(id)) return; + unknownInbounds.add(id); + log?.warn?.( + `tps-mail: stamp-reconcile-read-failed: ${id} actor=${agent} state=${state} path=${path} code=${code}; restore readable records and restart the account`, + ); + }; const onObligationReadError = (path: string, code: string) => { - unknownInbounds.add(path.endsWith(".json") ? basename(path, ".json") : "*"); - reportReadError("unknown", "unknown")(path, code); + reportReadError("unknown", path.endsWith(".json") ? basename(path, ".json") : "*")(path, code); }; - for (const rec of listObligations(mailDir, agent, onObligationReadError)) { + for (const rec of records ?? listObligations(mailDir, agent, onObligationReadError)) { + if (unknownInbounds.has("*") || unknownInbounds.has(rec.inboundId)) continue; if (rec.state !== "acked" && rec.state !== "failed") continue; let unreadable = false; const onReadError = (path: string, code: string) => { @@ -666,9 +672,13 @@ export function reconcileTerminalCurStamps(mailDir: string, agent: string, log: const kind = rec.state === "acked" ? "ack" : "nack"; const key = kind === "ack" ? "ackedAt" : "nackedAt"; const curPath = findCurPath(mailDir, agent, rec.inboundId, onReadError); - const missing = (path: string) => log?.warn?.( - `tps-mail: ${kind}-stamp-reconcile-failed: ${rec.inboundId} actor=${agent} state=${rec.state} path=${path} code=ENOENT; inspect the missing cur record; obligation retained`, - ); + const missing = (path: string) => { + unknownInbounds.add(rec.inboundId); + log?.warn?.( + `tps-mail: ${kind}-stamp-reconcile-failed: ${rec.inboundId} actor=${agent} state=${rec.state} path=${path} code=ENOENT; inspect the missing cur record; obligation retained; repair it and restart the account`, + ); + }; + if (unreadable) continue; if (!curPath) { if (!unreadable) missing(resolve(mailDir, agent, "cur")); continue; @@ -682,9 +692,10 @@ export function reconcileTerminalCurStamps(mailDir: string, agent: string, log: const patch = kind === "ack" ? { ackedAt: new Date().toISOString(), read: true } : { nackedAt: new Date().toISOString(), nackReason: rec.failure ?? "failed" }; - const r = patchMailFile(curPath, patch, key); + const r = patchMailFile(curPath, patch, key, rec.inboundId); if (r.ok) log?.info?.(`tps-mail: reconciled the ${kind} stamp for ${rec.inboundId} at ${curPath}`); else { + unknownInbounds.add(rec.inboundId); log?.warn?.( `tps-mail: ${kind}-stamp-reconcile-failed: ${rec.inboundId} actor=${agent} state=${rec.state} path=${r.path} code=${r.code}; ` + (r.reason === "record-missing" ? "inspect the missing cur record; obligation retained" : "restore writable records and restart the account; obligation retained"), @@ -1642,7 +1653,9 @@ function installYieldSubscription(api: any): boolean { function findCurPath(mailDir: string, agent: string, inboundId: string, onReadError?: (path: string, code: string) => void): string | null { const curDir = resolve(mailDir, agent, "cur"); try { - for (const name of readdirSync(curDir)) { + const names = readdirSync(curDir); + const matching = names.filter((name) => name === `${inboundId}.json` || name.endsWith(`-${inboundId}.json`)); + for (const name of matching.length ? matching : names) { if (!name.endsWith(".json")) continue; const p = resolve(curDir, name); const rec = readMailFile(p, onReadError); @@ -2372,12 +2385,17 @@ const gateway: ChannelGatewayAdapter = { } } catch { /* ignore */ } - // STAMP RECONCILIATION (cli#492): an acked or failed obligation whose - // cur/ record never received its stamp — the write failed AFTER the - // durable transition — is re-stamped here, under the same lock that - // write uses, BEFORE the recovery loop below decides whether to - // re-dispatch the record. - const unknownInbounds = reconcileTerminalCurStamps(account.mailDir, agentId, log); + // Attempt terminal stamps before recovery and retention. + const unknownInbounds = new Set(); + const startupFailure = (path: string, code: string, id = "*", state = "unknown") => { + if (unknownInbounds.has(id)) return; + unknownInbounds.add(id); + log?.warn?.(`tps-mail: startup-unresolved: ${id} actor=${agentId} state=${state} path=${path} code=${code}; repair the record and restart the account; obligation retained`); + }; + const startupRecords = listObligations(account.mailDir, agentId, (path, code) => + startupFailure(path, code, path.endsWith(".json") ? basename(path, ".json") : "*"), + ); + reconcileTerminalCurStamps(account.mailDir, agentId, log, startupRecords, unknownInbounds); // Crash recovery (at-least-once): re-dispatch cur/ records that were // promoted but never acked/nacked. cur/ is a DESTINATION, so the record @@ -2394,13 +2412,15 @@ const gateway: ChannelGatewayAdapter = { if (!filename.endsWith(".json")) continue; const curPath = resolve(curDir, filename); if (seenFiles.has(curPath)) continue; - const record = readMailFile(curPath); + const known = startupRecords.find((rec) => filename === `${rec.inboundId}.json` || filename.endsWith(`-${rec.inboundId}.json`)); + if (unknownInbounds.has("*") || (known && unknownInbounds.has(known.inboundId))) continue; + const record = readMailFile(curPath, (path, code) => startupFailure(path, code, known?.inboundId ?? basename(filename, ".json"), known?.state)); if (!record || record.ackedAt || record.nackedAt) continue; if (unknownInbounds.has("*") || unknownInbounds.has(record.id)) continue; void recoverUnackedCurRecord(agentId, curPath, record); } } - } catch { /* ignore */ } + } catch (err: any) { startupFailure(curDir, err?.code ?? "READ_FAILED"); } // Reap stranded tmp/*.promote scratch from an interrupted promote (the // catch only runs on a thrown error, so a kill leaves orphans no other @@ -2430,9 +2450,9 @@ const gateway: ChannelGatewayAdapter = { // below HOLDS any record still owing its nack (obligations.ts) while it // is inside the bounded hold, so an owed mail survives whether the retry // or the sweep runs first. - for (const rec of listObligations(account.mailDir, agentId)) { + for (const rec of startupRecords) { if (!isLive()) break; - if (!nackOwed(rec)) continue; + if (unknownInbounds.has("*") || unknownInbounds.has(rec.inboundId) || !nackOwed(rec)) continue; retryOwedNackInBackground( account.mailDir, agentId, @@ -2459,7 +2479,7 @@ const gateway: ChannelGatewayAdapter = { // a route cannot pin a record forever. try { const retentionDays = resolveObligationRetentionDays(pluginConfig, (cfg as any)?.channels?.[CHANNEL_ID]); - if (unknownInbounds.size === 0) { + if (!unknownInbounds.has("*")) { sweepTerminalObligations( account.mailDir, agentId, @@ -2467,6 +2487,9 @@ const gateway: ChannelGatewayAdapter = { log, Date.now(), resolveObligationNackHoldDays(retentionDays, pluginConfig, (cfg as any)?.channels?.[CHANNEL_ID], log), + unknownInbounds, + startupFailure, + startupRecords, ); } } catch (err: any) { @@ -2476,10 +2499,13 @@ const gateway: ChannelGatewayAdapter = { // RESTART RECOVERY (S2): in-memory timers die with the process, so // reconcile every durable obligation record against the maildir/outbox // and RE-ARM the deadline where work is still outstanding. - for (const rec of listObligations(account.mailDir, agentId)) { + for (const rec of startupRecords) { if (!isLive()) break; - if (TERMINAL_STATES.has(rec.state)) continue; - const recCurPath = findCurPath(account.mailDir, agentId, rec.inboundId); + if (unknownInbounds.has("*") || unknownInbounds.has(rec.inboundId) || TERMINAL_STATES.has(rec.state)) continue; + const onFailure = (path: string, code: string) => startupFailure(path, code, rec.inboundId, rec.state); + const recCurPath = findCurPath(account.mailDir, agentId, rec.inboundId, onFailure); + if (!recCurPath) onFailure(curDir, "ENOENT"); + if (unknownInbounds.has(rec.inboundId)) continue; const ctx = makeYieldCtx( account.mailDir, agentId, @@ -2493,7 +2519,12 @@ const gateway: ChannelGatewayAdapter = { rec.inboundEnvelopeId, ); yieldContexts.set(rec.obligationId, ctx); - await reconcileObligation(ctx, rec); + try { + await reconcileObligation(ctx, rec); + } catch (err: any) { + startupFailure(recCurPath!, err?.code ?? "RECONCILE_FAILED", rec.inboundId, rec.state); + yieldContexts.delete(rec.obligationId); + } } } catch (err: any) { log?.warn?.( diff --git a/plugins/openclaw-tps-mail/src/obligations.ts b/plugins/openclaw-tps-mail/src/obligations.ts index f4ea6d86..7cdf2eaf 100644 --- a/plugins/openclaw-tps-mail/src/obligations.ts +++ b/plugins/openclaw-tps-mail/src/obligations.ts @@ -197,7 +197,7 @@ export function readObligationResult(mailDir: string, agent: string, inboundId: const path = obligationPath(mailDir, agent, inboundId); try { const record: unknown = JSON.parse(readFileSync(path, "utf-8")); - if (!isObligationRecord(record)) { + if (!isObligationRecord(record) || record.inboundId !== inboundId) { return { status: "unverified", path, code: "INVALID_RECORD" }; } return { status: "found", record }; @@ -227,7 +227,7 @@ export function listObligations( if (!name.endsWith(".json") || name.startsWith(".")) continue; try { const record: unknown = JSON.parse(readFileSync(resolve(dir, name), "utf-8")); - if (!isObligationRecord(record)) { + if (!isObligationRecord(record) || `${record.inboundId}.json` !== name) { onReadError(resolve(dir, name), "INVALID_RECORD"); continue; } @@ -446,18 +446,37 @@ const ALL_STATES: ReadonlySet = new Set([ "failed", ]); -/** True when the agent's cur/ record for this inbound is still UNRESOLVED - * (present without ackedAt/nackedAt). Startup recovery may re-dispatch it, so - * its obligation must not be swept yet — a crash between ackObligation's - * `acked` transition and the cur/ `ackedAt` patch leaves exactly this shape. */ -function curRecordUnresolved(mailDir: string, agent: string, inboundId: string): boolean { - const p = resolve(mailDir, agent, "cur", `${inboundId}.json`); +function curRecordUnresolved(mailDir: string, agent: string, inboundId: string, onFailure: (path: string, code: string) => void, requireCur = false): boolean { + const curDirectory = resolve(mailDir, agent, "cur"); + let names: string[]; try { - const rec = JSON.parse(readFileSync(p, "utf-8")); - return !rec?.ackedAt && !rec?.nackedAt; - } catch { - return false; // absent/unreadable: nothing recovery can re-drive + names = readdirSync(curDirectory); + } catch (err: any) { + if (err?.code === "ENOENT" && !requireCur) return false; + onFailure(curDirectory, err?.code ?? "READ_FAILED"); + return true; } + const matching = names.filter((name) => name === `${inboundId}.json` || name.endsWith(`-${inboundId}.json`)); + for (const name of matching.length ? matching : names) { + if (!name.endsWith(".json")) continue; + const path = resolve(curDirectory, name); + try { + const rec = JSON.parse(readFileSync(path, "utf-8")); + if (!rec || typeof rec !== "object" || typeof rec.id !== "string") { + onFailure(path, "INVALID_RECORD"); + return true; + } + if (rec.id === inboundId) return !rec.ackedAt && !rec.nackedAt; + } catch (err: any) { + onFailure(path, err?.code ?? "INVALID_RECORD"); + return true; + } + } + if (requireCur) { + onFailure(curDirectory, "ENOENT"); + return true; + } + return false; } /** The record's OWN recorded last-transition time in ms. `lastTransitionAt` @@ -527,6 +546,9 @@ export function sweepTerminalObligations( log?: ObligationLog, nowMs: number = Date.now(), nackHoldDays: number = retentionDays * DEFAULT_NACK_HOLD_MULTIPLE, + unresolved = new Set(), + onFailure?: (path: string, code: string, id: string, state: string) => void, + heldRecords: ObligationRecord[] = [], ): RetentionResult { const res: RetentionResult = { removed: 0, @@ -544,6 +566,12 @@ export function sweepTerminalObligations( res.disabled = true; return res; } + const fail = (path: string, code: string, id: string, state: string) => { + if (unresolved.has(id)) return; + if (onFailure) onFailure(path, code, id, state); + else log?.warn?.(`tps-mail: retention-unresolved: ${id} actor=${agent} state=${state} path=${path} code=${code}; repair the record and restart the account; obligation retained`); + unresolved.add(id); + }; const dir = obligationsDir(mailDir, agent); let names: string[] = []; try { @@ -556,9 +584,8 @@ export function sweepTerminalObligations( // unreadable the agent's own receipts cannot be attributed either. const code = (err as NodeJS.ErrnoException)?.code; if (code !== "ENOENT") { - log?.warn?.( - `tps-mail: obligation retention: could not read ${dir}: ${err instanceof Error ? err.message : String(err)}; sweep skipped`, - ); + fail(dir, code ?? "READ_FAILED", "*", "unknown"); + if (!onFailure) log?.warn?.(`tps-mail: obligation retention: could not read ${dir}; sweep skipped`); return res; } } @@ -573,29 +600,66 @@ export function sweepTerminalObligations( // unique UUID, and the receipts live in this same store — so every receipt is // attributable to an obligation this sweep can see. There is no host-wide dir // to be careful of, and no inbound for two obligations to collide on. + const failedReceiptIds = new Set(); + const receiptFailureCodes = new Map(); + const receiptSnapshot = new Map(); + if (onFailure) { + const root = receiptsDir(mailDir, agent); + let receiptNames: string[] = []; + try { receiptNames = readdirSync(root); } + catch (err: any) { + if (err?.code !== "ENOENT") { + fail(root, err?.code ?? "READ_FAILED", "*", "unknown"); + return res; + } + } + for (const name of receiptNames) { + if (!name.endsWith(".json") || name.startsWith(".")) continue; + try { + const receipt = JSON.parse(readFileSync(resolve(root, name), "utf-8")); + if (!receipt || typeof receipt.obligationId !== "string") throw new Error("invalid receipt"); + receiptSnapshot.set(name, receipt); + } catch (err: any) { + const id = name.replace(/\.json$/, ""); + failedReceiptIds.add(id); + const code = err?.code ?? "INVALID_RECEIPT"; + receiptFailureCodes.set(id, code); + const owner = heldRecords.find((rec) => rec.obligationId === id); + fail(resolve(root, name), code, owner?.inboundId ?? `receipt:${name}`, owner?.state ?? "unknown"); + } + } + } const terminalObligationIds = new Set(); - const liveObligationIds = new Set(); + const liveObligationIds = new Set(heldRecords.filter((rec) => unresolved.has(rec.inboundId)).map((rec) => rec.obligationId)); + if (unresolved.size > 0) res.unreadable++; for (const name of names) { if (!name.endsWith(".json") || name.startsWith(".")) continue; const path = resolve(dir, name); + const fileId = name.replace(/\.json$/, ""); + if (unresolved.has("*") || unresolved.has(fileId)) { + res.heldForRecovery++; + continue; + } let record: unknown; try { record = JSON.parse(readFileSync(path, "utf-8")); - } catch { + } catch (err: any) { res.unreadable++; leftUnreadable.push(name); + fail(path, err?.code ?? "INVALID_RECORD", fileId, "unknown"); continue; } // Shape check: a parseable value that is not an object with a RECOGNIZED // state (null, no state, an unknown state) is a malformed record, not a // non-terminal one — reported as unreadable, never swept. - const state = (record as { state?: unknown } | null)?.state; - if (typeof record !== "object" || record === null || Array.isArray(record) || typeof state !== "string" || !ALL_STATES.has(state)) { + if (!isObligationRecord(record) || record.inboundId !== fileId) { res.unreadable++; leftUnreadable.push(name); + fail(path, "INVALID_RECORD", fileId, "unknown"); continue; } - if (!TERMINAL_STATES.has(state as ObligationState)) { + const state = record.state; + if (!TERMINAL_STATES.has(state)) { const liveObligationId = (record as { obligationId?: unknown }).obligationId; if (typeof liveObligationId === "string") liveObligationIds.add(liveObligationId); res.left++; // pending / delivering / posted / yielded are never deletable @@ -603,6 +667,28 @@ export function sweepTerminalObligations( } const snapshotObligationId = (record as { obligationId?: unknown }).obligationId; if (typeof snapshotObligationId === "string") terminalObligationIds.add(snapshotObligationId); + const inboundId = (record as { inboundId?: unknown }).inboundId; + const hold = () => { + if (typeof snapshotObligationId === "string") liveObligationIds.add(snapshotObligationId); + res.heldForRecovery++; + }; + if (typeof snapshotObligationId === "string" && failedReceiptIds.has(snapshotObligationId)) { + fail(resolve(receiptsDir(mailDir, agent), `${snapshotObligationId}.json`), receiptFailureCodes.get(snapshotObligationId) ?? "RECEIPT_READ_FAILED", fileId, state); + hold(); + continue; + } + const transitionMs = obligationLastTransitionMs(record); + if (transitionMs === null) { + res.unreadable++; + leftUnreadable.push(name); + fail(path, "INVALID_TIMESTAMP", fileId, state); + hold(); + continue; + } + if (typeof inboundId === "string" && curRecordUnresolved(mailDir, agent, inboundId, (p, code) => fail(p, code, fileId, state), Boolean(onFailure))) { + hold(); + continue; + } // cli#389 round 11, item 1: a record still OWING its nack mail is not // deletable while it is INSIDE the hold window — startup retries delivery // from this exact shape (`nackPending` with no `nackSentAt`), so sweeping it @@ -618,7 +704,6 @@ export function sweepTerminalObligations( res.heldForNack++; continue; } - res.abandonedForNack++; // cli#389 round 13, item 2: RELEASE the debt in the same pass that gives up // on it — clear `nackPending` and record `nackAbandonedAt`. Without the // clear, a record retention then KEEPS would be abandoned and logged @@ -630,30 +715,23 @@ export function sweepTerminalObligations( mailDir, agent, typeof recInbound === "string" ? recInbound : name.replace(/\.json$/, ""), - log, + { warn: () => {} }, new Date(nowMs).toISOString(), ); + if (!released) { + fail(path, "WRITE_FAILED", fileId, state); + hold(); + continue; + } + res.abandonedForNack++; log?.warn?.( `tps-mail: nack-abandoned: ${name} has owed its nack past the hold window (${nackHoldDays} day(s)); ` + - `the debt is released${released ? "" : " (the release could not be recorded, so a later sweep will abandon it again)"} ` + + `the debt is released ` + `and normal retention applies to the record`, ); // fall through to the normal terminal-retention rules below } - // A terminal record whose cur/ record is still unresolved is HELD until - // startup recovery resolves it (else a re-dispatch would open a fresh - // obligation and double-post). - const inboundId = (record as { inboundId?: unknown }).inboundId; - if (typeof inboundId === "string" && curRecordUnresolved(mailDir, agent, inboundId)) { - res.heldForRecovery++; - continue; - } - const t = obligationLastTransitionMs(record); - if (t === null) { - res.unreadable++; - leftUnreadable.push(name); - continue; - } + const t = transitionMs; if (t >= cutoff) { res.left++; continue; @@ -662,9 +740,8 @@ export function sweepTerminalObligations( unlinkSync(path); res.removed++; } catch (err) { - log?.warn?.( - `tps-mail: obligation retention: could not delete ${name}: ${err instanceof Error ? err.message : String(err)}; left in place`, - ); + fail(path, (err as NodeJS.ErrnoException).code ?? "DELETE_FAILED", fileId, state); + hold(); res.left++; } } @@ -689,7 +766,7 @@ export function sweepTerminalObligations( const receiptsRoot = receiptsDir(mailDir, agent); let receiptNames: string[]; try { - receiptNames = readdirSync(receiptsRoot); + receiptNames = onFailure ? [...receiptSnapshot.keys()] : readdirSync(receiptsRoot); } catch { // No receipts directory yet (no receipted delivery was ever made): done. receiptNames = []; @@ -697,11 +774,13 @@ export function sweepTerminalObligations( for (const name of receiptNames) { if (!name.endsWith(".json") || name.startsWith(".")) continue; const path = resolve(receiptsRoot, name); + if (onFailure && failedReceiptIds.has(name.replace(/\.json$/, ""))) continue; let receipt: unknown; try { - receipt = JSON.parse(readFileSync(path, "utf-8")); - } catch { + receipt = onFailure ? receiptSnapshot.get(name) : JSON.parse(readFileSync(path, "utf-8")); + } catch (err: any) { res.receiptsUnreadable++; + if (onFailure) fail(path, err?.code ?? "INVALID_RECEIPT", `receipt:${name}`, "unknown"); continue; } const obligationId = (receipt as { obligationId?: unknown } | null)?.obligationId; @@ -710,9 +789,15 @@ export function sweepTerminalObligations( // names no obligation id cannot be attributed to one, so it is left in // place rather than aged out on a guess. if (typeof obligationId !== "string" || obligationId.length === 0) continue; - const terminal = terminalObligationIds.has(obligationId); const live = liveObligationIds.has(obligationId); + if (live) continue; + const terminal = terminalObligationIds.has(obligationId); + if (onFailure && terminal) continue; const t = typeof ts === "string" ? Date.parse(ts) : Number.NaN; + if (onFailure && !terminal && !Number.isFinite(t)) { + fail(path, "INVALID_TIMESTAMP", `receipt:${name}`, "orphan"); + continue; + } const agedOrphan = !live && Number.isFinite(t) && t < cutoff; // FAIL SAFE (cli#389 round 6, item 3): an obligation record that could not // be read joins neither set, so a receipt for THAT obligation looks @@ -729,12 +814,16 @@ export function sweepTerminalObligations( unlinkSync(path); res.receiptsRemoved++; } catch (err) { + if (onFailure) { + fail(path, (err as NodeJS.ErrnoException).code ?? "DELETE_FAILED", `receipt:${name}`, "orphan"); + continue; + } log?.warn?.( `tps-mail: obligation retention: could not delete receipt ${name}: ${err instanceof Error ? err.message : String(err)}; left in place`, ); } } - if (res.receiptsUnreadable > 0) { + if (res.receiptsUnreadable > 0 && !onFailure) { log?.warn?.( `tps-mail: obligation retention: left ${res.receiptsUnreadable} unreadable receipt(s) in place (never deleted)`, ); @@ -747,7 +836,7 @@ export function sweepTerminalObligations( } // Logged ONCE: a single line for the unreadable/malformed records we left. - if (leftUnreadable.length > 0) { + if (leftUnreadable.length > 0 && !onFailure) { log?.warn?.( `tps-mail: obligation retention: left ${leftUnreadable.length} unreadable/malformed record(s) in place (never deleted): ${leftUnreadable.join(", ")}`, ); diff --git a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts index c9f62f62..6ac7a3de 100644 --- a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts +++ b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts @@ -16,9 +16,15 @@ let stampError: Error | undefined; let obligationReadFailure: string | undefined; let obligationReads = 0; let failObligationReads = Infinity; +let curReadFailure: string | undefined; +let curReads = 0; +let curReadsUntilFailure = 1; mock.module("node:fs", () => ({ ...realFs, readFileSync: (...args: any[]) => { + if (args[0] === curReadFailure && ++curReads >= curReadsUntilFailure) { + throw Object.assign(new Error("injected cur read failure"), { code: "EACCES" }); + } if (args[0] === obligationReadFailure && ++obligationReads <= failObligationReads) { throw Object.assign(new Error("injected obligation read failure"), { code: "EACCES" }); } @@ -87,6 +93,9 @@ afterEach(() => { obligationReadFailure = undefined; obligationReads = 0; failObligationReads = Infinity; + curReadFailure = undefined; + curReads = 0; + curReadsUntilFailure = 1; setStampRetryDelaysForTests(prevDelays); if (origHome === undefined) delete process.env.HOME; else process.env.HOME = origHome; if (origKeys === undefined) delete process.env.TPS_TEST_KEYS_DIR; else process.env.TPS_TEST_KEYS_DIR = origKeys; @@ -374,7 +383,7 @@ for (const failReads of [Infinity, 1]) { expect(realFs.readFileSync(path, "utf8")).toBe(bytes); expect(realFs.existsSync(cur.path)).toBe(true); expect(JSON.parse(realFs.readFileSync(cur.path, "utf8")).ackedAt).toBeUndefined(); - expect(second.logs.some((m) => m.includes("actor=anvil state=unknown") && m.includes(path) && m.includes("restore readable records and restart the account"))).toBe(true); + expect(second.logs.some((m) => m.includes("actor=anvil state=unknown") && m.includes(path) && m.includes("repair the record and restart the account"))).toBe(true); } finally { await second.stop(); obligationReadFailure = undefined; @@ -389,3 +398,52 @@ for (const failReads of [Infinity, 1]) { } }, 15000); } + +for (const state of ["acked", "failed"] as const) { + for (const stage of ["lookup-read", "reread", "locked-read", "recovery-read", "stamp-write"] as const) { + it(`startup ${state} ${stage} holds aged obligations across restarts without redispatch`, async () => { + const first = await boot(true); + try { + expect(await pollUntil(() => first.dispatch() !== null)).toBe(true); + if (state === "acked") await first.deliver("verdict"); + else first.skip("empty"); + first.settle(); + expect(await pollUntil(() => obligation(first.inboundId)?.state === state)).toBe(true); + expect(await pollUntil(() => !!readCur()?.record?.[state === "acked" ? "ackedAt" : "nackedAt"])).toBe(true); + } finally { await first.stop(); } + const cur = readCur()!; + delete cur.record.ackedAt; + delete cur.record.nackedAt; + realFs.writeFileSync(cur.path, JSON.stringify(cur.record)); + const path = resolve(mailDir, "anvil", ".obligations", `${first.inboundId}.json`); + const terminal = JSON.parse(realFs.readFileSync(path, "utf8")); + terminal.lastTransitionAt = new Date(0).toISOString(); + realFs.writeFileSync(path, JSON.stringify(terminal)); + const bytes = realFs.readFileSync(path, "utf8"); + for (let restart = 0; restart < 2; restart++) { + if (stage === "stamp-write") stampError = new Error("injected stamp failure"); + else { + curReadFailure = cur.path; + curReads = 0; + curReadsUntilFailure = stage === "lookup-read" ? 1 : stage === "reread" ? 2 : stage === "recovery-read" ? (restart === 0 ? 4 : 3) : 3; + } + const next = await boot(false); + try { + expect(await pollUntil(() => next.logs.some((m) => m.includes(`actor=anvil state=${state}`) && m.includes(first.inboundId)))).toBe(true); + await sleep(100); + expect(next.dispatch()).toBeNull(); + expect(next.logs.filter((m) => m.includes("actor=anvil") && m.includes(first.inboundId))).toHaveLength(1); + expect(realFs.readFileSync(path, "utf8")).toBe(bytes); + if (stage !== "recovery-read") expect(JSON.parse(realFs.readFileSync(cur.path, "utf8"))[state === "acked" ? "ackedAt" : "nackedAt"]).toBeUndefined(); + } finally { await next.stop(); } + } + stampError = undefined; + curReadFailure = undefined; + const repaired = await boot(false); + try { + expect(await pollUntil(() => !!readCur()?.record?.[state === "acked" ? "ackedAt" : "nackedAt"])).toBe(true); + expect(repaired.dispatch()).toBeNull(); + } finally { await repaired.stop(); } + }, 15000); + } +} diff --git a/plugins/openclaw-tps-mail/test/obligation-retention.test.ts b/plugins/openclaw-tps-mail/test/obligation-retention.test.ts index 439ea7eb..c25c5ef3 100644 --- a/plugins/openclaw-tps-mail/test/obligation-retention.test.ts +++ b/plugins/openclaw-tps-mail/test/obligation-retention.test.ts @@ -92,6 +92,12 @@ const receipt = (obligationId: string, replyToId: string, ts: string, agent: str /** Drive the plugin's startup (which runs the retention sweep) and wait until * `done()` or a deadline, then abort. */ async function runStartup(pluginConfig: Record, done: () => boolean): Promise { + const curDir = join(mailDir, AGENT, "cur"); + mkdirSync(curDir, { recursive: true }); + for (const name of readdirSync(obligationsDir(mailDir, AGENT))) { + const id = name.replace(/\.json$/, ""); + writeFileSync(join(curDir, `timestamp-${id}.json`), JSON.stringify({ id, ackedAt: "done", nackedAt: "done" })); + } mockApi.pluginConfig = pluginConfig; pluginModule.register(mockApi); controller = new AbortController(); diff --git a/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts b/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts index 4790bb67..b21721ad 100644 --- a/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts +++ b/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts @@ -11,12 +11,23 @@ let listFailure: string | undefined; let stampOnRead: { path: string; count: number } | undefined; let readsBeforeRemoval = 1; let uncodedWriteFailure = false; +let deleteFailure: string | undefined; +let renameFailure: string | undefined; +let replaceIdentity = false; mock.module("node:fs", () => ({ ...realFs, readdirSync: (...args: any[]) => { if (args[0] === listFailure) throw Object.assign(new Error("injected list failure"), { code: "EACCES" }); return (realFs.readdirSync as any)(...args); }, + unlinkSync: (...args: any[]) => { + if (args[0] === deleteFailure) throw Object.assign(new Error("injected delete failure"), { code: "EACCES" }); + return (realFs.unlinkSync as any)(...args); + }, + renameSync: (...args: any[]) => { + if (args[1] === renameFailure) throw Object.assign(new Error("injected rename failure"), { code: "EACCES" }); + return (realFs.renameSync as any)(...args); + }, openSync: (...args: any[]) => { if (uncodedWriteFailure && String(args[0]).includes(".ack-")) throw new Error("injected write failure"); return (realFs.openSync as any)(...args); @@ -26,6 +37,7 @@ mock.module("node:fs", () => ({ const result = (realFs.readFileSync as any)(...args); if (stampOnRead?.path === args[0] && --stampOnRead.count === 0) { const record = JSON.parse(String(result)); + if (replaceIdentity) record.id = "replacement"; record.ackedAt = "concurrent-ack"; record.nackedAt = "concurrent-nack"; realFs.writeFileSync(stampOnRead.path, JSON.stringify(record)); @@ -40,7 +52,7 @@ mock.module("node:fs", () => ({ })); const { patchMailFile, reconcileTerminalCurStamps } = await import("../src/index.js"); -const { createObligation, listObligations } = await import("../src/obligations.js"); +const { createObligation, listObligations, sweepTerminalObligations } = await import("../src/obligations.js"); const root = realFs.mkdtempSync(join(tmpdir(), "patch-mail-")); const path = join(root, "record.json"); @@ -49,7 +61,8 @@ afterEach(() => { readsBeforeRemoval = readsUntilFailure = 1; readFailure = listFailure = undefined; stampOnRead = undefined; - uncodedWriteFailure = false; + uncodedWriteFailure = replaceIdentity = false; + deleteFailure = renameFailure = undefined; realFs.rmSync(join(root, "anvil"), { recursive: true, force: true }); realFs.rmSync(path, { force: true }); }); @@ -85,7 +98,7 @@ function terminalFixture(state: "acked" | "failed") { realFs.mkdirSync(curDir, { recursive: true }); const obligationPath = join(obligationDir, "inbound.json"); const curPath = join(curDir, "timestamp-inbound.json"); - realFs.writeFileSync(obligationPath, JSON.stringify({ inboundId: "inbound", state, failure: "empty" })); + realFs.writeFileSync(obligationPath, JSON.stringify({ obligationId: "ob-inbound", inboundId: "inbound", state, failure: "empty", lastTransitionAt: new Date(0).toISOString() })); realFs.writeFileSync(curPath, JSON.stringify({ id: "inbound", body: "hello" })); const logs: string[] = []; const reconcile = () => reconcileTerminalCurStamps(root, "anvil", { warn: (message: string) => logs.push(message) }); @@ -211,3 +224,129 @@ test("creation refuses an unreadable existing terminal obligation", () => { expect(() => createObligation(root, "anvil", () => ({ inboundId: "inbound", state: "pending" }) as any)).toThrow("state=unknown"); expect(realFs.readFileSync(f.obligationPath, "utf8")).toBe(bytes); }); + +for (const state of ["acked", "failed"] as const) { + for (const stage of ["cur-list", "cur-lookup-read", "cur-reread", "locked-read", "stamp-write", "stamp-rename", "cur-missing", "identity-change"] as const) { + test(`${state}: ${stage} holds an aged timestamped inbound and continues healthy records`, () => { + const f = terminalFixture(state); + if (stage === "cur-list") listFailure = f.curDir; + if (stage.endsWith("read")) { + readFailure = f.curPath; + readsUntilFailure = stage === "cur-reread" ? 2 : stage === "locked-read" ? 3 : 1; + } + if (stage === "stamp-write") uncodedWriteFailure = true; + if (stage === "stamp-rename") renameFailure = f.curPath; + if (stage === "cur-missing") realFs.unlinkSync(f.curPath); + if (stage === "identity-change") { + replaceIdentity = true; + stampOnRead = { path: f.curPath, count: 2 }; + } + const bytes = realFs.readFileSync(f.obligationPath, "utf8"); + const unresolved = f.reconcile(); + expect(unresolved.has("inbound")).toBe(true); + const receiptDir = join(f.obligationDir, "receipts"); + realFs.mkdirSync(receiptDir); + const receiptPath = join(receiptDir, "ob-inbound.json"); + realFs.writeFileSync(receiptPath, JSON.stringify({ obligationId: "ob-inbound", ts: new Date(0).toISOString() })); + realFs.writeFileSync(join(f.obligationDir, "healthy.json"), JSON.stringify({ inboundId: "healthy", obligationId: "ob-healthy", state: "acked", lastTransitionAt: new Date(0).toISOString() })); + listFailure = readFailure = renameFailure = undefined; + uncodedWriteFailure = false; + realFs.writeFileSync(join(f.curDir, "timestamp-healthy.json"), JSON.stringify({ id: "healthy", ackedAt: "done" })); + sweepTerminalObligations(root, "anvil", 7, { warn: (m) => f.logs.push(m) }, Date.now(), 28, unresolved, undefined, [{ inboundId: "inbound", obligationId: "ob-inbound" } as any]); + expect(realFs.readFileSync(f.obligationPath, "utf8")).toBe(bytes); + expect(realFs.existsSync(receiptPath)).toBe(true); + expect(realFs.existsSync(join(f.obligationDir, "healthy.json"))).toBe(false); + expect(f.logs.filter((m) => m.includes("actor=anvil"))).toHaveLength(1); + expect(f.logs.find((m) => m.includes("actor=anvil"))).toContain(`state=${state}`); + expect(f.logs.find((m) => m.includes("actor=anvil"))).toContain("restart the account"); + replaceIdentity = false; + if (stage === "cur-missing" || stage === "identity-change") realFs.writeFileSync(f.curPath, JSON.stringify({ id: "inbound" })); + expect(f.reconcile().size).toBe(0); + expect(JSON.parse(realFs.readFileSync(f.curPath, "utf8"))[state === "acked" ? "ackedAt" : "nackedAt"]).toBeDefined(); + }); + } +} + +for (const stage of ["obligation-reread", "cur-retention-read", "cur-retention-list", "obligation-delete", "abandonment-write", "age", "identity", "cur-retention-missing"] as const) { + test(`retention ${stage} retains evidence, reports once, and continues`, () => { + const f = terminalFixture(stage === "abandonment-write" ? "failed" : "acked"); + const record = JSON.parse(realFs.readFileSync(f.obligationPath, "utf8")); + if (stage === "abandonment-write") record.nackPending = true; + if (stage === "age") record.lastTransitionAt = "invalid"; + if (stage === "identity") record.inboundId = "other"; + realFs.writeFileSync(f.obligationPath, JSON.stringify(record)); + realFs.writeFileSync(f.curPath, JSON.stringify({ id: "inbound", ackedAt: "done" })); + const bytes = realFs.readFileSync(f.obligationPath, "utf8"); + if (stage === "obligation-reread") readFailure = f.obligationPath; + if (stage === "cur-retention-read") readFailure = f.curPath; + if (stage === "cur-retention-list") listFailure = f.curDir; + if (stage === "cur-retention-missing") realFs.unlinkSync(f.curPath); + if (stage === "obligation-delete") deleteFailure = f.obligationPath; + if (stage === "abandonment-write") renameFailure = f.obligationPath; + const unresolved = new Set(); + sweepTerminalObligations(root, "anvil", 7, { warn: (m) => f.logs.push(m) }, Date.now(), 28, unresolved, (path, code, id, state) => { + f.logs.push(`actor=anvil state=${state} path=${path} code=${code}; repair and restart the account`); + unresolved.add(id); + }); + expect(unresolved.has("inbound")).toBe(true); + expect(realFs.readFileSync(f.obligationPath, "utf8")).toBe(bytes); + expect(f.logs.filter((m) => m.includes("actor=anvil"))).toHaveLength(1); + expect(f.logs.find((m) => m.includes("actor=anvil"))).toContain("restart the account"); + }); +} + +test("retention holds a timestamped cur record before abandoning its nack debt", () => { + const f = terminalFixture("failed"); + const record = JSON.parse(realFs.readFileSync(f.obligationPath, "utf8")); + record.nackPending = true; + realFs.writeFileSync(f.obligationPath, JSON.stringify(record)); + const bytes = realFs.readFileSync(f.obligationPath, "utf8"); + const result = sweepTerminalObligations(root, "anvil", 7); + expect(result.heldForRecovery).toBe(1); + expect(result.abandonedForNack).toBe(0); + expect(realFs.readFileSync(f.obligationPath, "utf8")).toBe(bytes); +}); + +for (const stage of ["receipt-list", "receipt-read", "receipt-delete", "receipt-age"] as const) { + test(`startup retention ${stage} reports an unresolved record and retains evidence`, () => { + const f = terminalFixture("acked"); + realFs.writeFileSync(f.curPath, JSON.stringify({ id: "inbound", ackedAt: "done" })); + const record = JSON.parse(realFs.readFileSync(f.obligationPath, "utf8")); + const receiptDir = join(f.obligationDir, "receipts"); + realFs.mkdirSync(receiptDir); + const receiptPath = join(receiptDir, "ob-inbound.json"); + realFs.writeFileSync(receiptPath, JSON.stringify({ obligationId: "ob-inbound", ts: new Date(0).toISOString() })); + if (stage === "receipt-list") listFailure = receiptDir; + if (stage === "receipt-read") readFailure = receiptPath; + if (stage === "receipt-age") { + realFs.unlinkSync(f.obligationPath); + realFs.writeFileSync(receiptPath, JSON.stringify({ obligationId: "ob-inbound", ts: "invalid" })); + } + if (stage === "receipt-delete") { + realFs.unlinkSync(f.obligationPath); + deleteFailure = receiptPath; + } + const unresolved = new Set(); + const onFailure = (path: string, code: string, id: string, state: string) => { + f.logs.push(`actor=anvil state=${state} path=${path} code=${code}; repair and restart the account`); + unresolved.add(id); + }; + sweepTerminalObligations(root, "anvil", 7, { warn: (m) => f.logs.push(m) }, Date.now(), 28, unresolved, onFailure, [record]); + expect(unresolved.size).toBe(1); + expect(realFs.existsSync(receiptPath)).toBe(true); + if (stage !== "receipt-delete" && stage !== "receipt-age") expect(realFs.existsSync(f.obligationPath)).toBe(true); + expect(f.logs.filter((m) => m.includes("actor=anvil"))).toHaveLength(1); + expect(f.logs.find((m) => m.includes("actor=anvil"))).toContain(`path=${stage === "receipt-list" ? receiptDir : receiptPath} code=${stage === "receipt-age" ? "INVALID_TIMESTAMP" : "EACCES"}`); + }); +} + +test("obligation filename and inboundId must agree before startup uses the record", () => { + const f = terminalFixture("acked"); + const record = JSON.parse(realFs.readFileSync(f.obligationPath, "utf8")); + record.inboundId = "other"; + realFs.writeFileSync(f.obligationPath, JSON.stringify(record)); + expect(f.reconcile().has("inbound")).toBe(true); + expect(f.logs).toHaveLength(1); + expect(f.logs[0]).toContain("INVALID_RECORD"); + expect(JSON.parse(realFs.readFileSync(f.curPath, "utf8")).ackedAt).toBeUndefined(); +}); From 5476ff183c3c30c2bb16f2948db5401f2fdca0a6 Mon Sep 17 00:00:00 2001 From: flint Date: Sat, 3 Oct 2026 19:45:48 -0700 Subject: [PATCH 07/17] fix(openclaw-tps-mail): live stamps require the expected inbound id; a mismatched obligation holds every identity it could represent Co-Authored-By: Claude Opus 5.5 --- plugins/openclaw-tps-mail/src/index.ts | 18 ++--- plugins/openclaw-tps-mail/src/obligations.ts | 12 ++-- .../test/cur-record-write.test.ts | 66 +++++++++++++++++++ .../test/patch-mail-file.test.ts | 8 +-- 4 files changed, 88 insertions(+), 16 deletions(-) diff --git a/plugins/openclaw-tps-mail/src/index.ts b/plugins/openclaw-tps-mail/src/index.ts index 43f2071c..961458f0 100644 --- a/plugins/openclaw-tps-mail/src/index.ts +++ b/plugins/openclaw-tps-mail/src/index.ts @@ -628,11 +628,11 @@ type CurRecordUpdate = | { ok: true } | { ok: false; reason: "record-missing" | "write-failed"; path: string; code: string }; -export function patchMailFile(path: string, patch: Partial, stampKey?: "ackedAt" | "nackedAt", inboundId?: string): CurRecordUpdate { +export function patchMailFile(path: string, patch: Partial, stampKey: "ackedAt" | "nackedAt" | undefined, inboundId: string): CurRecordUpdate { try { let alreadyStamped = false; const r = updateExistingRecord(path, (current) => { - if (inboundId !== undefined && current.id !== inboundId) { + if (current.id !== inboundId) { throw Object.assign(new Error("cur identity changed"), { code: "ID_MISMATCH" }); } if (stampKey && current[stampKey]) { @@ -658,8 +658,9 @@ export function reconcileTerminalCurStamps(mailDir: string, agent: string, log: `tps-mail: stamp-reconcile-read-failed: ${id} actor=${agent} state=${state} path=${path} code=${code}; restore readable records and restart the account`, ); }; - const onObligationReadError = (path: string, code: string) => { - reportReadError("unknown", path.endsWith(".json") ? basename(path, ".json") : "*")(path, code); + const onObligationReadError = (path: string, code: string, ids: string[]) => { + reportReadError("unknown", ids[0])(path, code); + for (const id of ids) unknownInbounds.add(id); }; for (const rec of records ?? listObligations(mailDir, agent, onObligationReadError)) { if (unknownInbounds.has("*") || unknownInbounds.has(rec.inboundId)) continue; @@ -968,7 +969,7 @@ function stampTerminalCur( attempt = 0, ): void { const key = kind === "ack" ? "ackedAt" : "nackedAt"; - const stamped = patchMailFile(ctx.curPath, patch, key); + const stamped = patchMailFile(ctx.curPath, patch, key, ctx.inboundId); if (stamped.ok) { if (attempt > 0) ctx.log?.info?.(`tps-mail: ${kind}-stamp-retry-ok: ${ctx.inboundId} at ${ctx.curPath} (retry ${attempt})`); return; @@ -2392,9 +2393,10 @@ const gateway: ChannelGatewayAdapter = { unknownInbounds.add(id); log?.warn?.(`tps-mail: startup-unresolved: ${id} actor=${agentId} state=${state} path=${path} code=${code}; repair the record and restart the account; obligation retained`); }; - const startupRecords = listObligations(account.mailDir, agentId, (path, code) => - startupFailure(path, code, path.endsWith(".json") ? basename(path, ".json") : "*"), - ); + const startupRecords = listObligations(account.mailDir, agentId, (path, code, ids) => { + startupFailure(path, code, ids[0]); + for (const id of ids) unknownInbounds.add(id); + }); reconcileTerminalCurStamps(account.mailDir, agentId, log, startupRecords, unknownInbounds); // Crash recovery (at-least-once): re-dispatch cur/ records that were diff --git a/plugins/openclaw-tps-mail/src/obligations.ts b/plugins/openclaw-tps-mail/src/obligations.ts index 7cdf2eaf..2c3ce6aa 100644 --- a/plugins/openclaw-tps-mail/src/obligations.ts +++ b/plugins/openclaw-tps-mail/src/obligations.ts @@ -210,7 +210,7 @@ export function readObligationResult(mailDir: string, agent: string, inboundId: export function listObligations( mailDir: string, agent: string, - onReadError: (path: string, code: string) => void = (path, code) => console.warn( + onReadError: (path: string, code: string, ids: string[]) => void = (path, code) => console.warn( `tps-mail: obligation-list-read-failed: actor=${agent} state=unknown path=${path} code=${code}; restore readable obligations and restart the account`, ), ): ObligationRecord[] { @@ -219,7 +219,7 @@ export function listObligations( try { names = readdirSync(dir); } catch (err: any) { - if (err?.code !== "ENOENT") onReadError(dir, err?.code ?? "READ_FAILED"); + if (err?.code !== "ENOENT") onReadError(dir, err?.code ?? "READ_FAILED", ["*"]); return []; } const out: ObligationRecord[] = []; @@ -228,12 +228,16 @@ export function listObligations( try { const record: unknown = JSON.parse(readFileSync(resolve(dir, name), "utf-8")); if (!isObligationRecord(record) || `${record.inboundId}.json` !== name) { - onReadError(resolve(dir, name), "INVALID_RECORD"); + const ids = [name.slice(0, -5)]; + if (record && typeof record === "object" && "inboundId" in record && typeof record.inboundId === "string") { + ids.push(record.inboundId); + } + onReadError(resolve(dir, name), "INVALID_RECORD", ids); continue; } out.push(record); } catch (err: any) { - onReadError(resolve(dir, name), err?.code ?? "INVALID_RECORD"); + onReadError(resolve(dir, name), err?.code ?? "INVALID_RECORD", [name.slice(0, -5)]); } } return out; diff --git a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts index 6ac7a3de..6fb9d69c 100644 --- a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts +++ b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts @@ -12,6 +12,7 @@ hashes.sha512 = (m: Uint8Array) => new Uint8Array(createHash("sha512").update(m) const realFs = { ...fs }; let removeOnTerminal: { path: string; state: string } | undefined; +let replaceOnTerminal: { path: string; state: string; bytes: string } | undefined; let stampError: Error | undefined; let obligationReadFailure: string | undefined; let obligationReads = 0; @@ -42,6 +43,12 @@ mock.module("node:fs", () => ({ removeOnTerminal = undefined; } } + if (replaceOnTerminal && String(args[0]).includes(".obligations/") && typeof args[1] === "string") { + if (JSON.parse(args[1]).state === replaceOnTerminal.state) { + realFs.writeFileSync(replaceOnTerminal.path, replaceOnTerminal.bytes); + replaceOnTerminal = undefined; + } + } return result; }, })); @@ -89,6 +96,7 @@ beforeEach(() => { afterEach(() => { removeOnTerminal = undefined; + replaceOnTerminal = undefined; stampError = undefined; obligationReadFailure = undefined; obligationReads = 0; @@ -289,6 +297,27 @@ describe("cli#492 — a failed cur/ stamp write is surfaced and retried", () => } }, 15000); + for (const alreadyStamped of [false, true]) { + it(`${kind}: a replacement identity is refused on the live stamp path (already stamped: ${alreadyStamped})`, async () => { + const h = await boot(true); + try { + expect(await pollUntil(() => h.dispatch() !== null)).toBe(true); + const cur = readCur()!; + const replacement = { ...cur.record, id: "replacement", ...(alreadyStamped ? { ackedAt: "existing-ack", nackedAt: "existing-nack" } : {}) }; + const bytes = JSON.stringify(replacement); + replaceOnTerminal = { path: cur.path, state, bytes }; + await finish(h); + expect(await pollUntil(() => failedLogs(h, `${kind}-stamp-failed`).some((m) => m.includes("code=ID_MISMATCH")))).toBe(true); + expect(obligation(h.inboundId)?.state).toBe(state); + await sleep(500); + expect(realFs.readFileSync(cur.path, "utf8")).toBe(bytes); + expect(failedLogs(h, `${kind}-stamp-failed`)).toHaveLength(4); + expect(h.logs.some((m) => m.includes(`${kind}-stamp-retry-ok`))).toBe(false); + } finally { await h.stop(); } + }, 15000); + + } + it(`${kind}: an exception without a code is reported and remains retryable`, async () => { const h = await boot(true); try { @@ -447,3 +476,40 @@ for (const state of ["acked", "failed"] as const) { }, 15000); } } + +for (const invalidState of [false, true]) { + it(`startup holds both identities of a mismatched obligation (invalid state: ${invalidState})`, async () => { + const first = await boot(true); + try { + expect(await pollUntil(() => first.dispatch() !== null)).toBe(true); + await first.deliver("verdict"); + first.settle(); + expect(await pollUntil(() => !!readCur()?.record?.ackedAt)).toBe(true); + } finally { await first.stop(); } + const cur = readCur()!; + delete cur.record.ackedAt; + realFs.writeFileSync(cur.path, JSON.stringify(cur.record)); + const dir = resolve(mailDir, "anvil", ".obligations"); + const original = resolve(dir, `${first.inboundId}.json`); + const record = JSON.parse(realFs.readFileSync(original, "utf8")); + record.lastTransitionAt = new Date(0).toISOString(); + if (invalidState) record.state = "invalid"; + const mismatched = resolve(dir, "filename-id.json"); + const bytes = JSON.stringify(record); + realFs.renameSync(original, mismatched); + realFs.writeFileSync(mismatched, bytes); + const filenameCur = resolve(mailDir, "anvil", "cur", "filename-id.json"); + realFs.writeFileSync(filenameCur, JSON.stringify({ ...cur.record, id: "filename-id" })); + const next = await boot(false); + try { + expect(await pollUntil(() => next.logs.some((m) => m.includes(mismatched) && m.includes("code=INVALID_RECORD")))).toBe(true); + await sleep(500); + expect(next.dispatch()).toBeNull(); + expect(next.logs.some((m) => m.includes(`delivering ${first.inboundId} `) || m.includes("delivering filename-id "))).toBe(false); + expect(realFs.readFileSync(mismatched, "utf8")).toBe(bytes); + expect(realFs.existsSync(cur.path)).toBe(true); + expect(realFs.existsSync(filenameCur)).toBe(true); + expect(JSON.parse(realFs.readFileSync(cur.path, "utf8")).ackedAt).toBeUndefined(); + } finally { next.settle(); await next.stop(); } + }, 15000); +} diff --git a/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts b/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts index b21721ad..5e5f429d 100644 --- a/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts +++ b/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts @@ -72,19 +72,19 @@ afterAll(() => realFs.rmSync(root, { recursive: true, force: true })); describe("patchMailFile", () => { test("patches an existing record", () => { realFs.writeFileSync(path, JSON.stringify({ id: "inbound", body: "hello" })); - expect(patchMailFile(path, { read: true })).toEqual({ ok: true }); + expect(patchMailFile(path, { read: true }, undefined, "inbound")).toEqual({ ok: true }); expect(JSON.parse(realFs.readFileSync(path, "utf-8"))).toEqual({ id: "inbound", body: "hello", read: true }); }); test("reports a missing record without creating it", () => { - expect(patchMailFile(path, { read: true })).toMatchObject({ ok: false, reason: "record-missing" }); + expect(patchMailFile(path, { read: true }, undefined, "inbound")).toMatchObject({ ok: false, reason: "record-missing" }); expect(realFs.existsSync(path)).toBe(false); }); test("reports a record removed after the read without recreating it", () => { realFs.writeFileSync(path, JSON.stringify({ id: "inbound", body: "hello" })); removeAfterRead = path; - expect(patchMailFile(path, { ackedAt: "receipt", read: true })).toMatchObject({ ok: false, reason: "record-missing" }); + expect(patchMailFile(path, { ackedAt: "receipt", read: true }, "ackedAt", "inbound")).toMatchObject({ ok: false, reason: "record-missing" }); expect(removeAfterRead).toBeUndefined(); expect(realFs.existsSync(path)).toBe(false); }); @@ -345,7 +345,7 @@ test("obligation filename and inboundId must agree before startup uses the recor const record = JSON.parse(realFs.readFileSync(f.obligationPath, "utf8")); record.inboundId = "other"; realFs.writeFileSync(f.obligationPath, JSON.stringify(record)); - expect(f.reconcile().has("inbound")).toBe(true); + expect([...f.reconcile()].sort()).toEqual(["inbound", "other"]); expect(f.logs).toHaveLength(1); expect(f.logs[0]).toContain("INVALID_RECORD"); expect(JSON.parse(realFs.readFileSync(f.curPath, "utf8")).ackedAt).toBeUndefined(); From 9082eef14eabe438f27c0f679be46d0e923cc3b6 Mon Sep 17 00:00:00 2001 From: flint Date: Sat, 3 Oct 2026 20:15:12 -0700 Subject: [PATCH 08/17] fix(openclaw-tps-mail): distinct remedies for an id mismatch and an unreadable directory; test titles say what they assert Co-Authored-By: Claude Opus 5.5 --- plugins/openclaw-tps-mail/src/index.ts | 14 ++++++++++++-- plugins/openclaw-tps-mail/src/obligations.ts | 1 + .../test/cur-record-write.test.ts | 2 +- .../openclaw-tps-mail/test/patch-mail-file.test.ts | 2 +- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/plugins/openclaw-tps-mail/src/index.ts b/plugins/openclaw-tps-mail/src/index.ts index 961458f0..40234cd3 100644 --- a/plugins/openclaw-tps-mail/src/index.ts +++ b/plugins/openclaw-tps-mail/src/index.ts @@ -699,7 +699,11 @@ export function reconcileTerminalCurStamps(mailDir: string, agent: string, log: unknownInbounds.add(rec.inboundId); log?.warn?.( `tps-mail: ${kind}-stamp-reconcile-failed: ${rec.inboundId} actor=${agent} state=${rec.state} path=${r.path} code=${r.code}; ` + - (r.reason === "record-missing" ? "inspect the missing cur record; obligation retained" : "restore writable records and restart the account; obligation retained"), + (r.reason === "record-missing" + ? "inspect the missing cur record; obligation retained" + : r.code === "ID_MISMATCH" + ? "the cur record has another id; restore the expected record or correct its path and restart the account; obligation retained" + : "restore writable records and restart the account; obligation retained"), ); } } @@ -979,6 +983,8 @@ function stampTerminalCur( `tps-mail: ${kind}-stamp-failed: ${ctx.inboundId} actor=${ctx.agent} state=${kind === "ack" ? "acked" : "failed"} path=${stamped.path} code=${stamped.code}; ` + (stamped.reason === "record-missing" ? "inspect the missing cur record; obligation retained" + : stamped.code === "ID_MISMATCH" + ? "the cur record has another id; restore the expected record or correct its path and restart the account; obligation retained" : delay === undefined ? "no retries left; restore writable records and restart the account; obligation retained" : `retry ${attempt + 1} of ${stampRetryDelaysMs.length} in ${delay}ms`), @@ -2391,7 +2397,11 @@ const gateway: ChannelGatewayAdapter = { const startupFailure = (path: string, code: string, id = "*", state = "unknown") => { if (unknownInbounds.has(id)) return; unknownInbounds.add(id); - log?.warn?.(`tps-mail: startup-unresolved: ${id} actor=${agentId} state=${state} path=${path} code=${code}; repair the record and restart the account; obligation retained`); + log?.warn?.( + id === "*" + ? `tps-mail: startup-unresolved: actor=${agentId} state=unknown path=${path} code=${code}; the directory could not be read; repair it and restart the account` + : `tps-mail: startup-unresolved: ${id} actor=${agentId} state=${state} path=${path} code=${code}; repair the record and restart the account; obligation retained`, + ); }; const startupRecords = listObligations(account.mailDir, agentId, (path, code, ids) => { startupFailure(path, code, ids[0]); diff --git a/plugins/openclaw-tps-mail/src/obligations.ts b/plugins/openclaw-tps-mail/src/obligations.ts index 2c3ce6aa..7264e542 100644 --- a/plugins/openclaw-tps-mail/src/obligations.ts +++ b/plugins/openclaw-tps-mail/src/obligations.ts @@ -573,6 +573,7 @@ export function sweepTerminalObligations( const fail = (path: string, code: string, id: string, state: string) => { if (unresolved.has(id)) return; if (onFailure) onFailure(path, code, id, state); + else if (id === "*") log?.warn?.(`tps-mail: retention-unresolved: actor=${agent} state=unknown path=${path} code=${code}; the directory could not be read; repair it and restart the account`); else log?.warn?.(`tps-mail: retention-unresolved: ${id} actor=${agent} state=${state} path=${path} code=${code}; repair the record and restart the account; obligation retained`); unresolved.add(id); }; diff --git a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts index 6fb9d69c..3d658629 100644 --- a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts +++ b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts @@ -128,7 +128,7 @@ function signedBody(from: string, to: string, body: string, seed: Buffer): strin )); } -/** The anvil cur/ record (promote() names it for the inbound id). */ +/** The anvil cur/ record (the first .json file in anvil/cur). */ function readCur(): { path: string; record: any } | null { const dir = resolve(mailDir, "anvil", "cur"); let names: string[]; diff --git a/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts b/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts index 5e5f429d..e0e877d8 100644 --- a/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts +++ b/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts @@ -268,7 +268,7 @@ for (const state of ["acked", "failed"] as const) { } for (const stage of ["obligation-reread", "cur-retention-read", "cur-retention-list", "obligation-delete", "abandonment-write", "age", "identity", "cur-retention-missing"] as const) { - test(`retention ${stage} retains evidence, reports once, and continues`, () => { + test(`retention ${stage} retains evidence and reports once`, () => { const f = terminalFixture(stage === "abandonment-write" ? "failed" : "acked"); const record = JSON.parse(realFs.readFileSync(f.obligationPath, "utf8")); if (stage === "abandonment-write") record.nackPending = true; From 73e57ab99ae5d2c65c97d361a3ae48de0b3190ac Mon Sep 17 00:00:00 2001 From: flint Date: Sat, 3 Oct 2026 21:01:55 -0700 Subject: [PATCH 09/17] fix(openclaw-tps-mail): a directory read failure is an explicit kind, never inferred from the id Co-Authored-By: Claude Opus 5.5 --- packages/pi-tps-mail/README.md | 12 ++------ packages/pi-tps-mail/src/watcher.ts | 6 +--- plugins/openclaw-tps-mail/src/index.ts | 19 +++++++------ plugins/openclaw-tps-mail/src/obligations.ts | 28 ++++++++++--------- .../test/cur-record-write.test.ts | 24 +++++++++++++++- .../test/patch-mail-file.test.ts | 28 ++++++++++++++++--- 6 files changed, 75 insertions(+), 42 deletions(-) diff --git a/packages/pi-tps-mail/README.md b/packages/pi-tps-mail/README.md index c53172f5..339eaef2 100644 --- a/packages/pi-tps-mail/README.md +++ b/packages/pi-tps-mail/README.md @@ -76,16 +76,8 @@ process.on("SIGTERM", () => watcher.stop()); Every `tps` invocation below runs as the agent (`TPS_AGENT_ID={agent}`, `TPS_MAIL_DIR={inboxRoot}/.tps/mail`). -1. Every 5 seconds (`pollIntervalMs`) it first finishes any reply its journal - still owes (step 3), then — when `new/` holds anything, or at least every - `rescanIntervalMs` — runs **`tps mail check {agent} --json`**. That is the - CLI's promotion path: it verifies each inbound's signed envelope (signature, - sender, recipient, replay, id shape), moves it to `cur/`, and dead-letters - what fails to `dlq/`; it also re-verifies and re-presents a `cur/` record - whose processing lease expired without an ack. **The watcher acts only on the - records that command returns** — never on a file it reads itself — so an - unsigned or forged inbound is never dispatched and never answered, and the - sender, the body and the thread all come from the verified envelope. +1. Runs **`tps mail check {agent} --json`** when `new/` holds anything or + `rescanIntervalMs` has elapsed. 2. For each verified inbound: - Spawns the launcher script with the verified message body as its argument, with a hard timeout (SIGTERM → 5s grace → SIGKILL). diff --git a/packages/pi-tps-mail/src/watcher.ts b/packages/pi-tps-mail/src/watcher.ts index af41dc68..b351b78b 100644 --- a/packages/pi-tps-mail/src/watcher.ts +++ b/packages/pi-tps-mail/src/watcher.ts @@ -5,11 +5,7 @@ // the agent: the CLI's own promotion path (promote()) verifies every new/ // record — signature, sender binding, recipient, replay, id shape — moves it to // cur/, and dead-letters what fails; it also re-verifies and re-presents a -// cur/ record whose processing lease expired without an ack. The watcher acts -// ONLY on the verified records that command prints: the sender, the body the -// launcher sees and the thread the reply signs (the envelope's messageId) all -// come from the verified envelope. An unsigned or forged inbound is never -// dispatched and never answered. +// cur/ record whose processing lease expired without an ack. // // THE REPLY JOURNAL. Before a reply is first sent, the watcher writes it to // `//.pi-tps-mail/replies/.json` (0600): the verified diff --git a/plugins/openclaw-tps-mail/src/index.ts b/plugins/openclaw-tps-mail/src/index.ts index 40234cd3..b872656c 100644 --- a/plugins/openclaw-tps-mail/src/index.ts +++ b/plugins/openclaw-tps-mail/src/index.ts @@ -77,6 +77,7 @@ import { scanForReceipt, sweepTerminalObligations, type ReceiptSignatureCheck, + type ReadFailureKind, transitionObligation, writeReceipt, type ObligationRecord, @@ -655,7 +656,7 @@ export function reconcileTerminalCurStamps(mailDir: string, agent: string, log: if (unknownInbounds.has(id)) return; unknownInbounds.add(id); log?.warn?.( - `tps-mail: stamp-reconcile-read-failed: ${id} actor=${agent} state=${state} path=${path} code=${code}; restore readable records and restart the account`, + `tps-mail: stamp-reconcile-read-failed: ${id} actor=${agent} state=${state} path=${path} code=${code}; repair the record and restart the account`, ); }; const onObligationReadError = (path: string, code: string, ids: string[]) => { @@ -703,7 +704,7 @@ export function reconcileTerminalCurStamps(mailDir: string, agent: string, log: ? "inspect the missing cur record; obligation retained" : r.code === "ID_MISMATCH" ? "the cur record has another id; restore the expected record or correct its path and restart the account; obligation retained" - : "restore writable records and restart the account; obligation retained"), + : "repair the record and restart the account; obligation retained"), ); } } @@ -986,7 +987,7 @@ function stampTerminalCur( : stamped.code === "ID_MISMATCH" ? "the cur record has another id; restore the expected record or correct its path and restart the account; obligation retained" : delay === undefined - ? "no retries left; restore writable records and restart the account; obligation retained" + ? "no retries left; repair the record and restart the account; obligation retained" : `retry ${attempt + 1} of ${stampRetryDelaysMs.length} in ${delay}ms`), ); if (delay === undefined || !isLiveContext(ctx)) return; @@ -2394,17 +2395,17 @@ const gateway: ChannelGatewayAdapter = { // Attempt terminal stamps before recovery and retention. const unknownInbounds = new Set(); - const startupFailure = (path: string, code: string, id = "*", state = "unknown") => { + const startupFailure = (path: string, code: string, id = "*", state = "unknown", kind: ReadFailureKind = "file") => { if (unknownInbounds.has(id)) return; unknownInbounds.add(id); log?.warn?.( - id === "*" + kind === "directory" ? `tps-mail: startup-unresolved: actor=${agentId} state=unknown path=${path} code=${code}; the directory could not be read; repair it and restart the account` - : `tps-mail: startup-unresolved: ${id} actor=${agentId} state=${state} path=${path} code=${code}; repair the record and restart the account; obligation retained`, + : `tps-mail: startup-unresolved: ${id} actor=${agentId} state=${state} path=${path} code=${code}; repair the record and restart the account`, ); }; - const startupRecords = listObligations(account.mailDir, agentId, (path, code, ids) => { - startupFailure(path, code, ids[0]); + const startupRecords = listObligations(account.mailDir, agentId, (path, code, ids, kind) => { + startupFailure(path, code, ids[0], "unknown", kind); for (const id of ids) unknownInbounds.add(id); }); reconcileTerminalCurStamps(account.mailDir, agentId, log, startupRecords, unknownInbounds); @@ -2432,7 +2433,7 @@ const gateway: ChannelGatewayAdapter = { void recoverUnackedCurRecord(agentId, curPath, record); } } - } catch (err: any) { startupFailure(curDir, err?.code ?? "READ_FAILED"); } + } catch (err: any) { startupFailure(curDir, err?.code ?? "READ_FAILED", "*", "unknown", "directory"); } // Reap stranded tmp/*.promote scratch from an interrupted promote (the // catch only runs on a thrown error, so a kill leaves orphans no other diff --git a/plugins/openclaw-tps-mail/src/obligations.ts b/plugins/openclaw-tps-mail/src/obligations.ts index 7264e542..fab3e380 100644 --- a/plugins/openclaw-tps-mail/src/obligations.ts +++ b/plugins/openclaw-tps-mail/src/obligations.ts @@ -161,6 +161,8 @@ export interface ObligationRecord { nackAbandonedAt?: string; } +export type ReadFailureKind = "directory" | "file"; + export interface ObligationLog { info?: (msg: string) => void; warn?: (msg: string) => void; @@ -210,8 +212,8 @@ export function readObligationResult(mailDir: string, agent: string, inboundId: export function listObligations( mailDir: string, agent: string, - onReadError: (path: string, code: string, ids: string[]) => void = (path, code) => console.warn( - `tps-mail: obligation-list-read-failed: actor=${agent} state=unknown path=${path} code=${code}; restore readable obligations and restart the account`, + onReadError: (path: string, code: string, ids: string[], kind: ReadFailureKind) => void = (path, code) => console.warn( + `tps-mail: obligation-list-read-failed: actor=${agent} state=unknown path=${path} code=${code}; repair the record and restart the account`, ), ): ObligationRecord[] { const dir = obligationsDir(mailDir, agent); @@ -219,7 +221,7 @@ export function listObligations( try { names = readdirSync(dir); } catch (err: any) { - if (err?.code !== "ENOENT") onReadError(dir, err?.code ?? "READ_FAILED", ["*"]); + if (err?.code !== "ENOENT") onReadError(dir, err?.code ?? "READ_FAILED", ["*"], "directory"); return []; } const out: ObligationRecord[] = []; @@ -232,12 +234,12 @@ export function listObligations( if (record && typeof record === "object" && "inboundId" in record && typeof record.inboundId === "string") { ids.push(record.inboundId); } - onReadError(resolve(dir, name), "INVALID_RECORD", ids); + onReadError(resolve(dir, name), "INVALID_RECORD", ids, "file"); continue; } out.push(record); } catch (err: any) { - onReadError(resolve(dir, name), err?.code ?? "INVALID_RECORD", [name.slice(0, -5)]); + onReadError(resolve(dir, name), err?.code ?? "INVALID_RECORD", [name.slice(0, -5)], "file"); } } return out; @@ -267,7 +269,7 @@ export function createObligation( const draft = make(); const result = readObligationResult(mailDir, agent, draft.inboundId); if (result.status === "unverified") { - const message = `tps-mail: obligation-create-read-failed: ${draft.inboundId} actor=${agent} state=unknown path=${result.path} code=${result.code}; restore readable records and restart the account; obligation retained`; + const message = `tps-mail: obligation-create-read-failed: ${draft.inboundId} actor=${agent} state=unknown path=${result.path} code=${result.code}; repair the record and restart the account`; log?.warn?.(message); throw new Error(message); } @@ -551,7 +553,7 @@ export function sweepTerminalObligations( nowMs: number = Date.now(), nackHoldDays: number = retentionDays * DEFAULT_NACK_HOLD_MULTIPLE, unresolved = new Set(), - onFailure?: (path: string, code: string, id: string, state: string) => void, + onFailure?: (path: string, code: string, id: string, state: string, kind: ReadFailureKind) => void, heldRecords: ObligationRecord[] = [], ): RetentionResult { const res: RetentionResult = { @@ -570,11 +572,11 @@ export function sweepTerminalObligations( res.disabled = true; return res; } - const fail = (path: string, code: string, id: string, state: string) => { + const fail = (path: string, code: string, id: string, state: string, kind: ReadFailureKind = "file") => { if (unresolved.has(id)) return; - if (onFailure) onFailure(path, code, id, state); - else if (id === "*") log?.warn?.(`tps-mail: retention-unresolved: actor=${agent} state=unknown path=${path} code=${code}; the directory could not be read; repair it and restart the account`); - else log?.warn?.(`tps-mail: retention-unresolved: ${id} actor=${agent} state=${state} path=${path} code=${code}; repair the record and restart the account; obligation retained`); + if (onFailure) onFailure(path, code, id, state, kind); + else if (kind === "directory") log?.warn?.(`tps-mail: retention-unresolved: actor=${agent} state=unknown path=${path} code=${code}; the directory could not be read; repair it and restart the account`); + else log?.warn?.(`tps-mail: retention-unresolved: ${id} actor=${agent} state=${state} path=${path} code=${code}; repair the record and restart the account`); unresolved.add(id); }; const dir = obligationsDir(mailDir, agent); @@ -589,7 +591,7 @@ export function sweepTerminalObligations( // unreadable the agent's own receipts cannot be attributed either. const code = (err as NodeJS.ErrnoException)?.code; if (code !== "ENOENT") { - fail(dir, code ?? "READ_FAILED", "*", "unknown"); + fail(dir, code ?? "READ_FAILED", "*", "unknown", "directory"); if (!onFailure) log?.warn?.(`tps-mail: obligation retention: could not read ${dir}; sweep skipped`); return res; } @@ -614,7 +616,7 @@ export function sweepTerminalObligations( try { receiptNames = readdirSync(root); } catch (err: any) { if (err?.code !== "ENOENT") { - fail(root, err?.code ?? "READ_FAILED", "*", "unknown"); + fail(root, err?.code ?? "READ_FAILED", "*", "unknown", "directory"); return res; } } diff --git a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts index 3d658629..388e73db 100644 --- a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts +++ b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts @@ -128,7 +128,7 @@ function signedBody(from: string, to: string, body: string, seed: Buffer): strin )); } -/** The anvil cur/ record (the first .json file in anvil/cur). */ +/** The anvil cur/ record (the first parseable .json file in anvil/cur). */ function readCur(): { path: string; record: any } | null { const dir = resolve(mailDir, "anvil", "cur"); let names: string[]; @@ -513,3 +513,25 @@ for (const invalidState of [false, true]) { } finally { next.settle(); await next.stop(); } }, 15000); } + +for (const failure of ["directory", "file"] as const) { + it(`startup distinguishes a ${failure} read failure from the literal star file`, async () => { + const dir = resolve(mailDir, "anvil", ".obligations"); + realFs.mkdirSync(resolve(mailDir, "anvil"), { recursive: true }); + const path = failure === "directory" ? dir : resolve(dir, "*.json"); + if (failure === "file") realFs.mkdirSync(dir); + realFs.writeFileSync(path, "{}"); + if (failure === "file") realFs.chmodSync(path, 0o000); + const h = await boot(false); + try { + expect(await pollUntil(() => h.logs.some((m) => m.includes("startup-unresolved") && m.includes(path)))).toBe(true); + const diagnostic = h.logs.find((m) => m.includes("startup-unresolved") && m.includes(path))!; + expect(diagnostic).toContain(failure === "directory" ? "the directory could not be read" : "repair the record"); + expect(diagnostic.includes("the directory could not be read")).toBe(failure === "directory"); + expect(diagnostic).not.toContain("obligation retained"); + } finally { + await h.stop(); + if (failure === "file") realFs.chmodSync(path, 0o644); + } + }); +} diff --git a/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts b/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts index e0e877d8..05f26db5 100644 --- a/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts +++ b/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts @@ -128,7 +128,7 @@ describe("terminal stamp reconciliation", () => { uncodedWriteFailure = true; f.reconcile(); expect(f.logs[0]).toContain(`actor=anvil state=${state} path=${f.curPath} code=WRITE_FAILED`); - expect(f.logs[0]).toContain("restore writable records and restart the account; obligation retained"); + expect(f.logs[0]).toContain("repair the record and restart the account; obligation retained"); uncodedWriteFailure = false; f.reconcile(); expect(JSON.parse(realFs.readFileSync(f.curPath, "utf8"))[key]).toBeDefined(); @@ -170,7 +170,7 @@ describe("terminal stamp reconciliation", () => { expect(f.logs[0]).toContain("stamp-reconcile-read-failed"); expect(f.logs[0]).toContain("actor=anvil state="); expect(f.logs[0]).toContain(`path=${target} code=EACCES`); - expect(f.logs[0]).toContain("restore readable records and restart the account"); + expect(f.logs[0]).toContain("repair the record and restart the account"); listFailure = readFailure = undefined; f.reconcile(); expect(JSON.parse(realFs.readFileSync(f.curPath, "utf8")).ackedAt).toBeDefined(); @@ -203,7 +203,7 @@ for (const invalid of [null, 7, "bad", [], { inboundId: "bad" }]) { expect(errors).toEqual([`${badPath}:INVALID_RECORD`]); expect(() => f.reconcile()).not.toThrow(); expect(f.logs[0]).toContain(`actor=anvil state=unknown path=${badPath} code=INVALID_RECORD`); - expect(f.logs[0]).toContain("restore readable records and restart the account"); + expect(f.logs[0]).toContain("repair the record and restart the account"); expect(JSON.parse(realFs.readFileSync(f.curPath, "utf8")).ackedAt).toBeDefined(); expect(realFs.readFileSync(badPath, "utf8")).toBe(JSON.stringify(invalid)); }); @@ -214,7 +214,7 @@ test("a null cur record is reported as unreadable", () => { realFs.writeFileSync(f.curPath, "null"); f.reconcile(); expect(f.logs[0]).toContain(`actor=anvil state=acked path=${f.curPath} code=INVALID_RECORD`); - expect(f.logs[0]).toContain("restore readable records and restart the account"); + expect(f.logs[0]).toContain("repair the record and restart the account"); }); test("creation refuses an unreadable existing terminal obligation", () => { @@ -350,3 +350,23 @@ test("obligation filename and inboundId must agree before startup uses the recor expect(f.logs[0]).toContain("INVALID_RECORD"); expect(JSON.parse(realFs.readFileSync(f.curPath, "utf8")).ackedAt).toBeUndefined(); }); + +for (const failure of ["directory", "file"] as const) { + test(`retention distinguishes a ${failure} read failure from the literal star file`, () => { + const dir = join(root, "anvil", ".obligations"); + realFs.mkdirSync(dir, { recursive: true }); + const path = failure === "directory" ? dir : join(dir, "*.json"); + if (failure === "directory") listFailure = path; + else { + realFs.writeFileSync(path, "{}"); + readFailure = path; + } + const logs: string[] = []; + sweepTerminalObligations(root, "anvil", 7, { warn: (m) => logs.push(m) }); + const diagnostic = logs.find((m) => m.includes("retention-unresolved"))!; + expect(diagnostic).toContain(`path=${path} code=EACCES`); + expect(diagnostic).toContain(failure === "directory" ? "the directory could not be read" : "repair the record"); + expect(diagnostic.includes("the directory could not be read")).toBe(failure === "directory"); + expect(diagnostic).not.toContain("obligation retained"); + }); +} From b99bdcf086ccfd697394b88f1dd8f06922794cea Mon Sep 17 00:00:00 2001 From: flint Date: Sat, 3 Oct 2026 21:28:35 -0700 Subject: [PATCH 10/17] fix(openclaw-tps-mail): diagnostics name the path, code and state and give one remedy Co-Authored-By: Claude Opus 5.5 --- plugins/openclaw-tps-mail/src/index.ts | 24 ++++++------------- plugins/openclaw-tps-mail/src/obligations.ts | 8 +++---- .../test/cur-record-write.test.ts | 9 ++++--- .../test/patch-mail-file.test.ts | 17 +++++++------ 4 files changed, 23 insertions(+), 35 deletions(-) diff --git a/plugins/openclaw-tps-mail/src/index.ts b/plugins/openclaw-tps-mail/src/index.ts index b872656c..14dfe057 100644 --- a/plugins/openclaw-tps-mail/src/index.ts +++ b/plugins/openclaw-tps-mail/src/index.ts @@ -656,7 +656,7 @@ export function reconcileTerminalCurStamps(mailDir: string, agent: string, log: if (unknownInbounds.has(id)) return; unknownInbounds.add(id); log?.warn?.( - `tps-mail: stamp-reconcile-read-failed: ${id} actor=${agent} state=${state} path=${path} code=${code}; repair the record and restart the account`, + `tps-mail: stamp-reconcile-read-failed: ${id} actor=${agent} state=${state} path=${path} code=${code}; fix the path named above and restart the account`, ); }; const onObligationReadError = (path: string, code: string, ids: string[]) => { @@ -677,7 +677,7 @@ export function reconcileTerminalCurStamps(mailDir: string, agent: string, log: const missing = (path: string) => { unknownInbounds.add(rec.inboundId); log?.warn?.( - `tps-mail: ${kind}-stamp-reconcile-failed: ${rec.inboundId} actor=${agent} state=${rec.state} path=${path} code=ENOENT; inspect the missing cur record; obligation retained; repair it and restart the account`, + `tps-mail: ${kind}-stamp-reconcile-failed: ${rec.inboundId} actor=${agent} state=${rec.state} path=${path} code=ENOENT; obligation retained; fix the path named above and restart the account`, ); }; if (unreadable) continue; @@ -700,11 +700,7 @@ export function reconcileTerminalCurStamps(mailDir: string, agent: string, log: unknownInbounds.add(rec.inboundId); log?.warn?.( `tps-mail: ${kind}-stamp-reconcile-failed: ${rec.inboundId} actor=${agent} state=${rec.state} path=${r.path} code=${r.code}; ` + - (r.reason === "record-missing" - ? "inspect the missing cur record; obligation retained" - : r.code === "ID_MISMATCH" - ? "the cur record has another id; restore the expected record or correct its path and restart the account; obligation retained" - : "repair the record and restart the account; obligation retained"), + `obligation retained; fix the path named above and restart the account`, ); } } @@ -982,13 +978,9 @@ function stampTerminalCur( const delay = stamped.reason === "record-missing" ? undefined : stampRetryDelaysMs[attempt]; ctx.log?.warn?.( `tps-mail: ${kind}-stamp-failed: ${ctx.inboundId} actor=${ctx.agent} state=${kind === "ack" ? "acked" : "failed"} path=${stamped.path} code=${stamped.code}; ` + - (stamped.reason === "record-missing" - ? "inspect the missing cur record; obligation retained" - : stamped.code === "ID_MISMATCH" - ? "the cur record has another id; restore the expected record or correct its path and restart the account; obligation retained" - : delay === undefined - ? "no retries left; repair the record and restart the account; obligation retained" - : `retry ${attempt + 1} of ${stampRetryDelaysMs.length} in ${delay}ms`), + (stamped.reason !== "record-missing" && delay !== undefined + ? `retry ${attempt + 1} of ${stampRetryDelaysMs.length} in ${delay}ms` + : `obligation retained; fix the path named above and restart the account`), ); if (delay === undefined || !isLiveContext(ctx)) return; const timer = accountTimer(ctx, () => stampTerminalCur(ctx, kind, patch, attempt + 1), delay); @@ -2399,9 +2391,7 @@ const gateway: ChannelGatewayAdapter = { if (unknownInbounds.has(id)) return; unknownInbounds.add(id); log?.warn?.( - kind === "directory" - ? `tps-mail: startup-unresolved: actor=${agentId} state=unknown path=${path} code=${code}; the directory could not be read; repair it and restart the account` - : `tps-mail: startup-unresolved: ${id} actor=${agentId} state=${state} path=${path} code=${code}; repair the record and restart the account`, + `tps-mail: startup-unresolved: ${kind === "directory" ? "" : id + " "}actor=${agentId} state=${kind === "directory" ? "unknown" : state} path=${path} code=${code}; fix the path named above and restart the account`, ); }; const startupRecords = listObligations(account.mailDir, agentId, (path, code, ids, kind) => { diff --git a/plugins/openclaw-tps-mail/src/obligations.ts b/plugins/openclaw-tps-mail/src/obligations.ts index fab3e380..77a871ff 100644 --- a/plugins/openclaw-tps-mail/src/obligations.ts +++ b/plugins/openclaw-tps-mail/src/obligations.ts @@ -213,7 +213,7 @@ export function listObligations( mailDir: string, agent: string, onReadError: (path: string, code: string, ids: string[], kind: ReadFailureKind) => void = (path, code) => console.warn( - `tps-mail: obligation-list-read-failed: actor=${agent} state=unknown path=${path} code=${code}; repair the record and restart the account`, + `tps-mail: obligation-list-read-failed: actor=${agent} state=unknown path=${path} code=${code}; fix the path named above and restart the account`, ), ): ObligationRecord[] { const dir = obligationsDir(mailDir, agent); @@ -269,7 +269,7 @@ export function createObligation( const draft = make(); const result = readObligationResult(mailDir, agent, draft.inboundId); if (result.status === "unverified") { - const message = `tps-mail: obligation-create-read-failed: ${draft.inboundId} actor=${agent} state=unknown path=${result.path} code=${result.code}; repair the record and restart the account`; + const message = `tps-mail: obligation-create-read-failed: ${draft.inboundId} actor=${agent} state=unknown path=${result.path} code=${result.code}; fix the path named above and restart the account`; log?.warn?.(message); throw new Error(message); } @@ -575,8 +575,8 @@ export function sweepTerminalObligations( const fail = (path: string, code: string, id: string, state: string, kind: ReadFailureKind = "file") => { if (unresolved.has(id)) return; if (onFailure) onFailure(path, code, id, state, kind); - else if (kind === "directory") log?.warn?.(`tps-mail: retention-unresolved: actor=${agent} state=unknown path=${path} code=${code}; the directory could not be read; repair it and restart the account`); - else log?.warn?.(`tps-mail: retention-unresolved: ${id} actor=${agent} state=${state} path=${path} code=${code}; repair the record and restart the account`); + else if (kind === "directory") log?.warn?.(`tps-mail: retention-unresolved: actor=${agent} state=unknown path=${path} code=${code}; fix the path named above and restart the account`); + else log?.warn?.(`tps-mail: retention-unresolved: ${id} actor=${agent} state=${state} path=${path} code=${code}; fix the path named above and restart the account`); unresolved.add(id); }; const dir = obligationsDir(mailDir, agent); diff --git a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts index 388e73db..dfbba3fb 100644 --- a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts +++ b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts @@ -287,7 +287,7 @@ describe("cli#492 — a failed cur/ stamp write is surfaced and retried", () => await finish(h); expect(await pollUntil(() => failedLogs(h, `${kind}-stamp-failed`).length > 0)).toBe(true); expect(failedLogs(h, `${kind}-stamp-failed`)[0]).toContain(`actor=anvil state=${state} path=${cur.path} code=ENOENT`); - expect(failedLogs(h, `${kind}-stamp-failed`)[0]).toContain("inspect the missing cur record; obligation retained"); + expect(failedLogs(h, `${kind}-stamp-failed`)[0]).toContain("obligation retained; fix the path named above and restart the account"); expect(obligation(h.inboundId)?.state).toBe(state); await sleep(400); expect(failedLogs(h, `${kind}-stamp-failed`).length).toBe(1); @@ -412,7 +412,7 @@ for (const failReads of [Infinity, 1]) { expect(realFs.readFileSync(path, "utf8")).toBe(bytes); expect(realFs.existsSync(cur.path)).toBe(true); expect(JSON.parse(realFs.readFileSync(cur.path, "utf8")).ackedAt).toBeUndefined(); - expect(second.logs.some((m) => m.includes("actor=anvil state=unknown") && m.includes(path) && m.includes("repair the record and restart the account"))).toBe(true); + expect(second.logs.some((m) => m.includes("actor=anvil state=unknown") && m.includes(path) && m.includes("fix the path named above and restart the account"))).toBe(true); } finally { await second.stop(); obligationReadFailure = undefined; @@ -526,9 +526,8 @@ for (const failure of ["directory", "file"] as const) { try { expect(await pollUntil(() => h.logs.some((m) => m.includes("startup-unresolved") && m.includes(path)))).toBe(true); const diagnostic = h.logs.find((m) => m.includes("startup-unresolved") && m.includes(path))!; - expect(diagnostic).toContain(failure === "directory" ? "the directory could not be read" : "repair the record"); - expect(diagnostic.includes("the directory could not be read")).toBe(failure === "directory"); - expect(diagnostic).not.toContain("obligation retained"); + expect(diagnostic).toContain("fix the path named above and restart the account"); + expect(diagnostic).not.toContain("obligation retained"); } finally { await h.stop(); if (failure === "file") realFs.chmodSync(path, 0o644); diff --git a/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts b/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts index 05f26db5..1ee1a94b 100644 --- a/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts +++ b/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts @@ -118,7 +118,7 @@ describe("terminal stamp reconciliation", () => { expect(removeAfterRead).toBeUndefined(); expect(f.logs).toHaveLength(1); expect(f.logs[0]).toContain(`${kind}-stamp-reconcile-failed: inbound actor=anvil state=${state} path=${f.curPath} code=ENOENT`); - expect(f.logs[0]).toContain("inspect the missing cur record; obligation retained"); + expect(f.logs[0]).toContain("obligation retained; fix the path named above and restart the account"); expect(realFs.existsSync(f.curPath)).toBe(false); expect(JSON.parse(realFs.readFileSync(f.obligationPath, "utf8")).state).toBe(state); }); @@ -128,7 +128,7 @@ describe("terminal stamp reconciliation", () => { uncodedWriteFailure = true; f.reconcile(); expect(f.logs[0]).toContain(`actor=anvil state=${state} path=${f.curPath} code=WRITE_FAILED`); - expect(f.logs[0]).toContain("repair the record and restart the account; obligation retained"); + expect(f.logs[0]).toContain("obligation retained; fix the path named above and restart the account"); uncodedWriteFailure = false; f.reconcile(); expect(JSON.parse(realFs.readFileSync(f.curPath, "utf8"))[key]).toBeDefined(); @@ -170,7 +170,7 @@ describe("terminal stamp reconciliation", () => { expect(f.logs[0]).toContain("stamp-reconcile-read-failed"); expect(f.logs[0]).toContain("actor=anvil state="); expect(f.logs[0]).toContain(`path=${target} code=EACCES`); - expect(f.logs[0]).toContain("repair the record and restart the account"); + expect(f.logs[0]).toContain("fix the path named above and restart the account"); listFailure = readFailure = undefined; f.reconcile(); expect(JSON.parse(realFs.readFileSync(f.curPath, "utf8")).ackedAt).toBeDefined(); @@ -203,7 +203,7 @@ for (const invalid of [null, 7, "bad", [], { inboundId: "bad" }]) { expect(errors).toEqual([`${badPath}:INVALID_RECORD`]); expect(() => f.reconcile()).not.toThrow(); expect(f.logs[0]).toContain(`actor=anvil state=unknown path=${badPath} code=INVALID_RECORD`); - expect(f.logs[0]).toContain("repair the record and restart the account"); + expect(f.logs[0]).toContain("fix the path named above and restart the account"); expect(JSON.parse(realFs.readFileSync(f.curPath, "utf8")).ackedAt).toBeDefined(); expect(realFs.readFileSync(badPath, "utf8")).toBe(JSON.stringify(invalid)); }); @@ -214,7 +214,7 @@ test("a null cur record is reported as unreadable", () => { realFs.writeFileSync(f.curPath, "null"); f.reconcile(); expect(f.logs[0]).toContain(`actor=anvil state=acked path=${f.curPath} code=INVALID_RECORD`); - expect(f.logs[0]).toContain("repair the record and restart the account"); + expect(f.logs[0]).toContain("fix the path named above and restart the account"); }); test("creation refuses an unreadable existing terminal obligation", () => { @@ -285,7 +285,7 @@ for (const stage of ["obligation-reread", "cur-retention-read", "cur-retention-l if (stage === "abandonment-write") renameFailure = f.obligationPath; const unresolved = new Set(); sweepTerminalObligations(root, "anvil", 7, { warn: (m) => f.logs.push(m) }, Date.now(), 28, unresolved, (path, code, id, state) => { - f.logs.push(`actor=anvil state=${state} path=${path} code=${code}; repair and restart the account`); + f.logs.push(`actor=anvil state=${state} path=${path} code=${code}; fix the path named above and restart the account`); unresolved.add(id); }); expect(unresolved.has("inbound")).toBe(true); @@ -328,7 +328,7 @@ for (const stage of ["receipt-list", "receipt-read", "receipt-delete", "receipt- } const unresolved = new Set(); const onFailure = (path: string, code: string, id: string, state: string) => { - f.logs.push(`actor=anvil state=${state} path=${path} code=${code}; repair and restart the account`); + f.logs.push(`actor=anvil state=${state} path=${path} code=${code}; fix the path named above and restart the account`); unresolved.add(id); }; sweepTerminalObligations(root, "anvil", 7, { warn: (m) => f.logs.push(m) }, Date.now(), 28, unresolved, onFailure, [record]); @@ -365,8 +365,7 @@ for (const failure of ["directory", "file"] as const) { sweepTerminalObligations(root, "anvil", 7, { warn: (m) => logs.push(m) }); const diagnostic = logs.find((m) => m.includes("retention-unresolved"))!; expect(diagnostic).toContain(`path=${path} code=EACCES`); - expect(diagnostic).toContain(failure === "directory" ? "the directory could not be read" : "repair the record"); - expect(diagnostic.includes("the directory could not be read")).toBe(failure === "directory"); + expect(diagnostic).toContain("fix the path named above and restart the account"); expect(diagnostic).not.toContain("obligation retained"); }); } From 2b77f9145df3f5b922d84e0a1ece0e1979d8e36c Mon Sep 17 00:00:00 2001 From: flint Date: Sat, 3 Oct 2026 21:38:39 -0700 Subject: [PATCH 11/17] fix(openclaw-tps-mail): the exhausted-retry diagnostic states no retries left Co-Authored-By: Claude Opus 5.5 --- plugins/openclaw-tps-mail/src/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/openclaw-tps-mail/src/index.ts b/plugins/openclaw-tps-mail/src/index.ts index 14dfe057..97422b0a 100644 --- a/plugins/openclaw-tps-mail/src/index.ts +++ b/plugins/openclaw-tps-mail/src/index.ts @@ -980,7 +980,8 @@ function stampTerminalCur( `tps-mail: ${kind}-stamp-failed: ${ctx.inboundId} actor=${ctx.agent} state=${kind === "ack" ? "acked" : "failed"} path=${stamped.path} code=${stamped.code}; ` + (stamped.reason !== "record-missing" && delay !== undefined ? `retry ${attempt + 1} of ${stampRetryDelaysMs.length} in ${delay}ms` - : `obligation retained; fix the path named above and restart the account`), + : (stamped.reason === "record-missing" ? "" : "no retries left; ") + + `obligation retained; fix the path named above and restart the account`), ); if (delay === undefined || !isLiveContext(ctx)) return; const timer = accountTimer(ctx, () => stampTerminalCur(ctx, kind, patch, attempt + 1), delay); From 202761cfe9fb4f195e092793c06d84aea8ec5877 Mon Sep 17 00:00:00 2001 From: flint Date: Sat, 3 Oct 2026 22:21:38 -0700 Subject: [PATCH 12/17] fix(openclaw-tps-mail): diagnostics name the path that failed; tests pin the unknown-vs-retained state Co-Authored-By: Claude Opus 5.5 --- packages/cli/src/utils/mail.ts | 14 ++++-- plugins/openclaw-tps-mail/src/index.ts | 4 +- plugins/openclaw-tps-mail/src/obligations.ts | 2 +- .../test/cur-record-write.test.ts | 43 ++++++++++++++++++- .../test/patch-mail-file.test.ts | 41 +++++++++++++++++- 5 files changed, 95 insertions(+), 9 deletions(-) diff --git a/packages/cli/src/utils/mail.ts b/packages/cli/src/utils/mail.ts index 12f28911..2784bcdb 100644 --- a/packages/cli/src/utils/mail.ts +++ b/packages/cli/src/utils/mail.ts @@ -11,7 +11,7 @@ import { } from "@tpsdev-ai/agent"; import { sanitizeIdentifier } from "../schema/sanitizer.js"; import { logEvent } from "./archive.js"; -import { acquireMailLock, acquireMailLockSync, type MailLock } from "./mail-lock.js"; +import { acquireMailLock, acquireMailLockSync, mailLockPath, type MailLock } from "./mail-lock.js"; import { createMailVerifyClient, type MailVerifyConfig } from "./mail-verify.js"; // cli#429: the ONE id shape rule, re-exported so the openclaw-tps-mail plugin @@ -189,10 +189,16 @@ export function updateExistingRecord( mutate: (record: T) => T | null, options: { snapshot?: T; afterWrite?: (record: T) => void; nonBlocking?: boolean } = {}, ): UpdateExistingResult { - const lock = acquireMailLockSync(dirname(dirname(path)), options.nonBlocking ? { timeoutMs: 0 } : {}); + const root = dirname(dirname(path)); + let lock: MailLock | null; + try { + lock = acquireMailLockSync(root, options.nonBlocking ? { timeoutMs: 0 } : {}); + } catch (err: any) { + throw Object.assign(err, { path: err?.path ?? mailLockPath(root) }); + } if (!lock) { if (options.nonBlocking) return { status: "busy" }; - throw new Error(`mail lock contention timeout for ${path}`); + throw Object.assign(new Error(`mail lock contention timeout for ${path}`), { path: mailLockPath(root) }); } const scratchPath = join(dirname(path), `.ack-${randomUUID()}.tmp`); let fd: number | undefined; @@ -215,7 +221,7 @@ export function updateExistingRecord( options.afterWrite?.(updated); return { status: "updated", record: updated }; } catch (err: any) { - if (!replaced && err?.code === "ENOENT") return { status: "gone" }; + if (!replaced && err?.code === "ENOENT" && err?.path === path) return { status: "gone" }; throw err; } finally { if (fd !== undefined) { try { closeSync(fd); } catch {} } diff --git a/plugins/openclaw-tps-mail/src/index.ts b/plugins/openclaw-tps-mail/src/index.ts index 97422b0a..783bc228 100644 --- a/plugins/openclaw-tps-mail/src/index.ts +++ b/plugins/openclaw-tps-mail/src/index.ts @@ -647,7 +647,7 @@ export function patchMailFile(path: string, patch: Partial, stampKe if (r.status === "gone") return { ok: false, reason: "record-missing", path, code: "ENOENT" }; return { ok: false, reason: "write-failed", path, code: r.status }; } catch (err: any) { - return { ok: false, reason: "write-failed", path, code: err?.code ?? "WRITE_FAILED" }; + return { ok: false, reason: "write-failed", path: err?.path ?? path, code: err?.code ?? "WRITE_FAILED" }; } } @@ -2392,7 +2392,7 @@ const gateway: ChannelGatewayAdapter = { if (unknownInbounds.has(id)) return; unknownInbounds.add(id); log?.warn?.( - `tps-mail: startup-unresolved: ${kind === "directory" ? "" : id + " "}actor=${agentId} state=${kind === "directory" ? "unknown" : state} path=${path} code=${code}; fix the path named above and restart the account`, + `tps-mail: startup-unresolved: ${kind === "directory" ? "" : id + " "}actor=${agentId} state=${kind === "directory" ? "unknown" : state} path=${path} code=${code}; ${kind === "directory" ? "" : "obligation retained; "}fix the path named above and restart the account`, ); }; const startupRecords = listObligations(account.mailDir, agentId, (path, code, ids, kind) => { diff --git a/plugins/openclaw-tps-mail/src/obligations.ts b/plugins/openclaw-tps-mail/src/obligations.ts index 77a871ff..737f0a3d 100644 --- a/plugins/openclaw-tps-mail/src/obligations.ts +++ b/plugins/openclaw-tps-mail/src/obligations.ts @@ -576,7 +576,7 @@ export function sweepTerminalObligations( if (unresolved.has(id)) return; if (onFailure) onFailure(path, code, id, state, kind); else if (kind === "directory") log?.warn?.(`tps-mail: retention-unresolved: actor=${agent} state=unknown path=${path} code=${code}; fix the path named above and restart the account`); - else log?.warn?.(`tps-mail: retention-unresolved: ${id} actor=${agent} state=${state} path=${path} code=${code}; fix the path named above and restart the account`); + else log?.warn?.(`tps-mail: retention-unresolved: ${id} actor=${agent} state=${state} path=${path} code=${code}; obligation retained; fix the path named above and restart the account`); unresolved.add(id); }; const dir = obligationsDir(mailDir, agent); diff --git a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts index dfbba3fb..85add52c 100644 --- a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts +++ b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts @@ -233,7 +233,7 @@ describe("cli#492 — a failed cur/ stamp write is surfaced and retried", () => expect(readCur()?.record?.[stampKey], "the stamp did NOT land").toBeUndefined(); expect( await pollUntil( - () => failedLogs(h, `${kind}-stamp-failed`).some((m) => m.includes(cur!.path) && m.includes("EACCES")), + () => failedLogs(h, `${kind}-stamp-failed`).some((m) => m.includes(`path=${resolve(mailDir, "anvil", "cur")}/.ack-`) && m.includes("EACCES")), 4000, ), "the failed write is logged by id, path and code", @@ -278,6 +278,41 @@ describe("cli#492 — a failed cur/ stamp write is surfaced and retried", () => await h2.stop(); }, 20000); + for (const stage of ["lock", "scratch"] as const) { + it(`${kind}: ${stage} acquisition reports the failing path in live and startup stamps`, async () => { + const h = await boot(true); + expect(await pollUntil(() => h.dispatch() !== null)).toBe(true); + const cur = readCur()!; + const blocked = stage === "lock" ? resolve(mailDir, "anvil") : resolve(mailDir, "anvil", "cur"); + const restore = () => realFs.chmodSync(blocked, 0o755); + realFs.chmodSync(blocked, 0o555); + try { + await finish(h); + expect(await pollUntil(() => failedLogs(h, `${kind}-stamp-failed`).length > 0)).toBe(true); + const diagnostic = failedLogs(h, `${kind}-stamp-failed`)[0]; + expect(diagnostic).toContain(stage === "lock" ? `path=${blocked}/.mail-lock.claim code=EACCES` : `path=${blocked}/.ack-`); + expect(diagnostic).not.toContain(`path=${cur.path}`); + expect(readCur()?.record?.[stampKey]).toBeUndefined(); + } finally { + await h.stop(); + restore(); + } + realFs.chmodSync(blocked, 0o555); + const next = await boot(false); + try { + expect(await pollUntil(() => next.logs.some((m) => m.includes(`${kind}-stamp-reconcile-failed`)))).toBe(true); + const diagnostic = next.logs.find((m) => m.includes(`${kind}-stamp-reconcile-failed`))!; + expect(diagnostic).toContain(stage === "lock" ? `path=${blocked}/.mail-lock.claim code=EACCES` : `path=${blocked}/.ack-`); + expect(diagnostic).not.toContain(`path=${cur.path}`); + expect(diagnostic).toContain("obligation retained"); + expect(readCur()?.record?.[stampKey]).toBeUndefined(); + } finally { + await next.stop(); + restore(); + } + }); + } + it(`${kind}: a missing record is reported after the durable transition`, async () => { const h = await boot(true); try { @@ -527,7 +562,13 @@ for (const failure of ["directory", "file"] as const) { expect(await pollUntil(() => h.logs.some((m) => m.includes("startup-unresolved") && m.includes(path)))).toBe(true); const diagnostic = h.logs.find((m) => m.includes("startup-unresolved") && m.includes(path))!; expect(diagnostic).toContain("fix the path named above and restart the account"); + if (failure === "directory") { + expect(diagnostic).toContain("startup-unresolved: actor=anvil state=unknown"); expect(diagnostic).not.toContain("obligation retained"); + } else { + expect(diagnostic).toContain("startup-unresolved: * actor=anvil state=unknown"); + expect(diagnostic).toContain("obligation retained"); + } } finally { await h.stop(); if (failure === "file") realFs.chmodSync(path, 0o644); diff --git a/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts b/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts index 1ee1a94b..d8e7e1fd 100644 --- a/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts +++ b/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts @@ -11,6 +11,7 @@ let listFailure: string | undefined; let stampOnRead: { path: string; count: number } | undefined; let readsBeforeRemoval = 1; let uncodedWriteFailure = false; +let scratchErrorCode: string | undefined; let deleteFailure: string | undefined; let renameFailure: string | undefined; let replaceIdentity = false; @@ -30,6 +31,7 @@ mock.module("node:fs", () => ({ }, openSync: (...args: any[]) => { if (uncodedWriteFailure && String(args[0]).includes(".ack-")) throw new Error("injected write failure"); + if (scratchErrorCode && String(args[0]).includes(".ack-")) throw Object.assign(new Error("injected scratch failure"), { code: scratchErrorCode, path: args[0] }); return (realFs.openSync as any)(...args); }, readFileSync: (...args: any[]) => { @@ -51,6 +53,7 @@ mock.module("node:fs", () => ({ }, })); +const { acquireMailLockSync, mailLockPath } = await import("@tpsdev-ai/agent"); const { patchMailFile, reconcileTerminalCurStamps } = await import("../src/index.js"); const { createObligation, listObligations, sweepTerminalObligations } = await import("../src/obligations.js"); const root = realFs.mkdtempSync(join(tmpdir(), "patch-mail-")); @@ -62,6 +65,7 @@ afterEach(() => { readFailure = listFailure = undefined; stampOnRead = undefined; uncodedWriteFailure = replaceIdentity = false; + scratchErrorCode = undefined; deleteFailure = renameFailure = undefined; realFs.rmSync(join(root, "anvil"), { recursive: true, force: true }); realFs.rmSync(path, { force: true }); @@ -70,6 +74,35 @@ afterEach(() => { afterAll(() => realFs.rmSync(root, { recursive: true, force: true })); describe("patchMailFile", () => { + test("a nested lock failure names the lock path", () => { + const f = terminalFixture("acked"); + const lock = acquireMailLockSync(join(root, "anvil"))!; + try { + expect(patchMailFile(f.curPath, { ackedAt: "done" }, "ackedAt", "inbound")).toMatchObject({ ok: false, reason: "write-failed", path: mailLockPath(join(root, "anvil")) }); + } finally { + lock.release(); + } + }); + + test("a lock timeout names the lock path", () => { + const f = terminalFixture("acked"); + const lockPath = mailLockPath(join(root, "anvil")); + realFs.mkdirSync(lockPath); + expect(patchMailFile(f.curPath, { ackedAt: "done" }, "ackedAt", "inbound")).toMatchObject({ ok: false, reason: "write-failed", path: lockPath }); + }); + + for (const code of ["EACCES", "ENOENT"]) { + test(`a scratch ${code} failure names the scratch path and preserves the record`, () => { + const f = terminalFixture("acked"); + scratchErrorCode = code; + const result = patchMailFile(f.curPath, { ackedAt: "done" }, "ackedAt", "inbound"); + expect(result).toMatchObject({ ok: false, reason: "write-failed", code }); + if (result.ok) throw new Error("expected scratch failure"); + expect(result.path).toStartWith(join(f.curDir, ".ack-")); + expect(JSON.parse(realFs.readFileSync(f.curPath, "utf8")).ackedAt).toBeUndefined(); + }); + } + test("patches an existing record", () => { realFs.writeFileSync(path, JSON.stringify({ id: "inbound", body: "hello" })); expect(patchMailFile(path, { read: true }, undefined, "inbound")).toEqual({ ok: true }); @@ -366,6 +399,12 @@ for (const failure of ["directory", "file"] as const) { const diagnostic = logs.find((m) => m.includes("retention-unresolved"))!; expect(diagnostic).toContain(`path=${path} code=EACCES`); expect(diagnostic).toContain("fix the path named above and restart the account"); - expect(diagnostic).not.toContain("obligation retained"); + if (failure === "directory") { + expect(diagnostic).toContain("retention-unresolved: actor=anvil state=unknown"); + expect(diagnostic).not.toContain("obligation retained"); + } else { + expect(diagnostic).toContain("retention-unresolved: * actor=anvil state=unknown"); + expect(diagnostic).toContain("obligation retained"); + } }); } From 977776b0150b5cb71b0491c41cd083f40f622e49 Mon Sep 17 00:00:00 2001 From: flint Date: Sat, 3 Oct 2026 22:55:27 -0700 Subject: [PATCH 13/17] fix(openclaw-tps-mail): a missing record's diagnostic names the record; stop-time cancellation is not claimed Co-Authored-By: Claude Opus 5.5 --- plugins/openclaw-tps-mail/src/index.ts | 2 +- plugins/openclaw-tps-mail/test/patch-mail-file.test.ts | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/plugins/openclaw-tps-mail/src/index.ts b/plugins/openclaw-tps-mail/src/index.ts index 783bc228..96e4564c 100644 --- a/plugins/openclaw-tps-mail/src/index.ts +++ b/plugins/openclaw-tps-mail/src/index.ts @@ -682,7 +682,7 @@ export function reconcileTerminalCurStamps(mailDir: string, agent: string, log: }; if (unreadable) continue; if (!curPath) { - if (!unreadable) missing(resolve(mailDir, agent, "cur")); + if (!unreadable) missing(resolve(mailDir, agent, "cur", `${rec.inboundId}.json`)); continue; } const cur = readMailFile(curPath, onReadError); diff --git a/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts b/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts index d8e7e1fd..40c89136 100644 --- a/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts +++ b/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts @@ -156,6 +156,14 @@ describe("terminal stamp reconciliation", () => { expect(JSON.parse(realFs.readFileSync(f.obligationPath, "utf8")).state).toBe(state); }); + test(`${kind}: a missing cur record is reported at its expected file path`, () => { + const f = terminalFixture(state); + realFs.unlinkSync(f.curPath); + f.reconcile(); + expect(f.logs).toHaveLength(1); + expect(f.logs[0]).toContain(`state=${state} path=${join(f.curDir, "inbound.json")} code=ENOENT`); + }); + test(`${kind}: an uncoded write failure is reported and the next run can stamp`, () => { const f = terminalFixture(state); uncodedWriteFailure = true; From f99464333d6f2f97bc1be784745f704bcfc23b0d Mon Sep 17 00:00:00 2001 From: flint Date: Sat, 3 Oct 2026 23:50:36 -0700 Subject: [PATCH 14/17] refactor(openclaw-tps-mail): one diagnostic builder fed with facts for every stamp/obligation message Co-Authored-By: Claude Opus 5.5 --- plugins/openclaw-tps-mail/src/diagnostics.ts | 29 +++++ plugins/openclaw-tps-mail/src/index.ts | 123 +++++++++--------- plugins/openclaw-tps-mail/src/obligations.ts | 86 +++++------- .../test/cur-record-write.test.ts | 83 ++++++++++-- .../test/diagnostics.test.ts | 19 +++ .../openclaw-tps-mail/test/locality.test.ts | 2 +- .../test/obligation-retention.test.ts | 6 +- .../test/patch-mail-file.test.ts | 48 ++++--- .../test/receipt-scan.test.ts | 24 ++++ .../test/reply-obligation.test.ts | 4 +- .../openclaw-tps-mail/test/startup.test.ts | 5 +- 11 files changed, 276 insertions(+), 153 deletions(-) create mode 100644 plugins/openclaw-tps-mail/src/diagnostics.ts create mode 100644 plugins/openclaw-tps-mail/test/diagnostics.test.ts diff --git a/plugins/openclaw-tps-mail/src/diagnostics.ts b/plugins/openclaw-tps-mail/src/diagnostics.ts new file mode 100644 index 00000000..c6739203 --- /dev/null +++ b/plugins/openclaw-tps-mail/src/diagnostics.ts @@ -0,0 +1,29 @@ +export type ObligationPresence = "retained" | "none" | "unknown"; + +export interface StampDiagnosticFacts { + kind: string; + actor: string; + id?: string; + path: string; + code: string; + obligation: ObligationPresence; + retriesExhausted?: boolean; +} + +export function formatStampDiagnostic(facts: StampDiagnosticFacts): string { + const fields = [ + `tps-mail: ${facts.kind}:`, + ...(facts.id === undefined ? [] : [facts.id]), + `actor=${facts.actor}`, + `path=${facts.path}`, + `code=${facts.code}`, + ]; + const state = facts.obligation === "retained" ? "obligation retained" + : facts.obligation === "unknown" ? "state unknown" : undefined; + return [ + fields.join(" "), + ...(state ? [state] : []), + ...(facts.retriesExhausted ? ["no retries left"] : []), + "fix the path named above and restart the account", + ].join("; "); +} diff --git a/plugins/openclaw-tps-mail/src/index.ts b/plugins/openclaw-tps-mail/src/index.ts index 96e4564c..5899fc90 100644 --- a/plugins/openclaw-tps-mail/src/index.ts +++ b/plugins/openclaw-tps-mail/src/index.ts @@ -54,9 +54,9 @@ import { randomUUID } from "node:crypto"; import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, writeFileSync, watch as fsWatch, type FSWatcher } from "node:fs"; import { homedir } from "node:os"; -import { basename, resolve } from "node:path"; +import { basename, dirname, resolve } from "node:path"; import type { Envelope, ChainEntry } from "@tpsdev-ai/agent"; -import { signEnvelope, verifyEnvelope, verifiedMailTier } from "@tpsdev-ai/agent"; +import { signEnvelope, verifyEnvelope, verifiedMailTier, mailLockPath } from "@tpsdev-ai/agent"; import { readAgentPrivateKey } from "@tpsdev-ai/cli/utils/agent-keys"; import { signForDelivery } from "@tpsdev-ai/cli/utils/mail-producer"; import { isValidEnvelopeId, mailRootForRecordPath, updateExistingRecord, promote, recoverPromoted, verifyRecordForMailbox, sweepStrandedPromoteScratch } from "@tpsdev-ai/cli/utils/mail"; @@ -69,6 +69,8 @@ import { listObligations, markNackSent, nackOwed, + obligationPath, + obligationsDir, newestSessionTranscript, readObligation, readObligationResult, @@ -86,6 +88,7 @@ import { type ReceiptScanDirs, } from "./obligations.js"; import type { OpenClawPluginApi } from "openclaw/plugin-sdk"; +import { formatStampDiagnostic, type ObligationPresence, type StampDiagnosticFacts } from "./diagnostics.js"; import { detectHostOpenClawVersion, evaluateHostSilentReplyGuard } from "./host-version.js"; import type { ChannelPlugin } from "openclaw/plugin-sdk/core"; import type { @@ -409,8 +412,9 @@ function persistReceiptAfterCommit( persistReceipt(mailDir, agent, message, route, branchId); } catch (err: any) { log?.warn?.( - `tps-mail: receipt-write-failed: the ${route} delivery committed, but its receipt was not written ` + - `(${err?.message ?? err}); the obligation resolves at its deadline`, + formatStampDiagnostic({ kind: "receipt-write-failed", actor: agent, id: message.headers?.["X-TPS-InReplyTo"], + path: resolve(receiptsDir(mailDir, agent), `${message.headers?.["X-TPS-Obligation"]}.json`), + code: err?.code ?? "WRITE_FAILED", obligation: "unknown" }), ); } } @@ -422,13 +426,14 @@ function persistReceiptAfterCommit( * never reported as failed and its inbound is never nacked. The logger itself is * guarded too: a throwing logger cannot fail a committed send either. */ -function postCommit(log: any, name: string, context: string, step: () => void): void { +function postCommit(log: any, name: string, context: string, step: () => void, facts?: Omit): void { try { step(); } catch (err: any) { try { log?.warn?.( - `tps-mail: ${name}: ${context} (${err?.message ?? err}); the delivery committed, so this is not a send failure`, + facts ? formatStampDiagnostic({ ...facts, kind: name, code: err?.code ?? "WRITE_FAILED" }) + : `tps-mail: ${name}: ${context} (${err?.message ?? err}); the delivery committed, so this is not a send failure`, ); } catch { /* a logger must never fail a committed send */ @@ -647,20 +652,23 @@ export function patchMailFile(path: string, patch: Partial, stampKe if (r.status === "gone") return { ok: false, reason: "record-missing", path, code: "ENOENT" }; return { ok: false, reason: "write-failed", path, code: r.status }; } catch (err: any) { - return { ok: false, reason: "write-failed", path: err?.path ?? path, code: err?.code ?? "WRITE_FAILED" }; + return { ok: false, reason: "write-failed", path: typeof err?.path === "string" + ? basename(err.path).startsWith(".ack-") ? path + : err.path.startsWith(`${mailLockPath(dirname(dirname(path)))}.`) ? mailLockPath(dirname(dirname(path))) : err.path + : path, code: err?.code ?? "WRITE_FAILED" }; } } export function reconcileTerminalCurStamps(mailDir: string, agent: string, log: any, records?: ObligationRecord[], unknownInbounds = new Set()): Set { - const reportReadError = (state: string, id: string) => (path: string, code: string) => { + const reportReadError = (obligation: ObligationPresence, id: string) => (path: string, code: string) => { if (unknownInbounds.has(id)) return; unknownInbounds.add(id); log?.warn?.( - `tps-mail: stamp-reconcile-read-failed: ${id} actor=${agent} state=${state} path=${path} code=${code}; fix the path named above and restart the account`, + formatStampDiagnostic({ kind: "stamp-reconcile-read-failed", actor: agent, id, path, code, obligation }), ); }; - const onObligationReadError = (path: string, code: string, ids: string[]) => { - reportReadError("unknown", ids[0])(path, code); + const onObligationReadError = (path: string, code: string, ids: string[], kind: ReadFailureKind) => { + reportReadError(kind === "directory" ? "unknown" : "retained", ids[0])(path, code); for (const id of ids) unknownInbounds.add(id); }; for (const rec of records ?? listObligations(mailDir, agent, onObligationReadError)) { @@ -669,7 +677,7 @@ export function reconcileTerminalCurStamps(mailDir: string, agent: string, log: let unreadable = false; const onReadError = (path: string, code: string) => { unreadable = true; - reportReadError(rec.state, rec.inboundId)(path, code); + reportReadError(path === resolve(mailDir, agent, "cur") ? "unknown" : "retained", rec.inboundId)(path, code); }; const kind = rec.state === "acked" ? "ack" : "nack"; const key = kind === "ack" ? "ackedAt" : "nackedAt"; @@ -677,7 +685,7 @@ export function reconcileTerminalCurStamps(mailDir: string, agent: string, log: const missing = (path: string) => { unknownInbounds.add(rec.inboundId); log?.warn?.( - `tps-mail: ${kind}-stamp-reconcile-failed: ${rec.inboundId} actor=${agent} state=${rec.state} path=${path} code=ENOENT; obligation retained; fix the path named above and restart the account`, + formatStampDiagnostic({ kind: `${kind}-stamp-reconcile-failed`, actor: agent, id: rec.inboundId, path, code: "ENOENT", obligation: "retained" }), ); }; if (unreadable) continue; @@ -699,8 +707,7 @@ export function reconcileTerminalCurStamps(mailDir: string, agent: string, log: else { unknownInbounds.add(rec.inboundId); log?.warn?.( - `tps-mail: ${kind}-stamp-reconcile-failed: ${rec.inboundId} actor=${agent} state=${rec.state} path=${r.path} code=${r.code}; ` + - `obligation retained; fix the path named above and restart the account`, + formatStampDiagnostic({ kind: `${kind}-stamp-reconcile-failed`, actor: agent, id: rec.inboundId, path: r.path, code: r.code, obligation: "retained" }), ); } } @@ -946,7 +953,7 @@ function readObligationForCleanup(ctx: YieldContext, obligationId: string) { const result = readObligationResult(ctx.mailDir, ctx.agent, ctx.inboundId); if (result.status === "unverified") { retryObligationRead(ctx, obligationId); - ctx.log?.warn?.(`tps-mail: obligation-read-unverified: ${ctx.inboundId} path=${result.path} code=${result.code}`); + ctx.log?.warn?.(formatStampDiagnostic({ kind: "obligation-read-unverified", actor: ctx.agent, id: ctx.inboundId, path: result.path, code: result.code, obligation: "unknown" })); } else if (result.status === "missing" || TERMINAL_STATES.has(result.record.state)) { releaseObligationState(obligationId); } @@ -977,11 +984,9 @@ function stampTerminalCur( } const delay = stamped.reason === "record-missing" ? undefined : stampRetryDelaysMs[attempt]; ctx.log?.warn?.( - `tps-mail: ${kind}-stamp-failed: ${ctx.inboundId} actor=${ctx.agent} state=${kind === "ack" ? "acked" : "failed"} path=${stamped.path} code=${stamped.code}; ` + - (stamped.reason !== "record-missing" && delay !== undefined - ? `retry ${attempt + 1} of ${stampRetryDelaysMs.length} in ${delay}ms` - : (stamped.reason === "record-missing" ? "" : "no retries left; ") + - `obligation retained; fix the path named above and restart the account`), + formatStampDiagnostic({ kind: `${kind}-stamp-failed`, actor: ctx.agent, id: ctx.inboundId, + path: stamped.path, code: stamped.code, obligation: "retained", + retriesExhausted: stamped.reason !== "record-missing" && delay === undefined }), ); if (delay === undefined || !isLiveContext(ctx)) return; const timer = accountTimer(ctx, () => stampTerminalCur(ctx, kind, patch, attempt + 1), delay); @@ -1001,7 +1006,7 @@ function ackObligation(ctx: YieldContext, obligationId: string, why: string): vo if (result.status === "unverified") return; const cur = result.status === "found" ? result.record : null; ctx.log?.warn?.( - `tps-mail: refusing to ack ${ctx.inboundId} — obligation is ${cur?.state ?? "gone"}; the inbound keeps no ackedAt`, + formatStampDiagnostic({ kind: "ack-refused", actor: ctx.agent, id: ctx.inboundId, path: obligationPath(ctx.mailDir, ctx.agent, ctx.inboundId), code: cur?.state ?? "OBLIGATION_MISSING", obligation: cur ? "retained" : "none" }), ); return; } @@ -1115,21 +1120,19 @@ async function settleObligation( if (result.status === "unverified") return "none"; const cur = result.status === "found" ? result.record : null; ctx.log?.warn?.( - `tps-mail: refusing to record the unconfirmed outcome for ${ctx.inboundId} — obligation is ${cur?.state ?? "gone"}; it stays as it is`, + formatStampDiagnostic({ kind: "unconfirmed-refused", actor: ctx.agent, id: ctx.inboundId, path: obligationPath(ctx.mailDir, ctx.agent, ctx.inboundId), code: cur?.state ?? "OBLIGATION_MISSING", obligation: cur ? "retained" : "none" }), ); return "none"; } } catch (err: any) { ctx.log?.warn?.( - `tps-mail: obligation-write-failed: could not record the unconfirmed outcome for ${ctx.inboundId} ` + - `(${err?.message ?? err}); one attempt, no retry — the obligation resolves on restart`, + formatStampDiagnostic({ kind: "obligation-write-failed", actor: ctx.agent, id: ctx.inboundId, path: obligationPath(ctx.mailDir, ctx.agent, ctx.inboundId), code: err?.code ?? "WRITE_FAILED", obligation: "retained" }), ); return "none"; } releaseObligationState(obligationId); ctx.log?.warn?.( - `tps-mail: obligation for ${ctx.inboundId} UNCONFIRMED: ${s.reason} — the reply was committed (${rec.state}) ` + - `and no receipt evidence was found by its deadline; NOT failed, no nack sent`, + formatStampDiagnostic({ kind: "obligation-unconfirmed", actor: ctx.agent, id: ctx.inboundId, path: obligationPath(ctx.mailDir, ctx.agent, ctx.inboundId), code: s.reason, obligation: "retained" }), ); return "unconfirmed"; } @@ -1157,8 +1160,7 @@ async function settleObligation( ); } catch (err: any) { ctx.log?.warn?.( - `tps-mail: obligation-write-failed: could not record the failure for ${ctx.inboundId} ` + - `(${err?.message ?? err}); one attempt, no retry — the obligation resolves on restart`, + formatStampDiagnostic({ kind: "obligation-write-failed", actor: ctx.agent, id: ctx.inboundId, path: obligationPath(ctx.mailDir, ctx.agent, ctx.inboundId), code: err?.code ?? "WRITE_FAILED", obligation: "retained" }), ); return "none"; } @@ -1167,8 +1169,7 @@ async function settleObligation( if (result.status === "unverified") return "none"; const cur = result.status === "found" ? result.record : null; ctx.log?.warn?.( - `tps-mail: refusing to record the failure for ${ctx.inboundId} — obligation is ${cur?.state ?? "gone"}; ` + - `the inbound keeps no nack stamp and no nack mail is sent`, + formatStampDiagnostic({ kind: "failure-refused", actor: ctx.agent, id: ctx.inboundId, path: obligationPath(ctx.mailDir, ctx.agent, ctx.inboundId), code: cur?.state ?? "OBLIGATION_MISSING", obligation: cur ? "retained" : "none" }), ); return "none"; } @@ -1180,13 +1181,10 @@ async function settleObligation( } // cli#389 round 10, item 1: AWAIT the one send, and record its outcome on the // record. The mail is owed until `nackSentAt` says otherwise (at-least-once). - const nackHandedOff = await deliverNack(ctx, failedOn); + await deliverNack(ctx, failedOn); if (!isLiveContext(ctx)) return "none"; ctx.log?.warn?.( - `tps-mail: obligation for ${ctx.inboundId} FAILED: ${failedOn} — nacked` + - (s.alreadyStamped ? ", the inbound already carried its nack" : "") + - (nackHandedOff ? ", sender notified" : ", the nack mail is still OWED (nackPending) — the next start retries delivery") + - ", never acked", + formatStampDiagnostic({ kind: "obligation-failed", actor: ctx.agent, id: ctx.inboundId, path: obligationPath(ctx.mailDir, ctx.agent, ctx.inboundId), code: failedOn, obligation: "retained" }), ); return "failed"; } @@ -1229,14 +1227,12 @@ async function deliverNack(ctx: YieldContext, reason: string, opts: { timeoutMs? // over again. The line here says which of the two happened. const recorded = markNackSent(ctx.mailDir, ctx.agent, ctx.inboundId, ctx.log); ctx.log?.warn?.( - `tps-mail: nack delivered to ${ctx.sender} for ${ctx.inboundId}` + - (recorded ? "" : " — the record could not be updated: it still owes the nack and a later start may send it again"), + formatStampDiagnostic({ kind: "nack-delivered", actor: ctx.agent, id: ctx.inboundId, path: obligationPath(ctx.mailDir, ctx.agent, ctx.inboundId), code: recorded ? "DELIVERED" : "NACK_SENT_WRITE_FAILED", obligation: "retained" }), ); return true; } ctx.log?.warn?.( - `tps-mail: nack-pending: the sender was NOT told about the failure of ${ctx.inboundId} (${reason}); ` + - `the obligation record keeps nackPending and the next start retries delivery`, + formatStampDiagnostic({ kind: "nack-pending", actor: ctx.agent, id: ctx.inboundId, path: obligationPath(ctx.mailDir, ctx.agent, ctx.inboundId), code: reason, obligation: "retained" }), ); return false; } @@ -1303,14 +1299,13 @@ function retryOwedNackInBackground( if (!isLiveContext(ctx)) return; if (outcome === "expired") { log?.warn?.( - `tps-mail: nack-retry-timeout: the owed nack for ${inboundId} to ${sender} did not complete within ${backstopMs}ms ` + - `(connect + ack); the transport was closed and the record keeps nackPending`, + formatStampDiagnostic({ kind: "nack-retry-timeout", actor: agentId, id: inboundId, path: obligationPath(mailDir, agentId, inboundId), code: "TIMEOUT", obligation: "retained" }), ); } }) .catch((err: any) => { if (!isLiveContext(ctx)) return; - log?.warn?.(`tps-mail: nack-retry-failed: the owed nack for ${inboundId} to ${sender} threw: ${err?.message ?? err}`); + log?.warn?.(formatStampDiagnostic({ kind: "nack-retry-failed", actor: agentId, id: inboundId, path: obligationPath(mailDir, agentId, inboundId), code: err?.code ?? "NACK_RETRY_FAILED", obligation: "retained" })); }) .finally(() => { if (timer) clearAccountTimer(accountId, timer); @@ -1340,15 +1335,13 @@ function markDelivering(mailDir: string, agent: string, inboundId: string, log: const current = readObligation(mailDir, agent, inboundId); if (current && TERMINAL_STATES.has(current.state)) { log?.warn?.( - `tps-mail: late-final-refused: obligation ${current.obligationId} for ${inboundId} is ${current.state}; ` + - `the late final is NOT delivered`, + formatStampDiagnostic({ kind: "late-final-refused", actor: agent, id: inboundId, path: obligationPath(mailDir, agent, inboundId), code: current.state, obligation: "retained" }), ); } return false; } catch (err: any) { log?.warn?.( - `tps-mail: obligation-write-failed: could not mark ${inboundId} delivering (${err?.message ?? err}); ` + - `nothing was sent`, + formatStampDiagnostic({ kind: "obligation-write-failed", actor: agent, id: inboundId, path: obligationPath(mailDir, agent, inboundId), code: err?.code ?? "WRITE_FAILED", obligation: "retained" }), ); return false; } @@ -1434,7 +1427,7 @@ function makeYieldCtx( */ function receiptSignatureCheck(ctx: YieldContext): ReceiptSignatureCheck { let client: ReturnType | null = null; - return async (envelope) => { + return async (envelope, path) => { if (!isLiveContext(ctx)) return false; try { client ??= createMailVerifyClient(ctx.agent); @@ -1446,8 +1439,7 @@ function receiptSignatureCheck(ctx: YieldContext): ReceiptSignatureCheck { } catch (err: any) { if (!isLiveContext(ctx)) return false; ctx.log?.warn?.( - `tps-mail: receipt-verify-unavailable: the reply carried by a receipt for ${ctx.inboundId} could not be verified ` + - `(${err?.message ?? err}); it is not evidence until a later scan verifies it`, + formatStampDiagnostic({ kind: "receipt-verify-unavailable", actor: ctx.agent, id: ctx.inboundId, path: path!, code: err?.code ?? "VERIFY_FAILED", obligation: "unknown" }), ); return false; } @@ -1791,7 +1783,7 @@ const gateway: ChannelGatewayAdapter = { } if (!existsSync(account.mailDir)) { - log?.warn?.(`tps-mail: mail directory does not exist: ${account.mailDir}`); + log?.warn?.(formatStampDiagnostic({ kind: "startup-mail-directory-missing", actor: account.accountId, path: account.mailDir, code: "ENOENT", obligation: "unknown" })); return; } @@ -1883,14 +1875,15 @@ const gateway: ChannelGatewayAdapter = { const recovered = await recoverPromoted(recipient, curPath); if (!isLive()) return; if (!recovered.ok) { + const rejectedPath = resolve(account.mailDir, recipient, "dlq", basename(curPath)); log?.warn?.( - `tps-mail: cur/ recovery refused ${record.id} (${recovered.class}): ${recovered.reason}`, + formatStampDiagnostic({ kind: "cur-recovery-refused", actor: recipient, id: record.id, path: !existsSync(curPath) && existsSync(rejectedPath) ? rejectedPath : curPath, code: recovered.class, obligation: "unknown" }), ); return; } await deliverPromoted(recipient, recovered.message, curPath); } catch (err: any) { - log?.warn?.(`tps-mail: cur/ recovery deferred for ${record.id}: ${err?.message ?? err}`); + log?.warn?.(formatStampDiagnostic({ kind: "cur-recovery-deferred", actor: recipient, id: record.id, path: curPath, code: err?.code ?? "RECOVERY_FAILED", obligation: "unknown" })); } } @@ -1998,7 +1991,7 @@ const gateway: ChannelGatewayAdapter = { log, ); } catch (err: any) { - log?.warn?.(`tps-mail: obligation creation deferred for ${msg.id}: ${err?.message ?? err}`); + log?.warn?.(formatStampDiagnostic({ kind: "obligation-create-failed", actor: recipient, id: msg.id, path: obligationPath(account.mailDir, recipient, msg.id), code: err?.code ?? "CREATE_FAILED", obligation: "unknown" })); return; } const obId = created.record.obligationId; @@ -2145,6 +2138,7 @@ const gateway: ChannelGatewayAdapter = { "obligation-posted-transition-failed", `the ${route!.kind} reply ${reply.id} committed to ${msg.from} but the obligation for ${msg.id} was not marked posted`, () => transitionObligation(account.mailDir, recipient, msg.id, "posted", { replyId: reply.id }, log), + { actor: recipient, id: msg.id, path: obligationPath(account.mailDir, recipient, msg.id), obligation: "retained" }, ); }; try { @@ -2261,6 +2255,7 @@ const gateway: ChannelGatewayAdapter = { r.kind === "local" ? undefined : r.branchId, log, ), + { actor: recipient, id: msg.id, path: resolve(receiptsDir(account.mailDir, recipient), `${obId}.json`), obligation: "unknown" }, ); } } @@ -2322,7 +2317,7 @@ const gateway: ChannelGatewayAdapter = { const live = readObligation(yieldCtx.mailDir, yieldCtx.agent, yieldCtx.inboundId); if (live && !TERMINAL_STATES.has(live.state)) { log?.warn?.( - `tps-mail: obligation for ${msg.id} is unresolved (no receipt evidence yet); deadline armed`, + formatStampDiagnostic({ kind: "obligation-unresolved", actor: recipient, id: msg.id, path: obligationPath(account.mailDir, recipient, msg.id), code: "NO_RECEIPT", obligation: "retained" }), ); } } @@ -2344,8 +2339,7 @@ const gateway: ChannelGatewayAdapter = { const rec = readObligation(yieldCtx.mailDir, yieldCtx.agent, yieldCtx.inboundId); if (rec && (rec.state === "delivering" || rec.state === "posted")) { log?.warn?.( - `tps-mail: post-commit-error:${yieldCtx.step ?? "unnamed-step"}: ${reason} — the delivery for ${msg.id} ` + - `committed (${rec.state}), so it is NOT failed and its inbound is NOT nacked; it resolves at its deadline`, + formatStampDiagnostic({ kind: `post-commit-error:${yieldCtx.step ?? "unnamed-step"}`, actor: recipient, id: msg.id, path: obligationPath(account.mailDir, recipient, msg.id), code: err?.code ?? "POST_COMMIT_FAILED", obligation: "retained" }), ); armDeadline(yieldCtx, obId); } else { @@ -2388,11 +2382,12 @@ const gateway: ChannelGatewayAdapter = { // Attempt terminal stamps before recovery and retention. const unknownInbounds = new Set(); - const startupFailure = (path: string, code: string, id = "*", state = "unknown", kind: ReadFailureKind = "file") => { + const startupFailure = (path: string, code: string, id = "*", state = "unknown", kind: ReadFailureKind = "file", obligation: ObligationPresence = kind === "directory" ? "unknown" : "retained") => { if (unknownInbounds.has(id)) return; unknownInbounds.add(id); log?.warn?.( - `tps-mail: startup-unresolved: ${kind === "directory" ? "" : id + " "}actor=${agentId} state=${kind === "directory" ? "unknown" : state} path=${path} code=${code}; ${kind === "directory" ? "" : "obligation retained; "}fix the path named above and restart the account`, + formatStampDiagnostic({ kind: "startup-unresolved", actor: agentId, + id: kind === "directory" ? undefined : id, path, code, obligation }), ); }; const startupRecords = listObligations(account.mailDir, agentId, (path, code, ids, kind) => { @@ -2418,7 +2413,7 @@ const gateway: ChannelGatewayAdapter = { if (seenFiles.has(curPath)) continue; const known = startupRecords.find((rec) => filename === `${rec.inboundId}.json` || filename.endsWith(`-${rec.inboundId}.json`)); if (unknownInbounds.has("*") || (known && unknownInbounds.has(known.inboundId))) continue; - const record = readMailFile(curPath, (path, code) => startupFailure(path, code, known?.inboundId ?? basename(filename, ".json"), known?.state)); + const record = readMailFile(curPath, (path, code) => startupFailure(path, code, known?.inboundId ?? basename(filename, ".json"), known?.state, "file", known ? "retained" : "unknown")); if (!record || record.ackedAt || record.nackedAt) continue; if (unknownInbounds.has("*") || unknownInbounds.has(record.id)) continue; void recoverUnackedCurRecord(agentId, curPath, record); @@ -2497,7 +2492,7 @@ const gateway: ChannelGatewayAdapter = { ); } } catch (err: any) { - log?.warn?.(`tps-mail: obligation retention sweep failed (ignored): ${err?.message ?? String(err)}`); + log?.warn?.(formatStampDiagnostic({ kind: "retention-sweep-failed", actor: agentId, path: err?.path ?? obligationsDir(account.mailDir, agentId), code: err?.code ?? "SWEEP_FAILED", obligation: "unknown" })); } // RESTART RECOVERY (S2): in-memory timers die with the process, so @@ -2506,9 +2501,9 @@ const gateway: ChannelGatewayAdapter = { for (const rec of startupRecords) { if (!isLive()) break; if (unknownInbounds.has("*") || unknownInbounds.has(rec.inboundId) || TERMINAL_STATES.has(rec.state)) continue; - const onFailure = (path: string, code: string) => startupFailure(path, code, rec.inboundId, rec.state); + const onFailure = (path: string, code: string) => startupFailure(path, code, rec.inboundId, rec.state, path === curDir ? "directory" : "file"); const recCurPath = findCurPath(account.mailDir, agentId, rec.inboundId, onFailure); - if (!recCurPath) onFailure(curDir, "ENOENT"); + if (!recCurPath) onFailure(resolve(curDir, `${rec.inboundId}.json`), "ENOENT"); if (unknownInbounds.has(rec.inboundId)) continue; const ctx = makeYieldCtx( account.mailDir, @@ -2532,7 +2527,7 @@ const gateway: ChannelGatewayAdapter = { } } catch (err: any) { log?.warn?.( - `tps-mail: failed to watch ${newDir}: ${err?.message ?? String(err)}`, + formatStampDiagnostic({ kind: "startup-watch-failed", actor: agentId, path: newDir, code: err?.code ?? "WATCH_FAILED", obligation: "unknown" }), ); } } diff --git a/plugins/openclaw-tps-mail/src/obligations.ts b/plugins/openclaw-tps-mail/src/obligations.ts index 737f0a3d..e53f7ade 100644 --- a/plugins/openclaw-tps-mail/src/obligations.ts +++ b/plugins/openclaw-tps-mail/src/obligations.ts @@ -1,3 +1,4 @@ +import { formatStampDiagnostic, type ObligationPresence } from "./diagnostics.js"; /** * obligations.ts — the durable reply-OBLIGATION store for the tps-mail plugin * (slice S2 of cli#392's follow-up). @@ -213,7 +214,7 @@ export function listObligations( mailDir: string, agent: string, onReadError: (path: string, code: string, ids: string[], kind: ReadFailureKind) => void = (path, code) => console.warn( - `tps-mail: obligation-list-read-failed: actor=${agent} state=unknown path=${path} code=${code}; fix the path named above and restart the account`, + formatStampDiagnostic({ kind: "obligation-list-read-failed", actor: agent, path, code, obligation: "unknown" }), ), ): ObligationRecord[] { const dir = obligationsDir(mailDir, agent); @@ -269,7 +270,7 @@ export function createObligation( const draft = make(); const result = readObligationResult(mailDir, agent, draft.inboundId); if (result.status === "unverified") { - const message = `tps-mail: obligation-create-read-failed: ${draft.inboundId} actor=${agent} state=unknown path=${result.path} code=${result.code}; fix the path named above and restart the account`; + const message = formatStampDiagnostic({ kind: "obligation-create-read-failed", actor: agent, id: draft.inboundId, path: result.path, code: result.code, obligation: "unknown" }); log?.warn?.(message); throw new Error(message); } @@ -365,8 +366,7 @@ export function markNackSent( return true; } catch (err) { log?.warn?.( - `tps-mail: obligation-write-failed: could not record nackSentAt for ${inboundId} ` + - `(${err instanceof Error ? err.message : String(err)}); the record keeps nackPending, so a later start may send the nack again`, + formatStampDiagnostic({ kind: "obligation-write-failed", actor: agent, id: inboundId, path: obligationPath(mailDir, agent, inboundId), code: (err as NodeJS.ErrnoException).code ?? "WRITE_FAILED", obligation: "retained" }), ); return false; } @@ -400,8 +400,7 @@ export function abandonOwedNack( return true; } catch (err) { log?.warn?.( - `tps-mail: obligation-write-failed: could not record the nack abandonment for ${inboundId} ` + - `(${err instanceof Error ? err.message : String(err)}); the record keeps nackPending, so a later sweep will abandon it again`, + formatStampDiagnostic({ kind: "obligation-write-failed", actor: agent, id: inboundId, path: obligationPath(mailDir, agent, inboundId), code: (err as NodeJS.ErrnoException).code ?? "WRITE_FAILED", obligation: "retained" }), ); return false; } @@ -452,14 +451,14 @@ const ALL_STATES: ReadonlySet = new Set([ "failed", ]); -function curRecordUnresolved(mailDir: string, agent: string, inboundId: string, onFailure: (path: string, code: string) => void, requireCur = false): boolean { +function curRecordUnresolved(mailDir: string, agent: string, inboundId: string, onFailure: (path: string, code: string, kind?: ReadFailureKind) => void, requireCur = false): boolean { const curDirectory = resolve(mailDir, agent, "cur"); let names: string[]; try { names = readdirSync(curDirectory); } catch (err: any) { if (err?.code === "ENOENT" && !requireCur) return false; - onFailure(curDirectory, err?.code ?? "READ_FAILED"); + onFailure(curDirectory, err?.code ?? "READ_FAILED", "directory"); return true; } const matching = names.filter((name) => name === `${inboundId}.json` || name.endsWith(`-${inboundId}.json`)); @@ -479,7 +478,7 @@ function curRecordUnresolved(mailDir: string, agent: string, inboundId: string, } } if (requireCur) { - onFailure(curDirectory, "ENOENT"); + onFailure(resolve(curDirectory, `${inboundId}.json`), "ENOENT"); return true; } return false; @@ -517,9 +516,7 @@ export function obligationLastTransitionMs(record: unknown): number | null { * reads those. * * Ages a record by its OWN recorded timestamp (`lastTransitionAt`, else - * `inboundTimestamp`), never the file mtime. Safe + best-effort: an - * unreadable/malformed record (or one whose timestamp cannot be parsed) is LEFT - * and logged ONCE; a deletion failure is logged and never blocks startup. + * `inboundTimestamp`), never the file mtime. * `retentionDays <= 0` disables the sweep. * * A terminal record still OWING ITS NACK MAIL (`nackPending` with no @@ -553,7 +550,7 @@ export function sweepTerminalObligations( nowMs: number = Date.now(), nackHoldDays: number = retentionDays * DEFAULT_NACK_HOLD_MULTIPLE, unresolved = new Set(), - onFailure?: (path: string, code: string, id: string, state: string, kind: ReadFailureKind) => void, + onFailure?: (path: string, code: string, id: string, state: string, kind: ReadFailureKind, obligation: ObligationPresence) => void, heldRecords: ObligationRecord[] = [], ): RetentionResult { const res: RetentionResult = { @@ -572,11 +569,11 @@ export function sweepTerminalObligations( res.disabled = true; return res; } - const fail = (path: string, code: string, id: string, state: string, kind: ReadFailureKind = "file") => { + const fail = (path: string, code: string, id: string, state: string, kind: ReadFailureKind = "file", obligation: ObligationPresence = kind === "directory" ? "unknown" : "retained") => { if (unresolved.has(id)) return; - if (onFailure) onFailure(path, code, id, state, kind); - else if (kind === "directory") log?.warn?.(`tps-mail: retention-unresolved: actor=${agent} state=unknown path=${path} code=${code}; fix the path named above and restart the account`); - else log?.warn?.(`tps-mail: retention-unresolved: ${id} actor=${agent} state=${state} path=${path} code=${code}; obligation retained; fix the path named above and restart the account`); + if (onFailure) onFailure(path, code, id, state, kind, obligation); + else log?.warn?.(formatStampDiagnostic({ kind: "retention-unresolved", actor: agent, + id: kind === "directory" ? undefined : id, path, code, obligation })); unresolved.add(id); }; const dir = obligationsDir(mailDir, agent); @@ -592,12 +589,10 @@ export function sweepTerminalObligations( const code = (err as NodeJS.ErrnoException)?.code; if (code !== "ENOENT") { fail(dir, code ?? "READ_FAILED", "*", "unknown", "directory"); - if (!onFailure) log?.warn?.(`tps-mail: obligation retention: could not read ${dir}; sweep skipped`); return res; } } const cutoff = nowMs - retentionDays * 24 * 60 * 60 * 1000; - const leftUnreadable: string[] = []; // WHICH OBLIGATIONS THE STORE HELD at the START of this sweep, by state. The // obligation loop below DELETES aged terminal records, so "is this receipt's // obligation live, terminal, or gone?" must be answered from a snapshot taken @@ -632,7 +627,7 @@ export function sweepTerminalObligations( const code = err?.code ?? "INVALID_RECEIPT"; receiptFailureCodes.set(id, code); const owner = heldRecords.find((rec) => rec.obligationId === id); - fail(resolve(root, name), code, owner?.inboundId ?? `receipt:${name}`, owner?.state ?? "unknown"); + fail(resolve(root, name), code, owner?.inboundId ?? `receipt:${name}`, owner?.state ?? "unknown", "file", owner ? "retained" : "unknown"); } } } @@ -652,7 +647,6 @@ export function sweepTerminalObligations( record = JSON.parse(readFileSync(path, "utf-8")); } catch (err: any) { res.unreadable++; - leftUnreadable.push(name); fail(path, err?.code ?? "INVALID_RECORD", fileId, "unknown"); continue; } @@ -661,7 +655,6 @@ export function sweepTerminalObligations( // non-terminal one — reported as unreadable, never swept. if (!isObligationRecord(record) || record.inboundId !== fileId) { res.unreadable++; - leftUnreadable.push(name); fail(path, "INVALID_RECORD", fileId, "unknown"); continue; } @@ -687,12 +680,11 @@ export function sweepTerminalObligations( const transitionMs = obligationLastTransitionMs(record); if (transitionMs === null) { res.unreadable++; - leftUnreadable.push(name); fail(path, "INVALID_TIMESTAMP", fileId, state); hold(); continue; } - if (typeof inboundId === "string" && curRecordUnresolved(mailDir, agent, inboundId, (p, code) => fail(p, code, fileId, state), Boolean(onFailure))) { + if (typeof inboundId === "string" && curRecordUnresolved(mailDir, agent, inboundId, (p, code, kind) => fail(p, code, fileId, state, kind), Boolean(onFailure))) { hold(); continue; } @@ -732,9 +724,8 @@ export function sweepTerminalObligations( } res.abandonedForNack++; log?.warn?.( - `tps-mail: nack-abandoned: ${name} has owed its nack past the hold window (${nackHoldDays} day(s)); ` + - `the debt is released ` + - `and normal retention applies to the record`, + formatStampDiagnostic({ kind: "nack-abandoned", actor: agent, id: fileId, path, + code: "NACK_HOLD_EXPIRED", obligation: "retained" }), ); // fall through to the normal terminal-retention rules below } @@ -787,7 +778,9 @@ export function sweepTerminalObligations( receipt = onFailure ? receiptSnapshot.get(name) : JSON.parse(readFileSync(path, "utf-8")); } catch (err: any) { res.receiptsUnreadable++; - if (onFailure) fail(path, err?.code ?? "INVALID_RECEIPT", `receipt:${name}`, "unknown"); + if (onFailure) fail(path, err?.code ?? "INVALID_RECEIPT", `receipt:${name}`, "unknown", "file", "unknown"); + else log?.warn?.(formatStampDiagnostic({ kind: "retention-receipt-read-failed", actor: agent, id: `receipt:${name}`, + path, code: err?.code ?? "INVALID_RECEIPT", obligation: "unknown" })); continue; } const obligationId = (receipt as { obligationId?: unknown } | null)?.obligationId; @@ -802,7 +795,7 @@ export function sweepTerminalObligations( if (onFailure && terminal) continue; const t = typeof ts === "string" ? Date.parse(ts) : Number.NaN; if (onFailure && !terminal && !Number.isFinite(t)) { - fail(path, "INVALID_TIMESTAMP", `receipt:${name}`, "orphan"); + fail(path, "INVALID_TIMESTAMP", `receipt:${name}`, "orphan", "file", res.unreadable > 0 ? "unknown" : "none"); continue; } const agedOrphan = !live && Number.isFinite(t) && t < cutoff; @@ -822,32 +815,15 @@ export function sweepTerminalObligations( res.receiptsRemoved++; } catch (err) { if (onFailure) { - fail(path, (err as NodeJS.ErrnoException).code ?? "DELETE_FAILED", `receipt:${name}`, "orphan"); + fail(path, (err as NodeJS.ErrnoException).code ?? "DELETE_FAILED", `receipt:${name}`, "orphan", "file", "none"); continue; } log?.warn?.( - `tps-mail: obligation retention: could not delete receipt ${name}: ${err instanceof Error ? err.message : String(err)}; left in place`, + formatStampDiagnostic({ kind: "retention-receipt-delete-failed", actor: agent, id: `receipt:${name}`, path, + code: (err as NodeJS.ErrnoException).code ?? "DELETE_FAILED", obligation: terminal ? "unknown" : "none" }), ); } } - if (res.receiptsUnreadable > 0 && !onFailure) { - log?.warn?.( - `tps-mail: obligation retention: left ${res.receiptsUnreadable} unreadable receipt(s) in place (never deleted)`, - ); - } - if (res.orphanReceiptsSkipped > 0) { - log?.warn?.( - `tps-mail: obligation retention: left ${res.orphanReceiptsSkipped} aged receipt(s) in place — ` + - `${res.unreadable} unreadable/malformed record(s) in the store make an orphan unprovable this pass`, - ); - } - - // Logged ONCE: a single line for the unreadable/malformed records we left. - if (leftUnreadable.length > 0 && !onFailure) { - log?.warn?.( - `tps-mail: obligation retention: left ${leftUnreadable.length} unreadable/malformed record(s) in place (never deleted): ${leftUnreadable.join(", ")}`, - ); - } log?.info?.( `tps-mail: obligation retention: removed ${res.removed} terminal record(s) older than ${retentionDays} day(s); kept ${res.left}` + (res.heldForRecovery > 0 ? `; held ${res.heldForRecovery} for unresolved cur/ recovery` : "") + @@ -982,7 +958,7 @@ const realFs: ReceiptScanFs = { * Flair, for one) counts as NOT verified — the candidate is not evidence, and * the obligation resolves by a later scan or at its deadline. */ -export type ReceiptSignatureCheck = (envelope: Envelope) => Promise; +export type ReceiptSignatureCheck = (envelope: Envelope, path?: string) => Promise; /** True for a value with the shape of a SIGNED v1 envelope (not yet verified). */ function isSignedEnvelopeShape(x: unknown): x is Envelope { @@ -1206,8 +1182,8 @@ export async function scanForReceipt( ): Promise { const { expectedReplyId, fs = realFs, threadMode = "legacy" } = opts; let malformed: { path: string; ownRecord: boolean } | null = null; - const verified = (envelope: Envelope | null): Promise => - isVerifiedReceiptReply(envelope, agent, recipient, replyToId, threadMode, checkSignature); + const verified = (envelope: Envelope | null, path: string): Promise => + isVerifiedReceiptReply(envelope, agent, recipient, replyToId, threadMode, (candidate) => checkSignature(candidate, path)); // (1) METADATA: the direct path, one file. A `direct` dir is NEVER listed — // the agent's receipts root holds a receipt per receipted delivery, so // walking it would parse every retained receipt on every scan (round 4). @@ -1227,7 +1203,7 @@ export async function scanForReceipt( (expectedReplyId === undefined || rec.replyId === expectedReplyId) && rec.obligationId === obligationId && rec.replyToId === replyToId && - (await verified(receiptEnvelope(rec.signedReply))) + (await verified(receiptEnvelope(rec.signedReply), direct)) ) { return { status: "found", path: direct }; } @@ -1270,7 +1246,7 @@ export async function scanForReceipt( if (record?.accountId !== accountId) continue; if (record?.from !== agent) continue; if (record?.replyToId !== replyToId) continue; - if (!(await verified(recordReceiptEnvelope(record)))) continue; + if (!(await verified(recordReceiptEnvelope(record), path))) continue; return { status: "found", path }; } // (2b) the BRIDGE SANDBOX RECORD (cli#389 round 5, item 2): the reduced @@ -1286,7 +1262,7 @@ export async function scanForReceipt( record.replyId.length > 0 && (expectedReplyId === undefined || record.replyId === expectedReplyId) && record?.from === agent && - (await verified(recordReceiptEnvelope(record))) + (await verified(recordReceiptEnvelope(record), path)) ) { return { status: "found", path }; } diff --git a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts index 85add52c..b718fa35 100644 --- a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts +++ b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts @@ -233,7 +233,7 @@ describe("cli#492 — a failed cur/ stamp write is surfaced and retried", () => expect(readCur()?.record?.[stampKey], "the stamp did NOT land").toBeUndefined(); expect( await pollUntil( - () => failedLogs(h, `${kind}-stamp-failed`).some((m) => m.includes(`path=${resolve(mailDir, "anvil", "cur")}/.ack-`) && m.includes("EACCES")), + () => failedLogs(h, `${kind}-stamp-failed`).some((m) => m.includes(`path=${readCur()!.path}`) && m.includes("EACCES")), 4000, ), "the failed write is logged by id, path and code", @@ -290,8 +290,8 @@ describe("cli#492 — a failed cur/ stamp write is surfaced and retried", () => await finish(h); expect(await pollUntil(() => failedLogs(h, `${kind}-stamp-failed`).length > 0)).toBe(true); const diagnostic = failedLogs(h, `${kind}-stamp-failed`)[0]; - expect(diagnostic).toContain(stage === "lock" ? `path=${blocked}/.mail-lock.claim code=EACCES` : `path=${blocked}/.ack-`); - expect(diagnostic).not.toContain(`path=${cur.path}`); + expect(diagnostic).toContain(stage === "lock" ? `path=${blocked}/.mail-lock code=EACCES` : `path=${cur.path} code=EACCES`); + if (stage === "lock") expect(diagnostic).not.toContain(`path=${cur.path}`); expect(readCur()?.record?.[stampKey]).toBeUndefined(); } finally { await h.stop(); @@ -302,8 +302,8 @@ describe("cli#492 — a failed cur/ stamp write is surfaced and retried", () => try { expect(await pollUntil(() => next.logs.some((m) => m.includes(`${kind}-stamp-reconcile-failed`)))).toBe(true); const diagnostic = next.logs.find((m) => m.includes(`${kind}-stamp-reconcile-failed`))!; - expect(diagnostic).toContain(stage === "lock" ? `path=${blocked}/.mail-lock.claim code=EACCES` : `path=${blocked}/.ack-`); - expect(diagnostic).not.toContain(`path=${cur.path}`); + expect(diagnostic).toContain(stage === "lock" ? `path=${blocked}/.mail-lock code=EACCES` : `path=${cur.path} code=EACCES`); + if (stage === "lock") expect(diagnostic).not.toContain(`path=${cur.path}`); expect(diagnostic).toContain("obligation retained"); expect(readCur()?.record?.[stampKey]).toBeUndefined(); } finally { @@ -321,7 +321,7 @@ describe("cli#492 — a failed cur/ stamp write is surfaced and retried", () => removeOnTerminal = { path: cur.path, state }; await finish(h); expect(await pollUntil(() => failedLogs(h, `${kind}-stamp-failed`).length > 0)).toBe(true); - expect(failedLogs(h, `${kind}-stamp-failed`)[0]).toContain(`actor=anvil state=${state} path=${cur.path} code=ENOENT`); + expect(failedLogs(h, `${kind}-stamp-failed`)[0]).toContain(`actor=anvil path=${cur.path} code=ENOENT`); expect(failedLogs(h, `${kind}-stamp-failed`)[0]).toContain("obligation retained; fix the path named above and restart the account"); expect(obligation(h.inboundId)?.state).toBe(state); await sleep(400); @@ -361,9 +361,10 @@ describe("cli#492 — a failed cur/ stamp write is surfaced and retried", () => await finish(h); expect(await pollUntil(() => failedLogs(h, `${kind}-stamp-failed`).length > 0)).toBe(true); const diagnostic = failedLogs(h, `${kind}-stamp-failed`)[0]; - expect(diagnostic).toContain(`actor=anvil state=${state}`); + expect(diagnostic).toContain(`actor=anvil`); expect(diagnostic).toContain("code=WRITE_FAILED"); - expect(diagnostic).toContain("retry 1"); + expect(diagnostic).toContain("obligation retained"); + expect(diagnostic).not.toContain("no retries left"); stampError = undefined; expect(await pollUntil(() => !!readCur()?.record?.[stampKey])).toBe(true); expect(obligation(h.inboundId)?.state).toBe(state); @@ -447,7 +448,7 @@ for (const failReads of [Infinity, 1]) { expect(realFs.readFileSync(path, "utf8")).toBe(bytes); expect(realFs.existsSync(cur.path)).toBe(true); expect(JSON.parse(realFs.readFileSync(cur.path, "utf8")).ackedAt).toBeUndefined(); - expect(second.logs.some((m) => m.includes("actor=anvil state=unknown") && m.includes(path) && m.includes("fix the path named above and restart the account"))).toBe(true); + expect(second.logs.some((m) => m.includes("actor=anvil") && m.includes(path) && m.includes("fix the path named above and restart the account"))).toBe(true); } finally { await second.stop(); obligationReadFailure = undefined; @@ -493,7 +494,7 @@ for (const state of ["acked", "failed"] as const) { } const next = await boot(false); try { - expect(await pollUntil(() => next.logs.some((m) => m.includes(`actor=anvil state=${state}`) && m.includes(first.inboundId)))).toBe(true); + expect(await pollUntil(() => next.logs.some((m) => m.includes(`actor=anvil`) && m.includes(first.inboundId)))).toBe(true); await sleep(100); expect(next.dispatch()).toBeNull(); expect(next.logs.filter((m) => m.includes("actor=anvil") && m.includes(first.inboundId))).toHaveLength(1); @@ -563,10 +564,10 @@ for (const failure of ["directory", "file"] as const) { const diagnostic = h.logs.find((m) => m.includes("startup-unresolved") && m.includes(path))!; expect(diagnostic).toContain("fix the path named above and restart the account"); if (failure === "directory") { - expect(diagnostic).toContain("startup-unresolved: actor=anvil state=unknown"); + expect(diagnostic).toContain("startup-unresolved: actor=anvil"); expect(diagnostic).not.toContain("obligation retained"); } else { - expect(diagnostic).toContain("startup-unresolved: * actor=anvil state=unknown"); + expect(diagnostic).toContain("startup-unresolved: * actor=anvil"); expect(diagnostic).toContain("obligation retained"); } } finally { @@ -575,3 +576,61 @@ for (const failure of ["directory", "file"] as const) { } }); } + +for (const state of ["pending", "yielded", "delivering", "posted"] as const) { + it(`startup reports the missing ${state} record`, async () => { + const dir = resolve(mailDir, "anvil", ".obligations"); + realFs.mkdirSync(dir, { recursive: true }); + realFs.mkdirSync(resolve(mailDir, "anvil", "cur")); + realFs.writeFileSync(resolve(dir, "missing.json"), JSON.stringify({ + inboundId: "missing", obligationId: "ob-missing", state, from: "flint", to: "anvil" })); + const h = await boot(false); + try { + expect(await pollUntil(() => h.logs.some((m) => m.includes("startup-unresolved")))).toBe(true); + expect(h.logs.find((m) => m.includes("startup-unresolved"))).toBe(`tps-mail: startup-unresolved: missing actor=anvil path=${resolve(mailDir, "anvil", "cur", "missing.json")} code=ENOENT; obligation retained; fix the path named above and restart the account`); + } finally { await h.stop(); } + }); +} + +it("startup reports the missing unconfirmed record during retention", async () => { + const dir = resolve(mailDir, "anvil", ".obligations"); + realFs.mkdirSync(dir, { recursive: true }); + realFs.mkdirSync(resolve(mailDir, "anvil", "cur")); + realFs.writeFileSync(resolve(dir, "missing.json"), JSON.stringify({ inboundId: "missing", + obligationId: "ob-missing", state: "unconfirmed", lastTransitionAt: new Date(0).toISOString() })); + const h = await boot(false); + try { + expect(await pollUntil(() => h.logs.some((m) => m.includes("startup-unresolved")))).toBe(true); + expect(h.logs.find((m) => m.includes("startup-unresolved"))).toBe(`tps-mail: startup-unresolved: missing actor=anvil path=${resolve(mailDir, "anvil", "cur", "missing.json")} code=ENOENT; obligation retained; fix the path named above and restart the account`); + } finally { await h.stop(); } +}); + +for (const stage of ["age", "delete"] as const) { + it(`startup orphan receipt ${stage} failure`, async () => { + const dir = resolve(mailDir, "anvil", ".obligations", "receipts"); + realFs.mkdirSync(dir, { recursive: true }); + const path = resolve(dir, "orphan.json"); + realFs.writeFileSync(path, JSON.stringify({ obligationId: "orphan", + ts: stage === "age" ? "invalid" : new Date(0).toISOString() })); + if (stage === "delete") realFs.chmodSync(dir, 0o555); + const h = await boot(false); + try { + expect(await pollUntil(() => h.logs.some((m) => m.includes("startup-unresolved")))).toBe(true); + expect(h.logs.find((m) => m.includes("startup-unresolved"))).toBe(`tps-mail: startup-unresolved: receipt:orphan.json actor=anvil path=${path} code=${stage === "age" ? "INVALID_TIMESTAMP" : "EACCES"}; fix the path named above and restart the account`); + expect(realFs.existsSync(path)).toBe(true); + } finally { await h.stop(); realFs.chmodSync(dir, 0o755); } + }); +} + +it("startup orphan receipt with an unreadable obligation", async () => { + const dir = resolve(mailDir, "anvil", ".obligations"); + realFs.mkdirSync(resolve(dir, "receipts"), { recursive: true }); + realFs.writeFileSync(resolve(dir, "broken.json"), "null"); + const path = resolve(dir, "receipts", "orphan.json"); + realFs.writeFileSync(path, JSON.stringify({ obligationId: "orphan", ts: "invalid" })); + const h = await boot(false); + try { + expect(await pollUntil(() => h.logs.some((m) => m.includes("receipt:orphan.json")))).toBe(true); + expect(h.logs.find((m) => m.includes("receipt:orphan.json"))).toBe(`tps-mail: startup-unresolved: receipt:orphan.json actor=anvil path=${path} code=INVALID_TIMESTAMP; state unknown; fix the path named above and restart the account`); + } finally { await h.stop(); } +}); diff --git a/plugins/openclaw-tps-mail/test/diagnostics.test.ts b/plugins/openclaw-tps-mail/test/diagnostics.test.ts new file mode 100644 index 00000000..c6dcb218 --- /dev/null +++ b/plugins/openclaw-tps-mail/test/diagnostics.test.ts @@ -0,0 +1,19 @@ +import { expect, test } from "bun:test"; +import { formatStampDiagnostic } from "../src/diagnostics.js"; + +for (const obligation of ["retained", "none", "unknown"] as const) { + for (const retriesExhausted of [false, true]) { + test(`${obligation}, exhausted=${retriesExhausted}`, () => { + expect(formatStampDiagnostic({ kind: "stamp-failed", actor: "anvil", id: "inbound", + path: "/mail/anvil/cur/inbound.json", code: "EACCES", obligation, retriesExhausted })).toBe( + "tps-mail: stamp-failed: inbound actor=anvil path=/mail/anvil/cur/inbound.json code=EACCES; " + + (obligation === "retained" ? "obligation retained; " : obligation === "unknown" ? "state unknown; " : "") + + (retriesExhausted ? "no retries left; " : "") + "fix the path named above and restart the account"); + }); + } +} + +test("optional id and retry flag", () => { + expect(formatStampDiagnostic({ kind: "read-failed", actor: "anvil", path: "/mail/anvil/cur", + code: "EACCES", obligation: "unknown" })).toBe("tps-mail: read-failed: actor=anvil path=/mail/anvil/cur code=EACCES; state unknown; fix the path named above and restart the account"); +}); diff --git a/plugins/openclaw-tps-mail/test/locality.test.ts b/plugins/openclaw-tps-mail/test/locality.test.ts index a7e62a06..9c5c8ddf 100644 --- a/plugins/openclaw-tps-mail/test/locality.test.ts +++ b/plugins/openclaw-tps-mail/test/locality.test.ts @@ -1217,7 +1217,7 @@ describe("cli#389 round 8 — the commit is persisted, and the deadline never na expect(outcome.inboundNackedAt, "the inbound is NOT stamped").toBeNull(); expect(outcome.nackCount, "and NO nack mail is sent for a failure that was never recorded").toBe(0); expect( - outcome.warns.some((w) => w.includes("refusing to record the failure")), + outcome.warns.some((w) => w.includes("failure-refused")), "the refusal is logged by name", ).toBe(true); } finally { diff --git a/plugins/openclaw-tps-mail/test/obligation-retention.test.ts b/plugins/openclaw-tps-mail/test/obligation-retention.test.ts index c25c5ef3..343aef16 100644 --- a/plugins/openclaw-tps-mail/test/obligation-retention.test.ts +++ b/plugins/openclaw-tps-mail/test/obligation-retention.test.ts @@ -166,7 +166,7 @@ describe("cli#401 — obligation retention", () => { expect(res.unreadable).toBe(1); expect(res.removed).toBe(0); expect(existsSync(join(obligationsDir(mailDir, AGENT), "broken.json"))).toBe(true); - const warns = logs.warn.filter((m) => m.includes("unreadable/malformed")); + const warns = logs.warn.filter((m) => m.includes("retention-unresolved")); expect(warns.length, "logged once").toBe(1); expect(warns[0]).toContain("broken.json"); }); @@ -254,7 +254,7 @@ describe("cli#401 — obligation retention", () => { writeFileSync(obligationsDir(mailDir, AGENT), "not a directory", "utf-8"); const res = sweepTerminalObligations(mailDir, AGENT, 7, { warn: (m) => logs.warn.push(m), info: (m) => logs.info.push(m) }); expect(res.removed).toBe(0); - expect(logs.warn.some((m) => m.includes("could not read")), "names the error").toBe(true); + expect(logs.warn.some((m) => m.includes("code=ENOTDIR")), "names the error").toBe(true); }); it("(i) malformed record SHAPES (null / no state / unknown state) are reported unreadable, not skipped", () => { @@ -266,7 +266,7 @@ describe("cli#401 — obligation retention", () => { const res = sweepTerminalObligations(mailDir, AGENT, 7, { warn: (m) => logs.warn.push(m), info: (m) => logs.info.push(m) }); expect(res.removed).toBe(0); expect(res.unreadable).toBe(3); - expect(logs.warn.filter((m) => m.includes("unreadable/malformed")).length, "logged once").toBe(1); + expect(logs.warn.filter((m) => m.includes("retention-unresolved")).length, "each path is reported").toBe(3); expect(existsSync(join(dir, "null.json")) && existsSync(join(dir, "no-state.json")) && existsSync(join(dir, "bad-state.json"))).toBe(true); }); diff --git a/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts b/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts index 40c89136..7c344f40 100644 --- a/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts +++ b/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts @@ -53,6 +53,7 @@ mock.module("node:fs", () => ({ }, })); +const { formatStampDiagnostic } = await import("../src/diagnostics.js"); const { acquireMailLockSync, mailLockPath } = await import("@tpsdev-ai/agent"); const { patchMailFile, reconcileTerminalCurStamps } = await import("../src/index.js"); const { createObligation, listObligations, sweepTerminalObligations } = await import("../src/obligations.js"); @@ -92,13 +93,15 @@ describe("patchMailFile", () => { }); for (const code of ["EACCES", "ENOENT"]) { - test(`a scratch ${code} failure names the scratch path and preserves the record`, () => { + test(`a scratch ${code} failure names the target path and preserves the record`, () => { const f = terminalFixture("acked"); scratchErrorCode = code; const result = patchMailFile(f.curPath, { ackedAt: "done" }, "ackedAt", "inbound"); expect(result).toMatchObject({ ok: false, reason: "write-failed", code }); if (result.ok) throw new Error("expected scratch failure"); - expect(result.path).toStartWith(join(f.curDir, ".ack-")); + expect(result.path).toBe(f.curPath); + f.reconcile(); + expect(f.logs[0]).toBe(`tps-mail: ack-stamp-reconcile-failed: inbound actor=anvil path=${f.curPath} code=${code}; obligation retained; fix the path named above and restart the account`); expect(JSON.parse(realFs.readFileSync(f.curPath, "utf8")).ackedAt).toBeUndefined(); }); } @@ -150,7 +153,7 @@ describe("terminal stamp reconciliation", () => { f.reconcile(); expect(removeAfterRead).toBeUndefined(); expect(f.logs).toHaveLength(1); - expect(f.logs[0]).toContain(`${kind}-stamp-reconcile-failed: inbound actor=anvil state=${state} path=${f.curPath} code=ENOENT`); + expect(f.logs[0]).toContain(`${kind}-stamp-reconcile-failed: inbound actor=anvil path=${f.curPath} code=ENOENT`); expect(f.logs[0]).toContain("obligation retained; fix the path named above and restart the account"); expect(realFs.existsSync(f.curPath)).toBe(false); expect(JSON.parse(realFs.readFileSync(f.obligationPath, "utf8")).state).toBe(state); @@ -161,14 +164,14 @@ describe("terminal stamp reconciliation", () => { realFs.unlinkSync(f.curPath); f.reconcile(); expect(f.logs).toHaveLength(1); - expect(f.logs[0]).toContain(`state=${state} path=${join(f.curDir, "inbound.json")} code=ENOENT`); + expect(f.logs[0]).toContain(`path=${join(f.curDir, "inbound.json")} code=ENOENT`); }); test(`${kind}: an uncoded write failure is reported and the next run can stamp`, () => { const f = terminalFixture(state); uncodedWriteFailure = true; f.reconcile(); - expect(f.logs[0]).toContain(`actor=anvil state=${state} path=${f.curPath} code=WRITE_FAILED`); + expect(f.logs[0]).toContain(`actor=anvil path=${f.curPath} code=WRITE_FAILED`); expect(f.logs[0]).toContain("obligation retained; fix the path named above and restart the account"); uncodedWriteFailure = false; f.reconcile(); @@ -209,7 +212,7 @@ describe("terminal stamp reconciliation", () => { } f.reconcile(); expect(f.logs[0]).toContain("stamp-reconcile-read-failed"); - expect(f.logs[0]).toContain("actor=anvil state="); + expect(f.logs[0]).toContain("actor=anvil"); expect(f.logs[0]).toContain(`path=${target} code=EACCES`); expect(f.logs[0]).toContain("fix the path named above and restart the account"); listFailure = readFailure = undefined; @@ -243,7 +246,7 @@ for (const invalid of [null, 7, "bad", [], { inboundId: "bad" }]) { expect(listObligations(root, "anvil", (path, code) => errors.push(`${path}:${code}`))).toHaveLength(1); expect(errors).toEqual([`${badPath}:INVALID_RECORD`]); expect(() => f.reconcile()).not.toThrow(); - expect(f.logs[0]).toContain(`actor=anvil state=unknown path=${badPath} code=INVALID_RECORD`); + expect(f.logs[0]).toContain(`actor=anvil path=${badPath} code=INVALID_RECORD`); expect(f.logs[0]).toContain("fix the path named above and restart the account"); expect(JSON.parse(realFs.readFileSync(f.curPath, "utf8")).ackedAt).toBeDefined(); expect(realFs.readFileSync(badPath, "utf8")).toBe(JSON.stringify(invalid)); @@ -254,7 +257,7 @@ test("a null cur record is reported as unreadable", () => { const f = terminalFixture("acked"); realFs.writeFileSync(f.curPath, "null"); f.reconcile(); - expect(f.logs[0]).toContain(`actor=anvil state=acked path=${f.curPath} code=INVALID_RECORD`); + expect(f.logs[0]).toContain(`actor=anvil path=${f.curPath} code=INVALID_RECORD`); expect(f.logs[0]).toContain("fix the path named above and restart the account"); }); @@ -262,7 +265,7 @@ test("creation refuses an unreadable existing terminal obligation", () => { const f = terminalFixture("acked"); const bytes = realFs.readFileSync(f.obligationPath, "utf8"); readFailure = f.obligationPath; - expect(() => createObligation(root, "anvil", () => ({ inboundId: "inbound", state: "pending" }) as any)).toThrow("state=unknown"); + expect(() => createObligation(root, "anvil", () => ({ inboundId: "inbound", state: "pending" }) as any)).toThrow("state unknown"); expect(realFs.readFileSync(f.obligationPath, "utf8")).toBe(bytes); }); @@ -298,7 +301,7 @@ for (const state of ["acked", "failed"] as const) { expect(realFs.existsSync(receiptPath)).toBe(true); expect(realFs.existsSync(join(f.obligationDir, "healthy.json"))).toBe(false); expect(f.logs.filter((m) => m.includes("actor=anvil"))).toHaveLength(1); - expect(f.logs.find((m) => m.includes("actor=anvil"))).toContain(`state=${state}`); + expect(f.logs.find((m) => m.includes("actor=anvil"))).toContain(`actor=anvil`); expect(f.logs.find((m) => m.includes("actor=anvil"))).toContain("restart the account"); replaceIdentity = false; if (stage === "cur-missing" || stage === "identity-change") realFs.writeFileSync(f.curPath, JSON.stringify({ id: "inbound" })); @@ -326,7 +329,7 @@ for (const stage of ["obligation-reread", "cur-retention-read", "cur-retention-l if (stage === "abandonment-write") renameFailure = f.obligationPath; const unresolved = new Set(); sweepTerminalObligations(root, "anvil", 7, { warn: (m) => f.logs.push(m) }, Date.now(), 28, unresolved, (path, code, id, state) => { - f.logs.push(`actor=anvil state=${state} path=${path} code=${code}; fix the path named above and restart the account`); + f.logs.push(`actor=anvil path=${path} code=${code}; fix the path named above and restart the account`); unresolved.add(id); }); expect(unresolved.has("inbound")).toBe(true); @@ -368,12 +371,15 @@ for (const stage of ["receipt-list", "receipt-read", "receipt-delete", "receipt- deleteFailure = receiptPath; } const unresolved = new Set(); - const onFailure = (path: string, code: string, id: string, state: string) => { - f.logs.push(`actor=anvil state=${state} path=${path} code=${code}; fix the path named above and restart the account`); + const onFailure = (path: string, code: string, id: string, state: string, kind: "directory" | "file", obligation: "retained" | "none" | "unknown") => { + f.logs.push(formatStampDiagnostic({ kind: "startup-unresolved", actor: "anvil", id: kind === "directory" ? undefined : id, path, code, obligation })); unresolved.add(id); }; sweepTerminalObligations(root, "anvil", 7, { warn: (m) => f.logs.push(m) }, Date.now(), 28, unresolved, onFailure, [record]); expect(unresolved.size).toBe(1); + if (stage === "receipt-delete" || stage === "receipt-age") { + expect(f.logs[0]).toBe(`tps-mail: startup-unresolved: receipt:ob-inbound.json actor=anvil path=${receiptPath} code=${stage === "receipt-age" ? "INVALID_TIMESTAMP" : "EACCES"}; fix the path named above and restart the account`); + } expect(realFs.existsSync(receiptPath)).toBe(true); if (stage !== "receipt-delete" && stage !== "receipt-age") expect(realFs.existsSync(f.obligationPath)).toBe(true); expect(f.logs.filter((m) => m.includes("actor=anvil"))).toHaveLength(1); @@ -408,11 +414,23 @@ for (const failure of ["directory", "file"] as const) { expect(diagnostic).toContain(`path=${path} code=EACCES`); expect(diagnostic).toContain("fix the path named above and restart the account"); if (failure === "directory") { - expect(diagnostic).toContain("retention-unresolved: actor=anvil state=unknown"); + expect(diagnostic).toContain("retention-unresolved: actor=anvil"); expect(diagnostic).not.toContain("obligation retained"); } else { - expect(diagnostic).toContain("retention-unresolved: * actor=anvil state=unknown"); + expect(diagnostic).toContain("retention-unresolved: * actor=anvil"); expect(diagnostic).toContain("obligation retained"); } }); } + +test("retention reports the expected missing unconfirmed record", () => { + const f = terminalFixture("acked"); + const record = JSON.parse(realFs.readFileSync(f.obligationPath, "utf8")); + record.state = "unconfirmed"; + realFs.writeFileSync(f.obligationPath, JSON.stringify(record)); + realFs.unlinkSync(f.curPath); + sweepTerminalObligations(root, "anvil", 7, undefined, Date.now(), 28, new Set(), + (path, code, id, state, kind, obligation) => f.logs.push(formatStampDiagnostic({ + kind: "startup-unresolved", actor: "anvil", id, path, code, obligation })), [record]); + expect(f.logs).toEqual([`tps-mail: startup-unresolved: inbound actor=anvil path=${join(f.curDir, "inbound.json")} code=ENOENT; obligation retained; fix the path named above and restart the account`]); +}); diff --git a/plugins/openclaw-tps-mail/test/receipt-scan.test.ts b/plugins/openclaw-tps-mail/test/receipt-scan.test.ts index befdac15..e0dacdcc 100644 --- a/plugins/openclaw-tps-mail/test/receipt-scan.test.ts +++ b/plugins/openclaw-tps-mail/test/receipt-scan.test.ts @@ -574,3 +574,27 @@ describe("receipt scan — cli#429 signed thread mode (real signatures)", () => expect(receiptThread({ inboundId: INBOUND })).toEqual({ threadId: INBOUND, mode: "legacy" }); }); }); + +for (const form of ["metadata", "posted", "bridge"] as const) { + it(`signature check receives the ${form} candidate path`, async () => { + let path: string; + if (form === "metadata") path = writeMetadataReceipt(metadata()); + else { + const record = reply(); + if (form === "bridge") { + delete record.headers; + delete record.accountId; + record.obligationId = OB_ID; + record.replyId = "reply-1"; + } + writeReply(record); + path = join(dir, "2026-05-26T00-00-00-reply-1.json"); + } + const paths: (string | undefined)[] = []; + const result = await scanForReceipt({ direct: form === "metadata" ? [receiptsRoot()] : [], + posted: form === "metadata" ? [] : [dir] }, OB_ID, INBOUND, AGENT, ACCOUNT, RECIPIENT, + async (envelope, candidatePath) => { paths.push(candidatePath); return check(envelope); }); + expect(result).toEqual({ status: "found", path }); + expect(paths).toEqual([path]); + }); +} diff --git a/plugins/openclaw-tps-mail/test/reply-obligation.test.ts b/plugins/openclaw-tps-mail/test/reply-obligation.test.ts index 027e947f..27a826cc 100644 --- a/plugins/openclaw-tps-mail/test/reply-obligation.test.ts +++ b/plugins/openclaw-tps-mail/test/reply-obligation.test.ts @@ -1125,11 +1125,11 @@ describe("cli#389 round 11 — an owed nack is never swept", () => { expect(arrived, "the owed mail is handed over").toBe(true); const logged = await pollUntil( - () => warned.join("\n").includes(`could not record nackSentAt for ${inboundId}`), + () => warned.some((m) => m.includes("obligation-write-failed") && m.includes(inboundId) && m.includes("code=")), 2000, ); expect(logged, "and the failed record write is logged BY NAME").toBe(true); - expect(warned.join("\n"), "the line names the consequence for the sender").toContain("later start"); + expect(warned.join("\n"), "the retained obligation is reported").toContain("obligation retained"); await h.stop(); } finally { chmodSync(obligationsDir, 0o700); // let the harness remove the tree diff --git a/plugins/openclaw-tps-mail/test/startup.test.ts b/plugins/openclaw-tps-mail/test/startup.test.ts index c69c9b3a..6299e081 100644 --- a/plugins/openclaw-tps-mail/test/startup.test.ts +++ b/plugins/openclaw-tps-mail/test/startup.test.ts @@ -464,10 +464,11 @@ describe("openclaw-tps-mail: seenFiles startup behavior", () => { const cfg = { bindings: [{ agentId, match: { channel: "tps-mail", accountId: "default" } }], }; + const warnings: string[] = []; const ctx = { account: { accountId: "default", mailDir: tempMailDir, enabled: true }, cfg, - log: { info: () => {}, warn: () => {}, error: () => {} }, + log: { info: () => {}, warn: (message: string) => warnings.push(message), error: () => {} }, channelRuntime, abortSignal: abortController.signal, }; @@ -481,6 +482,8 @@ describe("openclaw-tps-mail: seenFiles startup behavior", () => { expect(readdirSync(dlqDir).filter((f) => f.endsWith(".json")).length).toBe(1); const reason = readFileSync(resolve(dlqDir, `${filename}.reason`), "utf-8"); expect(reason).toContain("class: unverified"); + expect(await pollUntil(() => warnings.some((m) => m.includes("cur-recovery-refused")), 2000)).toBe(true); + expect(warnings.find((m) => m.includes("cur-recovery-refused"))).toBe(`tps-mail: cur-recovery-refused: msg-forged-001 actor=${agentId} path=${resolve(dlqDir, filename)} code=unverified; state unknown; fix the path named above and restart the account`); abortController.abort(); try { await startPromise; } catch { /* expected on abort */ } From 2ba3799ac8ffe270551a197d7ee870af104a81bf Mon Sep 17 00:00:00 2001 From: flint Date: Sun, 4 Oct 2026 00:40:57 -0700 Subject: [PATCH 15/17] fix(openclaw-tps-mail): diagnostics narrowed to the failed stamp write (#468); every other message is main's Co-Authored-By: Claude Opus 5.5 --- .../fixed-492-cur-record-stamp-reconcile.md | 2 +- plugins/openclaw-tps-mail/src/diagnostics.ts | 6 +- plugins/openclaw-tps-mail/src/index.ts | 254 +++++++--------- plugins/openclaw-tps-mail/src/obligations.ts | 274 +++++++----------- .../test/cur-record-write.test.ts | 177 +---------- .../test/diagnostics.test.ts | 17 +- .../openclaw-tps-mail/test/locality.test.ts | 2 +- .../test/obligation-retention.test.ts | 12 +- .../test/patch-mail-file.test.ts | 221 ++------------ .../test/receipt-scan.test.ts | 24 -- .../test/reply-obligation.test.ts | 4 +- .../openclaw-tps-mail/test/startup.test.ts | 5 +- 12 files changed, 264 insertions(+), 734 deletions(-) diff --git a/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md b/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md index af543e34..7868b429 100644 --- a/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md +++ b/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md @@ -1 +1 @@ -- **Ack/nack stamps use locked writes**. Stamps use the CLI's existing-only `updateExistingRecord` (#469) (Closes #492). +- **Report and reconcile failed cur/ stamps**. Retry failed ack/nack stamp writes while the account runs, and reconcile terminal stamps at startup. Hold failed reconciliation from recovery and retention. diff --git a/plugins/openclaw-tps-mail/src/diagnostics.ts b/plugins/openclaw-tps-mail/src/diagnostics.ts index c6739203..36edc69b 100644 --- a/plugins/openclaw-tps-mail/src/diagnostics.ts +++ b/plugins/openclaw-tps-mail/src/diagnostics.ts @@ -5,7 +5,7 @@ export interface StampDiagnosticFacts { actor: string; id?: string; path: string; - code: string; + code?: string; obligation: ObligationPresence; retriesExhausted?: boolean; } @@ -16,7 +16,7 @@ export function formatStampDiagnostic(facts: StampDiagnosticFacts): string { ...(facts.id === undefined ? [] : [facts.id]), `actor=${facts.actor}`, `path=${facts.path}`, - `code=${facts.code}`, + ...(facts.code === undefined ? [] : [`code=${facts.code}`]), ]; const state = facts.obligation === "retained" ? "obligation retained" : facts.obligation === "unknown" ? "state unknown" : undefined; @@ -24,6 +24,6 @@ export function formatStampDiagnostic(facts: StampDiagnosticFacts): string { fields.join(" "), ...(state ? [state] : []), ...(facts.retriesExhausted ? ["no retries left"] : []), - "fix the path named above and restart the account", + "resolve the failure and restart the account", ].join("; "); } diff --git a/plugins/openclaw-tps-mail/src/index.ts b/plugins/openclaw-tps-mail/src/index.ts index 5899fc90..6047c26f 100644 --- a/plugins/openclaw-tps-mail/src/index.ts +++ b/plugins/openclaw-tps-mail/src/index.ts @@ -69,8 +69,6 @@ import { listObligations, markNackSent, nackOwed, - obligationPath, - obligationsDir, newestSessionTranscript, readObligation, readObligationResult, @@ -79,7 +77,6 @@ import { scanForReceipt, sweepTerminalObligations, type ReceiptSignatureCheck, - type ReadFailureKind, transitionObligation, writeReceipt, type ObligationRecord, @@ -88,7 +85,7 @@ import { type ReceiptScanDirs, } from "./obligations.js"; import type { OpenClawPluginApi } from "openclaw/plugin-sdk"; -import { formatStampDiagnostic, type ObligationPresence, type StampDiagnosticFacts } from "./diagnostics.js"; +import { formatStampDiagnostic, type ObligationPresence } from "./diagnostics.js"; import { detectHostOpenClawVersion, evaluateHostSilentReplyGuard } from "./host-version.js"; import type { ChannelPlugin } from "openclaw/plugin-sdk/core"; import type { @@ -266,17 +263,17 @@ function findBoundAgents(cfg: any, accountId: string): string[] { // ─── TPS mail envelope helpers ─────────────────────────────────────────────── -function readMailFile(filePath: string, onReadError?: (path: string, code: string) => void): TpsMailBody | null { +function readMailFile(filePath: string, onReadError?: (path: string, code?: string) => void): TpsMailBody | null { try { const raw = readFileSync(filePath, "utf-8"); const record: unknown = JSON.parse(raw); - if (!record || typeof record !== "object" || Array.isArray(record) || typeof (record as TpsMailBody).id !== "string") { - onReadError?.(filePath, "INVALID_RECORD"); + if (onReadError && (!record || typeof record !== "object" || Array.isArray(record) || typeof (record as TpsMailBody).id !== "string")) { + onReadError?.(filePath); return null; } return record as TpsMailBody; } catch (err: any) { - onReadError?.(filePath, err?.code ?? "INVALID_RECORD"); + onReadError?.(filePath, err?.code); return null; } } @@ -412,9 +409,8 @@ function persistReceiptAfterCommit( persistReceipt(mailDir, agent, message, route, branchId); } catch (err: any) { log?.warn?.( - formatStampDiagnostic({ kind: "receipt-write-failed", actor: agent, id: message.headers?.["X-TPS-InReplyTo"], - path: resolve(receiptsDir(mailDir, agent), `${message.headers?.["X-TPS-Obligation"]}.json`), - code: err?.code ?? "WRITE_FAILED", obligation: "unknown" }), + `tps-mail: receipt-write-failed: the ${route} delivery committed, but its receipt was not written ` + + `(${err?.message ?? err}); the obligation resolves at its deadline`, ); } } @@ -426,14 +422,13 @@ function persistReceiptAfterCommit( * never reported as failed and its inbound is never nacked. The logger itself is * guarded too: a throwing logger cannot fail a committed send either. */ -function postCommit(log: any, name: string, context: string, step: () => void, facts?: Omit): void { +function postCommit(log: any, name: string, context: string, step: () => void): void { try { step(); } catch (err: any) { try { log?.warn?.( - facts ? formatStampDiagnostic({ ...facts, kind: name, code: err?.code ?? "WRITE_FAILED" }) - : `tps-mail: ${name}: ${context} (${err?.message ?? err}); the delivery committed, so this is not a send failure`, + `tps-mail: ${name}: ${context} (${err?.message ?? err}); the delivery committed, so this is not a send failure`, ); } catch { /* a logger must never fail a committed send */ @@ -632,7 +627,7 @@ function routeFor(mailDir: string, cfg: any, accountId: string, to: string): Mai type CurRecordUpdate = | { ok: true } - | { ok: false; reason: "record-missing" | "write-failed"; path: string; code: string }; + | { ok: false; reason: "record-missing" | "write-failed"; path: string; code?: string }; export function patchMailFile(path: string, patch: Partial, stampKey: "ackedAt" | "nackedAt" | undefined, inboundId: string): CurRecordUpdate { try { @@ -650,54 +645,44 @@ export function patchMailFile(path: string, patch: Partial, stampKe if (r.status === "changed" && alreadyStamped) return { ok: true }; if (r.status === "updated") return { ok: true }; if (r.status === "gone") return { ok: false, reason: "record-missing", path, code: "ENOENT" }; - return { ok: false, reason: "write-failed", path, code: r.status }; + return { ok: false, reason: "write-failed", path, code: undefined }; } catch (err: any) { - return { ok: false, reason: "write-failed", path: typeof err?.path === "string" - ? basename(err.path).startsWith(".ack-") ? path - : err.path.startsWith(`${mailLockPath(dirname(dirname(path)))}.`) ? mailLockPath(dirname(dirname(path))) : err.path - : path, code: err?.code ?? "WRITE_FAILED" }; + return { ok: false, reason: "write-failed", path: typeof err?.path === "string" && (err.path === mailLockPath(dirname(dirname(path))) || err.path.startsWith(`${mailLockPath(dirname(dirname(path)))}.`)) + ? mailLockPath(dirname(dirname(path))) : path, code: typeof err?.code === "string" ? err.code : undefined }; } } +function stampObligationPresence(mailDir: string, agent: string, inboundId: string): ObligationPresence { + const result = readObligationResult(mailDir, agent, inboundId); + return result.status === "found" ? "retained" : result.status === "missing" ? "none" : "unknown"; +} + export function reconcileTerminalCurStamps(mailDir: string, agent: string, log: any, records?: ObligationRecord[], unknownInbounds = new Set()): Set { - const reportReadError = (obligation: ObligationPresence, id: string) => (path: string, code: string) => { + const reportReadError = (obligation: ObligationPresence, id: string) => (path: string, code?: string) => { if (unknownInbounds.has(id)) return; unknownInbounds.add(id); log?.warn?.( formatStampDiagnostic({ kind: "stamp-reconcile-read-failed", actor: agent, id, path, code, obligation }), ); }; - const onObligationReadError = (path: string, code: string, ids: string[], kind: ReadFailureKind) => { - reportReadError(kind === "directory" ? "unknown" : "retained", ids[0])(path, code); + const onObligationReadError = (_path: string, _code: string | undefined, ids: string[]) => { for (const id of ids) unknownInbounds.add(id); }; for (const rec of records ?? listObligations(mailDir, agent, onObligationReadError)) { if (unknownInbounds.has("*") || unknownInbounds.has(rec.inboundId)) continue; if (rec.state !== "acked" && rec.state !== "failed") continue; let unreadable = false; - const onReadError = (path: string, code: string) => { + const onReadError = (path: string, code?: string) => { unreadable = true; - reportReadError(path === resolve(mailDir, agent, "cur") ? "unknown" : "retained", rec.inboundId)(path, code); + if (path === resolve(mailDir, agent, "cur")) unknownInbounds.add(rec.inboundId); + else reportReadError(stampObligationPresence(mailDir, agent, rec.inboundId), rec.inboundId)(path, code); }; const kind = rec.state === "acked" ? "ack" : "nack"; const key = kind === "ack" ? "ackedAt" : "nackedAt"; const curPath = findCurPath(mailDir, agent, rec.inboundId, onReadError); - const missing = (path: string) => { - unknownInbounds.add(rec.inboundId); - log?.warn?.( - formatStampDiagnostic({ kind: `${kind}-stamp-reconcile-failed`, actor: agent, id: rec.inboundId, path, code: "ENOENT", obligation: "retained" }), - ); - }; - if (unreadable) continue; - if (!curPath) { - if (!unreadable) missing(resolve(mailDir, agent, "cur", `${rec.inboundId}.json`)); - continue; - } + if (unreadable || !curPath) continue; const cur = readMailFile(curPath, onReadError); - if (!cur) { - if (!unreadable) missing(curPath); - continue; - } + if (!cur) continue; if (cur[key]) continue; const patch = kind === "ack" ? { ackedAt: new Date().toISOString(), read: true } @@ -707,7 +692,7 @@ export function reconcileTerminalCurStamps(mailDir: string, agent: string, log: else { unknownInbounds.add(rec.inboundId); log?.warn?.( - formatStampDiagnostic({ kind: `${kind}-stamp-reconcile-failed`, actor: agent, id: rec.inboundId, path: r.path, code: r.code, obligation: "retained" }), + formatStampDiagnostic({ kind: `${kind}-stamp-reconcile-failed`, actor: agent, id: rec.inboundId, path: r.path, code: r.code, obligation: stampObligationPresence(mailDir, agent, rec.inboundId) }), ); } } @@ -953,7 +938,7 @@ function readObligationForCleanup(ctx: YieldContext, obligationId: string) { const result = readObligationResult(ctx.mailDir, ctx.agent, ctx.inboundId); if (result.status === "unverified") { retryObligationRead(ctx, obligationId); - ctx.log?.warn?.(formatStampDiagnostic({ kind: "obligation-read-unverified", actor: ctx.agent, id: ctx.inboundId, path: result.path, code: result.code, obligation: "unknown" })); + ctx.log?.warn?.(`tps-mail: obligation-read-unverified: ${ctx.inboundId} path=${result.path} code=${result.code}`); } else if (result.status === "missing" || TERMINAL_STATES.has(result.record.state)) { releaseObligationState(obligationId); } @@ -985,7 +970,7 @@ function stampTerminalCur( const delay = stamped.reason === "record-missing" ? undefined : stampRetryDelaysMs[attempt]; ctx.log?.warn?.( formatStampDiagnostic({ kind: `${kind}-stamp-failed`, actor: ctx.agent, id: ctx.inboundId, - path: stamped.path, code: stamped.code, obligation: "retained", + path: stamped.path, code: stamped.code, obligation: stampObligationPresence(ctx.mailDir, ctx.agent, ctx.inboundId), retriesExhausted: stamped.reason !== "record-missing" && delay === undefined }), ); if (delay === undefined || !isLiveContext(ctx)) return; @@ -993,6 +978,7 @@ function stampTerminalCur( if (typeof (timer as any).unref === "function") (timer as any).unref(); } + function ackObligation(ctx: YieldContext, obligationId: string, why: string): void { if (!isLiveContext(ctx)) return; // Only stamp the inbound when the ACK TRANSITION actually landed. A terminal @@ -1006,7 +992,7 @@ function ackObligation(ctx: YieldContext, obligationId: string, why: string): vo if (result.status === "unverified") return; const cur = result.status === "found" ? result.record : null; ctx.log?.warn?.( - formatStampDiagnostic({ kind: "ack-refused", actor: ctx.agent, id: ctx.inboundId, path: obligationPath(ctx.mailDir, ctx.agent, ctx.inboundId), code: cur?.state ?? "OBLIGATION_MISSING", obligation: cur ? "retained" : "none" }), + `tps-mail: refusing to ack ${ctx.inboundId} — obligation is ${cur?.state ?? "gone"}; the inbound keeps no ackedAt`, ); return; } @@ -1120,19 +1106,21 @@ async function settleObligation( if (result.status === "unverified") return "none"; const cur = result.status === "found" ? result.record : null; ctx.log?.warn?.( - formatStampDiagnostic({ kind: "unconfirmed-refused", actor: ctx.agent, id: ctx.inboundId, path: obligationPath(ctx.mailDir, ctx.agent, ctx.inboundId), code: cur?.state ?? "OBLIGATION_MISSING", obligation: cur ? "retained" : "none" }), + `tps-mail: refusing to record the unconfirmed outcome for ${ctx.inboundId} — obligation is ${cur?.state ?? "gone"}; it stays as it is`, ); return "none"; } } catch (err: any) { ctx.log?.warn?.( - formatStampDiagnostic({ kind: "obligation-write-failed", actor: ctx.agent, id: ctx.inboundId, path: obligationPath(ctx.mailDir, ctx.agent, ctx.inboundId), code: err?.code ?? "WRITE_FAILED", obligation: "retained" }), + `tps-mail: obligation-write-failed: could not record the unconfirmed outcome for ${ctx.inboundId} ` + + `(${err?.message ?? err}); one attempt, no retry — the obligation resolves on restart`, ); return "none"; } releaseObligationState(obligationId); ctx.log?.warn?.( - formatStampDiagnostic({ kind: "obligation-unconfirmed", actor: ctx.agent, id: ctx.inboundId, path: obligationPath(ctx.mailDir, ctx.agent, ctx.inboundId), code: s.reason, obligation: "retained" }), + `tps-mail: obligation for ${ctx.inboundId} UNCONFIRMED: ${s.reason} — the reply was committed (${rec.state}) ` + + `and no receipt evidence was found by its deadline; NOT failed, no nack sent`, ); return "unconfirmed"; } @@ -1160,7 +1148,8 @@ async function settleObligation( ); } catch (err: any) { ctx.log?.warn?.( - formatStampDiagnostic({ kind: "obligation-write-failed", actor: ctx.agent, id: ctx.inboundId, path: obligationPath(ctx.mailDir, ctx.agent, ctx.inboundId), code: err?.code ?? "WRITE_FAILED", obligation: "retained" }), + `tps-mail: obligation-write-failed: could not record the failure for ${ctx.inboundId} ` + + `(${err?.message ?? err}); one attempt, no retry — the obligation resolves on restart`, ); return "none"; } @@ -1169,7 +1158,8 @@ async function settleObligation( if (result.status === "unverified") return "none"; const cur = result.status === "found" ? result.record : null; ctx.log?.warn?.( - formatStampDiagnostic({ kind: "failure-refused", actor: ctx.agent, id: ctx.inboundId, path: obligationPath(ctx.mailDir, ctx.agent, ctx.inboundId), code: cur?.state ?? "OBLIGATION_MISSING", obligation: cur ? "retained" : "none" }), + `tps-mail: refusing to record the failure for ${ctx.inboundId} — obligation is ${cur?.state ?? "gone"}; ` + + `the inbound keeps no nack stamp and no nack mail is sent`, ); return "none"; } @@ -1181,10 +1171,13 @@ async function settleObligation( } // cli#389 round 10, item 1: AWAIT the one send, and record its outcome on the // record. The mail is owed until `nackSentAt` says otherwise (at-least-once). - await deliverNack(ctx, failedOn); + const nackHandedOff = await deliverNack(ctx, failedOn); if (!isLiveContext(ctx)) return "none"; ctx.log?.warn?.( - formatStampDiagnostic({ kind: "obligation-failed", actor: ctx.agent, id: ctx.inboundId, path: obligationPath(ctx.mailDir, ctx.agent, ctx.inboundId), code: failedOn, obligation: "retained" }), + `tps-mail: obligation for ${ctx.inboundId} FAILED: ${failedOn} — nacked` + + (s.alreadyStamped ? ", the inbound already carried its nack" : "") + + (nackHandedOff ? ", sender notified" : ", the nack mail is still OWED (nackPending) — the next start retries delivery") + + ", never acked", ); return "failed"; } @@ -1227,12 +1220,14 @@ async function deliverNack(ctx: YieldContext, reason: string, opts: { timeoutMs? // over again. The line here says which of the two happened. const recorded = markNackSent(ctx.mailDir, ctx.agent, ctx.inboundId, ctx.log); ctx.log?.warn?.( - formatStampDiagnostic({ kind: "nack-delivered", actor: ctx.agent, id: ctx.inboundId, path: obligationPath(ctx.mailDir, ctx.agent, ctx.inboundId), code: recorded ? "DELIVERED" : "NACK_SENT_WRITE_FAILED", obligation: "retained" }), + `tps-mail: nack delivered to ${ctx.sender} for ${ctx.inboundId}` + + (recorded ? "" : " — the record could not be updated: it still owes the nack and a later start may send it again"), ); return true; } ctx.log?.warn?.( - formatStampDiagnostic({ kind: "nack-pending", actor: ctx.agent, id: ctx.inboundId, path: obligationPath(ctx.mailDir, ctx.agent, ctx.inboundId), code: reason, obligation: "retained" }), + `tps-mail: nack-pending: the sender was NOT told about the failure of ${ctx.inboundId} (${reason}); ` + + `the obligation record keeps nackPending and the next start retries delivery`, ); return false; } @@ -1299,13 +1294,14 @@ function retryOwedNackInBackground( if (!isLiveContext(ctx)) return; if (outcome === "expired") { log?.warn?.( - formatStampDiagnostic({ kind: "nack-retry-timeout", actor: agentId, id: inboundId, path: obligationPath(mailDir, agentId, inboundId), code: "TIMEOUT", obligation: "retained" }), + `tps-mail: nack-retry-timeout: the owed nack for ${inboundId} to ${sender} did not complete within ${backstopMs}ms ` + + `(connect + ack); the transport was closed and the record keeps nackPending`, ); } }) .catch((err: any) => { if (!isLiveContext(ctx)) return; - log?.warn?.(formatStampDiagnostic({ kind: "nack-retry-failed", actor: agentId, id: inboundId, path: obligationPath(mailDir, agentId, inboundId), code: err?.code ?? "NACK_RETRY_FAILED", obligation: "retained" })); + log?.warn?.(`tps-mail: nack-retry-failed: the owed nack for ${inboundId} to ${sender} threw: ${err?.message ?? err}`); }) .finally(() => { if (timer) clearAccountTimer(accountId, timer); @@ -1335,13 +1331,15 @@ function markDelivering(mailDir: string, agent: string, inboundId: string, log: const current = readObligation(mailDir, agent, inboundId); if (current && TERMINAL_STATES.has(current.state)) { log?.warn?.( - formatStampDiagnostic({ kind: "late-final-refused", actor: agent, id: inboundId, path: obligationPath(mailDir, agent, inboundId), code: current.state, obligation: "retained" }), + `tps-mail: late-final-refused: obligation ${current.obligationId} for ${inboundId} is ${current.state}; ` + + `the late final is NOT delivered`, ); } return false; } catch (err: any) { log?.warn?.( - formatStampDiagnostic({ kind: "obligation-write-failed", actor: agent, id: inboundId, path: obligationPath(mailDir, agent, inboundId), code: err?.code ?? "WRITE_FAILED", obligation: "retained" }), + `tps-mail: obligation-write-failed: could not mark ${inboundId} delivering (${err?.message ?? err}); ` + + `nothing was sent`, ); return false; } @@ -1427,7 +1425,7 @@ function makeYieldCtx( */ function receiptSignatureCheck(ctx: YieldContext): ReceiptSignatureCheck { let client: ReturnType | null = null; - return async (envelope, path) => { + return async (envelope) => { if (!isLiveContext(ctx)) return false; try { client ??= createMailVerifyClient(ctx.agent); @@ -1439,7 +1437,8 @@ function receiptSignatureCheck(ctx: YieldContext): ReceiptSignatureCheck { } catch (err: any) { if (!isLiveContext(ctx)) return false; ctx.log?.warn?.( - formatStampDiagnostic({ kind: "receipt-verify-unavailable", actor: ctx.agent, id: ctx.inboundId, path: path!, code: err?.code ?? "VERIFY_FAILED", obligation: "unknown" }), + `tps-mail: receipt-verify-unavailable: the reply carried by a receipt for ${ctx.inboundId} could not be verified ` + + `(${err?.message ?? err}); it is not evidence until a later scan verifies it`, ); return false; } @@ -1643,19 +1642,19 @@ function installYieldSubscription(api: any): boolean { } /** The cur/ path for an inbound id (cur filenames are timestamp-id, not the id). */ -function findCurPath(mailDir: string, agent: string, inboundId: string, onReadError?: (path: string, code: string) => void): string | null { +function findCurPath(mailDir: string, agent: string, inboundId: string, onReadError?: (path: string, code?: string) => void): string | null { const curDir = resolve(mailDir, agent, "cur"); try { const names = readdirSync(curDir); - const matching = names.filter((name) => name === `${inboundId}.json` || name.endsWith(`-${inboundId}.json`)); - for (const name of matching.length ? matching : names) { + const matching = onReadError ? names.filter((name) => name === `${inboundId}.json` || name.endsWith(`-${inboundId}.json`)) : []; + for (const name of onReadError ? matching : names) { if (!name.endsWith(".json")) continue; const p = resolve(curDir, name); const rec = readMailFile(p, onReadError); if (rec?.id === inboundId) return p; } } catch (err: any) { - if (err?.code !== "ENOENT") onReadError?.(curDir, err?.code ?? "READ_FAILED"); + if (err?.code !== "ENOENT") onReadError?.(curDir, err?.code); } return null; } @@ -1783,7 +1782,7 @@ const gateway: ChannelGatewayAdapter = { } if (!existsSync(account.mailDir)) { - log?.warn?.(formatStampDiagnostic({ kind: "startup-mail-directory-missing", actor: account.accountId, path: account.mailDir, code: "ENOENT", obligation: "unknown" })); + log?.warn?.(`tps-mail: mail directory does not exist: ${account.mailDir}`); return; } @@ -1875,15 +1874,14 @@ const gateway: ChannelGatewayAdapter = { const recovered = await recoverPromoted(recipient, curPath); if (!isLive()) return; if (!recovered.ok) { - const rejectedPath = resolve(account.mailDir, recipient, "dlq", basename(curPath)); log?.warn?.( - formatStampDiagnostic({ kind: "cur-recovery-refused", actor: recipient, id: record.id, path: !existsSync(curPath) && existsSync(rejectedPath) ? rejectedPath : curPath, code: recovered.class, obligation: "unknown" }), + `tps-mail: cur/ recovery refused ${record.id} (${recovered.class}): ${recovered.reason}`, ); return; } await deliverPromoted(recipient, recovered.message, curPath); } catch (err: any) { - log?.warn?.(formatStampDiagnostic({ kind: "cur-recovery-deferred", actor: recipient, id: record.id, path: curPath, code: err?.code ?? "RECOVERY_FAILED", obligation: "unknown" })); + log?.warn?.(`tps-mail: cur/ recovery deferred for ${record.id}: ${err?.message ?? err}`); } } @@ -1969,31 +1967,25 @@ const gateway: ChannelGatewayAdapter = { // The obligation record is created HERE, keyed on the inbound id — a // replayed inbound finds its record and opens NO second obligation. const obligationId = randomUUID(); - let created: ReturnType; - try { - created = createObligation( - account.mailDir, - recipient, - () => ({ - obligationId, - inboundId: msg.id, - // cli#429: the durable thread id — the inbound's SIGNED envelope id - // (promote()/recoverPromoted() stamp it, and the id rule holds there). - ...(isValidEnvelopeId(msg.envelopeId) ? { inboundEnvelopeId: msg.envelopeId } : {}), - inboundTimestamp: msg.timestamp, - from: msg.from, - to: recipient, - accountId: account.accountId, - state: "pending", - deadlineAt: null, - attempts: 1, - }), - log, - ); - } catch (err: any) { - log?.warn?.(formatStampDiagnostic({ kind: "obligation-create-failed", actor: recipient, id: msg.id, path: obligationPath(account.mailDir, recipient, msg.id), code: err?.code ?? "CREATE_FAILED", obligation: "unknown" })); - return; - } + const created = createObligation( + account.mailDir, + recipient, + () => ({ + obligationId, + inboundId: msg.id, + // cli#429: the durable thread id — the inbound's SIGNED envelope id + // (promote()/recoverPromoted() stamp it, and the id rule holds there). + ...(isValidEnvelopeId(msg.envelopeId) ? { inboundEnvelopeId: msg.envelopeId } : {}), + inboundTimestamp: msg.timestamp, + from: msg.from, + to: recipient, + accountId: account.accountId, + state: "pending", + deadlineAt: null, + attempts: 1, + }), + log, + ); const obId = created.record.obligationId; const yieldCtx = makeYieldCtx( account.mailDir, @@ -2138,7 +2130,6 @@ const gateway: ChannelGatewayAdapter = { "obligation-posted-transition-failed", `the ${route!.kind} reply ${reply.id} committed to ${msg.from} but the obligation for ${msg.id} was not marked posted`, () => transitionObligation(account.mailDir, recipient, msg.id, "posted", { replyId: reply.id }, log), - { actor: recipient, id: msg.id, path: obligationPath(account.mailDir, recipient, msg.id), obligation: "retained" }, ); }; try { @@ -2255,7 +2246,6 @@ const gateway: ChannelGatewayAdapter = { r.kind === "local" ? undefined : r.branchId, log, ), - { actor: recipient, id: msg.id, path: resolve(receiptsDir(account.mailDir, recipient), `${obId}.json`), obligation: "unknown" }, ); } } @@ -2317,7 +2307,7 @@ const gateway: ChannelGatewayAdapter = { const live = readObligation(yieldCtx.mailDir, yieldCtx.agent, yieldCtx.inboundId); if (live && !TERMINAL_STATES.has(live.state)) { log?.warn?.( - formatStampDiagnostic({ kind: "obligation-unresolved", actor: recipient, id: msg.id, path: obligationPath(account.mailDir, recipient, msg.id), code: "NO_RECEIPT", obligation: "retained" }), + `tps-mail: obligation for ${msg.id} is unresolved (no receipt evidence yet); deadline armed`, ); } } @@ -2339,7 +2329,8 @@ const gateway: ChannelGatewayAdapter = { const rec = readObligation(yieldCtx.mailDir, yieldCtx.agent, yieldCtx.inboundId); if (rec && (rec.state === "delivering" || rec.state === "posted")) { log?.warn?.( - formatStampDiagnostic({ kind: `post-commit-error:${yieldCtx.step ?? "unnamed-step"}`, actor: recipient, id: msg.id, path: obligationPath(account.mailDir, recipient, msg.id), code: err?.code ?? "POST_COMMIT_FAILED", obligation: "retained" }), + `tps-mail: post-commit-error:${yieldCtx.step ?? "unnamed-step"}: ${reason} — the delivery for ${msg.id} ` + + `committed (${rec.state}), so it is NOT failed and its inbound is NOT nacked; it resolves at its deadline`, ); armDeadline(yieldCtx, obId); } else { @@ -2380,21 +2371,7 @@ const gateway: ChannelGatewayAdapter = { } } catch { /* ignore */ } - // Attempt terminal stamps before recovery and retention. - const unknownInbounds = new Set(); - const startupFailure = (path: string, code: string, id = "*", state = "unknown", kind: ReadFailureKind = "file", obligation: ObligationPresence = kind === "directory" ? "unknown" : "retained") => { - if (unknownInbounds.has(id)) return; - unknownInbounds.add(id); - log?.warn?.( - formatStampDiagnostic({ kind: "startup-unresolved", actor: agentId, - id: kind === "directory" ? undefined : id, path, code, obligation }), - ); - }; - const startupRecords = listObligations(account.mailDir, agentId, (path, code, ids, kind) => { - startupFailure(path, code, ids[0], "unknown", kind); - for (const id of ids) unknownInbounds.add(id); - }); - reconcileTerminalCurStamps(account.mailDir, agentId, log, startupRecords, unknownInbounds); + const unknownInbounds = reconcileTerminalCurStamps(account.mailDir, agentId, log); // Crash recovery (at-least-once): re-dispatch cur/ records that were // promoted but never acked/nacked. cur/ is a DESTINATION, so the record @@ -2411,15 +2388,12 @@ const gateway: ChannelGatewayAdapter = { if (!filename.endsWith(".json")) continue; const curPath = resolve(curDir, filename); if (seenFiles.has(curPath)) continue; - const known = startupRecords.find((rec) => filename === `${rec.inboundId}.json` || filename.endsWith(`-${rec.inboundId}.json`)); - if (unknownInbounds.has("*") || (known && unknownInbounds.has(known.inboundId))) continue; - const record = readMailFile(curPath, (path, code) => startupFailure(path, code, known?.inboundId ?? basename(filename, ".json"), known?.state, "file", known ? "retained" : "unknown")); - if (!record || record.ackedAt || record.nackedAt) continue; - if (unknownInbounds.has("*") || unknownInbounds.has(record.id)) continue; + const record = readMailFile(curPath); + if (!record || record.ackedAt || record.nackedAt || unknownInbounds.has("*") || unknownInbounds.has(record.id)) continue; void recoverUnackedCurRecord(agentId, curPath, record); } } - } catch (err: any) { startupFailure(curDir, err?.code ?? "READ_FAILED", "*", "unknown", "directory"); } + } catch { /* ignore */ } // Reap stranded tmp/*.promote scratch from an interrupted promote (the // catch only runs on a thrown error, so a kill leaves orphans no other @@ -2449,9 +2423,9 @@ const gateway: ChannelGatewayAdapter = { // below HOLDS any record still owing its nack (obligations.ts) while it // is inside the bounded hold, so an owed mail survives whether the retry // or the sweep runs first. - for (const rec of startupRecords) { + for (const rec of listObligations(account.mailDir, agentId)) { if (!isLive()) break; - if (unknownInbounds.has("*") || unknownInbounds.has(rec.inboundId) || !nackOwed(rec)) continue; + if (!nackOwed(rec)) continue; retryOwedNackInBackground( account.mailDir, agentId, @@ -2478,33 +2452,26 @@ const gateway: ChannelGatewayAdapter = { // a route cannot pin a record forever. try { const retentionDays = resolveObligationRetentionDays(pluginConfig, (cfg as any)?.channels?.[CHANNEL_ID]); - if (!unknownInbounds.has("*")) { - sweepTerminalObligations( - account.mailDir, - agentId, - retentionDays, - log, - Date.now(), - resolveObligationNackHoldDays(retentionDays, pluginConfig, (cfg as any)?.channels?.[CHANNEL_ID], log), - unknownInbounds, - startupFailure, - startupRecords, - ); - } + if (!unknownInbounds.has("*")) sweepTerminalObligations( + account.mailDir, + agentId, + retentionDays, + log, + Date.now(), + resolveObligationNackHoldDays(retentionDays, pluginConfig, (cfg as any)?.channels?.[CHANNEL_ID], log), + unknownInbounds, + ); } catch (err: any) { - log?.warn?.(formatStampDiagnostic({ kind: "retention-sweep-failed", actor: agentId, path: err?.path ?? obligationsDir(account.mailDir, agentId), code: err?.code ?? "SWEEP_FAILED", obligation: "unknown" })); + log?.warn?.(`tps-mail: obligation retention sweep failed (ignored): ${err?.message ?? String(err)}`); } // RESTART RECOVERY (S2): in-memory timers die with the process, so // reconcile every durable obligation record against the maildir/outbox // and RE-ARM the deadline where work is still outstanding. - for (const rec of startupRecords) { + for (const rec of listObligations(account.mailDir, agentId)) { if (!isLive()) break; - if (unknownInbounds.has("*") || unknownInbounds.has(rec.inboundId) || TERMINAL_STATES.has(rec.state)) continue; - const onFailure = (path: string, code: string) => startupFailure(path, code, rec.inboundId, rec.state, path === curDir ? "directory" : "file"); - const recCurPath = findCurPath(account.mailDir, agentId, rec.inboundId, onFailure); - if (!recCurPath) onFailure(resolve(curDir, `${rec.inboundId}.json`), "ENOENT"); - if (unknownInbounds.has(rec.inboundId)) continue; + if (TERMINAL_STATES.has(rec.state) || unknownInbounds.has("*") || unknownInbounds.has(rec.inboundId)) continue; + const recCurPath = findCurPath(account.mailDir, agentId, rec.inboundId); const ctx = makeYieldCtx( account.mailDir, agentId, @@ -2518,16 +2485,11 @@ const gateway: ChannelGatewayAdapter = { rec.inboundEnvelopeId, ); yieldContexts.set(rec.obligationId, ctx); - try { - await reconcileObligation(ctx, rec); - } catch (err: any) { - startupFailure(recCurPath!, err?.code ?? "RECONCILE_FAILED", rec.inboundId, rec.state); - yieldContexts.delete(rec.obligationId); - } + await reconcileObligation(ctx, rec); } } catch (err: any) { log?.warn?.( - formatStampDiagnostic({ kind: "startup-watch-failed", actor: agentId, path: newDir, code: err?.code ?? "WATCH_FAILED", obligation: "unknown" }), + `tps-mail: failed to watch ${newDir}: ${err?.message ?? String(err)}`, ); } } diff --git a/plugins/openclaw-tps-mail/src/obligations.ts b/plugins/openclaw-tps-mail/src/obligations.ts index e53f7ade..6f76332d 100644 --- a/plugins/openclaw-tps-mail/src/obligations.ts +++ b/plugins/openclaw-tps-mail/src/obligations.ts @@ -1,4 +1,3 @@ -import { formatStampDiagnostic, type ObligationPresence } from "./diagnostics.js"; /** * obligations.ts — the durable reply-OBLIGATION store for the tps-mail plugin * (slice S2 of cli#392's follow-up). @@ -162,8 +161,6 @@ export interface ObligationRecord { nackAbandonedAt?: string; } -export type ReadFailureKind = "directory" | "file"; - export interface ObligationLog { info?: (msg: string) => void; warn?: (msg: string) => void; @@ -177,13 +174,6 @@ export function obligationPath(mailDir: string, agent: string, inboundId: string return resolve(obligationsDir(mailDir, agent), `${inboundId}.json`); } -function isObligationRecord(record: unknown): record is ObligationRecord { - if (!record || typeof record !== "object" || Array.isArray(record)) return false; - const value = record as Partial; - return typeof value.inboundId === "string" && typeof value.state === "string" && - ALL_STATES.has(value.state); -} - export function readObligation(mailDir: string, agent: string, inboundId: string): ObligationRecord | null { const p = obligationPath(mailDir, agent, inboundId); try { @@ -199,8 +189,8 @@ export function readObligationResult(mailDir: string, agent: string, inboundId: | { status: "unverified"; path: string; code: string } { const path = obligationPath(mailDir, agent, inboundId); try { - const record: unknown = JSON.parse(readFileSync(path, "utf-8")); - if (!isObligationRecord(record) || record.inboundId !== inboundId) { + const record = JSON.parse(readFileSync(path, "utf-8")) as ObligationRecord; + if (!record || typeof record.state !== "string") { return { status: "unverified", path, code: "INVALID_RECORD" }; } return { status: "found", record }; @@ -210,37 +200,30 @@ export function readObligationResult(mailDir: string, agent: string, inboundId: } } -export function listObligations( - mailDir: string, - agent: string, - onReadError: (path: string, code: string, ids: string[], kind: ReadFailureKind) => void = (path, code) => console.warn( - formatStampDiagnostic({ kind: "obligation-list-read-failed", actor: agent, path, code, obligation: "unknown" }), - ), -): ObligationRecord[] { +export function listObligations(mailDir: string, agent: string, onReadError?: (path: string, code: string | undefined, ids: string[], kind: "directory" | "file") => void): ObligationRecord[] { const dir = obligationsDir(mailDir, agent); let names: string[]; try { names = readdirSync(dir); } catch (err: any) { - if (err?.code !== "ENOENT") onReadError(dir, err?.code ?? "READ_FAILED", ["*"], "directory"); + if (err?.code !== "ENOENT") onReadError?.(dir, err?.code, ["*"], "directory"); return []; } const out: ObligationRecord[] = []; for (const name of names) { if (!name.endsWith(".json") || name.startsWith(".")) continue; try { - const record: unknown = JSON.parse(readFileSync(resolve(dir, name), "utf-8")); - if (!isObligationRecord(record) || `${record.inboundId}.json` !== name) { + const record = JSON.parse(readFileSync(resolve(dir, name), "utf-8")) as ObligationRecord; + if (onReadError && (!record || typeof record.inboundId !== "string" || `${record.inboundId}.json` !== name || !ALL_STATES.has(record.state))) { const ids = [name.slice(0, -5)]; - if (record && typeof record === "object" && "inboundId" in record && typeof record.inboundId === "string") { - ids.push(record.inboundId); - } - onReadError(resolve(dir, name), "INVALID_RECORD", ids, "file"); + if (typeof record?.inboundId === "string") ids.push(record.inboundId); + onReadError(resolve(dir, name), undefined, ids, "file"); continue; } out.push(record); } catch (err: any) { - onReadError(resolve(dir, name), err?.code ?? "INVALID_RECORD", [name.slice(0, -5)], "file"); + onReadError?.(resolve(dir, name), err?.code, [name.slice(0, -5)], "file"); + // A torn record is not readable truth; skip it rather than crash recovery. } } return out; @@ -268,14 +251,8 @@ export function createObligation( log?: ObligationLog, ): { created: boolean; record: ObligationRecord } { const draft = make(); - const result = readObligationResult(mailDir, agent, draft.inboundId); - if (result.status === "unverified") { - const message = formatStampDiagnostic({ kind: "obligation-create-read-failed", actor: agent, id: draft.inboundId, path: result.path, code: result.code, obligation: "unknown" }); - log?.warn?.(message); - throw new Error(message); - } - if (result.status === "found") { - const existing = result.record; + const existing = readObligation(mailDir, agent, draft.inboundId); + if (existing) { log?.info?.( `tps-mail: obligation for inbound ${draft.inboundId} already exists (${existing.obligationId}); not creating a second`, ); @@ -366,7 +343,8 @@ export function markNackSent( return true; } catch (err) { log?.warn?.( - formatStampDiagnostic({ kind: "obligation-write-failed", actor: agent, id: inboundId, path: obligationPath(mailDir, agent, inboundId), code: (err as NodeJS.ErrnoException).code ?? "WRITE_FAILED", obligation: "retained" }), + `tps-mail: obligation-write-failed: could not record nackSentAt for ${inboundId} ` + + `(${err instanceof Error ? err.message : String(err)}); the record keeps nackPending, so a later start may send the nack again`, ); return false; } @@ -400,7 +378,8 @@ export function abandonOwedNack( return true; } catch (err) { log?.warn?.( - formatStampDiagnostic({ kind: "obligation-write-failed", actor: agent, id: inboundId, path: obligationPath(mailDir, agent, inboundId), code: (err as NodeJS.ErrnoException).code ?? "WRITE_FAILED", obligation: "retained" }), + `tps-mail: obligation-write-failed: could not record the nack abandonment for ${inboundId} ` + + `(${err instanceof Error ? err.message : String(err)}); the record keeps nackPending, so a later sweep will abandon it again`, ); return false; } @@ -451,37 +430,18 @@ const ALL_STATES: ReadonlySet = new Set([ "failed", ]); -function curRecordUnresolved(mailDir: string, agent: string, inboundId: string, onFailure: (path: string, code: string, kind?: ReadFailureKind) => void, requireCur = false): boolean { - const curDirectory = resolve(mailDir, agent, "cur"); - let names: string[]; +/** True when the agent's cur/ record for this inbound is still UNRESOLVED + * (present without ackedAt/nackedAt). Startup recovery may re-dispatch it, so + * its obligation must not be swept yet — a crash between ackObligation's + * `acked` transition and the cur/ `ackedAt` patch leaves exactly this shape. */ +function curRecordUnresolved(mailDir: string, agent: string, inboundId: string): boolean { + const p = resolve(mailDir, agent, "cur", `${inboundId}.json`); try { - names = readdirSync(curDirectory); - } catch (err: any) { - if (err?.code === "ENOENT" && !requireCur) return false; - onFailure(curDirectory, err?.code ?? "READ_FAILED", "directory"); - return true; - } - const matching = names.filter((name) => name === `${inboundId}.json` || name.endsWith(`-${inboundId}.json`)); - for (const name of matching.length ? matching : names) { - if (!name.endsWith(".json")) continue; - const path = resolve(curDirectory, name); - try { - const rec = JSON.parse(readFileSync(path, "utf-8")); - if (!rec || typeof rec !== "object" || typeof rec.id !== "string") { - onFailure(path, "INVALID_RECORD"); - return true; - } - if (rec.id === inboundId) return !rec.ackedAt && !rec.nackedAt; - } catch (err: any) { - onFailure(path, err?.code ?? "INVALID_RECORD"); - return true; - } - } - if (requireCur) { - onFailure(resolve(curDirectory, `${inboundId}.json`), "ENOENT"); - return true; + const rec = JSON.parse(readFileSync(p, "utf-8")); + return !rec?.ackedAt && !rec?.nackedAt; + } catch { + return false; // absent/unreadable: nothing recovery can re-drive } - return false; } /** The record's OWN recorded last-transition time in ms. `lastTransitionAt` @@ -516,7 +476,9 @@ export function obligationLastTransitionMs(record: unknown): number | null { * reads those. * * Ages a record by its OWN recorded timestamp (`lastTransitionAt`, else - * `inboundTimestamp`), never the file mtime. + * `inboundTimestamp`), never the file mtime. Safe + best-effort: an + * unreadable/malformed record (or one whose timestamp cannot be parsed) is LEFT + * and logged ONCE; a deletion failure is logged and never blocks startup. * `retentionDays <= 0` disables the sweep. * * A terminal record still OWING ITS NACK MAIL (`nackPending` with no @@ -550,8 +512,6 @@ export function sweepTerminalObligations( nowMs: number = Date.now(), nackHoldDays: number = retentionDays * DEFAULT_NACK_HOLD_MULTIPLE, unresolved = new Set(), - onFailure?: (path: string, code: string, id: string, state: string, kind: ReadFailureKind, obligation: ObligationPresence) => void, - heldRecords: ObligationRecord[] = [], ): RetentionResult { const res: RetentionResult = { removed: 0, @@ -569,13 +529,6 @@ export function sweepTerminalObligations( res.disabled = true; return res; } - const fail = (path: string, code: string, id: string, state: string, kind: ReadFailureKind = "file", obligation: ObligationPresence = kind === "directory" ? "unknown" : "retained") => { - if (unresolved.has(id)) return; - if (onFailure) onFailure(path, code, id, state, kind, obligation); - else log?.warn?.(formatStampDiagnostic({ kind: "retention-unresolved", actor: agent, - id: kind === "directory" ? undefined : id, path, code, obligation })); - unresolved.add(id); - }; const dir = obligationsDir(mailDir, agent); let names: string[] = []; try { @@ -588,11 +541,14 @@ export function sweepTerminalObligations( // unreadable the agent's own receipts cannot be attributed either. const code = (err as NodeJS.ErrnoException)?.code; if (code !== "ENOENT") { - fail(dir, code ?? "READ_FAILED", "*", "unknown", "directory"); + log?.warn?.( + `tps-mail: obligation retention: could not read ${dir}: ${err instanceof Error ? err.message : String(err)}; sweep skipped`, + ); return res; } } const cutoff = nowMs - retentionDays * 24 * 60 * 60 * 1000; + const leftUnreadable: string[] = []; // WHICH OBLIGATIONS THE STORE HELD at the START of this sweep, by state. The // obligation loop below DELETES aged terminal records, so "is this receipt's // obligation live, terminal, or gone?" must be answered from a snapshot taken @@ -602,92 +558,43 @@ export function sweepTerminalObligations( // unique UUID, and the receipts live in this same store — so every receipt is // attributable to an obligation this sweep can see. There is no host-wide dir // to be careful of, and no inbound for two obligations to collide on. - const failedReceiptIds = new Set(); - const receiptFailureCodes = new Map(); - const receiptSnapshot = new Map(); - if (onFailure) { - const root = receiptsDir(mailDir, agent); - let receiptNames: string[] = []; - try { receiptNames = readdirSync(root); } - catch (err: any) { - if (err?.code !== "ENOENT") { - fail(root, err?.code ?? "READ_FAILED", "*", "unknown", "directory"); - return res; - } - } - for (const name of receiptNames) { - if (!name.endsWith(".json") || name.startsWith(".")) continue; - try { - const receipt = JSON.parse(readFileSync(resolve(root, name), "utf-8")); - if (!receipt || typeof receipt.obligationId !== "string") throw new Error("invalid receipt"); - receiptSnapshot.set(name, receipt); - } catch (err: any) { - const id = name.replace(/\.json$/, ""); - failedReceiptIds.add(id); - const code = err?.code ?? "INVALID_RECEIPT"; - receiptFailureCodes.set(id, code); - const owner = heldRecords.find((rec) => rec.obligationId === id); - fail(resolve(root, name), code, owner?.inboundId ?? `receipt:${name}`, owner?.state ?? "unknown", "file", owner ? "retained" : "unknown"); - } - } - } const terminalObligationIds = new Set(); - const liveObligationIds = new Set(heldRecords.filter((rec) => unresolved.has(rec.inboundId)).map((rec) => rec.obligationId)); - if (unresolved.size > 0) res.unreadable++; + const liveObligationIds = new Set(); for (const name of names) { if (!name.endsWith(".json") || name.startsWith(".")) continue; const path = resolve(dir, name); - const fileId = name.replace(/\.json$/, ""); - if (unresolved.has("*") || unresolved.has(fileId)) { - res.heldForRecovery++; - continue; - } let record: unknown; try { record = JSON.parse(readFileSync(path, "utf-8")); - } catch (err: any) { + } catch { res.unreadable++; - fail(path, err?.code ?? "INVALID_RECORD", fileId, "unknown"); + leftUnreadable.push(name); continue; } // Shape check: a parseable value that is not an object with a RECOGNIZED // state (null, no state, an unknown state) is a malformed record, not a // non-terminal one — reported as unreadable, never swept. - if (!isObligationRecord(record) || record.inboundId !== fileId) { + const state = (record as { state?: unknown } | null)?.state; + if (typeof record !== "object" || record === null || Array.isArray(record) || typeof state !== "string" || !ALL_STATES.has(state)) { res.unreadable++; - fail(path, "INVALID_RECORD", fileId, "unknown"); + leftUnreadable.push(name); continue; } - const state = record.state; - if (!TERMINAL_STATES.has(state)) { + if (!TERMINAL_STATES.has(state as ObligationState)) { const liveObligationId = (record as { obligationId?: unknown }).obligationId; if (typeof liveObligationId === "string") liveObligationIds.add(liveObligationId); res.left++; // pending / delivering / posted / yielded are never deletable continue; } - const snapshotObligationId = (record as { obligationId?: unknown }).obligationId; - if (typeof snapshotObligationId === "string") terminalObligationIds.add(snapshotObligationId); - const inboundId = (record as { inboundId?: unknown }).inboundId; - const hold = () => { - if (typeof snapshotObligationId === "string") liveObligationIds.add(snapshotObligationId); + const heldInboundId = (record as { inboundId?: unknown }).inboundId; + if (typeof heldInboundId === "string" && unresolved.has(heldInboundId)) { + const obligationId = (record as { obligationId?: unknown }).obligationId; + if (typeof obligationId === "string") liveObligationIds.add(obligationId); res.heldForRecovery++; - }; - if (typeof snapshotObligationId === "string" && failedReceiptIds.has(snapshotObligationId)) { - fail(resolve(receiptsDir(mailDir, agent), `${snapshotObligationId}.json`), receiptFailureCodes.get(snapshotObligationId) ?? "RECEIPT_READ_FAILED", fileId, state); - hold(); - continue; - } - const transitionMs = obligationLastTransitionMs(record); - if (transitionMs === null) { - res.unreadable++; - fail(path, "INVALID_TIMESTAMP", fileId, state); - hold(); - continue; - } - if (typeof inboundId === "string" && curRecordUnresolved(mailDir, agent, inboundId, (p, code, kind) => fail(p, code, fileId, state, kind), Boolean(onFailure))) { - hold(); continue; } + const snapshotObligationId = (record as { obligationId?: unknown }).obligationId; + if (typeof snapshotObligationId === "string") terminalObligationIds.add(snapshotObligationId); // cli#389 round 11, item 1: a record still OWING its nack mail is not // deletable while it is INSIDE the hold window — startup retries delivery // from this exact shape (`nackPending` with no `nackSentAt`), so sweeping it @@ -703,6 +610,7 @@ export function sweepTerminalObligations( res.heldForNack++; continue; } + res.abandonedForNack++; // cli#389 round 13, item 2: RELEASE the debt in the same pass that gives up // on it — clear `nackPending` and record `nackAbandonedAt`. Without the // clear, a record retention then KEEPS would be abandoned and logged @@ -714,22 +622,30 @@ export function sweepTerminalObligations( mailDir, agent, typeof recInbound === "string" ? recInbound : name.replace(/\.json$/, ""), - { warn: () => {} }, + log, new Date(nowMs).toISOString(), ); - if (!released) { - fail(path, "WRITE_FAILED", fileId, state); - hold(); - continue; - } - res.abandonedForNack++; log?.warn?.( - formatStampDiagnostic({ kind: "nack-abandoned", actor: agent, id: fileId, path, - code: "NACK_HOLD_EXPIRED", obligation: "retained" }), + `tps-mail: nack-abandoned: ${name} has owed its nack past the hold window (${nackHoldDays} day(s)); ` + + `the debt is released${released ? "" : " (the release could not be recorded, so a later sweep will abandon it again)"} ` + + `and normal retention applies to the record`, ); // fall through to the normal terminal-retention rules below } - const t = transitionMs; + // A terminal record whose cur/ record is still unresolved is HELD until + // startup recovery resolves it (else a re-dispatch would open a fresh + // obligation and double-post). + const inboundId = (record as { inboundId?: unknown }).inboundId; + if (typeof inboundId === "string" && curRecordUnresolved(mailDir, agent, inboundId)) { + res.heldForRecovery++; + continue; + } + const t = obligationLastTransitionMs(record); + if (t === null) { + res.unreadable++; + leftUnreadable.push(name); + continue; + } if (t >= cutoff) { res.left++; continue; @@ -738,8 +654,9 @@ export function sweepTerminalObligations( unlinkSync(path); res.removed++; } catch (err) { - fail(path, (err as NodeJS.ErrnoException).code ?? "DELETE_FAILED", fileId, state); - hold(); + log?.warn?.( + `tps-mail: obligation retention: could not delete ${name}: ${err instanceof Error ? err.message : String(err)}; left in place`, + ); res.left++; } } @@ -764,7 +681,7 @@ export function sweepTerminalObligations( const receiptsRoot = receiptsDir(mailDir, agent); let receiptNames: string[]; try { - receiptNames = onFailure ? [...receiptSnapshot.keys()] : readdirSync(receiptsRoot); + receiptNames = readdirSync(receiptsRoot); } catch { // No receipts directory yet (no receipted delivery was ever made): done. receiptNames = []; @@ -772,15 +689,11 @@ export function sweepTerminalObligations( for (const name of receiptNames) { if (!name.endsWith(".json") || name.startsWith(".")) continue; const path = resolve(receiptsRoot, name); - if (onFailure && failedReceiptIds.has(name.replace(/\.json$/, ""))) continue; let receipt: unknown; try { - receipt = onFailure ? receiptSnapshot.get(name) : JSON.parse(readFileSync(path, "utf-8")); - } catch (err: any) { + receipt = JSON.parse(readFileSync(path, "utf-8")); + } catch { res.receiptsUnreadable++; - if (onFailure) fail(path, err?.code ?? "INVALID_RECEIPT", `receipt:${name}`, "unknown", "file", "unknown"); - else log?.warn?.(formatStampDiagnostic({ kind: "retention-receipt-read-failed", actor: agent, id: `receipt:${name}`, - path, code: err?.code ?? "INVALID_RECEIPT", obligation: "unknown" })); continue; } const obligationId = (receipt as { obligationId?: unknown } | null)?.obligationId; @@ -789,15 +702,9 @@ export function sweepTerminalObligations( // names no obligation id cannot be attributed to one, so it is left in // place rather than aged out on a guess. if (typeof obligationId !== "string" || obligationId.length === 0) continue; - const live = liveObligationIds.has(obligationId); - if (live) continue; const terminal = terminalObligationIds.has(obligationId); - if (onFailure && terminal) continue; + const live = liveObligationIds.has(obligationId); const t = typeof ts === "string" ? Date.parse(ts) : Number.NaN; - if (onFailure && !terminal && !Number.isFinite(t)) { - fail(path, "INVALID_TIMESTAMP", `receipt:${name}`, "orphan", "file", res.unreadable > 0 ? "unknown" : "none"); - continue; - } const agedOrphan = !live && Number.isFinite(t) && t < cutoff; // FAIL SAFE (cli#389 round 6, item 3): an obligation record that could not // be read joins neither set, so a receipt for THAT obligation looks @@ -814,16 +721,29 @@ export function sweepTerminalObligations( unlinkSync(path); res.receiptsRemoved++; } catch (err) { - if (onFailure) { - fail(path, (err as NodeJS.ErrnoException).code ?? "DELETE_FAILED", `receipt:${name}`, "orphan", "file", "none"); - continue; - } log?.warn?.( - formatStampDiagnostic({ kind: "retention-receipt-delete-failed", actor: agent, id: `receipt:${name}`, path, - code: (err as NodeJS.ErrnoException).code ?? "DELETE_FAILED", obligation: terminal ? "unknown" : "none" }), + `tps-mail: obligation retention: could not delete receipt ${name}: ${err instanceof Error ? err.message : String(err)}; left in place`, ); } } + if (res.receiptsUnreadable > 0) { + log?.warn?.( + `tps-mail: obligation retention: left ${res.receiptsUnreadable} unreadable receipt(s) in place (never deleted)`, + ); + } + if (res.orphanReceiptsSkipped > 0) { + log?.warn?.( + `tps-mail: obligation retention: left ${res.orphanReceiptsSkipped} aged receipt(s) in place — ` + + `${res.unreadable} unreadable/malformed record(s) in the store make an orphan unprovable this pass`, + ); + } + + // Logged ONCE: a single line for the unreadable/malformed records we left. + if (leftUnreadable.length > 0) { + log?.warn?.( + `tps-mail: obligation retention: left ${leftUnreadable.length} unreadable/malformed record(s) in place (never deleted): ${leftUnreadable.join(", ")}`, + ); + } log?.info?.( `tps-mail: obligation retention: removed ${res.removed} terminal record(s) older than ${retentionDays} day(s); kept ${res.left}` + (res.heldForRecovery > 0 ? `; held ${res.heldForRecovery} for unresolved cur/ recovery` : "") + @@ -958,7 +878,7 @@ const realFs: ReceiptScanFs = { * Flair, for one) counts as NOT verified — the candidate is not evidence, and * the obligation resolves by a later scan or at its deadline. */ -export type ReceiptSignatureCheck = (envelope: Envelope, path?: string) => Promise; +export type ReceiptSignatureCheck = (envelope: Envelope) => Promise; /** True for a value with the shape of a SIGNED v1 envelope (not yet verified). */ function isSignedEnvelopeShape(x: unknown): x is Envelope { @@ -1182,8 +1102,8 @@ export async function scanForReceipt( ): Promise { const { expectedReplyId, fs = realFs, threadMode = "legacy" } = opts; let malformed: { path: string; ownRecord: boolean } | null = null; - const verified = (envelope: Envelope | null, path: string): Promise => - isVerifiedReceiptReply(envelope, agent, recipient, replyToId, threadMode, (candidate) => checkSignature(candidate, path)); + const verified = (envelope: Envelope | null): Promise => + isVerifiedReceiptReply(envelope, agent, recipient, replyToId, threadMode, checkSignature); // (1) METADATA: the direct path, one file. A `direct` dir is NEVER listed — // the agent's receipts root holds a receipt per receipted delivery, so // walking it would parse every retained receipt on every scan (round 4). @@ -1203,7 +1123,7 @@ export async function scanForReceipt( (expectedReplyId === undefined || rec.replyId === expectedReplyId) && rec.obligationId === obligationId && rec.replyToId === replyToId && - (await verified(receiptEnvelope(rec.signedReply), direct)) + (await verified(receiptEnvelope(rec.signedReply))) ) { return { status: "found", path: direct }; } @@ -1246,7 +1166,7 @@ export async function scanForReceipt( if (record?.accountId !== accountId) continue; if (record?.from !== agent) continue; if (record?.replyToId !== replyToId) continue; - if (!(await verified(recordReceiptEnvelope(record), path))) continue; + if (!(await verified(recordReceiptEnvelope(record)))) continue; return { status: "found", path }; } // (2b) the BRIDGE SANDBOX RECORD (cli#389 round 5, item 2): the reduced @@ -1262,7 +1182,7 @@ export async function scanForReceipt( record.replyId.length > 0 && (expectedReplyId === undefined || record.replyId === expectedReplyId) && record?.from === agent && - (await verified(recordReceiptEnvelope(record), path)) + (await verified(recordReceiptEnvelope(record))) ) { return { status: "found", path }; } diff --git a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts index b718fa35..4f830808 100644 --- a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts +++ b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts @@ -248,7 +248,7 @@ describe("cli#492 — a failed cur/ stamp write is surfaced and retried", () => await h.stop(); }, 20000); - it(`${kind}: a persistent failure stops after the bound and the next account start re-stamps the record`, async () => { + it(`${kind}: persistent failures exhaust live retries`, async () => { const h = await boot(true); expect(await pollUntil(() => h.dispatch() !== null, 4000), "dispatch started").toBe(true); const cur = readCur(); @@ -322,7 +322,7 @@ describe("cli#492 — a failed cur/ stamp write is surfaced and retried", () => await finish(h); expect(await pollUntil(() => failedLogs(h, `${kind}-stamp-failed`).length > 0)).toBe(true); expect(failedLogs(h, `${kind}-stamp-failed`)[0]).toContain(`actor=anvil path=${cur.path} code=ENOENT`); - expect(failedLogs(h, `${kind}-stamp-failed`)[0]).toContain("obligation retained; fix the path named above and restart the account"); + expect(failedLogs(h, `${kind}-stamp-failed`)[0]).toContain("obligation retained; resolve the failure and restart the account"); expect(obligation(h.inboundId)?.state).toBe(state); await sleep(400); expect(failedLogs(h, `${kind}-stamp-failed`).length).toBe(1); @@ -362,7 +362,7 @@ describe("cli#492 — a failed cur/ stamp write is surfaced and retried", () => expect(await pollUntil(() => failedLogs(h, `${kind}-stamp-failed`).length > 0)).toBe(true); const diagnostic = failedLogs(h, `${kind}-stamp-failed`)[0]; expect(diagnostic).toContain(`actor=anvil`); - expect(diagnostic).toContain("code=WRITE_FAILED"); + expect(diagnostic).not.toContain("code="); expect(diagnostic).toContain("obligation retained"); expect(diagnostic).not.toContain("no retries left"); stampError = undefined; @@ -421,51 +421,8 @@ describe("cli#492 — a failed cur/ stamp write is surfaced and retried", () => }, 15000); }); -for (const failReads of [Infinity, 1]) { - it(`startup retains an unreadable terminal obligation without redispatch (failed reads: ${failReads})`, async () => { - const first = await boot(true); - expect(await pollUntil(() => first.dispatch() !== null)).toBe(true); - await first.deliver("verdict"); - first.settle(); - expect(await pollUntil(() => !!readCur()?.record?.ackedAt)).toBe(true); - await first.stop(); - const cur = readCur()!; - delete cur.record.ackedAt; - realFs.writeFileSync(cur.path, JSON.stringify(cur.record)); - const path = resolve(mailDir, "anvil", ".obligations", `${first.inboundId}.json`); - const terminal = JSON.parse(realFs.readFileSync(path, "utf8")); - terminal.lastTransitionAt = new Date(0).toISOString(); - realFs.writeFileSync(path, JSON.stringify(terminal)); - const bytes = realFs.readFileSync(path, "utf8"); - obligationReadFailure = path; - failObligationReads = failReads; - const second = await boot(false); - try { - expect(await pollUntil(() => second.logs.some((m) => m.includes(path) && m.includes("code=EACCES")))).toBe(true); - await sleep(300); - expect(second.dispatch()).toBeNull(); - expect(second.logs.some((m) => m.includes(`delivering ${first.inboundId} `))).toBe(false); - expect(realFs.readFileSync(path, "utf8")).toBe(bytes); - expect(realFs.existsSync(cur.path)).toBe(true); - expect(JSON.parse(realFs.readFileSync(cur.path, "utf8")).ackedAt).toBeUndefined(); - expect(second.logs.some((m) => m.includes("actor=anvil") && m.includes(path) && m.includes("fix the path named above and restart the account"))).toBe(true); - } finally { - await second.stop(); - obligationReadFailure = undefined; - } - const third = await boot(false); - try { - expect(await pollUntil(() => !!readCur()?.record?.ackedAt)).toBe(true); - expect(third.dispatch()).toBeNull(); - - } finally { - await third.stop(); - } - }, 15000); -} - for (const state of ["acked", "failed"] as const) { - for (const stage of ["lookup-read", "reread", "locked-read", "recovery-read", "stamp-write"] as const) { + for (const stage of ["lookup-read", "reread", "locked-read", "stamp-write"] as const) { it(`startup ${state} ${stage} holds aged obligations across restarts without redispatch`, async () => { const first = await boot(true); try { @@ -490,7 +447,7 @@ for (const state of ["acked", "failed"] as const) { else { curReadFailure = cur.path; curReads = 0; - curReadsUntilFailure = stage === "lookup-read" ? 1 : stage === "reread" ? 2 : stage === "recovery-read" ? (restart === 0 ? 4 : 3) : 3; + curReadsUntilFailure = stage === "lookup-read" ? 1 : stage === "reread" ? 2 : 3; } const next = await boot(false); try { @@ -499,7 +456,7 @@ for (const state of ["acked", "failed"] as const) { expect(next.dispatch()).toBeNull(); expect(next.logs.filter((m) => m.includes("actor=anvil") && m.includes(first.inboundId))).toHaveLength(1); expect(realFs.readFileSync(path, "utf8")).toBe(bytes); - if (stage !== "recovery-read") expect(JSON.parse(realFs.readFileSync(cur.path, "utf8"))[state === "acked" ? "ackedAt" : "nackedAt"]).toBeUndefined(); + expect(JSON.parse(realFs.readFileSync(cur.path, "utf8"))[state === "acked" ? "ackedAt" : "nackedAt"]).toBeUndefined(); } finally { await next.stop(); } } stampError = undefined; @@ -512,125 +469,3 @@ for (const state of ["acked", "failed"] as const) { }, 15000); } } - -for (const invalidState of [false, true]) { - it(`startup holds both identities of a mismatched obligation (invalid state: ${invalidState})`, async () => { - const first = await boot(true); - try { - expect(await pollUntil(() => first.dispatch() !== null)).toBe(true); - await first.deliver("verdict"); - first.settle(); - expect(await pollUntil(() => !!readCur()?.record?.ackedAt)).toBe(true); - } finally { await first.stop(); } - const cur = readCur()!; - delete cur.record.ackedAt; - realFs.writeFileSync(cur.path, JSON.stringify(cur.record)); - const dir = resolve(mailDir, "anvil", ".obligations"); - const original = resolve(dir, `${first.inboundId}.json`); - const record = JSON.parse(realFs.readFileSync(original, "utf8")); - record.lastTransitionAt = new Date(0).toISOString(); - if (invalidState) record.state = "invalid"; - const mismatched = resolve(dir, "filename-id.json"); - const bytes = JSON.stringify(record); - realFs.renameSync(original, mismatched); - realFs.writeFileSync(mismatched, bytes); - const filenameCur = resolve(mailDir, "anvil", "cur", "filename-id.json"); - realFs.writeFileSync(filenameCur, JSON.stringify({ ...cur.record, id: "filename-id" })); - const next = await boot(false); - try { - expect(await pollUntil(() => next.logs.some((m) => m.includes(mismatched) && m.includes("code=INVALID_RECORD")))).toBe(true); - await sleep(500); - expect(next.dispatch()).toBeNull(); - expect(next.logs.some((m) => m.includes(`delivering ${first.inboundId} `) || m.includes("delivering filename-id "))).toBe(false); - expect(realFs.readFileSync(mismatched, "utf8")).toBe(bytes); - expect(realFs.existsSync(cur.path)).toBe(true); - expect(realFs.existsSync(filenameCur)).toBe(true); - expect(JSON.parse(realFs.readFileSync(cur.path, "utf8")).ackedAt).toBeUndefined(); - } finally { next.settle(); await next.stop(); } - }, 15000); -} - -for (const failure of ["directory", "file"] as const) { - it(`startup distinguishes a ${failure} read failure from the literal star file`, async () => { - const dir = resolve(mailDir, "anvil", ".obligations"); - realFs.mkdirSync(resolve(mailDir, "anvil"), { recursive: true }); - const path = failure === "directory" ? dir : resolve(dir, "*.json"); - if (failure === "file") realFs.mkdirSync(dir); - realFs.writeFileSync(path, "{}"); - if (failure === "file") realFs.chmodSync(path, 0o000); - const h = await boot(false); - try { - expect(await pollUntil(() => h.logs.some((m) => m.includes("startup-unresolved") && m.includes(path)))).toBe(true); - const diagnostic = h.logs.find((m) => m.includes("startup-unresolved") && m.includes(path))!; - expect(diagnostic).toContain("fix the path named above and restart the account"); - if (failure === "directory") { - expect(diagnostic).toContain("startup-unresolved: actor=anvil"); - expect(diagnostic).not.toContain("obligation retained"); - } else { - expect(diagnostic).toContain("startup-unresolved: * actor=anvil"); - expect(diagnostic).toContain("obligation retained"); - } - } finally { - await h.stop(); - if (failure === "file") realFs.chmodSync(path, 0o644); - } - }); -} - -for (const state of ["pending", "yielded", "delivering", "posted"] as const) { - it(`startup reports the missing ${state} record`, async () => { - const dir = resolve(mailDir, "anvil", ".obligations"); - realFs.mkdirSync(dir, { recursive: true }); - realFs.mkdirSync(resolve(mailDir, "anvil", "cur")); - realFs.writeFileSync(resolve(dir, "missing.json"), JSON.stringify({ - inboundId: "missing", obligationId: "ob-missing", state, from: "flint", to: "anvil" })); - const h = await boot(false); - try { - expect(await pollUntil(() => h.logs.some((m) => m.includes("startup-unresolved")))).toBe(true); - expect(h.logs.find((m) => m.includes("startup-unresolved"))).toBe(`tps-mail: startup-unresolved: missing actor=anvil path=${resolve(mailDir, "anvil", "cur", "missing.json")} code=ENOENT; obligation retained; fix the path named above and restart the account`); - } finally { await h.stop(); } - }); -} - -it("startup reports the missing unconfirmed record during retention", async () => { - const dir = resolve(mailDir, "anvil", ".obligations"); - realFs.mkdirSync(dir, { recursive: true }); - realFs.mkdirSync(resolve(mailDir, "anvil", "cur")); - realFs.writeFileSync(resolve(dir, "missing.json"), JSON.stringify({ inboundId: "missing", - obligationId: "ob-missing", state: "unconfirmed", lastTransitionAt: new Date(0).toISOString() })); - const h = await boot(false); - try { - expect(await pollUntil(() => h.logs.some((m) => m.includes("startup-unresolved")))).toBe(true); - expect(h.logs.find((m) => m.includes("startup-unresolved"))).toBe(`tps-mail: startup-unresolved: missing actor=anvil path=${resolve(mailDir, "anvil", "cur", "missing.json")} code=ENOENT; obligation retained; fix the path named above and restart the account`); - } finally { await h.stop(); } -}); - -for (const stage of ["age", "delete"] as const) { - it(`startup orphan receipt ${stage} failure`, async () => { - const dir = resolve(mailDir, "anvil", ".obligations", "receipts"); - realFs.mkdirSync(dir, { recursive: true }); - const path = resolve(dir, "orphan.json"); - realFs.writeFileSync(path, JSON.stringify({ obligationId: "orphan", - ts: stage === "age" ? "invalid" : new Date(0).toISOString() })); - if (stage === "delete") realFs.chmodSync(dir, 0o555); - const h = await boot(false); - try { - expect(await pollUntil(() => h.logs.some((m) => m.includes("startup-unresolved")))).toBe(true); - expect(h.logs.find((m) => m.includes("startup-unresolved"))).toBe(`tps-mail: startup-unresolved: receipt:orphan.json actor=anvil path=${path} code=${stage === "age" ? "INVALID_TIMESTAMP" : "EACCES"}; fix the path named above and restart the account`); - expect(realFs.existsSync(path)).toBe(true); - } finally { await h.stop(); realFs.chmodSync(dir, 0o755); } - }); -} - -it("startup orphan receipt with an unreadable obligation", async () => { - const dir = resolve(mailDir, "anvil", ".obligations"); - realFs.mkdirSync(resolve(dir, "receipts"), { recursive: true }); - realFs.writeFileSync(resolve(dir, "broken.json"), "null"); - const path = resolve(dir, "receipts", "orphan.json"); - realFs.writeFileSync(path, JSON.stringify({ obligationId: "orphan", ts: "invalid" })); - const h = await boot(false); - try { - expect(await pollUntil(() => h.logs.some((m) => m.includes("receipt:orphan.json")))).toBe(true); - expect(h.logs.find((m) => m.includes("receipt:orphan.json"))).toBe(`tps-mail: startup-unresolved: receipt:orphan.json actor=anvil path=${path} code=INVALID_TIMESTAMP; state unknown; fix the path named above and restart the account`); - } finally { await h.stop(); } -}); diff --git a/plugins/openclaw-tps-mail/test/diagnostics.test.ts b/plugins/openclaw-tps-mail/test/diagnostics.test.ts index c6dcb218..0d446a8e 100644 --- a/plugins/openclaw-tps-mail/test/diagnostics.test.ts +++ b/plugins/openclaw-tps-mail/test/diagnostics.test.ts @@ -4,16 +4,23 @@ import { formatStampDiagnostic } from "../src/diagnostics.js"; for (const obligation of ["retained", "none", "unknown"] as const) { for (const retriesExhausted of [false, true]) { test(`${obligation}, exhausted=${retriesExhausted}`, () => { - expect(formatStampDiagnostic({ kind: "stamp-failed", actor: "anvil", id: "inbound", + expect(formatStampDiagnostic({ kind: "ack-stamp-failed", actor: "anvil", id: "inbound", path: "/mail/anvil/cur/inbound.json", code: "EACCES", obligation, retriesExhausted })).toBe( - "tps-mail: stamp-failed: inbound actor=anvil path=/mail/anvil/cur/inbound.json code=EACCES; " + + "tps-mail: ack-stamp-failed: inbound actor=anvil path=/mail/anvil/cur/inbound.json code=EACCES; " + (obligation === "retained" ? "obligation retained; " : obligation === "unknown" ? "state unknown; " : "") + - (retriesExhausted ? "no retries left; " : "") + "fix the path named above and restart the account"); + (retriesExhausted ? "no retries left; " : "") + "resolve the failure and restart the account"); }); } } test("optional id and retry flag", () => { - expect(formatStampDiagnostic({ kind: "read-failed", actor: "anvil", path: "/mail/anvil/cur", - code: "EACCES", obligation: "unknown" })).toBe("tps-mail: read-failed: actor=anvil path=/mail/anvil/cur code=EACCES; state unknown; fix the path named above and restart the account"); + expect(formatStampDiagnostic({ kind: "stamp-reconcile-read-failed", actor: "anvil", path: "/mail/anvil/cur/inbound.json", + code: "EACCES", obligation: "unknown" })).toBe("tps-mail: stamp-reconcile-read-failed: actor=anvil path=/mail/anvil/cur/inbound.json code=EACCES; state unknown; resolve the failure and restart the account"); +}); + +test("an uncoded stamp failure omits code", () => { + const message = formatStampDiagnostic({ kind: "ack-stamp-failed", actor: "anvil", + path: "/mail/anvil/cur/inbound.json", obligation: "unknown" }); + expect(message).not.toContain("code="); + expect(message).toContain("state unknown"); }); diff --git a/plugins/openclaw-tps-mail/test/locality.test.ts b/plugins/openclaw-tps-mail/test/locality.test.ts index 9c5c8ddf..a7e62a06 100644 --- a/plugins/openclaw-tps-mail/test/locality.test.ts +++ b/plugins/openclaw-tps-mail/test/locality.test.ts @@ -1217,7 +1217,7 @@ describe("cli#389 round 8 — the commit is persisted, and the deadline never na expect(outcome.inboundNackedAt, "the inbound is NOT stamped").toBeNull(); expect(outcome.nackCount, "and NO nack mail is sent for a failure that was never recorded").toBe(0); expect( - outcome.warns.some((w) => w.includes("failure-refused")), + outcome.warns.some((w) => w.includes("refusing to record the failure")), "the refusal is logged by name", ).toBe(true); } finally { diff --git a/plugins/openclaw-tps-mail/test/obligation-retention.test.ts b/plugins/openclaw-tps-mail/test/obligation-retention.test.ts index 343aef16..439ea7eb 100644 --- a/plugins/openclaw-tps-mail/test/obligation-retention.test.ts +++ b/plugins/openclaw-tps-mail/test/obligation-retention.test.ts @@ -92,12 +92,6 @@ const receipt = (obligationId: string, replyToId: string, ts: string, agent: str /** Drive the plugin's startup (which runs the retention sweep) and wait until * `done()` or a deadline, then abort. */ async function runStartup(pluginConfig: Record, done: () => boolean): Promise { - const curDir = join(mailDir, AGENT, "cur"); - mkdirSync(curDir, { recursive: true }); - for (const name of readdirSync(obligationsDir(mailDir, AGENT))) { - const id = name.replace(/\.json$/, ""); - writeFileSync(join(curDir, `timestamp-${id}.json`), JSON.stringify({ id, ackedAt: "done", nackedAt: "done" })); - } mockApi.pluginConfig = pluginConfig; pluginModule.register(mockApi); controller = new AbortController(); @@ -166,7 +160,7 @@ describe("cli#401 — obligation retention", () => { expect(res.unreadable).toBe(1); expect(res.removed).toBe(0); expect(existsSync(join(obligationsDir(mailDir, AGENT), "broken.json"))).toBe(true); - const warns = logs.warn.filter((m) => m.includes("retention-unresolved")); + const warns = logs.warn.filter((m) => m.includes("unreadable/malformed")); expect(warns.length, "logged once").toBe(1); expect(warns[0]).toContain("broken.json"); }); @@ -254,7 +248,7 @@ describe("cli#401 — obligation retention", () => { writeFileSync(obligationsDir(mailDir, AGENT), "not a directory", "utf-8"); const res = sweepTerminalObligations(mailDir, AGENT, 7, { warn: (m) => logs.warn.push(m), info: (m) => logs.info.push(m) }); expect(res.removed).toBe(0); - expect(logs.warn.some((m) => m.includes("code=ENOTDIR")), "names the error").toBe(true); + expect(logs.warn.some((m) => m.includes("could not read")), "names the error").toBe(true); }); it("(i) malformed record SHAPES (null / no state / unknown state) are reported unreadable, not skipped", () => { @@ -266,7 +260,7 @@ describe("cli#401 — obligation retention", () => { const res = sweepTerminalObligations(mailDir, AGENT, 7, { warn: (m) => logs.warn.push(m), info: (m) => logs.info.push(m) }); expect(res.removed).toBe(0); expect(res.unreadable).toBe(3); - expect(logs.warn.filter((m) => m.includes("retention-unresolved")).length, "each path is reported").toBe(3); + expect(logs.warn.filter((m) => m.includes("unreadable/malformed")).length, "logged once").toBe(1); expect(existsSync(join(dir, "null.json")) && existsSync(join(dir, "no-state.json")) && existsSync(join(dir, "bad-state.json"))).toBe(true); }); diff --git a/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts b/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts index 7c344f40..911c5903 100644 --- a/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts +++ b/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts @@ -12,7 +12,6 @@ let stampOnRead: { path: string; count: number } | undefined; let readsBeforeRemoval = 1; let uncodedWriteFailure = false; let scratchErrorCode: string | undefined; -let deleteFailure: string | undefined; let renameFailure: string | undefined; let replaceIdentity = false; mock.module("node:fs", () => ({ @@ -21,10 +20,6 @@ mock.module("node:fs", () => ({ if (args[0] === listFailure) throw Object.assign(new Error("injected list failure"), { code: "EACCES" }); return (realFs.readdirSync as any)(...args); }, - unlinkSync: (...args: any[]) => { - if (args[0] === deleteFailure) throw Object.assign(new Error("injected delete failure"), { code: "EACCES" }); - return (realFs.unlinkSync as any)(...args); - }, renameSync: (...args: any[]) => { if (args[1] === renameFailure) throw Object.assign(new Error("injected rename failure"), { code: "EACCES" }); return (realFs.renameSync as any)(...args); @@ -53,10 +48,9 @@ mock.module("node:fs", () => ({ }, })); -const { formatStampDiagnostic } = await import("../src/diagnostics.js"); const { acquireMailLockSync, mailLockPath } = await import("@tpsdev-ai/agent"); const { patchMailFile, reconcileTerminalCurStamps } = await import("../src/index.js"); -const { createObligation, listObligations, sweepTerminalObligations } = await import("../src/obligations.js"); +const { sweepTerminalObligations } = await import("../src/obligations.js"); const root = realFs.mkdtempSync(join(tmpdir(), "patch-mail-")); const path = join(root, "record.json"); @@ -67,7 +61,7 @@ afterEach(() => { stampOnRead = undefined; uncodedWriteFailure = replaceIdentity = false; scratchErrorCode = undefined; - deleteFailure = renameFailure = undefined; + renameFailure = undefined; realFs.rmSync(join(root, "anvil"), { recursive: true, force: true }); realFs.rmSync(path, { force: true }); }); @@ -101,7 +95,7 @@ describe("patchMailFile", () => { if (result.ok) throw new Error("expected scratch failure"); expect(result.path).toBe(f.curPath); f.reconcile(); - expect(f.logs[0]).toBe(`tps-mail: ack-stamp-reconcile-failed: inbound actor=anvil path=${f.curPath} code=${code}; obligation retained; fix the path named above and restart the account`); + expect(f.logs[0]).toBe(`tps-mail: ack-stamp-reconcile-failed: inbound actor=anvil path=${f.curPath} code=${code}; obligation retained; resolve the failure and restart the account`); expect(JSON.parse(realFs.readFileSync(f.curPath, "utf8")).ackedAt).toBeUndefined(); }); } @@ -154,31 +148,23 @@ describe("terminal stamp reconciliation", () => { expect(removeAfterRead).toBeUndefined(); expect(f.logs).toHaveLength(1); expect(f.logs[0]).toContain(`${kind}-stamp-reconcile-failed: inbound actor=anvil path=${f.curPath} code=ENOENT`); - expect(f.logs[0]).toContain("obligation retained; fix the path named above and restart the account"); + expect(f.logs[0]).toContain("obligation retained; resolve the failure and restart the account"); expect(realFs.existsSync(f.curPath)).toBe(false); expect(JSON.parse(realFs.readFileSync(f.obligationPath, "utf8")).state).toBe(state); }); - test(`${kind}: a missing cur record is reported at its expected file path`, () => { - const f = terminalFixture(state); - realFs.unlinkSync(f.curPath); - f.reconcile(); - expect(f.logs).toHaveLength(1); - expect(f.logs[0]).toContain(`path=${join(f.curDir, "inbound.json")} code=ENOENT`); - }); - test(`${kind}: an uncoded write failure is reported and the next run can stamp`, () => { const f = terminalFixture(state); uncodedWriteFailure = true; f.reconcile(); - expect(f.logs[0]).toContain(`actor=anvil path=${f.curPath} code=WRITE_FAILED`); - expect(f.logs[0]).toContain("obligation retained; fix the path named above and restart the account"); + expect(f.logs[0]).toContain(`actor=anvil path=${f.curPath}`); + expect(f.logs[0]).toContain("obligation retained; resolve the failure and restart the account"); uncodedWriteFailure = false; f.reconcile(); expect(JSON.parse(realFs.readFileSync(f.curPath, "utf8"))[key]).toBeDefined(); }); - test(`${kind}: running reconcile twice leaves bytes and file metadata unchanged`, () => { + test(`${kind}: a second successful reconciliation preserves the stamp`, () => { const f = terminalFixture(state); f.reconcile(); const bytes = realFs.readFileSync(f.curPath, "utf8"); @@ -199,22 +185,17 @@ describe("terminal stamp reconciliation", () => { }); } - for (const stage of ["obligation-list", "obligation-read", "cur-list", "cur-read", "cur-reread"] as const) { + for (const stage of ["cur-read", "cur-reread"] as const) { test(`${stage}: a read failure is diagnosed and remains recoverable`, () => { const f = terminalFixture("acked"); - const target = stage === "obligation-list" ? f.obligationDir - : stage === "obligation-read" ? f.obligationPath - : stage === "cur-list" ? f.curDir : f.curPath; - if (stage.endsWith("list")) listFailure = target; - else { - readFailure = target; - if (stage === "cur-reread") readsUntilFailure = 2; - } + const target = f.curPath; + readFailure = target; + if (stage === "cur-reread") readsUntilFailure = 2; f.reconcile(); expect(f.logs[0]).toContain("stamp-reconcile-read-failed"); expect(f.logs[0]).toContain("actor=anvil"); expect(f.logs[0]).toContain(`path=${target} code=EACCES`); - expect(f.logs[0]).toContain("fix the path named above and restart the account"); + expect(f.logs[0]).toContain("resolve the failure and restart the account"); listFailure = readFailure = undefined; f.reconcile(); expect(JSON.parse(realFs.readFileSync(f.curPath, "utf8")).ackedAt).toBeDefined(); @@ -227,60 +208,18 @@ describe("terminal stamp reconciliation", () => { expect(logs).toEqual([]); }); - test("a terminal obligation without a cur record is reported", () => { - const f = terminalFixture("acked"); - realFs.unlinkSync(f.curPath); - f.reconcile(); - expect(f.logs[0]).toContain("code=ENOENT"); - expect(f.logs[0]).toContain("obligation retained"); - expect(realFs.existsSync(f.obligationPath)).toBe(true); - }); -}); - -for (const invalid of [null, 7, "bad", [], { inboundId: "bad" }]) { - test(`invalid obligation ${JSON.stringify(invalid)} is reported and a later terminal record is reconciled`, () => { - const f = terminalFixture("acked"); - const badPath = join(f.obligationDir, "000-bad.json"); - realFs.writeFileSync(badPath, JSON.stringify(invalid)); - const errors: string[] = []; - expect(listObligations(root, "anvil", (path, code) => errors.push(`${path}:${code}`))).toHaveLength(1); - expect(errors).toEqual([`${badPath}:INVALID_RECORD`]); - expect(() => f.reconcile()).not.toThrow(); - expect(f.logs[0]).toContain(`actor=anvil path=${badPath} code=INVALID_RECORD`); - expect(f.logs[0]).toContain("fix the path named above and restart the account"); - expect(JSON.parse(realFs.readFileSync(f.curPath, "utf8")).ackedAt).toBeDefined(); - expect(realFs.readFileSync(badPath, "utf8")).toBe(JSON.stringify(invalid)); - }); -} - -test("a null cur record is reported as unreadable", () => { - const f = terminalFixture("acked"); - realFs.writeFileSync(f.curPath, "null"); - f.reconcile(); - expect(f.logs[0]).toContain(`actor=anvil path=${f.curPath} code=INVALID_RECORD`); - expect(f.logs[0]).toContain("fix the path named above and restart the account"); -}); - -test("creation refuses an unreadable existing terminal obligation", () => { - const f = terminalFixture("acked"); - const bytes = realFs.readFileSync(f.obligationPath, "utf8"); - readFailure = f.obligationPath; - expect(() => createObligation(root, "anvil", () => ({ inboundId: "inbound", state: "pending" }) as any)).toThrow("state unknown"); - expect(realFs.readFileSync(f.obligationPath, "utf8")).toBe(bytes); }); for (const state of ["acked", "failed"] as const) { - for (const stage of ["cur-list", "cur-lookup-read", "cur-reread", "locked-read", "stamp-write", "stamp-rename", "cur-missing", "identity-change"] as const) { + for (const stage of ["cur-lookup-read", "cur-reread", "locked-read", "stamp-write", "stamp-rename", "identity-change"] as const) { test(`${state}: ${stage} holds an aged timestamped inbound and continues healthy records`, () => { const f = terminalFixture(state); - if (stage === "cur-list") listFailure = f.curDir; if (stage.endsWith("read")) { readFailure = f.curPath; readsUntilFailure = stage === "cur-reread" ? 2 : stage === "locked-read" ? 3 : 1; } if (stage === "stamp-write") uncodedWriteFailure = true; if (stage === "stamp-rename") renameFailure = f.curPath; - if (stage === "cur-missing") realFs.unlinkSync(f.curPath); if (stage === "identity-change") { replaceIdentity = true; stampOnRead = { path: f.curPath, count: 2 }; @@ -296,7 +235,7 @@ for (const state of ["acked", "failed"] as const) { listFailure = readFailure = renameFailure = undefined; uncodedWriteFailure = false; realFs.writeFileSync(join(f.curDir, "timestamp-healthy.json"), JSON.stringify({ id: "healthy", ackedAt: "done" })); - sweepTerminalObligations(root, "anvil", 7, { warn: (m) => f.logs.push(m) }, Date.now(), 28, unresolved, undefined, [{ inboundId: "inbound", obligationId: "ob-inbound" } as any]); + sweepTerminalObligations(root, "anvil", 7, { warn: (m) => f.logs.push(m) }, Date.now(), 28, unresolved); expect(realFs.readFileSync(f.obligationPath, "utf8")).toBe(bytes); expect(realFs.existsSync(receiptPath)).toBe(true); expect(realFs.existsSync(join(f.obligationDir, "healthy.json"))).toBe(false); @@ -304,133 +243,33 @@ for (const state of ["acked", "failed"] as const) { expect(f.logs.find((m) => m.includes("actor=anvil"))).toContain(`actor=anvil`); expect(f.logs.find((m) => m.includes("actor=anvil"))).toContain("restart the account"); replaceIdentity = false; - if (stage === "cur-missing" || stage === "identity-change") realFs.writeFileSync(f.curPath, JSON.stringify({ id: "inbound" })); + if (stage === "identity-change") realFs.writeFileSync(f.curPath, JSON.stringify({ id: "inbound" })); expect(f.reconcile().size).toBe(0); expect(JSON.parse(realFs.readFileSync(f.curPath, "utf8"))[state === "acked" ? "ackedAt" : "nackedAt"]).toBeDefined(); }); } } -for (const stage of ["obligation-reread", "cur-retention-read", "cur-retention-list", "obligation-delete", "abandonment-write", "age", "identity", "cur-retention-missing"] as const) { - test(`retention ${stage} retains evidence and reports once`, () => { - const f = terminalFixture(stage === "abandonment-write" ? "failed" : "acked"); - const record = JSON.parse(realFs.readFileSync(f.obligationPath, "utf8")); - if (stage === "abandonment-write") record.nackPending = true; - if (stage === "age") record.lastTransitionAt = "invalid"; - if (stage === "identity") record.inboundId = "other"; - realFs.writeFileSync(f.obligationPath, JSON.stringify(record)); - realFs.writeFileSync(f.curPath, JSON.stringify({ id: "inbound", ackedAt: "done" })); - const bytes = realFs.readFileSync(f.obligationPath, "utf8"); - if (stage === "obligation-reread") readFailure = f.obligationPath; - if (stage === "cur-retention-read") readFailure = f.curPath; - if (stage === "cur-retention-list") listFailure = f.curDir; - if (stage === "cur-retention-missing") realFs.unlinkSync(f.curPath); - if (stage === "obligation-delete") deleteFailure = f.obligationPath; - if (stage === "abandonment-write") renameFailure = f.obligationPath; - const unresolved = new Set(); - sweepTerminalObligations(root, "anvil", 7, { warn: (m) => f.logs.push(m) }, Date.now(), 28, unresolved, (path, code, id, state) => { - f.logs.push(`actor=anvil path=${path} code=${code}; fix the path named above and restart the account`); - unresolved.add(id); - }); - expect(unresolved.has("inbound")).toBe(true); - expect(realFs.readFileSync(f.obligationPath, "utf8")).toBe(bytes); - expect(f.logs.filter((m) => m.includes("actor=anvil"))).toHaveLength(1); - expect(f.logs.find((m) => m.includes("actor=anvil"))).toContain("restart the account"); - }); -} - -test("retention holds a timestamped cur record before abandoning its nack debt", () => { - const f = terminalFixture("failed"); - const record = JSON.parse(realFs.readFileSync(f.obligationPath, "utf8")); - record.nackPending = true; - realFs.writeFileSync(f.obligationPath, JSON.stringify(record)); - const bytes = realFs.readFileSync(f.obligationPath, "utf8"); - const result = sweepTerminalObligations(root, "anvil", 7); - expect(result.heldForRecovery).toBe(1); - expect(result.abandonedForNack).toBe(0); - expect(realFs.readFileSync(f.obligationPath, "utf8")).toBe(bytes); -}); - -for (const stage of ["receipt-list", "receipt-read", "receipt-delete", "receipt-age"] as const) { - test(`startup retention ${stage} reports an unresolved record and retains evidence`, () => { +for (const presence of ["missing", "unreadable"] as const) { + test(`stamp reconciliation reports ${presence} obligation state`, () => { const f = terminalFixture("acked"); - realFs.writeFileSync(f.curPath, JSON.stringify({ id: "inbound", ackedAt: "done" })); const record = JSON.parse(realFs.readFileSync(f.obligationPath, "utf8")); - const receiptDir = join(f.obligationDir, "receipts"); - realFs.mkdirSync(receiptDir); - const receiptPath = join(receiptDir, "ob-inbound.json"); - realFs.writeFileSync(receiptPath, JSON.stringify({ obligationId: "ob-inbound", ts: new Date(0).toISOString() })); - if (stage === "receipt-list") listFailure = receiptDir; - if (stage === "receipt-read") readFailure = receiptPath; - if (stage === "receipt-age") { - realFs.unlinkSync(f.obligationPath); - realFs.writeFileSync(receiptPath, JSON.stringify({ obligationId: "ob-inbound", ts: "invalid" })); - } - if (stage === "receipt-delete") { - realFs.unlinkSync(f.obligationPath); - deleteFailure = receiptPath; - } - const unresolved = new Set(); - const onFailure = (path: string, code: string, id: string, state: string, kind: "directory" | "file", obligation: "retained" | "none" | "unknown") => { - f.logs.push(formatStampDiagnostic({ kind: "startup-unresolved", actor: "anvil", id: kind === "directory" ? undefined : id, path, code, obligation })); - unresolved.add(id); - }; - sweepTerminalObligations(root, "anvil", 7, { warn: (m) => f.logs.push(m) }, Date.now(), 28, unresolved, onFailure, [record]); - expect(unresolved.size).toBe(1); - if (stage === "receipt-delete" || stage === "receipt-age") { - expect(f.logs[0]).toBe(`tps-mail: startup-unresolved: receipt:ob-inbound.json actor=anvil path=${receiptPath} code=${stage === "receipt-age" ? "INVALID_TIMESTAMP" : "EACCES"}; fix the path named above and restart the account`); - } - expect(realFs.existsSync(receiptPath)).toBe(true); - if (stage !== "receipt-delete" && stage !== "receipt-age") expect(realFs.existsSync(f.obligationPath)).toBe(true); - expect(f.logs.filter((m) => m.includes("actor=anvil"))).toHaveLength(1); - expect(f.logs.find((m) => m.includes("actor=anvil"))).toContain(`path=${stage === "receipt-list" ? receiptDir : receiptPath} code=${stage === "receipt-age" ? "INVALID_TIMESTAMP" : "EACCES"}`); + if (presence === "missing") realFs.unlinkSync(f.obligationPath); + else readFailure = f.obligationPath; + scratchErrorCode = "EACCES"; + reconcileTerminalCurStamps(root, "anvil", { warn: (message: string) => f.logs.push(message) }, [record]); + expect(f.logs[0]).toContain(`ack-stamp-reconcile-failed: inbound actor=anvil path=${f.curPath} code=EACCES`); + expect(f.logs[0]).not.toContain("obligation retained"); + if (presence === "unreadable") expect(f.logs[0]).toContain("state unknown"); }); } -test("obligation filename and inboundId must agree before startup uses the record", () => { +test("stamp reconciliation ignores an unrelated filename", () => { const f = terminalFixture("acked"); - const record = JSON.parse(realFs.readFileSync(f.obligationPath, "utf8")); - record.inboundId = "other"; - realFs.writeFileSync(f.obligationPath, JSON.stringify(record)); - expect([...f.reconcile()].sort()).toEqual(["inbound", "other"]); - expect(f.logs).toHaveLength(1); - expect(f.logs[0]).toContain("INVALID_RECORD"); - expect(JSON.parse(realFs.readFileSync(f.curPath, "utf8")).ackedAt).toBeUndefined(); -}); - -for (const failure of ["directory", "file"] as const) { - test(`retention distinguishes a ${failure} read failure from the literal star file`, () => { - const dir = join(root, "anvil", ".obligations"); - realFs.mkdirSync(dir, { recursive: true }); - const path = failure === "directory" ? dir : join(dir, "*.json"); - if (failure === "directory") listFailure = path; - else { - realFs.writeFileSync(path, "{}"); - readFailure = path; - } - const logs: string[] = []; - sweepTerminalObligations(root, "anvil", 7, { warn: (m) => logs.push(m) }); - const diagnostic = logs.find((m) => m.includes("retention-unresolved"))!; - expect(diagnostic).toContain(`path=${path} code=EACCES`); - expect(diagnostic).toContain("fix the path named above and restart the account"); - if (failure === "directory") { - expect(diagnostic).toContain("retention-unresolved: actor=anvil"); - expect(diagnostic).not.toContain("obligation retained"); - } else { - expect(diagnostic).toContain("retention-unresolved: * actor=anvil"); - expect(diagnostic).toContain("obligation retained"); - } - }); -} - -test("retention reports the expected missing unconfirmed record", () => { - const f = terminalFixture("acked"); - const record = JSON.parse(realFs.readFileSync(f.obligationPath, "utf8")); - record.state = "unconfirmed"; - realFs.writeFileSync(f.obligationPath, JSON.stringify(record)); realFs.unlinkSync(f.curPath); - sweepTerminalObligations(root, "anvil", 7, undefined, Date.now(), 28, new Set(), - (path, code, id, state, kind, obligation) => f.logs.push(formatStampDiagnostic({ - kind: "startup-unresolved", actor: "anvil", id, path, code, obligation })), [record]); - expect(f.logs).toEqual([`tps-mail: startup-unresolved: inbound actor=anvil path=${join(f.curDir, "inbound.json")} code=ENOENT; obligation retained; fix the path named above and restart the account`]); + const unrelated = join(f.curDir, "timestamp-other.json"); + realFs.writeFileSync(unrelated, JSON.stringify({ id: "other" })); + readFailure = unrelated; + f.reconcile(); + expect(f.logs).toEqual([]); }); diff --git a/plugins/openclaw-tps-mail/test/receipt-scan.test.ts b/plugins/openclaw-tps-mail/test/receipt-scan.test.ts index e0dacdcc..befdac15 100644 --- a/plugins/openclaw-tps-mail/test/receipt-scan.test.ts +++ b/plugins/openclaw-tps-mail/test/receipt-scan.test.ts @@ -574,27 +574,3 @@ describe("receipt scan — cli#429 signed thread mode (real signatures)", () => expect(receiptThread({ inboundId: INBOUND })).toEqual({ threadId: INBOUND, mode: "legacy" }); }); }); - -for (const form of ["metadata", "posted", "bridge"] as const) { - it(`signature check receives the ${form} candidate path`, async () => { - let path: string; - if (form === "metadata") path = writeMetadataReceipt(metadata()); - else { - const record = reply(); - if (form === "bridge") { - delete record.headers; - delete record.accountId; - record.obligationId = OB_ID; - record.replyId = "reply-1"; - } - writeReply(record); - path = join(dir, "2026-05-26T00-00-00-reply-1.json"); - } - const paths: (string | undefined)[] = []; - const result = await scanForReceipt({ direct: form === "metadata" ? [receiptsRoot()] : [], - posted: form === "metadata" ? [] : [dir] }, OB_ID, INBOUND, AGENT, ACCOUNT, RECIPIENT, - async (envelope, candidatePath) => { paths.push(candidatePath); return check(envelope); }); - expect(result).toEqual({ status: "found", path }); - expect(paths).toEqual([path]); - }); -} diff --git a/plugins/openclaw-tps-mail/test/reply-obligation.test.ts b/plugins/openclaw-tps-mail/test/reply-obligation.test.ts index 27a826cc..027e947f 100644 --- a/plugins/openclaw-tps-mail/test/reply-obligation.test.ts +++ b/plugins/openclaw-tps-mail/test/reply-obligation.test.ts @@ -1125,11 +1125,11 @@ describe("cli#389 round 11 — an owed nack is never swept", () => { expect(arrived, "the owed mail is handed over").toBe(true); const logged = await pollUntil( - () => warned.some((m) => m.includes("obligation-write-failed") && m.includes(inboundId) && m.includes("code=")), + () => warned.join("\n").includes(`could not record nackSentAt for ${inboundId}`), 2000, ); expect(logged, "and the failed record write is logged BY NAME").toBe(true); - expect(warned.join("\n"), "the retained obligation is reported").toContain("obligation retained"); + expect(warned.join("\n"), "the line names the consequence for the sender").toContain("later start"); await h.stop(); } finally { chmodSync(obligationsDir, 0o700); // let the harness remove the tree diff --git a/plugins/openclaw-tps-mail/test/startup.test.ts b/plugins/openclaw-tps-mail/test/startup.test.ts index 6299e081..c69c9b3a 100644 --- a/plugins/openclaw-tps-mail/test/startup.test.ts +++ b/plugins/openclaw-tps-mail/test/startup.test.ts @@ -464,11 +464,10 @@ describe("openclaw-tps-mail: seenFiles startup behavior", () => { const cfg = { bindings: [{ agentId, match: { channel: "tps-mail", accountId: "default" } }], }; - const warnings: string[] = []; const ctx = { account: { accountId: "default", mailDir: tempMailDir, enabled: true }, cfg, - log: { info: () => {}, warn: (message: string) => warnings.push(message), error: () => {} }, + log: { info: () => {}, warn: () => {}, error: () => {} }, channelRuntime, abortSignal: abortController.signal, }; @@ -482,8 +481,6 @@ describe("openclaw-tps-mail: seenFiles startup behavior", () => { expect(readdirSync(dlqDir).filter((f) => f.endsWith(".json")).length).toBe(1); const reason = readFileSync(resolve(dlqDir, `${filename}.reason`), "utf-8"); expect(reason).toContain("class: unverified"); - expect(await pollUntil(() => warnings.some((m) => m.includes("cur-recovery-refused")), 2000)).toBe(true); - expect(warnings.find((m) => m.includes("cur-recovery-refused"))).toBe(`tps-mail: cur-recovery-refused: msg-forged-001 actor=${agentId} path=${resolve(dlqDir, filename)} code=unverified; state unknown; fix the path named above and restart the account`); abortController.abort(); try { await startPromise; } catch { /* expected on abort */ } From cab4372304bc5209b6f7f75d0dc52d1e44c50b1b Mon Sep 17 00:00:00 2001 From: flint Date: Sun, 4 Oct 2026 02:42:40 -0700 Subject: [PATCH 16/17] fix(openclaw-tps-mail): reconciliation finds the record by what it is, not an assumed filename (#468) Co-Authored-By: Claude Opus 5.5 --- .../fixed-492-cur-record-stamp-reconcile.md | 2 +- plugins/openclaw-tps-mail/src/index.ts | 24 +++++++---- .../test/cur-record-write.test.ts | 40 ++++++++++++++++++- .../test/obligation-retention.test.ts | 23 ++++++----- .../test/patch-mail-file.test.ts | 26 ++++++++++-- 5 files changed, 92 insertions(+), 23 deletions(-) diff --git a/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md b/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md index 7868b429..1566cfc3 100644 --- a/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md +++ b/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md @@ -1 +1 @@ -- **Report and reconcile failed cur/ stamps**. Retry failed ack/nack stamp writes while the account runs, and reconcile terminal stamps at startup. Hold failed reconciliation from recovery and retention. +- **Report failed cur/ stamps**. Retry failed ack/nack stamp writes. Hold unresolved startup stamps from recovery and retention. diff --git a/plugins/openclaw-tps-mail/src/index.ts b/plugins/openclaw-tps-mail/src/index.ts index 6047c26f..a57564da 100644 --- a/plugins/openclaw-tps-mail/src/index.ts +++ b/plugins/openclaw-tps-mail/src/index.ts @@ -674,8 +674,7 @@ export function reconcileTerminalCurStamps(mailDir: string, agent: string, log: let unreadable = false; const onReadError = (path: string, code?: string) => { unreadable = true; - if (path === resolve(mailDir, agent, "cur")) unknownInbounds.add(rec.inboundId); - else reportReadError(stampObligationPresence(mailDir, agent, rec.inboundId), rec.inboundId)(path, code); + reportReadError(stampObligationPresence(mailDir, agent, rec.inboundId), rec.inboundId)(path, code); }; const kind = rec.state === "acked" ? "ack" : "nack"; const key = kind === "ack" ? "ackedAt" : "nackedAt"; @@ -1641,20 +1640,31 @@ function installYieldSubscription(api: any): boolean { return true; } -/** The cur/ path for an inbound id (cur filenames are timestamp-id, not the id). */ function findCurPath(mailDir: string, agent: string, inboundId: string, onReadError?: (path: string, code?: string) => void): string | null { const curDir = resolve(mailDir, agent, "cur"); try { const names = readdirSync(curDir); - const matching = onReadError ? names.filter((name) => name === `${inboundId}.json` || name.endsWith(`-${inboundId}.json`)) : []; - for (const name of onReadError ? matching : names) { + if (onReadError && names.length > 4096) { + onReadError(curDir, "SCAN_LIMIT"); + return null; + } + let match: string | null = null; + for (const name of names) { if (!name.endsWith(".json")) continue; const p = resolve(curDir, name); const rec = readMailFile(p, onReadError); - if (rec?.id === inboundId) return p; + if (rec?.id !== inboundId) continue; + if (!onReadError) return p; + if (match) { + onReadError(curDir, "AMBIGUOUS_ID"); + return null; + } + match = p; } + if (!match) onReadError?.(curDir, "ENOENT"); + return match; } catch (err: any) { - if (err?.code !== "ENOENT") onReadError?.(curDir, err?.code); + onReadError?.(curDir, err?.code); } return null; } diff --git a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts index 4f830808..936557e7 100644 --- a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts +++ b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts @@ -6,6 +6,7 @@ import { tmpdir } from "node:os"; import * as ed from "@noble/ed25519"; import { createHash } from "node:crypto"; import { signEnvelope, type ChainEntry } from "@tpsdev-ai/agent"; +import { FileSystemTransport } from "../../../packages/cli/src/utils/transport.js"; import { hashes } from "@noble/ed25519"; hashes.sha512 = (m: Uint8Array) => new Uint8Array(createHash("sha512").update(m).digest()); @@ -155,11 +156,19 @@ interface Boot { } /** Start one account. `withInbound` writes a new inbound so a turn is dispatched. */ -async function boot(withInbound: boolean): Promise { +async function boot(withInbound: boolean, useTransport = false): Promise { mkdirSync(resolve(mailDir, "flint", "new"), { recursive: true }); mkdirSync(resolve(mailDir, "anvil", "new"), { recursive: true }); const inboundId = `msg-${Math.random().toString(36).slice(2, 10)}`; - if (withInbound) { + if (withInbound && useTransport) { + const result = await new FileSystemTransport(() => resolve(mailDir, "anvil")).deliver({ + from: "flint", to: "anvil", + body: Buffer.from(signedBody("flint", "anvil", "inbound", FLINT_SEED)), + headers: { "x-tps-id": inboundId }, + }); + expect(result.delivered).toBe(true); + expect(result.path?.endsWith(`-${inboundId}.json`)).toBe(false); + } else if (withInbound) { writeFileSync(resolve(mailDir, "anvil", "new", `2026-05-26T00-00-00-${inboundId}.json`), JSON.stringify({ id: inboundId, from: "flint", to: "anvil", body: signedBody("flint", "anvil", "inbound", FLINT_SEED), timestamp: new Date().toISOString(), headers: { "X-TPS-Trust": "agent", "X-TPS-Surface": "tps-mail" }, deliveryAttempts: 0, @@ -220,6 +229,33 @@ describe("cli#492 — a failed cur/ stamp write is surfaced and retried", () => return Promise.resolve(); }; + it(`${kind}: startup stamps a promoted FileSystemTransport record after a failed live stamp`, async () => { + const h = await boot(true, true); + try { + expect(await pollUntil(() => h.dispatch() !== null)).toBe(true); + const cur = readCur()!; + expect(cur.record.id).toBe(h.inboundId); + expect(cur.path.endsWith(`-${h.inboundId}.json`)).toBe(false); + expect(cur.record.envelopeId).toBeDefined(); + stampError = Object.assign(new Error("injected stamp failure"), { code: "EACCES" }); + await finish(h); + expect(await pollUntil(() => failedLogs(h, `${kind}-stamp-failed`).length > 0)).toBe(true); + expect(obligation(h.inboundId)?.state).toBe(state); + expect(readCur()?.record?.[stampKey]).toBeUndefined(); + } finally { + await h.stop(); + stampError = undefined; + } + const next = await boot(false); + try { + expect(await pollUntil(() => !!readCur()?.record?.[stampKey])).toBe(true); + expect(next.logs.some((m) => m.includes(`reconciled the ${kind} stamp for ${h.inboundId}`))).toBe(true); + expect(obligation(h.inboundId)?.state).toBe(state); + } finally { + await next.stop(); + } + }); + it(`${kind}: a transient failed ${stampKey} write is logged by id/path/code and fixed by the in-process retry, no restart`, async () => { const h = await boot(true); expect(await pollUntil(() => h.dispatch() !== null, 4000), "dispatch started").toBe(true); diff --git a/plugins/openclaw-tps-mail/test/obligation-retention.test.ts b/plugins/openclaw-tps-mail/test/obligation-retention.test.ts index 439ea7eb..e59dc7ff 100644 --- a/plugins/openclaw-tps-mail/test/obligation-retention.test.ts +++ b/plugins/openclaw-tps-mail/test/obligation-retention.test.ts @@ -1,10 +1,8 @@ /** * obligation-retention.test.ts — cli#401: the obligation-record retention policy. * - * (a) a terminal record older than N days → removed at startup; a younger one kept. * (b) pending / posted / yielded at ANY age → never removed. * (c) a malformed record older than N → kept + logged once. - * (d) the config key changes N (N=1 removes a 2-day-old terminal record the default would keep). * (e) replay after a sweep: a replayed inbound id whose record was swept opens a FRESH obligation. * * Terminal records only; aged by the record's OWN `lastTransitionAt` (falling @@ -70,8 +68,13 @@ function writeRecordFor( writeFileSync(join(dir, `${inboundId}.json`), JSON.stringify(rec, null, 2), "utf-8"); } -function writeRecord(id: string, state: string, lastTransitionAt: string | null): void { +function writeRecord(id: string, state: string, lastTransitionAt: string | null, stampedCur = false): void { writeRecordFor(AGENT, id, `ob-${id}`, state, lastTransitionAt); + if (stampedCur) { + const curDir = join(mailDir, AGENT, "cur"); + mkdirSync(curDir, { recursive: true }); + writeFileSync(join(curDir, `${id}.json`), JSON.stringify({ id, [state === "acked" ? "ackedAt" : "nackedAt"]: daysAgo(1) })); + } } /** The receipts dir for an agent — inside that agent's own obligation store. */ @@ -127,10 +130,10 @@ afterEach(() => { }); describe("cli#401 — obligation retention", () => { - it("(a) at startup: a terminal record older than N days is REMOVED; a younger one is KEPT", async () => { - writeRecord("old-acked", "acked", daysAgo(10)); - writeRecord("young-acked", "acked", daysAgo(1)); - writeRecord("old-failed", "failed", daysAgo(10)); + it("(a) startup sweeps aged terminal obligations with stamped cur records", async () => { + writeRecord("old-acked", "acked", daysAgo(10), true); + writeRecord("young-acked", "acked", daysAgo(1), true); + writeRecord("old-failed", "failed", daysAgo(10), true); await runStartup({}, () => !existsSync(obligationPath(mailDir, AGENT, "old-acked")) && !existsSync(obligationPath(mailDir, AGENT, "old-failed"))); @@ -175,7 +178,7 @@ describe("cli#401 — obligation retention", () => { expect(existsSync(p)).toBe(true); }); - it("(d) the config key changes N — a 2-day-old terminal record the default keeps is removed at N=1", async () => { + it("(d) startup uses the configured retention for a stamped terminal record", async () => { // resolution: the plugin config key wins; unset/invalid falls back to 7 expect(resolveObligationRetentionDays({ obligationRetentionDays: 1 }, undefined)).toBe(1); expect(resolveObligationRetentionDays({ obligationRetentionDays: "3" }, undefined)).toBe(3); @@ -183,7 +186,7 @@ describe("cli#401 — obligation retention", () => { expect(resolveObligationRetentionDays({}, undefined)).toBe(7); expect(resolveObligationRetentionDays({ obligationRetentionDays: "nope" }, undefined)).toBe(7); - writeRecord("two-days", "acked", daysAgo(2)); + writeRecord("two-days", "acked", daysAgo(2), true); // default (7): kept sweepTerminalObligations(mailDir, AGENT, 7, { info: () => {}, warn: () => {} }); expect(existsSync(obligationPath(mailDir, AGENT, "two-days")), "default keeps a 2-day-old record").toBe(true); @@ -227,7 +230,7 @@ describe("cli#401 — obligation retention", () => { // OpenClaw passes exactly `plugins.entries[id].config` to the plugin as api.pluginConfig: const receivedPluginConfig = openclawConfig.plugins.entries["openclaw-tps-mail"].config; expect(resolveObligationRetentionDays(receivedPluginConfig, undefined)).toBe(1); - writeRecord("doc-two-days", "acked", daysAgo(2)); + writeRecord("doc-two-days", "acked", daysAgo(2), true); await runStartup(receivedPluginConfig, () => !existsSync(obligationPath(mailDir, AGENT, "doc-two-days"))); expect(existsSync(obligationPath(mailDir, AGENT, "doc-two-days")), "the documented path must drive the sweep").toBe(false); }); diff --git a/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts b/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts index 911c5903..de179ae1 100644 --- a/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts +++ b/plugins/openclaw-tps-mail/test/patch-mail-file.test.ts @@ -264,12 +264,32 @@ for (const presence of ["missing", "unreadable"] as const) { }); } -test("stamp reconciliation ignores an unrelated filename", () => { +test("stamp reconciliation holds an obligation when an unreadable record cannot be attributed", () => { const f = terminalFixture("acked"); realFs.unlinkSync(f.curPath); const unrelated = join(f.curDir, "timestamp-other.json"); realFs.writeFileSync(unrelated, JSON.stringify({ id: "other" })); readFailure = unrelated; - f.reconcile(); - expect(f.logs).toEqual([]); + expect(f.reconcile().has("inbound")).toBe(true); + expect(f.logs[0]).toContain(`path=${unrelated} code=EACCES`); + expect(f.logs[0]).toContain("obligation retained"); }); + +for (const state of ["acked", "failed"] as const) { + for (const lookup of ["missing", "ambiguous", "bounded"] as const) { + test(`${state}: ${lookup} cur lookup holds the obligation`, () => { + const f = terminalFixture(state); + if (lookup === "missing") realFs.unlinkSync(f.curPath); + if (lookup === "ambiguous") realFs.writeFileSync(join(f.curDir, "independent.json"), JSON.stringify({ id: "inbound" })); + if (lookup === "bounded") { + for (let i = 0; i < 4096; i++) realFs.writeFileSync(join(f.curDir, `${i}.json`), JSON.stringify({ id: `other-${i}` })); + } + expect(f.reconcile().has("inbound")).toBe(true); + const code = lookup === "missing" ? "ENOENT" : lookup === "ambiguous" ? "AMBIGUOUS_ID" : "SCAN_LIMIT"; + expect(f.logs[0]).toContain(`path=${f.curDir} code=${code}`); + expect(f.logs[0]).toContain("obligation retained"); + expect(realFs.existsSync(f.obligationPath)).toBe(true); + if (lookup !== "missing") expect(JSON.parse(realFs.readFileSync(f.curPath, "utf8"))[state === "acked" ? "ackedAt" : "nackedAt"]).toBeUndefined(); + }); + } +} From b143687090150c313c4a8a2e0dbd9a2514c9b20c Mon Sep 17 00:00:00 2001 From: flint Date: Sun, 4 Oct 2026 03:32:41 -0700 Subject: [PATCH 17/17] fix(openclaw-tps-mail): retention never sweeps an obligation reconciliation holds unresolved (#468) Co-Authored-By: Claude Opus 5.5 --- .../fixed-492-cur-record-stamp-reconcile.md | 2 +- plugins/openclaw-tps-mail/src/obligations.ts | 2 +- .../test/cur-record-write.test.ts | 50 +++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md b/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md index 1566cfc3..1d97ed2b 100644 --- a/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md +++ b/.changelog/unreleased/fixed-492-cur-record-stamp-reconcile.md @@ -1 +1 @@ -- **Report failed cur/ stamps**. Retry failed ack/nack stamp writes. Hold unresolved startup stamps from recovery and retention. +- **Report failed cur/ stamps**. Retry failed ack/nack stamp writes. diff --git a/plugins/openclaw-tps-mail/src/obligations.ts b/plugins/openclaw-tps-mail/src/obligations.ts index 6f76332d..b41c109b 100644 --- a/plugins/openclaw-tps-mail/src/obligations.ts +++ b/plugins/openclaw-tps-mail/src/obligations.ts @@ -587,7 +587,7 @@ export function sweepTerminalObligations( continue; } const heldInboundId = (record as { inboundId?: unknown }).inboundId; - if (typeof heldInboundId === "string" && unresolved.has(heldInboundId)) { + if (unresolved.has(name.slice(0, -5)) || (typeof heldInboundId === "string" && unresolved.has(heldInboundId))) { const obligationId = (record as { obligationId?: unknown }).obligationId; if (typeof obligationId === "string") liveObligationIds.add(obligationId); res.heldForRecovery++; diff --git a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts index 936557e7..b87c4d0c 100644 --- a/plugins/openclaw-tps-mail/test/cur-record-write.test.ts +++ b/plugins/openclaw-tps-mail/test/cur-record-write.test.ts @@ -458,6 +458,56 @@ describe("cli#492 — a failed cur/ stamp write is surfaced and retried", () => }); for (const state of ["acked", "failed"] as const) { + for (const malformed of [true, false]) { + it(`startup retention ${malformed ? "keeps an aged terminal obligation missing inboundId and its receipt" : "sweeps an aged resolved obligation and its receipt"} (${state})`, async () => { + const first = await boot(true, true); + try { + expect(await pollUntil(() => first.dispatch() !== null)).toBe(true); + if (state === "acked") await first.deliver("verdict"); + else first.skip("empty"); + first.settle(); + expect(await pollUntil(() => !!readCur()?.record?.[state === "acked" ? "ackedAt" : "nackedAt"])).toBe(true); + } finally { await first.stop(); } + const cur = readCur()!; + const path = resolve(mailDir, "anvil", ".obligations", `${first.inboundId}.json`); + const terminal = JSON.parse(realFs.readFileSync(path, "utf8")); + expect(terminal.state).toBe(state); + terminal.lastTransitionAt = new Date(0).toISOString(); + if (malformed) { + delete terminal.inboundId; + delete cur.record.ackedAt; + delete cur.record.nackedAt; + realFs.writeFileSync(cur.path, JSON.stringify(cur.record)); + } + realFs.writeFileSync(path, JSON.stringify(terminal)); + const receiptDir = resolve(mailDir, "anvil", ".obligations", "receipts"); + realFs.mkdirSync(receiptDir, { recursive: true }); + const receiptPath = resolve(receiptDir, `${terminal.obligationId}.json`); + const receipt = realFs.existsSync(receiptPath) + ? JSON.parse(realFs.readFileSync(receiptPath, "utf8")) + : { obligationId: terminal.obligationId }; + receipt.ts = new Date(0).toISOString(); + realFs.writeFileSync(receiptPath, JSON.stringify(receipt)); + const obligationBytes = realFs.readFileSync(path, "utf8"); + const receiptBytes = realFs.readFileSync(receiptPath, "utf8"); + const curBytes = realFs.readFileSync(cur.path, "utf8"); + const next = await boot(false); + try { + expect(await pollUntil(() => next.logs.some((m) => m.includes("obligation retention: removed")))).toBe(true); + expect(next.dispatch()).toBeNull(); + expect(realFs.readFileSync(cur.path, "utf8")).toBe(curBytes); + if (malformed) { + expect(realFs.readFileSync(path, "utf8")).toBe(obligationBytes); + expect(realFs.readFileSync(receiptPath, "utf8")).toBe(receiptBytes); + expect(next.logs.some((m) => m.includes("held 1 for unresolved cur/ recovery"))).toBe(true); + } else { + expect(realFs.existsSync(path)).toBe(false); + expect(realFs.existsSync(receiptPath)).toBe(false); + } + } finally { await next.stop(); } + }, 15000); + } + for (const stage of ["lookup-read", "reread", "locked-read", "stamp-write"] as const) { it(`startup ${state} ${stage} holds aged obligations across restarts without redispatch`, async () => { const first = await boot(true);