From 625142becd285b8c77c3cd0ea93b89693da863b6 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Sat, 7 Nov 2026 16:15:15 -0800 Subject: [PATCH 01/75] feat(sdk): replace broker transport with node spawn --- docs/AGENT-RELAY-TRANSPORT.md | 121 ++-- docs/MIGRATION-NODE-SPAWN.md | 46 ++ docs/SURFACE.md | 72 +-- docs/agent-communication.md | 122 +--- examples/issue-swarm/README.md | 29 +- packages/sdk/bun.lock | 41 +- packages/sdk/package-lock.json | 40 +- packages/sdk/package.json | 7 +- packages/sdk/src/agent-relay-receipt.ts | 123 +--- packages/sdk/src/agent-relay-transport.ts | 403 +++++------- packages/sdk/src/authored-broker.ts | 80 +-- .../sdk/src/communication/broker-preflight.ts | 26 +- .../sdk/src/communication/broker-worker.ts | 198 ------ packages/sdk/src/communication/environment.ts | 2 +- packages/sdk/src/communication/preflight.ts | 30 +- packages/sdk/src/communication/pty.ts | 51 -- packages/sdk/src/communication/relay.ts | 122 +--- packages/sdk/src/communication/worker.ts | 114 +--- packages/sdk/src/index.ts | 17 + packages/sdk/src/node-spawn-local.ts | 222 +++++++ packages/sdk/src/node-spawn-remote.ts | 171 +++++ packages/sdk/src/node-spawn-result.ts | 114 ++++ packages/sdk/src/node-spawn-runtime.ts | 67 ++ packages/sdk/src/node-spawn-state.ts | 67 ++ packages/sdk/src/node-spawn.ts | 92 +++ packages/sdk/src/relayflowd-path.ts | 2 +- packages/sdk/src/worker-cli.ts | 15 +- packages/sdk/src/worker.ts | 37 +- .../sdk/tests/agent-relay-hardening.test.ts | 123 ++-- .../sdk/tests/agent-relay-transport.test.ts | 594 ++++-------------- packages/sdk/tests/authored-broker.test.ts | 77 --- packages/sdk/tests/broker-environment.test.ts | 25 - packages/sdk/tests/broker-flow.test.ts | 134 ---- packages/sdk/tests/broker-preflight.test.ts | 50 +- packages/sdk/tests/broker-retirement.test.ts | 31 + packages/sdk/tests/broker-worker.test.ts | 226 ------- .../tests/check-helper-communication.test.ts | 2 +- ...ommunication-environment-preflight.test.ts | 19 +- ...mmunication-feature-secret-scoping.test.ts | 72 --- .../tests/communication-mixed-resume.test.ts | 57 -- .../sdk/tests/communication-refusal.test.ts | 6 +- .../sdk/tests/communication-review.test.ts | 2 +- .../sdk/tests/communication-worker.test.ts | 135 ---- packages/sdk/tests/node-spawn-ports.test.ts | 72 +++ packages/sdk/tsconfig.tests.json | 1 + packages/surface/AUTHORING.md | 16 +- packages/surface/src/context.ts | 16 +- 47 files changed, 1540 insertions(+), 2549 deletions(-) create mode 100644 docs/MIGRATION-NODE-SPAWN.md delete mode 100644 packages/sdk/src/communication/broker-worker.ts delete mode 100644 packages/sdk/src/communication/pty.ts create mode 100644 packages/sdk/src/node-spawn-local.ts create mode 100644 packages/sdk/src/node-spawn-remote.ts create mode 100644 packages/sdk/src/node-spawn-result.ts create mode 100644 packages/sdk/src/node-spawn-runtime.ts create mode 100644 packages/sdk/src/node-spawn-state.ts create mode 100644 packages/sdk/src/node-spawn.ts delete mode 100644 packages/sdk/tests/authored-broker.test.ts delete mode 100644 packages/sdk/tests/broker-environment.test.ts delete mode 100644 packages/sdk/tests/broker-flow.test.ts create mode 100644 packages/sdk/tests/broker-retirement.test.ts delete mode 100644 packages/sdk/tests/broker-worker.test.ts delete mode 100644 packages/sdk/tests/communication-feature-secret-scoping.test.ts delete mode 100644 packages/sdk/tests/communication-mixed-resume.test.ts delete mode 100644 packages/sdk/tests/communication-worker.test.ts create mode 100644 packages/sdk/tests/node-spawn-ports.test.ts diff --git a/docs/AGENT-RELAY-TRANSPORT.md b/docs/AGENT-RELAY-TRANSPORT.md index 873929b63..759bd626d 100644 --- a/docs/AGENT-RELAY-TRANSPORT.md +++ b/docs/AGENT-RELAY-TRANSPORT.md @@ -1,58 +1,63 @@ -# Relay agent task transport - -An agent step with `transport: 'relay'` waits for a durable final task receipt. -Spawn readiness and invocation acceptance do not complete the Flows step. - -This adapter requires the task contract in AgentWorkforce/relaycast#436 -(`c80202f2dfe7a15a6cef5b81f2452b1d7acd6add`) and its provider in -AgentWorkforce/relay#1772 (`6a06fd176df94ca1054d5b7965bd777b7d881eb6`; -subsequent proof-only commit `a8577736dd32c0bcf51a37f88ac0b04fb056da3f`). -These changes need to be admitted and deployed in that order before a live -Cloud proof can validate this adapter. The independent schema-validation fix in AgentWorkforce/flows#398 is also an -explicit validate dependency; its patch is not included here. Local wire fixtures do not establish -that the deployed service supports the contract. - -Configure a pre-provisioned `RELAY_AGENT_TOKEN` for the caller in the provider's -workspace and, if needed, `RELAY_BASE_URL` (default `https://cast.agentrelay.com`). -A workspace API key cannot substitute for the agent token. HTTPS is required -except for literal loopback IP addresses used in local development. The Relay provider -must explicitly enable `AGENT_RELAY_TASK_PROVIDER=1` with persistent state. -The worker uses its injected `agent_result` tool to submit the final output or -failure and waits for the provider's durable acknowledgment before exiting. - -Flows resolves the caller with `GET /v1/agent` and invokes -`POST /v1/actions/task.run/invoke`. The journal idempotency key identifies the -invocation; the input carries run, step, dispatch identity, model, working -directory, result schema, wake context, and a 24-hour task deadline. The adapter -uses the pinned engine's `action-invoke-v1` deterministic invocation ID so it -can reconcile a lost POST acknowledgment without another POST. - -Before dispatch, Flows exclusively creates and syncs a private claim under its -durable data directory's `relay-tasks/`. It binds the canonical engine origin, -workspace, caller, invocation, journal identity, and full input. The data -directory must survive runner restarts; it includes the task prompt and needs -the same protection as the journal. Tokens are never stored in the claim. -A corrupt claim or changed caller, endpoint, or input fails closed. Token -rotation for the same caller remains possible. - -Only the original claim creator sends a POST. Resumed or ambiguous dispatches -use `GET /v1/actions/task.run/invocations/:id` exclusively. If the task cannot be -confirmed, Flows reports an explicit failure; it never creates another task -under a replacement principal. A terminal failure is immutable for that -journal identity, including on a retry of the same dispatch. Starting a new -logical task requires a new journal identity. - -While polling, Flows renews the existing worker lease. Lease loss aborts polling -and prevents a stale journal completion. Execution identity must remain stable across every observed receipt. Generation -and acceptance time may appear together once and must then remain stable; final receipts must match the caller, invocation, -input, run, step, and dispatch. A completed receipt supplies the exact JSON -output, including arrays, scalars, and null. A failed receipt supplies an -explicit failure reason. Receipt correlation is retained in the step's -trajectory metadata. Known credential values are redacted from error details. - -Optional accounting counters `tokens_input` and `tokens_output` feed existing -model pricing. A priced task without the required usage still fails the -existing usage check. Cancellation of local polling does not invent a remote -cancellation API; the durable provider and engine task deadline govern the -remote execution. Historical terminal receipts can still be recovered after -the deadline. +# Agent Relay node-spawn transport + +An agent step with `transport: 'relay'` runs through Agent Relay's native node +spawn control. Flows no longer invokes the Relaycast `task.run` action and does +not start, connect to, or embed the retired Relay broker. + +## Runtime contract + +The worker needs a dedicated publisher `RELAY_AGENT_TOKEN`; a workspace key +cannot replace it. `RELAY_BASE_URL` defaults to +`https://cast.agentrelay.com`. Optional placement constraints are +`RELAYFLOW_NODE_NAME`, `RELAYFLOW_NODE_REPO`, and +`RELAYFLOW_NODE_SANDBOX_ONLY=1`. + +The remote path uses `@agent-relay/sdk >=17.2.1 <18`: + +1. `placement.spawn` selects a live node advertising `spawn:`. +2. Flows requests `confirm: true` and `verifyReady: true` and refuses an + unconfirmed task, identity, node, launch, or readiness result. +3. The spawned agent reports one versioned result envelope to the publisher's + durable inbox. Flows verifies the immutable sender id and name plus run, + step, and dispatch correlation. +4. Flows invokes the node's `release` capability on the same node with the + exact agent id, then records spawn, delivery, and release evidence. +5. Only after `stepComplete` is durably accepted by the journal does Flows ack + the inbox delivery. Relay is transport; the journal is execution authority. + +Spawn readiness is not task completion. A result without the exact sender and +attempt correlation cannot complete a step. + +## Replay and cancellation + +Before dispatch, Flows creates and fsyncs a private claim under +`/node-spawns/`. It binds publisher identity, Relay origin, journal +identity, stable worker name, and full spawn input. Tokens are never stored. + +The current fleet placement SDK has no caller-provided idempotency key. Flows +therefore sends at most one spawn for a newly created claim. A recorded +invocation is reconciled with `commands.getInvocation`; an existing claim with +no recorded acknowledgement is the ambiguous crash window and fails closed. +It is never answered by a blind second spawn. The stable worker name provides +additional collision protection at the node. + +Lease loss or timeout aborts the inbox wait and attempts exact release. A +failed cleanup remains auditable in the durable claim. A result delivery stays +unacked until its terminal journal append succeeds, so a worker crash cannot +turn Relay acknowledgement into fabricated execution truth. + +## Local Desktop bridge + +The SDK contains a local `NodeSpawnPort` adapter for the Desktop core's bridge +v2 methods: `spawn.targets`, `spawn.invoke`, `spawn.status`, and +`spawn.release`. It connects to the private same-user `core.sock`, validates +socket ownership/mode, performs the `hello` handshake, passes no credentials, +and requires an explicit caller name. + +It is not selected by the CLI yet. Desktop currently authorizes only an +existing live coding-session registration; there is no headless Flows service +caller or delegation contract. Flows does not borrow another session's name. +Local admission therefore stays fail-closed until Desktop supplies that +contract. + +See [the migration note](MIGRATION-NODE-SPAWN.md) for removed broker surfaces. diff --git a/docs/MIGRATION-NODE-SPAWN.md b/docs/MIGRATION-NODE-SPAWN.md new file mode 100644 index 000000000..182c4c928 --- /dev/null +++ b/docs/MIGRATION-NODE-SPAWN.md @@ -0,0 +1,46 @@ +# Migrating from Relay broker execution to node spawn + +This release removes the frozen `@agent-relay/harness-driver` dependency and +all broker/PTy launch paths from `@relayflows/sdk`. + +## One-shot remote agents + +Keep `transport: 'relay'`, install `@agent-relay/sdk` in the supported range +`>=17.2.1 <18`, and provide a dedicated publisher `RELAY_AGENT_TOKEN`. The +transport now uses fleet node placement and the durable result inbox instead +of Relaycast `task.run`. + +Custom executable paths and arbitrary injected environment variables do not +carry over. The selected node must advertise `spawn:` for the declared +harness. Use `RELAYFLOW_NODE_NAME` for an exact node or +`RELAYFLOW_NODE_REPO` for repository placement. + +## Removed surfaces + +- `transport: 'broker'`, `keepAlive`, `f.message`, and `f.release` for retained + sessions fail before child-run admission. +- YAML `communication.links` fails preflight. +- `examples/issue-swarm` is retained only as a labelled migration reference; + it is not runnable on this release. +- `RELAYFLOW_RELAY_BROKER_BIN`, broker connection files, helper sockets, and + broker health checks are no longer read. + +There is deliberately no compatibility fallback to broker 17.2.1. Retained +local sessions and linked communication remain unavailable until Desktop has a +supported headless Flows caller/delegation surface. An operator-supplied or +borrowed coding-session name is not a substitute. + +## Hosted and self-hosted execution + +Hosted workers use the agent-token fleet path. Self-hosted workers use the +same path for remote nodes. Selecting the local Desktop bridge is withheld +until the service-caller contract exists. + +Self-hosted Flows licensing is owned by relaycontrol's signed offline licence +design. This SDK does not invent an entitlement store; deployments must fail +closed until a signed `flows` entitlement attestation is available. Scheduled +self-hosted submissions remain Kubernetes CronJobs, not `relaycron`. + +Relay CLI migration is a separate follow-up: `relay flows` already consumes +the v2 SDK, while legacy `node workflow run` and the cloud worker must not move +until the Relay and Cloud owners approve that change. diff --git a/docs/SURFACE.md b/docs/SURFACE.md index af9b2a5c1..63e1bfccd 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -412,64 +412,20 @@ flow-wide `FlowHeader.workspace` / `tools.fs` scopes. The chief harness above remains an aspirational example; this option does not make that entire harness executable today. -### Interactive broker sessions - -`f.agent(name, { transport: "broker", task, cli, model, cwd, skills, keepAlive, -channel, timeout })` starts an interactive PTY on the existing local Relay broker. -Install the optional `@agent-relay/harness-driver` and `@agent-relay/sdk` packages -and start the broker with `agent-relay up` in the run root. The SDK connects using -that root's connection file (or `AGENT_RELAY_STATE_DIR`). It does not start a -private broker. Preflight checks the optional packages and a read-only broker -health request before admitting the step. - -```ts -const implementer = await f.agent("implement-42", { - transport: "broker", cli: "codex", task: "Implement issue #42 and open a PR.", - cwd: "/worktrees/issue-42", skills: ["review-fix-signoff-loop"], keepAlive: true, -}); -const fixed = await f.message(implementer.agentName!, "Address the review findings.", { - timeout: "10m", -}); -await f.release(implementer.agentName!); -``` - -Broker `cwd` accepts an existing absolute directory or a path relative to the run -root, including an external git worktree. `skills` names come from the canonical -`AgentWorkforce/skills` repository. The `skills` CLI installs full directories, -including supporting files, in the agent's project before spawning. Supported -skill targets are Claude Code, Codex, Gemini CLI, Cursor, OpenCode, and Devin -(through its Windsurf-compatible skill directory); an -unsupported target or installation failure fails the step. - -The broker name contains the sanitized flow and authored agent name plus a -run/step hash. The default channel and broker `team` are `flow-`; `channel` -overrides only the channel. Relay Desktop can group `/api/spawned` entries by -team. Run, flow, and step attribution also appears in `RELAYFLOW_RUN_ID`, -`RELAYFLOW_FLOW_NAME`, and `RELAYFLOW_STEP_NAME` in the harness environment. The -broker spawn API has no arbitrary metadata field. - -The prompt supplies a scoped completion helper and a turn ID. The agent calls -`complete TURN_ID JSON_SUMMARY`; `AgentResult.summary` contains that JSON text, -`agentName` identifies the broker session, and `artifacts` is empty. Stale and -duplicate turn completions are refused. `keepAlive: true` retains the session for -`f.message`; otherwise completion releases it. `f.release` journals an explicit, -idempotent release, so a later step can safely remove the worktree. Closing the -flow's local worker releases remaining sessions and disconnects from the broker. -Retained sessions require a local worker stream and reject attached named or -predicate gates before spawning. Validate their results in a later step so the -agent name remains available for explicit cleanup. An idle PTY exit retires the -session; subsequent messages fail immediately. - -`timeout` accepts milliseconds or a duration, defaults to 60 minutes, and is -bounded at 60 minutes. `f.message` inherits the agent timeout unless overridden. -A timeout resolves with `completionReason: "timeout"` and releases the session. -Transport errors fail the step. Broker sessions report no token or dollar usage; -preflight warns that those budget ceilings cannot bound their spend, including -when the declared model has a frozen price. Broker sessions use `maxIterations: 1` and -`transportRetries: 0`; `resume`, `recoveryMode`, `workspace`, and `permissions` -are refused. Completed calls replay from the journal. A lost worker cannot -reattach a retained session's completion socket: a later message fails closed -instead of silently spawning a replacement. +### Native node-spawn sessions + +`transport: "relay"` uses Agent Relay fleet placement and a durable publisher +inbox. The node must advertise `spawn:`; spawn readiness is recorded as +evidence but does not complete the step. The exact spawned identity reports a +versioned result, the node releases that same identity, the worker appends the +terminal journal fact, and only then acknowledges the Relay delivery. + +The retired `transport: "broker"` retained-session surface (`keepAlive`, +`f.message`, and `f.release`) is no longer admitted. It has no PTY or local +broker fallback. Native retained sessions remain fail-closed until Agent Relay +Desktop exposes an authorized headless Flows caller/delegation contract; Flows +will not borrow a live coding session's registration. See +[the migration note](MIGRATION-NODE-SPAWN.md). ### Agent retry and recovery controls diff --git a/docs/agent-communication.md b/docs/agent-communication.md index 8ea781aad..e3eb5fe5b 100644 --- a/docs/agent-communication.md +++ b/docs/agent-communication.md @@ -1,111 +1,19 @@ -# Agent communication +# Agent communication status -A YAML/JSON flow opts in with explicit, directed links: +The former communication runtime launched interactive broker-managed PTYs and +injected a local helper socket. That path was removed with the Relay broker. +Flows does not fall back to it. -```yaml -communication: - timeoutMs: 300000 - links: - - { from: designer, to: reviewer } - - { from: reviewer, to: designer } -``` +Existing YAML `communication.links` declarations are still parsed so old +bundles receive a precise admission error. Preflight refuses them before a run +is admitted. Native node-spawn communication will require two contracts that +do not yet exist together: -See [the runnable conversation](../workflows/agent-communication.flow.yaml). -With no links, ordinary agent steps use the existing runner: no communication -worker, optional Relay package import, or broker is started. DAG dependencies -alone do not opt in. Both linked steps must be agents that can run concurrently; -ancestral dependencies and shared writable surfaces are rejected. +- a supported headless local service caller/delegation contract from Agent + Relay Desktop; and +- a journal-authoritative send/receive/ack protocol that works without a PTY, + helper socket, arbitrary environment injection, or direct broker events. -Install `@agent-relay/harness-driver` and `@agent-relay/sdk` (12.3.1 or newer in 12.x) in the SDK's -runtime environment. They are optional peers, so ordinary SDK installs do not -pull them in. For workspace development, use the built packages from `../relay`. -Set `RELAY_API_KEY` to the existing workspace key and run: - -```bash -flows run workflows/agent-communication.flow.yaml --local-agent -``` - -`flows check` verifies the workspace-key format, optional package resolution, -Node availability, and executable broker before submission. These are local -checks; they do not prove the workspace key is accepted by the service. -Missing prerequisites produce a `probe_failed` refusal, rather than a daemon -protocol error. Ordinary flows do not perform these probes. - -Use the canonical workspace setup in `scripts/run-workflow.sh` when running a -watchable conversation. The run report includes a scoped observer URL; readable -conversation projections go to `wf-`. Workspace keys never belong in URLs. - -The kernel journal remains authoritative. Each directed link has one writable -channel, allowing concurrent dispatch. A small helper journals sends; the -receiving worker records a durable delivery and forwards it through Relaycast. -The existing Relay broker injects it into the declared CLI's managed PTY session. Agents -do not call a receive/poll tool. They explicitly acknowledge processing and call -complete. Injection/verification receipts do **not** acknowledge processing. -Observer channel creation and publication run independently. Their failures emit -`communication_projection_failed` diagnostics without failing or blocking the -conversation. A failed observer publication can be absent from the workspace; -the journal still holds the message. Direct injection and journal failures -remain execution failures. - -Sends use stable semantic message IDs. A reset attempt receives its conversation -history from the journal; unacknowledged messages are injected again. Injection -is at-least-once across attempts, not exactly-once model execution. History over -64 channel facts or 24 KB refuses reset rather than silently discarding context. -Use `flows resume --local-agent --data-dir ` for recovery. - -The bridge is CLI-independent: it passes the declared executable, model, working -directory, and helper environment to Relay's PTY harness. Relay owns CLI-specific -launch flags and injection; Flows does not impose a CLI allowlist or add Claude -flags to other tools. Different CLI tools can participate in the same topology. -The CLI must support an interactive session with a shell/terminal tool. Claude, -Codex, Gemini, Cursor, OpenCode, Droid, Aider, Goose, Grok, Pi, DeepAgents, and custom -interactive executables use the same path. Availability of a binary alone is -not proof of authentication or model access: for tools without a Flows auth -probe, preflight emits `managed_cli_unverified`. A login prompt, unavailable -model, or failed task does not count as successful completion. - -Current scope is local sessions with stream surfaces. This does not add -the public TypeScript `f.channel` API or hosted worker provisioning. Node must be -on PATH for the helper. Interactive sessions report unmetered usage: dollar and -token ceilings cannot bound their spend. Preflight warns; `timeoutMs` bounds the -session (default five minutes, maximum fifteen). Each attempt releases its owned -agent identity; the last session closes the run's broker and publisher. - -The broker receives a filtered environment. Managed CLIs receive their known -provider authentication variables (multi-provider tools receive the supported -provider variables), plus a unique helper session token; unrelated ambient -credentials and broker administration keys are excluded. Custom CLIs can use -their own login/configuration files. Arbitrary environment variables are not -forwarded. Helper requests without the matching token are rejected before any -journal operation. Tokens are never included in the prompt or journal. - -Local agents share the operator's OS user and home directory. This is a trusted -local execution mode, not a sandbox: environment filtering and session tokens -do not prevent a hostile same-user process from reading accessible credentials -or another process's environment. Declared permissions remain advisory until -the separate permission-enforcement work lands. - -Communication workers opt into the journal protocol's `worker.attach.required_streams` -filter using their step's receipt stream. Holding extra pins alone is not enough -to reserve them for unrelated work. Resume reattaches both the ordinary local -worker and the linked workers, so mixed flows retain capacity for each peer. -Use the matching daemon build with this SDK; an older daemon refuses the new -attach field rather than silently ignoring the eligibility constraint. - -After the example completes, verify its recorded exchange: - -```bash -python scripts/verify-agent-communication.py /runs/.sqlite3 -``` - -The verifier checks message IDs, append/delivery/ack ordering, concurrent starts, -injection receipts, both agent completions, and the run's successful completion. - -A mixed-CLI example is [Claude → Codex → Cursor](../workflows/mixed-cli-communication.flow.yaml). -Those three tools have completed a live journaled exchange. Gemini was blocked -by missing local Google authentication; OpenCode's default free model exhausted -its quota; its configured OpenAI key was also rejected on a second attempt. -Routing tests cover all names above and a custom executable; those -are not claims of live end-to-end coverage for every installed tool. - -Verify the mixed example with `python scripts/verify-mixed-cli-communication.py /runs/.sqlite3`. +Until those contracts land, split communicating work into ordinary journaled +steps or use one-shot `transport: 'relay'` agents. Relay messages may carry +task/result transport, but they do not replace journal facts. diff --git a/examples/issue-swarm/README.md b/examples/issue-swarm/README.md index 78b02348a..17bb07e04 100644 --- a/examples/issue-swarm/README.md +++ b/examples/issue-swarm/README.md @@ -1,5 +1,11 @@ # issue-swarm +> **Legacy example:** this source demonstrates the retired retained-session +> broker API (`transport: 'broker'`, `keepAlive`, `f.message`, and `f.release`). +> Current releases refuse it before child-run admission. It remains as migration +> reference until a native retained-session design is approved; do not deploy or +> run it. See [the node-spawn migration note](../../docs/MIGRATION-NODE-SPAWN.md). + Open labeled GitHub issues become independently implemented pull requests. Each issue gets a branch, worktree, and persistent broker implementer. Three reviewers use `adversarial-review` with maintainability, git history, and code structure @@ -7,28 +13,9 @@ lenses. Findings return to the same implementer. A bounded monitor sends CI, conflict, and unresolved review-thread repairs using its `pr-comments` skill. The flow never merges. -Run from a checkout of the target repository with its `origin` configured: - -```sh -flows run examples/issue-swarm/issue-swarm.flow.ts \ - --input '{"label":"agent","count":3,"monitorMinutes":30}' -``` - -The source is self-contained and deploys as-is for manual runs by name: - -```sh -flows deploy examples/issue-swarm/issue-swarm.flow.ts --repo owner/name -flows run issue-swarm --input '{"label":"agent","count":3}' -``` - The example requires the surface/SDK broker transport with `f.message` timeouts -and `f.release`. Install and authenticate `gh`, `git`, Node 22+, Claude Code and -Codex (or the selected CLIs), and start the local Agent Relay broker. The broker -installs named skills from the canonical AgentWorkforce skills package. Broker -agents are ordinary sessions visible in Relay Desktop; the returned agent name -identifies the session receiving follow-up messages. The monitor has write -authority to push branches and address PR comments, so run on repositories where -those actions are intended. +and `f.release`; those surfaces no longer have an execution backend. The +remaining description documents the former behavior for migration planning. | Input | Default | Meaning | | --- | --- | --- | diff --git a/packages/sdk/bun.lock b/packages/sdk/bun.lock index f0afee965..8dcd18060 100644 --- a/packages/sdk/bun.lock +++ b/packages/sdk/bun.lock @@ -6,41 +6,46 @@ "name": "@relayflows/sdk", "dependencies": { "@modelcontextprotocol/sdk": "^1.30.0", - "@relayfile/adapter-core": "0.6.2", - "@relayfile/relay-helpers": "0.4.12", - "@relayflows/surface": "2.0.29", + "@relayfile/adapter-core": "0.8.5", + "@relayfile/relay-helpers": "0.6.1", + "@relayflows/surface": "2.0.65", "@types/js-yaml": "^4.0.9", + "acorn": "^8.18.0", "ai-hist": "0.4.1", "ajv": "^8.17.1", "ajv-draft-04": "^1.0.0", "js-yaml": "^5.4.1", - "re2js": "^2.8.6", + "re2js": "2.8.6", + "typescript": "^5.6.0", + "undici": "^7.29.1", "yaml": "^2.5.1", }, "devDependencies": { "@agent-relay/cli-surface": "^12.2.4", + "@agent-relay/sdk": "17.2.1", "@types/node": "^22.7.0", - "typescript": "^5.6.0", + "bluebird": "^3.7.2", + "esbuild": "^0.21.5", "vitest": "^2.1.0", }, "peerDependencies": { - "@agent-relay/harness-driver": ">=12.3.1 <13", - "@agent-relay/sdk": ">=12.3.1 <13", + "@agent-relay/sdk": ">=17.2.1 <18", }, "optionalPeers": [ - "@agent-relay/harness-driver", "@agent-relay/sdk", ], }, }, "overrides": { "@relayflows/surface": { - "@relayfile/relay-helpers": "0.4.12", + "@relayfile/relay-helpers": "0.6.1", }, }, "packages": { "@agent-relay/cli-surface": ["@agent-relay/cli-surface@12.2.4", "", {}, "sha512-DqJXout26UOLNXi+yTgAD53ek9TXGoSw+5LX8wRwqWeiItiBdw5oHE54UeIzjdbVxNZyk58Dw1494goHN/+AUg=="], + "@agent-relay/sdk": ["@agent-relay/sdk@17.2.1", "", { "dependencies": { "@relaycast/sdk": "^8.0.7", "@relaycast/types": "^8.0.7", "zod": "^4.4.3" } }, "sha512-HkcrbmkqmQLF7t7pNGvl+sjHH3K4ZyqE2jl4nP0qsRLzc1J2ZS+HO+km/FurW9RhbBPFRy16Ev1G3JN1FHIVpA=="], + "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.21.5", "", { "os": "aix", "cpu": "ppc64" }, "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ=="], "@esbuild/android-arm": ["@esbuild/android-arm@0.21.5", "", { "os": "android", "cpu": "arm" }, "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg=="], @@ -97,11 +102,15 @@ "@napi-rs/lzma-linux-x64-gnu": ["@napi-rs/lzma-linux-x64-gnu@1.5.1", "", { "os": "linux", "cpu": "x64" }, "sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ=="], - "@relayfile/adapter-core": ["@relayfile/adapter-core@0.6.2", "", { "dependencies": { "@scalar/postman-to-openapi": "^0.6.0", "cheerio": "^1.2.0", "minimatch": "^10.0.3", "yaml": "^2.8.1" }, "peerDependencies": { "@relayfile/sdk": ">=0.6.0 <1" }, "bin": { "adapter-core": "dist/src/cli.js" } }, "sha512-fVBwiK1W0af4wQ4w/Jr7GA1VBH1qz9Qw1lm90MrOJN0zQfVq1YRwRu9fPk4rp50TQDveLBKE0+MoaLMUmqr5Jg=="], + "@relaycast/sdk": ["@relaycast/sdk@8.16.0", "", { "dependencies": { "@relaycast/types": "8.16.0", "zod": "^4.3.6" } }, "sha512-c3jXSDR2WAJCaiNsd92N/3sADLDHkFraw4Ogey2sKLtq48D/p/TQLnD9BE5r9Rk5zoZ0Y8Ls4mzceCeVGekFBQ=="], + + "@relaycast/types": ["@relaycast/types@8.16.0", "", { "dependencies": { "zod": "^4.3.6" } }, "sha512-yLho3/MMMCu4+GEt0L67XU/n5HUatq9mDEsNlOZUPcAjHTCbbTNjN7MRfb6y91wKeLvzha0ggHJ3DzxR4mH4GQ=="], - "@relayfile/adapter-linear": ["@relayfile/adapter-linear@0.4.13", "", { "dependencies": { "@relayfile/adapter-core": "^0.6.1" }, "peerDependencies": { "@relayfile/sdk": ">=0.6.0 <1" } }, "sha512-cJtjrmhK2dV5o7QD61N2vuvl4XnMGjrcWX5zrM9otneVVA983GHRKdHv014Hq7QSIpG5FMSuLGPziYn6vFJI+A=="], + "@relayfile/adapter-core": ["@relayfile/adapter-core@0.8.5", "", { "dependencies": { "@scalar/postman-to-openapi": "^0.6.0", "cheerio": "^1.2.0", "minimatch": "^10.0.3", "yaml": "^2.8.1" }, "peerDependencies": { "@relayfile/sdk": ">=0.6.0 <1" }, "bin": { "adapter-core": "dist/src/cli.js" } }, "sha512-J10VAyHz1P7wQDKKmVCJD+2kORQs1icP7CtqG1u8r0wO73S76UStimPsCzlGl9UhVhzqdkCZ7wf+GKH9f4N6Aw=="], - "@relayfile/adapter-reddit": ["@relayfile/adapter-reddit@0.2.10", "", { "dependencies": { "@relayfile/adapter-core": "^0.6.1" }, "peerDependencies": { "@relayfile/sdk": ">=0.6.0 <1" } }, "sha512-FBGpWqJgOAC7nSE3SUAPil+pYxpBne1higJmSqGIrycbiIvBPxbluUsTb5Ct7XSyvDB3QmMbOMog5CDaVLhnKQ=="], + "@relayfile/adapter-linear": ["@relayfile/adapter-linear@0.6.1", "", { "dependencies": { "@relayfile/adapter-core": "^0.8.5" }, "peerDependencies": { "@relayfile/sdk": ">=0.6.0 <1" } }, "sha512-BeTSUcm0+vN4krpESFiAfvjJ/WesxKAKhpdQM6uuNYWyl3efNXMj5Qe5nDGCSO2QHMyhKAZYjCyzzgqB9YnqQw=="], + + "@relayfile/adapter-reddit": ["@relayfile/adapter-reddit@0.4.1", "", { "dependencies": { "@relayfile/adapter-core": "^0.8.5" }, "peerDependencies": { "@relayfile/sdk": ">=0.6.0 <1" } }, "sha512-Mz57v1vIE5sEy5F4Z7ix550Ga6l615lgQi0QDc8kgjWiUD9QqZW+Wm+iG+96S4vIQueM70pm60/fitrGGnIGAw=="], "@relayfile/cli-darwin-arm64": ["@relayfile/cli-darwin-arm64@0.10.69", "", { "os": "darwin", "cpu": "arm64" }, "sha512-zJpbOfz4Iq2273TkQHvzOmQ06tA2jga6Pxa0WSYBHTfxMwWE7tk9gGwPrtU4ppOiyON6jvQuTQ3byBhwOwlLsQ=="], @@ -125,11 +134,11 @@ "@relayfile/mount-linux-x64": ["@relayfile/mount-linux-x64@0.10.69", "", { "os": "linux", "cpu": "x64" }, "sha512-aTy6ZiYgF+8SuSF9hGDgJU/z9K6rh+UUAl6Emp/valiw3FmM0fLOt2rAt1/KyXtuhgJE09/XoPRYBLa8UyIeUg=="], - "@relayfile/relay-helpers": ["@relayfile/relay-helpers@0.4.12", "", { "dependencies": { "@relayfile/adapter-core": "^0.6.1", "@relayfile/adapter-linear": "^0.4.13", "@relayfile/adapter-reddit": "^0.2.10" } }, "sha512-Izjl33HLCjxDFVZHK/+f+e/73IUiEg534QNBJGkRYZXjH6LvPKfGiPXViiY5zYLxFtgnzNQa+/TI6MB4X3BnEQ=="], + "@relayfile/relay-helpers": ["@relayfile/relay-helpers@0.6.1", "", { "dependencies": { "@relayfile/adapter-core": "^0.8.5", "@relayfile/adapter-linear": "^0.6.1", "@relayfile/adapter-reddit": "^0.4.1" } }, "sha512-C43jTVFwsqoiyGoS2RczvGgpSyeu9+v7q9nDKZj2NJp9WBqRHTZd6DlU2jH/OB1ovnY4cIGbH3o7TMdNLc50aQ=="], "@relayfile/sdk": ["@relayfile/sdk@0.10.69", "", { "dependencies": { "@relayfile/core": "0.10.69", "ignore": "^7.0.5", "tar": "^7.5.10" }, "optionalDependencies": { "@relayfile/cli-darwin-arm64": "0.10.69", "@relayfile/cli-darwin-x64": "0.10.69", "@relayfile/cli-linux-arm64": "0.10.69", "@relayfile/cli-linux-x64": "0.10.69", "@relayfile/cli-win32-arm64": "0.10.69", "@relayfile/cli-win32-x64": "0.10.69", "@relayfile/mount-darwin-arm64": "0.10.69", "@relayfile/mount-darwin-x64": "0.10.69", "@relayfile/mount-linux-arm64": "0.10.69", "@relayfile/mount-linux-x64": "0.10.69" } }, "sha512-yKImAt0XrR7lqJTaWXCv4NlCyZe1YFxh1qEBKoKCPYhEiF31Rrbarq+PZlQybg4r2Kbiff4XtSERMy31iHvoSw=="], - "@relayflows/surface": ["@relayflows/surface@2.0.29", "", { "dependencies": { "ai-hist": "0.4.1" }, "peerDependencies": { "@relayfile/relay-helpers": "0.4.12" } }, "sha512-DpxA5hJWkHvRYR0uxkpPbjiakYNvFRgaeMbAc5wizgC68DQwxelM3PxF6yrvkBCdndqRNjMK+OqD9tZMMJRrrQ=="], + "@relayflows/surface": ["@relayflows/surface@2.0.65", "", { "dependencies": { "ai-hist": "0.4.1" }, "peerDependencies": { "@relayfile/relay-helpers": "0.6.1" } }, "sha512-QGMnlZ+4Jvm2GsB6NyFZ7Wosb9/roHooYUTAqxv8nOmQ2RY9GslGuH7dOGcQN3kY87amkFrAIEY7yizOWtjetw=="], "@rollup/rollup-android-arm-eabi": ["@rollup/rollup-android-arm-eabi@4.63.0", "", { "os": "android", "cpu": "arm" }, "sha512-70TeIFezKKy65LgAVyQh+w94/gjWhvPWaLaGGeMEgVrPkQhuj/M5bAYYZzIFUj9Y69oHyTm5Um/R6gcLh4A8JA=="], @@ -209,6 +218,8 @@ "accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="], + "acorn": ["acorn@8.19.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-oJlA3XiRm7Cyk6qFD2Jn8ak9B7jSy0qy00ADO3+8dpT0LSjFihQYv4C02LFCSYJV3Q37xYwwRy5m+IpIiUzqWw=="], + "ai-hist": ["ai-hist@0.4.1", "", { "dependencies": { "@modelcontextprotocol/sdk": "^1.29.0", "sql.js": "^1.13.0", "zod": "^4.4.3" }, "bin": { "ai-hist-mcp": "dist/mcp-server.js" } }, "sha512-qn/jXFtWoY4timtzRj1DO5RgqPJA8UoDAm9Qe3bJ9wM3ph+McK+kY/6H2YDUs67MHzWjGlSc1KAe6cdmFbDfeQ=="], "ajv": ["ajv@8.17.1", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g=="], @@ -223,6 +234,8 @@ "balanced-match": ["balanced-match@4.0.4", "", {}, "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA=="], + "bluebird": ["bluebird@3.7.2", "", {}, "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="], + "body-parser": ["body-parser@2.3.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^2.0.0", "debug": "^4.4.3", "http-errors": "^2.0.1", "iconv-lite": "^0.7.2", "on-finished": "^2.4.1", "qs": "^6.15.2", "raw-body": "^3.0.2", "type-is": "^2.1.0" } }, "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw=="], "boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="], diff --git a/packages/sdk/package-lock.json b/packages/sdk/package-lock.json index b91f32fc8..9dcac116e 100644 --- a/packages/sdk/package-lock.json +++ b/packages/sdk/package-lock.json @@ -29,19 +29,16 @@ }, "devDependencies": { "@agent-relay/cli-surface": "^12.2.4", + "@agent-relay/sdk": "17.2.1", "@types/node": "^22.7.0", "bluebird": "^3.7.2", "esbuild": "^0.21.5", "vitest": "^2.1.0" }, "peerDependencies": { - "@agent-relay/harness-driver": ">=12.3.1 <13 || >=15.1.0 <18", - "@agent-relay/sdk": ">=12.3.1 <13 || >=15.1.0 <18" + "@agent-relay/sdk": ">=17.2.1 <18" }, "peerDependenciesMeta": { - "@agent-relay/harness-driver": { - "optional": true - }, "@agent-relay/sdk": { "optional": true } @@ -56,6 +53,20 @@ "node": ">=22.0.0" } }, + "node_modules/@agent-relay/sdk": { + "version": "17.2.1", + "resolved": "https://registry.npmjs.org/@agent-relay/sdk/-/sdk-17.2.1.tgz", + "integrity": "sha512-HkcrbmkqmQLF7t7pNGvl+sjHH3K4ZyqE2jl4nP0qsRLzc1J2ZS+HO+km/FurW9RhbBPFRy16Ev1G3JN1FHIVpA==", + "dev": true, + "dependencies": { + "@relaycast/sdk": "^8.0.7", + "@relaycast/types": "^8.0.7", + "zod": "^4.4.3" + }, + "engines": { + "node": ">=22.0.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", @@ -536,6 +547,25 @@ "node": "^22.20 || ^24.12 || >=25" } }, + "node_modules/@relaycast/sdk": { + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/@relaycast/sdk/-/sdk-8.16.0.tgz", + "integrity": "sha512-c3jXSDR2WAJCaiNsd92N/3sADLDHkFraw4Ogey2sKLtq48D/p/TQLnD9BE5r9Rk5zoZ0Y8Ls4mzceCeVGekFBQ==", + "dev": true, + "dependencies": { + "@relaycast/types": "8.16.0", + "zod": "^4.3.6" + } + }, + "node_modules/@relaycast/types": { + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/@relaycast/types/-/types-8.16.0.tgz", + "integrity": "sha512-yLho3/MMMCu4+GEt0L67XU/n5HUatq9mDEsNlOZUPcAjHTCbbTNjN7MRfb6y91wKeLvzha0ggHJ3DzxR4mH4GQ==", + "dev": true, + "dependencies": { + "zod": "^4.3.6" + } + }, "node_modules/@relayfile/adapter-core": { "version": "0.8.5", "resolved": "https://registry.npmjs.org/@relayfile/adapter-core/-/adapter-core-0.8.5.tgz", diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 0eab0629b..2626f0d16 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -64,6 +64,7 @@ }, "devDependencies": { "@agent-relay/cli-surface": "^12.2.4", + "@agent-relay/sdk": "17.2.1", "@types/node": "^22.7.0", "bluebird": "^3.7.2", "esbuild": "^0.21.5", @@ -75,13 +76,9 @@ "directory": "packages/sdk" }, "peerDependencies": { - "@agent-relay/harness-driver": ">=12.3.1 <13 || >=15.1.0 <18", - "@agent-relay/sdk": ">=12.3.1 <13 || >=15.1.0 <18" + "@agent-relay/sdk": ">=17.2.1 <18" }, "peerDependenciesMeta": { - "@agent-relay/harness-driver": { - "optional": true - }, "@agent-relay/sdk": { "optional": true } diff --git a/packages/sdk/src/agent-relay-receipt.ts b/packages/sdk/src/agent-relay-receipt.ts index 958e6f30c..1909f9a27 100644 --- a/packages/sdk/src/agent-relay-receipt.ts +++ b/packages/sdk/src/agent-relay-receipt.ts @@ -1,9 +1,5 @@ -import { canonicalize } from "./canonical.js"; - -export const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value); -export const nonempty = (value: unknown): value is string => - typeof value === "string" && value.length > 0; +import type { NodeResultDelivery, NodeResultEnvelope } from './node-spawn-result.js'; +import type { NodeSpawnReceipt } from './node-spawn.js'; export interface RelayTaskExecution { execution_id: string; @@ -11,110 +7,31 @@ export interface RelayTaskExecution { step_id: string; dispatch_id: string; deadline: string; - worker_generation?: string; - accepted_at?: string; + node_id?: string; + node_name: string; + agent_id: string; + agent_name: string; + delivery_id: string; + message_id: string; + release_invocation_id?: string; accounting?: Record; } + +/** Journal evidence for a task executed through native node spawn. */ export interface RelayTaskReceipt { invocation_id: string; - action_name: "task.run"; - status: "pending" | "dispatched" | "running" | "completed" | "failed"; + action_name: 'spawn'; + status: 'completed' | 'failed'; task_execution: RelayTaskExecution; output: unknown; error: string | null; - completed_at: string | null; + completed_at: string; } -/** GET is authoritative. A spawn/POST acknowledgment is never a task result. */ -export function readTaskReceipt( - value: unknown, - invocationId: string, - input: Record, - previous?: RelayTaskReceipt, -): RelayTaskReceipt { - const context = input.task_context as Record; - if ( - !isRecord(value) || - value.invocation_id !== invocationId || - value.action_name !== "task.run" || - canonicalize(value.input) !== canonicalize(input) || - !isRecord(value.task_execution) - ) { - throw new Error("Relay task receipt has mismatched invocation or input"); - } - const execution = value.task_execution; - const validDate = (date: unknown): date is string => - nonempty(date) && Number.isFinite(Date.parse(date)); - if ( - !nonempty(execution.execution_id) || - !validDate(execution.deadline) || - ["run_id", "step_id", "dispatch_id"].some( - (key) => execution[key] !== context[key], - ) || - !["pending", "dispatched", "running", "completed", "failed"].includes( - String(value.status), - ) - ) { - throw new Error("Relay task receipt has invalid execution correlation"); - } - const accepted = - nonempty(execution.worker_generation) && validDate(execution.accepted_at); - if ( - (value.status === "running" || value.status === "completed") && - !accepted - ) { - throw new Error("Relay task receipt is missing durable acceptance"); - } - if ( - (execution.worker_generation !== undefined || - execution.accepted_at !== undefined) && - !accepted - ) { - throw new Error("Relay task receipt has incomplete durable acceptance"); - } - if ( - previous !== undefined && - (previous.task_execution.execution_id !== execution.execution_id || - previous.task_execution.deadline !== execution.deadline || - (previous.task_execution.worker_generation !== undefined && - (previous.task_execution.worker_generation !== - execution.worker_generation || - previous.task_execution.accepted_at !== execution.accepted_at))) - ) { - throw new Error( - "Relay task receipt changed its execution, accepted generation/time, or deadline", - ); - } - if ( - execution.accounting !== undefined && - (!isRecord(execution.accounting) || - Object.values(execution.accounting).some( - (n) => typeof n !== "number" || !Number.isFinite(n) || n < 0, - )) - ) { - throw new Error("Relay task receipt has invalid accounting"); - } - if ( - value.status === "completed" && - (!Object.hasOwn(value, "output") || - value.error !== null || - !validDate(value.completed_at)) - ) { - throw new Error("Relay task receipt has no authoritative final output"); - } - if ( - value.status === "failed" && - (!nonempty(value.error) || !validDate(value.completed_at)) - ) { - throw new Error("Relay task failure receipt has no reason"); - } - return { - invocation_id: invocationId, - action_name: "task.run", - status: value.status as RelayTaskReceipt["status"], - task_execution: execution as unknown as RelayTaskExecution, - output: value.output, - error: value.error as string | null, - completed_at: value.completed_at as string | null, - }; +/** The delivery callback is process-local and must never enter journal output. */ +export interface RelayTaskResult { + receipt: RelayTaskReceipt; + delivery: NodeResultDelivery; + spawn: NodeSpawnReceipt; + envelope: NodeResultEnvelope; } diff --git a/packages/sdk/src/agent-relay-transport.ts b/packages/sdk/src/agent-relay-transport.ts index 73e8ffac4..cdc8ead7f 100644 --- a/packages/sdk/src/agent-relay-transport.ts +++ b/packages/sdk/src/agent-relay-transport.ts @@ -1,16 +1,11 @@ -import { createHash } from "node:crypto"; -import { isIP } from "node:net"; -import { setTimeout as delay } from "node:timers/promises"; -import { claimRelayTask } from "./agent-relay-state.js"; -import { canonicalize } from "./canonical.js"; -import { - isRecord, - nonempty, - readTaskReceipt, - type RelayTaskReceipt, -} from "./agent-relay-receipt.js"; +import { createHash } from 'node:crypto'; +import { claimNodeSpawn, recordNodeSpawn, type NodeSpawnClaim } from './node-spawn-state.js'; +import { nodeResultInstruction, waitForNodeResult } from './node-spawn-result.js'; +import { createRemoteNodeSpawnRuntime, type NodeSpawnRuntime } from './node-spawn-runtime.js'; +import { NodeSpawnError, nonempty, validIdempotencyKey } from './node-spawn.js'; +import type { RelayTaskResult } from './agent-relay-receipt.js'; -export type AgentTransport = "direct" | "relay"; +export type AgentTransport = 'direct' | 'relay'; export interface AgentRelayTaskRequest { cli: string; task: string; @@ -21,253 +16,185 @@ export interface AgentRelayTaskRequest { stepId: string; idempotencyKey: string; dataDir: string; - /** Engine task contract ceiling; bounded independently of the renewing lease. */ + /** Engine task contract ceiling; bounded independently of the worker lease. */ timeoutMs?: number; } export interface AgentRelayEnv { baseUrl?: string; agentToken?: string; + node?: string; + repo?: string; + sandboxOnly?: boolean; } export class AgentRelayTransportError extends Error { - constructor( - message: string, - public readonly cause?: unknown, - ) { + constructor(message: string, public readonly cause?: unknown) { super(message); - this.name = "AgentRelayTransportError"; + this.name = 'AgentRelayTransportError'; } } -export function readAgentRelayEnv( - env: NodeJS.ProcessEnv = process.env, -): Required { + +export function readAgentRelayEnv(env: NodeJS.ProcessEnv = process.env): Required> { const agentToken = env.RELAY_AGENT_TOKEN?.trim(); - if (!agentToken) - throw new AgentRelayTransportError( - "Relay task transport requires a pre-provisioned RELAY_AGENT_TOKEN.", - ); - return { - baseUrl: env.RELAY_BASE_URL?.trim() || "https://cast.agentrelay.com", - agentToken, - }; + if (!agentToken) throw new AgentRelayTransportError( + 'Node-spawn transport requires a dedicated publisher RELAY_AGENT_TOKEN.'); + return { baseUrl: env.RELAY_BASE_URL?.trim() || 'https://cast.agentrelay.com', agentToken }; } -/** Exact Relaycast #436 HTTP contract. Only terminal GET receipts can return. */ +/** + * Dispatches exactly once through an Agent Relay node and waits on the + * publisher's durable inbox. AgentWorker acks the returned delivery only + * after stepComplete has durably entered the journal. + */ export async function runAgentRelayTask( request: AgentRelayTaskRequest, - options: AgentRelayEnv & { - fetch?: typeof fetch; - signal?: AbortSignal; - pollMs?: number; - } = {}, -): Promise { - const env = options.agentToken - ? { - baseUrl: options.baseUrl || "https://cast.agentrelay.com", - agentToken: options.agentToken, - } - : { ...readAgentRelayEnv(), ...options }; - const base = new URL(env.baseUrl!); - if ( - base.username || - base.password || - base.search || - base.hash || - base.pathname !== "/" || - !["https:", "http:"].includes(base.protocol) - ) - throw new AgentRelayTransportError("Invalid Relay task base URL"); - const loopback = - base.hostname === "[::1]" || - (isIP(base.hostname) === 4 && base.hostname.startsWith("127.")); - if (base.protocol !== "https:" && !loopback) { - throw new AgentRelayTransportError( - "Relay task credentials require HTTPS outside literal loopback addresses", - ); - } - const baseUrl = base.origin; + options: AgentRelayEnv & { runtime?: NodeSpawnRuntime; signal?: AbortSignal } = {}, +): Promise { + options.signal?.throwIfAborted(); const timeoutMs = request.timeoutMs ?? 86_400_000; - if ( - !Number.isSafeInteger(timeoutMs) || - timeoutMs < 1 || - timeoutMs > 86_400_000 || - ![ - request.runId, - request.stepId, - request.idempotencyKey, - request.dataDir, - request.cli, - ].every(nonempty) - ) { - throw new AgentRelayTransportError( - "Relay task requires durable dispatch identity and a valid deadline", - ); + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 86_400_000 + || ![request.runId, request.stepId, request.dataDir, request.cli].every(nonempty) + || !validIdempotencyKey(request.idempotencyKey)) { + throw new AgentRelayTransportError('Node-spawn task requires durable dispatch identity and a valid deadline'); } - const pollMs = options.pollMs ?? 1000; - if (!Number.isFinite(pollMs) || pollMs < 1) - throw new AgentRelayTransportError("Invalid Relay task polling interval"); - const doFetch = options.fetch ?? globalThis.fetch; - const outer = options.signal ?? new AbortController().signal; - let deadline = Date.now() + timeoutMs + 30_000; - let missingDeadline = Date.now() + 30_000; - async function http( - path: string, - body?: unknown, - ): Promise> { - for (;;) { - outer.throwIfAborted(); - if (Date.now() >= deadline) - throw new AgentRelayTransportError( - "Relay task status unavailable before its reconciliation deadline", - ); - const bounded = AbortSignal.any([ - outer, - AbortSignal.timeout( - Math.min(15_000, Math.max(1, deadline - Date.now())), - ), - ]); - let response: Response | undefined; - let value: unknown; - try { - response = await doFetch(new URL(path, baseUrl), { - method: body === undefined ? "GET" : "POST", - redirect: "error", - signal: bounded, - headers: { - authorization: `Bearer ${env.agentToken}`, - "content-type": "application/json", - ...(body === undefined - ? {} - : { "Idempotency-Key": request.idempotencyKey }), - }, - ...(body === undefined ? {} : { body: canonicalize(body) }), - }); - if ( - response.status === 404 && - body === undefined && - path.includes("/invocations/") - ) { - if (Date.now() >= missingDeadline) - throw new AgentRelayTransportError( - "Relay task dispatch remains unconfirmed; no repeat POST is permitted", - ); - } else if ( - !response.ok && - response.status !== 429 && - response.status < 500 - ) { - // Never copy untrusted response bodies, URLs, or tokens into diagnostics. - throw new AgentRelayTransportError( - `Relay task request refused with HTTP ${response.status}`, - ); - } - if (response.ok) value = await response.json(); - if (body !== undefined && !response.ok) return {}; // ambiguous POST: GET only below - } catch (error) { - outer.throwIfAborted(); - if (error instanceof AgentRelayTransportError) throw error; - if (body !== undefined) return {}; // response loss or invalid JSON never causes another POST - if (response?.ok && !bounded.aborted) - throw new AgentRelayTransportError( - "Relay task response was not valid JSON", - ); - } - outer.throwIfAborted(); - if (response?.ok && value !== undefined) { - if (!isRecord(value) || value.ok !== true || !isRecord(value.data)) - throw new AgentRelayTransportError( - "Relay task response has an invalid data envelope", - ); - return value.data; - } - await delay(pollMs, undefined, { signal: outer }); - } - } - // Resolving the agent is read-only. Pin identity before any invocation so a - // restarted runner cannot create a second task using another caller's key. - const agent = await http("/v1/agent"); - if (!nonempty(agent.id)) - throw new AgentRelayTransportError("Relay task caller identity is missing"); - if (!nonempty(agent.workspace_id)) - throw new AgentRelayTransportError( - "Relay task workspace identity is missing", - ); - // Pinned action-invoke-v1 identity contract from Relaycast #436. - const invocationId = - "inv_idem_" + - createHash("sha256") - .update( - [ - "action-invoke-v1", - agent.workspace_id, - agent.id, - "task.run", - request.idempotencyKey, - ].join("\0"), - ) - .digest("hex"); - const input = { - cli: request.cli, - task: request.task, - ...(request.model === undefined ? {} : { model: request.model }), - ...(request.worker_cwd === undefined - ? {} - : { worker_cwd: request.worker_cwd }), - ...(request.result_schema === undefined - ? {} - : { result_schema: request.result_schema }), - task_context: { - run_id: request.runId, - step_id: request.stepId, - dispatch_id: request.idempotencyKey, - timeout_ms: timeoutMs, - }, - }; - const { claim, created } = await claimRelayTask(request.dataDir, { - version: 1, - baseUrl, - callerId: agent.id, - workspaceId: agent.workspace_id, - invocationId, + const runtime = options.runtime ?? await createRemoteNodeSpawnRuntime({ + ...process.env, + ...(options.baseUrl === undefined ? {} : { RELAY_BASE_URL: options.baseUrl }), + ...(options.agentToken === undefined ? {} : { RELAY_AGENT_TOKEN: options.agentToken }), + ...(options.node === undefined ? {} : { RELAYFLOW_NODE_NAME: options.node }), + ...(options.repo === undefined ? {} : { RELAYFLOW_NODE_REPO: options.repo }), + ...(options.sandboxOnly === undefined ? {} : { RELAYFLOW_NODE_SANDBOX_ONLY: options.sandboxOnly ? '1' : '0' }), + }); + const correlation = { runId: request.runId, stepId: request.stepId, dispatchId: request.idempotencyKey }; + const agentName = stableAgentName(request); + const task = [ + request.task, + request.result_schema === undefined ? undefined + : `The final output must satisfy this JSON Schema: ${JSON.stringify(request.result_schema)}`, + nodeResultInstruction(correlation), + ].filter(nonempty).join('\n\n'); + const proposed: NodeSpawnClaim = { + version: 2, + baseUrl: runtime.baseUrl, + publisherId: runtime.publisher.id, + publisherName: runtime.publisher.name, runId: request.runId, stepId: request.stepId, idempotencyKey: request.idempotencyKey, - input, + agentName, + request: { + cli: request.cli, task, ...(request.model === undefined ? {} : { model: request.model }), + ...(request.worker_cwd === undefined ? {} : { worker_cwd: request.worker_cwd }), + ...(runtime.node === undefined ? {} : { node: runtime.node }), + ...(runtime.repo === undefined ? {} : { repo: runtime.repo }), + ...(runtime.sandboxOnly === undefined ? {} : { sandbox_only: runtime.sandboxOnly }), + }, startedAt: Date.now(), - }); - // A restarted runner may recover a terminal receipt after the task deadline. - // Give that read one bounded window; never reopen or extend the remote task. - deadline = Math.max( - claim.startedAt + timeoutMs + 30_000, - Date.now() + 15_000, - ); - missingDeadline = claim.startedAt + 30_000; - if (created) { - const ack = await http("/v1/actions/task.run/invoke", { input }); - if ( - Object.keys(ack).length && - (ack.invocation_id !== invocationId || - ack.action_name !== "task.run" || - canonicalize(ack.input) !== canonicalize(input)) - ) { - throw new AgentRelayTransportError( - "Relay task acknowledgment has mismatched invocation or input", - ); - } + }; + const claimed = await claimNodeSpawn(request.dataDir, proposed); + let claim = claimed.claim; + if (!claimed.created && claim.spawn === undefined) { + throw new AgentRelayTransportError( + 'A prior node-spawn dispatch ended in the ambiguous pre-ack window; refusing a second spawn'); } - let previous: RelayTaskReceipt | undefined; - for (;;) { - const value = await http( - `/v1/actions/task.run/invocations/${encodeURIComponent(invocationId)}`, - ); - if (value.caller_id !== agent.id) - throw new AgentRelayTransportError( - "Relay task receipt belongs to another caller", - ); - const receipt = readTaskReceipt(value, invocationId, input, previous); - outer.throwIfAborted(); - if (receipt.status === "completed" || receipt.status === "failed") - return receipt; - previous = receipt; - await delay(pollMs, undefined, { signal: outer }); + try { + if (claim.spawn === undefined) { + const spawn = await runtime.port.spawn({ + name: agentName, + harness: request.cli, + task, + model: request.model, + directory: request.worker_cwd, + idempotencyKey: request.idempotencyKey, + node: runtime.node, + repo: runtime.repo, + sandboxOnly: runtime.sandboxOnly, + timeoutMs: Math.min(timeoutMs + 30_000, 86_430_000), + }); + claim = { ...claim, spawn }; + await recordNodeSpawn(claimed.path, claim); + } else { + const status = await runtime.port.status('spawn', claim.spawn.id); + if (status.status === 'failed') throw new NodeSpawnError('The recorded node spawn failed', status.errorCode); + } + const spawn = claim.spawn; + if (spawn === undefined) throw new AgentRelayTransportError('Node-spawn claim lost its confirmed receipt'); + const timeout = AbortSignal.timeout(timeoutMs); + const signal = options.signal === undefined ? timeout : AbortSignal.any([options.signal, timeout]); + const delivery = await waitForNodeResult(runtime.inbox, { + agentId: spawn.agentId, + agentName: spawn.name, + ...correlation, + }, signal); + signal.throwIfAborted(); + let releaseError: unknown; + if (claim.release === undefined) { + try { + const released = await runtime.port.release({ receipt: spawn, + idempotencyKey: releaseKey(request.idempotencyKey) }); + claim = { ...claim, release: { invocationId: released.id, status: released.status } }; + await recordNodeSpawn(claimed.path, claim); + } catch (error) { releaseError = error; } + } + signal.throwIfAborted(); + const envelope = delivery.envelope; + const error = releaseError === undefined + ? envelope.status === 'failed' ? envelope.error! : null + : `Spawned agent release was not confirmed: ${safeError(releaseError)}`; + const status = error === null ? 'completed' : 'failed'; + return { + spawn, + delivery, + envelope, + receipt: { + invocation_id: spawn.id, + action_name: 'spawn', + status, + task_execution: { + execution_id: spawn.id, + run_id: request.runId, + step_id: request.stepId, + dispatch_id: request.idempotencyKey, + deadline: new Date(claim.startedAt + timeoutMs).toISOString(), + node_id: spawn.nodeId, + node_name: spawn.nodeName, + agent_id: spawn.agentId, + agent_name: spawn.name, + delivery_id: delivery.inboxItemId, + message_id: delivery.messageId, + ...(claim.release === undefined ? {} : { release_invocation_id: claim.release.invocationId }), + ...(envelope.accounting === undefined ? {} : { accounting: envelope.accounting }), + }, + output: envelope.output, + error, + completed_at: new Date().toISOString(), + }, + }; + } catch (error) { + if (claim.spawn !== undefined && claim.release === undefined) { + try { + const released = await runtime.port.release({ receipt: claim.spawn, + idempotencyKey: releaseKey(request.idempotencyKey) }); + claim = { ...claim, release: { invocationId: released.id, status: released.status } }; + await recordNodeSpawn(claimed.path, claim); + } catch { /* Durable claim retains exact cleanup identity for audit/recovery. */ } + } + options.signal?.throwIfAborted(); + throw new AgentRelayTransportError(safeError(error), error); } } + +function stableAgentName(request: AgentRelayTaskRequest): string { + const digest = createHash('sha256') + .update(JSON.stringify([request.runId, request.stepId, request.idempotencyKey])) + .digest('hex').slice(0, 24); + const label = request.stepId.toLowerCase().replace(/[^a-z0-9_-]+/g, '-').replace(/^-+|-+$/g, '') || 'step'; + return `rf-${label.slice(0, 30)}-${digest}`; +} + +function releaseKey(key: string): string { + return `release_${createHash('sha256').update(key).digest('hex').slice(0, 32)}`; +} + +function safeError(error: unknown): string { + return error instanceof Error && error.message ? error.message : 'Node-spawn task failed'; +} diff --git a/packages/sdk/src/authored-broker.ts b/packages/sdk/src/authored-broker.ts index 27578dafd..7166ebd90 100644 --- a/packages/sdk/src/authored-broker.ts +++ b/packages/sdk/src/authored-broker.ts @@ -1,73 +1,23 @@ -import { randomUUID } from 'node:crypto'; import type { AgentOptions, AgentResult, NamedGate } from '@relayflows/surface'; -import { parseAgentStepTimeout } from './compile.js'; import type { StepSpec } from './spec.js'; -import { BROKER_TYPE, brokerAgentName, brokerOptionsError, type BrokerInstruction } from './communication/broker-spec.js'; import { AuthoredFlowExecutionError } from './authored-flow-error.js'; -type Declaration = { instruction: BrokerInstruction; cli?: string; model?: string; secrets?: string[] }; +const RETIRED = 'transport: broker was removed with the Relay broker. Use transport: relay for one-shot fleet execution. ' + + 'Retained local sessions require an authorized Desktop service-caller contract and remain fail-closed.'; -/** Broker options stay in a journaled instruction; the kernel only sees agent steps. */ -export function authoredBroker(definition: { name: string }, rootRunId: string | undefined, - localStream: string | undefined, run: (step: StepSpec) => Promise) { - const owner = rootRunId ?? randomUUID(); - const declarations = new Map(); - const invalid = (message: string): never => { throw new AuthoredFlowExecutionError('agent_cli_unresolved', message); }; - const execute = async (id: string, declaration: Declaration, verification?: NamedGate): Promise => { - const { instruction, cli, model, secrets } = declaration; - const output = await run({ id, type: 'agent', instruction: JSON.stringify(instruction), cli, model, - timeoutMs: instruction.timeoutMs, - // Retained sessions must return to the same worker, including after a child completes. - surfaces: localStream === undefined ? undefined : { streams: [{ stream: localStream }] }, - maxIterations: 1, transportRetries: 0, - ...(secrets?.length ? { secrets } : {}), - ...(verification === undefined ? {} : { verification }) }); - if (!output || typeof output !== 'object' || !('completionReason' in output)) { - throw new AuthoredFlowExecutionError('journal_protocol_violation', 'Broker step produced an invalid result'); - } - return output as AgentResult; - }; - const lookup = (name: string): Declaration => declarations.get(name) ?? invalid(`No retained broker agent named ${name} in this flow`); +/** Compatibility boundary for authored flows compiled before broker retirement. */ +export function authoredBroker( + _definition: { name: string }, + _rootRunId: string | undefined, + _localStream: string | undefined, + _run: (step: StepSpec) => Promise, +) { + const refused = (): never => { throw new AuthoredFlowExecutionError('agent_cli_unresolved', RETIRED); }; return { - async agent(id: string, name: string, options: AgentOptions, verification?: NamedGate, - secrets?: string[]): Promise { - const problem = brokerOptionsError(options); - if (problem) invalid(`f.agent options: ${problem}`); - if (options.keepAlive && localStream === undefined) invalid('Retained broker agents require a local agent worker stream'); - if (options.keepAlive && verification !== undefined) invalid('Retained broker agents do not support gates; validate their result in a later step'); - if (typeof options.task !== 'string' || !options.task.trim()) invalid('f.agent task must be nonempty'); - if (options.workspace !== undefined || options.permissions !== undefined) invalid('Broker agents use cwd; workspace and permissions are not supported'); - if ((options.maxIterations ?? 1) !== 1 || (options.transportRetries ?? 0) !== 0 - || options.recoveryMode !== undefined) invalid('Broker sessions require maxIterations 1, transportRetries 0, and no recoveryMode'); - if (typeof name !== 'string' || !name.trim()) invalid('Broker agent name must be nonempty'); - const instruction: BrokerInstruction = { type: BROKER_TYPE, operation: 'spawn', owner, - flow: definition.name, step: name, agentName: brokerAgentName(definition.name, name, owner, id), - task: options.task, cwd: options.cwd, skills: options.skills ?? [], - channel: options.channel ?? `flow-${owner}`, keepAlive: options.keepAlive ?? false, - timeoutMs: options.timeout === undefined ? 3_600_000 : parseAgentStepTimeout(options.timeout) }; - const declaration = { instruction, cli: options.cli, model: options.model, - ...(secrets?.length ? { secrets } : {}) }; - const result = await execute(id, declaration, verification); - if (instruction.keepAlive && result.completionReason === 'success') declarations.set(instruction.agentName, declaration); - return result; - }, - async message(id: string, name: string, text: string, options?: { timeout?: string | number }, - verification?: NamedGate): Promise { - const declaration = lookup(name); - if (typeof text !== 'string' || !text.trim()) invalid('f.message text must be nonempty'); - const result = await execute(id, { ...declaration, instruction: { ...declaration.instruction, - operation: 'message', task: text, timeoutMs: options?.timeout === undefined - ? declaration.instruction.timeoutMs : parseAgentStepTimeout(options.timeout) } }, verification); - return result; - }, - async release(id: string, name: string): Promise { - const declaration = lookup(name); - await run({ id, type: 'agent', cli: declaration.cli, model: declaration.model, - instruction: JSON.stringify({ ...declaration.instruction, operation: 'release' }), - surfaces: localStream === undefined ? undefined : { streams: [{ stream: localStream }] }, - ...(declaration.secrets?.length ? { secrets: declaration.secrets } : {}), - maxIterations: 1, transportRetries: 0 }); - // Keep the declaration so an explicit release remains idempotent. - }, + agent: async (_id: string, _name: string, _options: AgentOptions, _verification?: NamedGate, + _secrets?: string[]): Promise => refused(), + message: async (_id: string, _name: string, _text: string, _options?: { timeout?: string | number }, + _verification?: NamedGate): Promise => refused(), + release: async (_id: string, _name: string): Promise => refused(), }; } diff --git a/packages/sdk/src/communication/broker-preflight.ts b/packages/sdk/src/communication/broker-preflight.ts index 514e0db60..51984e286 100644 --- a/packages/sdk/src/communication/broker-preflight.ts +++ b/packages/sdk/src/communication/broker-preflight.ts @@ -1,22 +1,6 @@ -import { createRequire } from 'node:module'; -import { spawnSync } from 'node:child_process'; -import { pathToFileURL } from 'node:url'; - -const require = createRequire(import.meta.url); - -/** Read-only SDK connection probe: never starts a broker or an agent harness. */ -export function checkBrokerEnvironment(cwd = process.cwd()): void { - let driver: string; - try { - require.resolve('@agent-relay/sdk'); - driver = require.resolve('@agent-relay/harness-driver'); - } catch { - throw new Error('Broker agents require optional @agent-relay/harness-driver and @agent-relay/sdk packages (>=12.3.1).'); - } - const probe = spawnSync(process.execPath, ['--input-type=module', '-e', - 'const { HarnessDriverClient } = await import(process.argv[1]); const client = HarnessDriverClient.connect({ cwd: process.argv[2] }); try { await client.healthCheck(); } finally { await client.shutdown(); }', - pathToFileURL(driver).href, cwd], { encoding: 'utf8', timeout: 10_000, maxBuffer: 64 * 1024 }); - if (probe.status !== 0) { - throw new Error(`Broker agents require a reachable local Relay broker for ${JSON.stringify(cwd)}. Run agent-relay up in the run root or set AGENT_RELAY_STATE_DIR to its connection directory.`); - } +/** Compatibility preflight for the retired `transport: broker` surface. */ +export function checkBrokerEnvironment(_cwd = process.cwd()): never { + throw new Error( + 'transport: broker was removed. Use transport: relay for one-shot fleet execution; retained local sessions remain fail-closed.', + ); } diff --git a/packages/sdk/src/communication/broker-worker.ts b/packages/sdk/src/communication/broker-worker.ts deleted file mode 100644 index 9940b9595..000000000 --- a/packages/sdk/src/communication/broker-worker.ts +++ /dev/null @@ -1,198 +0,0 @@ -import { realpath, stat } from 'node:fs/promises'; -import { resolve } from 'node:path'; -import type { JournalClient } from '../journal-client.js'; -import type { StepDispatchEvent } from '../protocol.js'; -import type { ResolvedKernelAgentStep } from '../resolved-cli-identity.js'; -import { isLeaseLost, withWorkerLease } from '../worker-lease.js'; -import { isReadInterruptionError } from '../journal-read-policy.js'; -import { workerInstruction } from '../worker-input.js'; -import { connectLocalBroker, type LocalBroker } from './relay.js'; -import { spawnFlowPty, shellQuote } from './pty.js'; -import { installBrokerSkills } from './skills.js'; -import { openCommunicationTools } from './tools.js'; -import type { BrokerInstruction } from './broker-spec.js'; - -interface Turn { - id: string; - completed?: boolean; - resolve(summary: string): void; - reject(error: unknown): void; -} -interface Session { - owner: string; - broker: LocalBroker; - pty?: Awaited>; - tools?: Awaited>; - turn?: Turn; - unsubscribe?: () => void; - exited?: string; - releasing?: Promise; -} - -/** Worker-owned live resources. Only journaled agent dispatches start a turn. */ -export class BrokerSessions { - private readonly sessions = new Map(); - constructor(private readonly runRoot: string = process.cwd(), private readonly environment?: NodeJS.ProcessEnv) {} - - async close(): Promise { - const results = await Promise.allSettled([...this.sessions.keys()].map(name => this.release(name))); - const errors = results.flatMap(result => result.status === 'rejected' ? [result.reason] : []); - if (errors.length) throw new AggregateError(errors, 'Failed to release flow broker sessions'); - } - - private async release(name: string): Promise { - const session = this.sessions.get(name); - if (!session) return; - if (session.releasing) return session.releasing; - session.releasing = Promise.resolve().then(() => this.releaseSession(name, session)); - try { await session.releasing; } - finally { session.releasing = undefined; } - } - - private async releaseSession(name: string, session: Session): Promise { - session.turn?.reject(new Error('Broker session released')); - await session.pty?.handle.release('Flow broker session ended', { deleteIdentity: true }); - session.unsubscribe?.(); - try { await session.tools?.close(); } - finally { try { await session.pty?.close(); } finally { await session.broker.shutdown(); } } - this.sessions.delete(name); - } - - async complete(client: JournalClient, dispatch: StepDispatchEvent, instruction: BrokerInstruction, dataDir: string): Promise { - let output: unknown; - let reason: 'success' | 'worker_error' | 'timeout' = 'success'; - try { - output = await withWorkerLease(client, dispatch, signal => this.run(dispatch, instruction, dataDir, signal, - client.supportsFeature?.('step_secret_scoping') ?? true)); - } catch (error) { - if (isLeaseLost(error) || isReadInterruptionError(error)) throw error; - reason = error instanceof Error && error.name === 'TimeoutError' ? 'timeout' : 'worker_error'; - output = { error: error instanceof Error ? error.message : String(error) }; - } - try { - await client.stepComplete(dispatch.run_id, dispatch.step_id, dispatch.attempt, dispatch.idempotency_key, - reason, { output, usage: { tokens_in: 0, tokens_out: 0, dollars_unmetered: true }, - started_pins: dispatch.pins, end_pins: dispatch.pins }); - } catch (error) { - await this.release(instruction.agentName); - throw error; - } - } - - private async run(dispatch: StepDispatchEvent, instruction: BrokerInstruction, dataDir: string, lease: AbortSignal, - scopeSecrets = true): Promise { - const { agentName: name, owner } = instruction; - let session = this.sessions.get(name); - if (session && session.owner !== owner) throw new Error('Broker agent belongs to another flow'); - if (session?.turn) throw new Error('Broker agent already has work in progress'); - if (instruction.operation === 'release') { - await this.release(name); - return { released: true }; - } - if (session?.exited) { - await this.release(name); - throw new Error(`Broker agent ${name}: ${session.exited}`); - } - if (instruction.operation === 'message' && !session) { - throw new Error(`Broker agent ${name} is no longer available; its worker session was lost or released`); - } - if (instruction.operation === 'spawn' && session) throw new Error(`Broker agent ${name} already exists`); - const deadline = AbortSignal.timeout(instruction.timeoutMs); - const signal = AbortSignal.any([lease, deadline]); - const turnId = `${dispatch.run_id}:${dispatch.step_id}:${dispatch.attempt}`; - let succeed!: (summary: string) => void; - let fail!: (error: unknown) => void; - const result = new Promise((yes, no) => { succeed = yes; fail = no; }); - void result.catch(() => {}); - let abortWait!: (error: unknown) => void; - const aborted = new Promise((_, reject) => { abortWait = reject; }); - void aborted.catch(() => {}); - const abort = () => { fail(signal.reason); abortWait(signal.reason); }; - signal.addEventListener('abort', abort, { once: true }); - if (signal.aborted) abort(); - let succeeded = false; - try { - if (instruction.operation === 'spawn') { - const spec = dispatch.spec as ResolvedKernelAgentStep; - if (!spec.cli) throw new Error('Broker agent requires a declared CLI'); - const cwd = await realpath(resolve(this.runRoot, instruction.cwd ?? '.')); - if (!(await stat(cwd)).isDirectory()) throw new Error(`Broker cwd is not a directory: ${cwd}`); - signal.throwIfAborted(); - await installBrokerSkills(instruction.skills, spec.cli_identity ?? spec.cli, cwd, signal); - const broker = await connectLocalBroker(this.runRoot); - session = { owner, broker, turn: { id: turnId, resolve: succeed, reject: fail } }; - this.sessions.set(name, session); - const current = session; - session.tools = await openCommunicationTools(async ({ operation, values }) => { - if (operation !== 'complete' || values.length !== 2 || values[0] !== current.turn?.id || current.turn?.completed) { - throw new Error('Use complete CURRENT_TURN_ID JSON_SUMMARY; stale and duplicate completions are refused'); - } - const summary = JSON.stringify(JSON.parse(values[1]!)); - const turn = current.turn; - turn!.completed = true; - // Allow the helper's response to flush before releasing the PTY. - setTimeout(() => turn!.resolve(summary), 100); - return { completed: true }; - }); - session.unsubscribe = broker.onEvent(event => { - if (event.name !== name || !['agent_exit', 'agent_exited', 'agent_released'].includes(event.kind)) return; - current.exited = event.kind; - if (!current.turn?.completed) current.turn?.reject(new Error(`Broker agent ${name}: ${event.kind}`)); - // An idle exit retires resources immediately. A failed release stays - // in the map and is retried (and surfaced) on the next call or close. - if (!current.turn) void this.release(name).catch(() => {}); - }); - broker.connectEvents(); - const task = `${workerInstruction(instruction.task, dispatch)}\n\n${this.completionPrompt(session, turnId)}\n${instruction.keepAlive ? 'Stay alive after completing; later flow steps can send more work.' : 'The flow releases this session after completion.'}`; - const spawned = spawnFlowPty(broker, { name, cli: spec.cli, identity: spec.cli_identity, - model: spec.model, task, cwd, dataDir, skipRelayPrompt: false, channels: [instruction.channel], team: `flow-${owner}`, - environment: this.environment, socket: session.tools.path, token: session.tools.token, - secrets: spec.secrets, scopeSecrets, - metadata: { runId: owner, flow: instruction.flow, step: instruction.step } }); - let pty: Awaited> | undefined; - try { - pty = await Promise.race([spawned, aborted]); - signal.throwIfAborted(); - session.pty = pty; - } catch (error) { - // Spawn can resolve after the timeout. The session release below - // only sees a handle that was stored in time. - if (session.pty === undefined) void spawned.then(async late => { - try { await late.handle.release('Flow broker session ended', { deleteIdentity: true }); } - catch { /* The timed-out step is already journaled. */ } - try { await late.close(); } - catch { /* A failed link removal must not reject this detached cleanup. */ } - }, () => {}); - throw error; - } - const ready = await Promise.race([session.pty.handle.waitForReady(Math.min(instruction.timeoutMs, 90_000)), - result.then(() => ({ reason: 'ready' }))]); - if (ready.reason !== 'ready') throw new Error(`Broker agent did not become ready: ${ready.reason}`); - } else { - session!.turn = { id: turnId, resolve: succeed, reject: fail }; - signal.throwIfAborted(); - const receipt = await Promise.race([session!.broker.sendMessage({ to: name, from: `flow-${owner}`, - text: `${instruction.task}\n\n${this.completionPrompt(session!, turnId)}`, mode: 'wait', - data: { runId: owner, flow: instruction.flow, step: instruction.step, turnId } }), aborted]); - if (!receipt.targets.includes(name) || receipt.event_id === 'unsupported_operation') { - throw new Error(`Broker did not accept the message for ${name}`); - } - } - const summary = await result; - signal.throwIfAborted(); - succeeded = true; - return { completionReason: 'success', summary, artifacts: [], agentName: name }; - } catch (error) { - if (signal.aborted) throw signal.reason; - throw error; - } finally { - signal.removeEventListener('abort', abort); - if (session) session.turn = undefined; - if (!succeeded || !instruction.keepAlive || session?.exited) await this.release(name); - } - } - - private completionPrompt(session: Session, turn: string): string { - return `Report completion with your shell tool: node ${shellQuote(session.tools!.helperPath)} complete ${shellQuote(turn)} 'JSON_SUMMARY'. The summary must be valid JSON (object, array, or string). Call exactly once for this turn, after the work is finished.`; - } -} diff --git a/packages/sdk/src/communication/environment.ts b/packages/sdk/src/communication/environment.ts index 7a65d450d..d1dfdb97a 100644 --- a/packages/sdk/src/communication/environment.ts +++ b/packages/sdk/src/communication/environment.ts @@ -53,7 +53,7 @@ export function agentEnvironment(cli: string, source: NodeJS.ProcessEnv = proces const platform = platformAgentNames(source, cliName); const allowed = scopeSecrets ? validateStepSecrets(grants, source) : []; if (allowed.some(name => relayNames.includes(name))) { - throw new Error('Relay broker credentials cannot be granted to a communication agent'); + throw new Error('Agent Relay control credentials cannot be granted to a communication agent'); } for (const name of names ?? []) { if (!allowed.includes(name) && !platform.has(name) diff --git a/packages/sdk/src/communication/preflight.ts b/packages/sdk/src/communication/preflight.ts index c3abfa5f6..72a7104b0 100644 --- a/packages/sdk/src/communication/preflight.ts +++ b/packages/sdk/src/communication/preflight.ts @@ -1,34 +1,12 @@ -import { accessSync, constants } from 'node:fs'; -import { createRequire } from 'node:module'; -import { spawnSync } from 'node:child_process'; -import { pathToFileURL } from 'node:url'; import type { FlowSpec, KernelRunSpec } from '../spec.js'; import { communicationInstruction } from './spec.js'; -const require = createRequire(import.meta.url); export class CommunicationEnvironmentError extends Error {} -/** Local-only probes: no broker, workspace creation, or optional runtime import in this process. */ +/** Refuse before admission; no hidden fallback may restart the retired broker. */ export function checkCommunicationEnvironment(spec: FlowSpec | KernelRunSpec): void { if (!spec.steps.some(step => step.type === 'agent' && communicationInstruction(step.instruction))) return; - if (!process.env.RELAY_API_KEY?.trim().startsWith('rk_live_')) { - throw new CommunicationEnvironmentError('Agent communication requires RELAY_API_KEY for an existing workspace.'); - } - let driverPath: string; - try { - require.resolve('@agent-relay/sdk'); - require.resolve('@agent-relay/harness-driver'); - driverPath = require.resolve('@agent-relay/harness-driver/broker-path'); - } catch { - throw new CommunicationEnvironmentError('Agent communication requires optional @agent-relay/harness-driver and @agent-relay/sdk packages (>=12.3.1).'); - } - const probe = spawnSync('node', ['--input-type=module', '-e', - 'const {getBrokerBinaryPath} = await import(process.argv[1]); const path = process.env.RELAYFLOW_RELAY_BROKER_BIN || getBrokerBinaryPath(); if (!path) process.exit(1); process.stdout.write(path);', - pathToFileURL(driverPath).href], { encoding: 'utf8', timeout: 10_000, maxBuffer: 64 * 1024 }); - try { - if (probe.status !== 0 || !probe.stdout.trim()) throw new Error(); - accessSync(probe.stdout.trim(), constants.X_OK); - } catch { - throw new CommunicationEnvironmentError('Agent communication requires Node.js and an executable Relay broker; install Agent Relay or set RELAYFLOW_RELAY_BROKER_BIN.'); - } + throw new CommunicationEnvironmentError( + 'Agent communication through broker/PTy was removed. Native node-spawn communication requires the pending authorized local service-caller contract.', + ); } diff --git a/packages/sdk/src/communication/pty.ts b/packages/sdk/src/communication/pty.ts deleted file mode 100644 index 88fb31075..000000000 --- a/packages/sdk/src/communication/pty.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { mkdir, mkdtemp, rm, symlink } from 'node:fs/promises'; -import { basename, join } from 'node:path'; -import { prepareDevinRuntime } from '../devin-runtime.js'; -import { agentEnvironment } from './environment.js'; -import { resolveCliModel } from '../cli-adapter.js'; -import type { RelayBroker } from './relay.js'; - -export const shellQuote = (value: string) => "'" + value.replaceAll("'", "'\"'\"'") + "'"; - -/** Both communication steps and retained sessions use Relay's PTY launch contract. */ -export async function spawnFlowPty(broker: Pick, options: { - name: string; cli: string; identity?: string; model?: string; task: string; - skipRelayPrompt?: boolean; - cwd: string; dataDir: string; channels: string[]; team?: string; - environment?: NodeJS.ProcessEnv; socket: string; token: string; - secrets?: string[]; scopeSecrets?: boolean; - metadata?: { runId: string; flow: string; step: string }; -}) { - const identity = options.identity ?? options.cli; - const argv0 = basename(identity); - let command = options.cli; - let link: string | undefined; - const environment = agentEnvironment(identity, options.environment ?? process.env, - options.secrets, options.scopeSecrets ?? true); - const prepared = argv0.replace(/\.exe$/i, '') === 'devin' ? prepareDevinRuntime(environment) : undefined; - try { - if (basename(command) !== argv0) { - const root = join(options.dataDir, 'communication'); - await mkdir(root, { recursive: true, mode: 0o700 }); - link = await mkdtemp(join(root, 'cli-')); - command = join(link, argv0); - await symlink(options.cli, command, 'file'); - } - const handle = await broker.spawnPty({ name: options.name, cli: argv0.replace(/\.exe$/i, ''), - task: options.task, channels: options.channels, skipRelayPrompt: options.skipRelayPrompt ?? true, - ...(options.team === undefined ? {} : { team: options.team }), - model: resolveCliModel(identity, options.model), cwd: options.cwd, - harnessConfig: { runtime: 'pty', command: shellQuote(command), args: [], cwd: options.cwd, - env: { ...(prepared?.env ?? environment), - RELAYFLOW_COMMUNICATION_SOCKET: options.socket, RELAYFLOW_COMMUNICATION_TOKEN: options.token, - ...(options.metadata === undefined ? {} : { - RELAYFLOW_RUN_ID: options.metadata.runId, RELAYFLOW_FLOW_NAME: options.metadata.flow, - RELAYFLOW_STEP_NAME: options.metadata.step, - }) }, delivery: { mode: 'pty-injection', format: 'relay-block' } } }); - return { handle, close: async () => { prepared?.close(); if (link) await rm(link, { recursive: true, force: true }); } }; - } catch (error) { - prepared?.close(); - if (link) await rm(link, { recursive: true, force: true }); - throw error; - } -} diff --git a/packages/sdk/src/communication/relay.ts b/packages/sdk/src/communication/relay.ts index 89dd6d9ce..d75440564 100644 --- a/packages/sdk/src/communication/relay.ts +++ b/packages/sdk/src/communication/relay.ts @@ -1,113 +1,39 @@ -import { randomUUID, createHash } from 'node:crypto'; -import { mkdir } from 'node:fs/promises'; -import { join } from 'node:path'; -import { createProjection } from './projection.js'; -import { brokerEnvironment } from './environment.js'; - -export interface RelayEvent { kind: string; name?: string; event_id?: string; delivery_id?: string; verification?: string; [key: string]: unknown } -export interface RelayHandle { - name: string; - generation?: string; - waitForReady(timeout?: number): Promise<{ reason: string }>; - release(reason?: string, options?: { deleteIdentity?: boolean }): Promise; -} -export interface RelayBroker { - spawnPty(input: Record): Promise; - onEvent(callback: (event: RelayEvent) => void): () => void; - shutdown(): Promise; -} +/** + * Compatibility types for journal communication helpers. Runtime broker + * construction was intentionally removed; no call here starts a daemon. + */ export interface RelayMessaging { - agents: { register(input: { name: string; type: string }): Promise<{ token: string }>; delete(name: string): Promise }; - channels: { create(input: { name: string }): Promise; get(name: string): Promise; join(name: string): Promise }; messages: { - direct(input: { to: string; text: string; idempotencyKey: string; metadata: Record; mode: 'wait' }): Promise<{ id: string }>; - send(input: { channel: string; text: string; idempotencyKey: string; metadata: Record }): Promise; + direct(input: Record): Promise; + send(input: Record): Promise; + }; + channels: { + create(input: Record): Promise; + get(name: string): Promise; + join(name: string): Promise; }; } -interface RelayModules { - HarnessDriverClient: { spawn(options: Record): Promise; - connect(options: { cwd: string }): LocalBroker }; - RelaycastMessagingClient: new (options: Record) => RelayMessaging; -} -export async function loadRelayModules(): Promise { - // Variable imports keep optional packages out of ordinary execution and bundles. - const driver = '@agent-relay/harness-driver'; - const sdk = '@agent-relay/sdk'; - try { - const [harness, messaging] = await Promise.all([import(driver), import(sdk)]); - return { HarnessDriverClient: harness.HarnessDriverClient, RelaycastMessagingClient: messaging.RelaycastMessagingClient }; - } catch { - throw new Error('Agent communication requires optional @agent-relay/harness-driver and @agent-relay/sdk packages (>=12.3.1).'); - } -} -export function relayWorkspaceKey(env = process.env): string { - const key = env.RELAY_API_KEY?.trim(); - if (!key?.startsWith('rk_live_')) throw new Error('Agent communication requires RELAY_API_KEY for an existing workspace.'); - return key; -} + export interface RelayRuntime { - broker: RelayBroker; - messaging: RelayMessaging; prefix: string; - channel: string; - project: ReturnType; + messaging: RelayMessaging; + project(input: Record): void; close(): Promise; } -const runtimes = new Map }>(); -export async function acquireRelayRuntime(dataDir: string, runId: string): Promise { - const key = `${dataDir}\0${runId}`; - let entry = runtimes.get(key); - if (!entry) { - entry = { references: 0, runtime: createRuntime(dataDir, runId) }; - runtimes.set(key, entry); - } - entry.references++; - try { - const runtime = await entry.runtime; - let closed = false; - return { ...runtime, close: async () => { - if (closed) return; - closed = true; - if (--entry!.references === 0) { runtimes.delete(key); await runtime.close(); } - } }; - } catch (error) { - if (--entry.references === 0) runtimes.delete(key); - throw error; - } -} -async function createRuntime(dataDir: string, runId: string): Promise { - const workspaceKey = relayWorkspaceKey(); - const { HarnessDriverClient, RelaycastMessagingClient } = await loadRelayModules(); - const hash = createHash('sha256').update(runId).digest('hex').slice(0, 12); - const prefix = `flow-${hash}-${randomUUID().slice(0, 8)}`; - const channel = `wf-${runId.toLowerCase()}`; - const options = { workspaceKey, ...(process.env.RELAY_BASE_URL ? { baseUrl: process.env.RELAY_BASE_URL } : {}) }; - const workspace = new RelaycastMessagingClient(options); - const publisher = `${prefix}-journal`; - const registration = await workspace.agents.register({ name: publisher, type: 'agent' }); - let broker: RelayBroker | undefined; + +export async function loadRelayModules(): Promise<{ RelaycastMessagingClient: unknown }> { try { - const messaging = new RelaycastMessagingClient({ ...options, agentToken: registration.token }); - const cwd = join(dataDir, 'communication', prefix); - await mkdir(cwd, { recursive: true, mode: 0o700 }); - broker = await HarnessDriverClient.spawn({ workspaceKey, brokerName: prefix, cwd, channels: [], - ...(process.env.RELAYFLOW_RELAY_BROKER_BIN ? { binaryPath: process.env.RELAYFLOW_RELAY_BROKER_BIN } : {}), - env: brokerEnvironment(process.env) }); - return { broker, messaging, prefix, channel, project: createProjection(messaging, channel, runId), close: async () => { - try { await broker!.shutdown(); } finally { await workspace.agents.delete(publisher); } - } }; + const sdk = await import('@agent-relay/sdk'); + return { RelaycastMessagingClient: sdk.RelaycastMessagingClient }; } catch (error) { - try { await broker?.shutdown(); } finally { await workspace.agents.delete(publisher); } - throw error; + throw new Error('Agent Relay node spawn requires optional @agent-relay/sdk >=17.2.1 <18.', { cause: error }); } } -/** Connected clients disconnect on shutdown; they never terminate the user's broker. */ -export interface LocalBroker extends RelayBroker { - connectEvents(): void; - sendMessage(input: { to: string; text: string; from: string; mode: 'wait'; data: Record }): Promise<{ event_id: string; targets: string[] }>; +export async function acquireRelayRuntime(_dataDir: string, _runId: string): Promise { + throw new Error('Broker-backed communication runtime was removed; native node-spawn communication is not admitted yet'); } -export async function connectLocalBroker(cwd: string): Promise { - const { HarnessDriverClient } = await loadRelayModules(); - return HarnessDriverClient.connect({ cwd }); + +export async function connectLocalBroker(_cwd: string): Promise { + throw new Error('Local Relay broker connections were removed'); } diff --git a/packages/sdk/src/communication/worker.ts b/packages/sdk/src/communication/worker.ts index f7b6a17e6..9783a9010 100644 --- a/packages/sdk/src/communication/worker.ts +++ b/packages/sdk/src/communication/worker.ts @@ -1,113 +1,5 @@ -import { communicationHistory } from './history.js'; -import { setTimeout as delay } from 'node:timers/promises'; -import type { JournalClient } from '../journal-client.js'; -import type { StepDispatchEvent } from '../protocol.js'; -import type { ResolvedKernelAgentStep } from '../resolved-cli-identity.js'; -import { isLeaseLost, withWorkerLease } from '../worker-lease.js'; -import { isReadInterruptionError } from '../journal-read-policy.js'; -import { workerInstruction } from '../worker-input.js'; -import { resolveAgentCwd } from '../agent-cwd.js'; -import { channelName, type CommunicationInstruction } from './spec.js'; -import { acquireRelayRuntime, type RelayHandle } from './relay.js'; -import { CommunicationSession } from './session.js'; -import { openCommunicationTools } from './tools.js'; -import { spawnFlowPty, shellQuote } from './pty.js'; - +/** The old communication runtime required a broker-managed PTY. */ export function requireCommunicationCli(cli: string | undefined): void { - if (!cli?.trim()) throw new Error('Agent communication requires a declared CLI executable'); -} -export async function completeCommunicationDispatch(client: JournalClient, dispatch: StepDispatchEvent, - instruction: CommunicationInstruction, dataDir: string, runRoot?: string, - environment?: NodeJS.ProcessEnv): Promise { - const spec = dispatch.spec as ResolvedKernelAgentStep; - requireCommunicationCli(spec.cli); - let output: unknown; - let completionReason: 'success' | 'worker_error' = 'success'; - try { - output = await withWorkerLease(client, dispatch, signal => { - // A lost lease stops this attempt's history reads instead of letting - // them run out the budget beside the kernel's retry. - const releaseReads = client.scopeReads?.(signal) ?? (() => {}); - return run(client, dispatch, instruction, spec, dataDir, signal, runRoot, environment).finally(releaseReads); - }); - } catch (error) { - // A lost lease or an unanswered read belongs to the kernel's sweep and - // retry: never complete the attempt here as a worker failure. - if (isLeaseLost(error) || isReadInterruptionError(error)) throw error; - completionReason = 'worker_error'; - output = { error: error instanceof Error ? error.message : String(error) }; - } - await client.stepComplete(dispatch.run_id, dispatch.step_id, dispatch.attempt, dispatch.idempotency_key, - completionReason, { output, usage: { tokens_in: 0, tokens_out: 0, dollars_unmetered: true }, - started_pins: dispatch.pins, end_pins: dispatch.pins }); -} -async function run(client: JournalClient, dispatch: StepDispatchEvent, instruction: CommunicationInstruction, - spec: ResolvedKernelAgentStep, dataDir: string, lease: AbortSignal, runRoot?: string, - environment?: NodeJS.ProcessEnv): Promise { - // Same contract as the CLI worker: a declared directory is resolved and held - // inside the same run root the CLI worker measures against, before anything - // is spawned. Thrown, not reported, because `completeCommunicationDispatch` - // already turns a throw here into a `worker_error` completion carrying the - // message. - const root = runRoot ?? process.cwd(); - const directory = resolveAgentCwd(root, spec.cwd) ?? root; - const controller = new AbortController(); - const signal = AbortSignal.any([lease, controller.signal, AbortSignal.timeout(instruction.timeoutMs)]); - const relay = await acquireRelayRuntime(dataDir, dispatch.run_id); - let handle: RelayHandle | undefined; - let tools: Awaited> | undefined; - let pumping: Promise | undefined; - let receipts = Promise.resolve(); - let unsubscribe: (() => void) | undefined; - let pty: Awaited> | undefined; - try { - let resolve!: (value: unknown) => void; - let reject!: (reason: unknown) => void; - const result = new Promise((yes, no) => { resolve = yes; reject = no; }); - // Install the rejection handler before spawn/readiness can fail. - void result.catch(() => {}); - const abort = () => reject(signal.reason); - signal.addEventListener('abort', abort, { once: true }); - if (signal.aborted) abort(); - const session = new CommunicationSession(client, dispatch, instruction, relay, summary => resolve({ summary }), reject); - tools = await openCommunicationTools(request => session.invoke(request)); - const helper = `node ${shellQuote(tools.helperPath)}`; - const history = await communicationHistory(client, dispatch, instruction); - const prompt = `${workerInstruction(instruction.instruction, dispatch)}${history}\n\nYou are flow agent ${dispatch.step_id}. Concurrent peers send messages automatically into your session through Relay. Do not poll for messages. Use your shell/terminal execution tool with ${helper} for flow communication:\n- send PEER STABLE_ID 'message text' (outgoing peers: ${instruction.outgoing.join(', ') || 'none'})\n- ack PEER DELIVERY_SEQ only after processing that message (incoming peers: ${instruction.incoming.join(', ') || 'none'})\n- complete 'summary' when your task and conversation are finished.\nUse these helpers for all peer communication; do not use Relay MCP messaging tools. Stay available for injected messages until the conversation is complete. Use stable semantic message IDs so retries deduplicate sends.`; - const name = `${relay.prefix}-${dispatch.step_id}`; - unsubscribe = relay.broker.onEvent(event => { - if (event.name !== name || !['delivery_injected', 'delivery_verified'].includes(event.kind)) return; - receipts = receipts.then(async () => { - await client.streamAppend(dispatch.run_id, channelName(dispatch.step_id, '$receipts'), { - kind: event.kind, delivery_id: event.delivery_id, verification: event.verification, - attempt: dispatch.attempt, processing_ack: false, - }); - }); - void receipts.catch(reject); - }); - pty = await spawnFlowPty(relay.broker, { name, cli: spec.cli!, identity: spec.cli_identity, - model: spec.model, task: prompt, channels: [], cwd: directory, dataDir, - environment, socket: tools.path, token: tools.token, - secrets: spec.secrets, scopeSecrets: client.supportsFeature?.('step_secret_scoping') ?? true }); - handle = pty.handle; - const ready = await handle.waitForReady(Math.min(instruction.timeoutMs, 90_000)); - if (ready.reason !== 'ready') throw new Error(`Communication agent did not become ready: ${ready.reason}`); - pumping = (async () => { - while (!signal.aborted) { await session.pump(); await delay(200, undefined, { signal }); } - })(); - void pumping.catch(error => { if (!controller.signal.aborted) reject(error); }); - try { return await result; } finally { signal.removeEventListener('abort', abort); } - } finally { - controller.abort(); - unsubscribe?.(); - try { await pumping?.catch(error => { if (error?.name !== 'AbortError') throw error; }); } - finally { - try { await receipts; } - finally { try { await handle?.release('Flow communication attempt ended', { deleteIdentity: true }); } - finally { try { await tools?.close(); } finally { - try { await relay.close(); } - finally { await pty?.close(); } - } } } - } - } + if (!cli?.trim()) throw new Error('Agent communication requires a declared CLI'); + throw new Error('Agent communication through broker/PTy was removed; native node-spawn communication is not admitted yet'); } diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index adbcc075f..515cd8d01 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -241,6 +241,23 @@ export { JOURNAL_WRITE_FAILED, PROTOCOL_VERSION } from './protocol.js'; export { JournalClient, JournalRequestTimeoutError, type JournalClientOptions } from './journal-client.js'; export { AgentWorker, type AgentWorkerOptions } from './worker.js'; +export { + NodeSpawnError, + type NodeReleaseRequest, + type NodeSpawnAction, + type NodeSpawnInvocation, + type NodeSpawnPort, + type NodeSpawnReceipt, + type NodeSpawnRequest, + type NodeSpawnTarget, +} from './node-spawn.js'; +export { LocalNodeSpawnPort, connectCore, coreSocketPath, type CoreClient } from './node-spawn-local.js'; +export { RemoteNodeSpawnPort, type RemoteNodeSpawnClient } from './node-spawn-remote.js'; +export { + createRemoteNodeSpawnRuntime, + type NodeSpawnPublisher, + type NodeSpawnRuntime, +} from './node-spawn-runtime.js'; export { validateWorkPackage, diff --git a/packages/sdk/src/node-spawn-local.ts b/packages/sdk/src/node-spawn-local.ts new file mode 100644 index 000000000..d47744bb5 --- /dev/null +++ b/packages/sdk/src/node-spawn-local.ts @@ -0,0 +1,222 @@ +import { createHash } from 'node:crypto'; +import { existsSync, lstatSync, statSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { dirname, isAbsolute, join } from 'node:path'; +import { createConnection, type Socket } from 'node:net'; +import { + NodeSpawnError, + type NodeReleaseRequest, + type NodeSpawnAction, + type NodeSpawnInvocation, + type NodeSpawnPort, + type NodeSpawnReceipt, + type NodeSpawnRequest, + type NodeSpawnTarget, + validIdempotencyKey, +} from './node-spawn.js'; + +const BRIDGE_VERSION = 2; +const DEFAULT_TIMEOUT_MS = 15_000; + +export interface CoreClient { + call(method: string, params?: Record): Promise; + close(): void; +} + +export function coreSocketPath(env: NodeJS.ProcessEnv = process.env, home = homedir()): string { + const fallback = join(home, '.agentworkforce', 'desktop', 'core.sock'); + const configured = env.AGENT_RELAY_CORE_SOCKET?.trim(); + if (!configured) return fallback; + return isAbsolute(configured) ? configured : join(dirname(fallback), configured); +} + +function pipeName(socket: string): string { + let key = socket.replace(/\//g, '\\').toLowerCase(); + if (key.startsWith('\\\\?\\')) key = key.slice(4); + while (key.length > 1 && key.endsWith('\\')) key = key.slice(0, -1); + return `\\\\.\\pipe\\agent-relay-${createHash('sha256').update(key).digest('hex').slice(0, 16)}`; +} + +function verifySocket(socket: string): void { + if (process.platform === 'win32' || typeof process.getuid !== 'function') return; + const directory = dirname(socket); + const entry = lstatSync(directory); + const target = statSync(directory); + if (entry.uid !== process.getuid() || target.uid !== process.getuid() + || !target.isDirectory() || (target.mode & 0o022) !== 0) { + throw new NodeSpawnError('The Agent Relay core socket directory is not private to this user', 'untrusted_core_socket'); + } + const file = lstatSync(socket); + if (!file.isSocket() || file.uid !== process.getuid() || (file.mode & 0o077) !== 0) { + throw new NodeSpawnError('The Agent Relay core socket is not private to this user', 'untrusted_core_socket'); + } +} + +/** Connects to Desktop's credential-holding core bridge; never starts a daemon. */ +export async function connectCore(socket = coreSocketPath(), timeoutMs = DEFAULT_TIMEOUT_MS): Promise { + if (process.platform !== 'win32') { + if (!existsSync(socket)) return undefined; + verifySocket(socket); + } + const stream = await openSocket(process.platform === 'win32' ? pipeName(socket) : socket, timeoutMs); + if (!stream) return undefined; + let nextId = 1; + let buffer = ''; + let closed = false; + const pending = new Map(); + const fail = (error: Error) => { + closed = true; + for (const waiter of pending.values()) { clearTimeout(waiter.timer); waiter.reject(error); } + pending.clear(); + }; + stream.setEncoding('utf8'); + stream.on('data', (chunk: string) => { + buffer += chunk; + for (let newline = buffer.indexOf('\n'); newline >= 0; newline = buffer.indexOf('\n')) { + const line = buffer.slice(0, newline); + buffer = buffer.slice(newline + 1); + let message: { id?: unknown; result?: unknown; error?: { message?: string } }; + try { message = JSON.parse(line); } catch { continue; } + if (typeof message.id !== 'number') continue; + const waiter = pending.get(message.id); + if (!waiter) continue; + pending.delete(message.id); + clearTimeout(waiter.timer); + if (message.error) waiter.reject(new NodeSpawnError(message.error.message ?? 'The Agent Relay core refused the request')); + else waiter.resolve(message.result ?? null); + } + }); + stream.on('error', error => fail(error)); + stream.on('close', () => fail(new NodeSpawnError('The Agent Relay core closed the connection'))); + const client: CoreClient = { + call(method: string, params: Record = {}): Promise { + if (closed) return Promise.reject(new NodeSpawnError('The Agent Relay core connection is closed')); + const id = nextId++; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + pending.delete(id); + reject(new NodeSpawnError(`The Agent Relay core did not answer ${method} in time`, 'core_timeout')); + }, timeoutMs); + pending.set(id, { resolve: value => resolve(value as T), reject, timer }); + stream.write(`${JSON.stringify({ jsonrpc: '2.0', id, method, params })}\n`); + }); + }, + close() { closed = true; stream.destroy(); }, + }; + try { + const hello = await client.call<{ bridge_version?: unknown }>('hello', { bridge_version: BRIDGE_VERSION }); + if (hello.bridge_version !== BRIDGE_VERSION) { + throw new NodeSpawnError('The process at the core socket does not speak Agent Relay bridge v2', 'core_version_mismatch'); + } + } catch (error) { client.close(); throw error; } + return client; +} + +async function openSocket(address: string, timeoutMs: number): Promise { + return new Promise(resolve => { + const socket = createConnection(address); + const timer = setTimeout(() => { socket.destroy(); resolve(undefined); }, timeoutMs); + socket.once('connect', () => { clearTimeout(timer); resolve(socket); }); + socket.once('error', () => { clearTimeout(timer); resolve(undefined); }); + }); +} + +interface CoreInvocation { + id: string; + agent_id?: string; + status: string; + node_id?: string; + name?: string; + spawned: boolean; + ready: boolean; + released?: boolean; + task_unconfirmed: boolean; + error_code?: string; +} + +/** + * Local native-spawn adapter. The caller name is deliberately explicit: this + * class never borrows a Desktop registration or provisions a service caller. + */ +export class LocalNodeSpawnPort implements NodeSpawnPort { + constructor(private readonly core: CoreClient, private readonly callerName: string) { + if (!callerName.trim()) throw new NodeSpawnError('Local node spawn requires an explicitly authorized caller name', 'caller_required'); + } + + async targets(): Promise { + const result = await this.core.call<{ targets: Array> }>('spawn.targets'); + if (!Array.isArray(result.targets)) throw new NodeSpawnError('The core returned invalid spawn targets'); + return result.targets.map(value => ({ + id: String(value.id), name: String(value.name), live: value.live === true, + handlersLive: value.handlers_live === true, stale: value.stale === true, + capabilities: Array.isArray(value.capabilities) ? value.capabilities.filter(v => typeof v === 'string') : [], + activeAgents: Number(value.active_agents), reservedAgents: Number(value.reserved_agents), maxAgents: Number(value.max_agents), + })); + } + + async spawn(request: NodeSpawnRequest): Promise { + if (!request.directory || !isAbsolute(request.directory)) throw new NodeSpawnError('Local node spawn requires an absolute directory', 'spawn_invalid_input'); + if (!validIdempotencyKey(request.idempotencyKey)) throw new NodeSpawnError('Invalid node-spawn idempotency key', 'spawn_invalid_input'); + const targets = (await this.targets()).filter(candidate => candidate.live && candidate.handlersLive + && !candidate.stale && candidate.capabilities.includes(`spawn:${request.harness}`)); + const target = request.node === undefined + ? targets[0] + : targets.find(candidate => candidate.name === request.node); + if (!target?.id) throw new NodeSpawnError('No eligible local Agent Relay spawn target', 'no_eligible_node'); + const raw = await this.core.call('spawn.invoke', { + caller_name: this.callerName, node_id: target.id, node_name: target.name, + name: request.name, harness: request.harness, task: request.task, + directory: request.directory, idempotency_key: request.idempotencyKey, + }); + // spawn.invoke is the action receipt; spawn.status is also the boundary + // that enriches it from the core's exact same-workspace registration. + const settled = await this.wait('spawn', raw.id, request.timeoutMs); + if (settled.status !== 'completed' || !settled.spawned || !settled.ready || !settled.agentId || !settled.name) { + throw new NodeSpawnError('The local node did not prove a ready spawned agent', settled.errorCode); + } + return { ...settled, action: 'spawn', nodeName: target.name, name: settled.name, + agentId: settled.agentId, spawned: true, ready: true }; + } + + async status(action: NodeSpawnAction, invocationId: string): Promise { + const raw = await this.core.call('spawn.status', { + caller_name: this.callerName, action, invocation_id: invocationId, + }); + return mapCoreInvocation(action, raw); + } + + async release(request: NodeReleaseRequest): Promise { + if (!validIdempotencyKey(request.idempotencyKey)) throw new NodeSpawnError('Invalid node-release idempotency key', 'spawn_invalid_input'); + const receipt = request.receipt; + if (!receipt.nodeId) throw new NodeSpawnError('Local release requires the exact spawn node id', 'release_identity_mismatch'); + const raw = await this.core.call('spawn.release', { + caller_name: this.callerName, node_id: receipt.nodeId, node_name: receipt.nodeName, + name: receipt.name, expected_agent_id: receipt.agentId, idempotency_key: request.idempotencyKey, + }); + const settled = raw.status === 'completed' ? mapCoreInvocation('release', raw) : await this.wait('release', raw.id); + if (settled.status !== 'completed' || settled.released !== true) { + throw new NodeSpawnError('The local node did not confirm exact agent release', settled.errorCode ?? 'release_unconfirmed'); + } + return settled; + } + + private async wait(action: NodeSpawnAction, id: string, timeoutMs = 360_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const value = await this.status(action, id); + if (value.status === 'completed' || value.status === 'failed') return value; + await new Promise(resolve => setTimeout(resolve, 500)); + } + throw new NodeSpawnError(`Timed out waiting for local ${action} confirmation`, `${action}_unconfirmed`); + } +} + +function mapCoreInvocation(action: NodeSpawnAction, value: CoreInvocation): NodeSpawnInvocation { + return { + id: value.id, action, status: value.status as NodeSpawnInvocation['status'], + nodeId: value.node_id, name: value.name, agentId: value.agent_id, + spawned: value.spawned, ready: value.ready, taskUnconfirmed: value.task_unconfirmed, + released: value.released, + errorCode: value.error_code, + }; +} diff --git a/packages/sdk/src/node-spawn-remote.ts b/packages/sdk/src/node-spawn-remote.ts new file mode 100644 index 000000000..0965e89cd --- /dev/null +++ b/packages/sdk/src/node-spawn-remote.ts @@ -0,0 +1,171 @@ +import { setTimeout as delay } from 'node:timers/promises'; +import { + NodeSpawnError, + isRecord, + nonempty, + terminalStatus, + type NodeReleaseRequest, + type NodeSpawnAction, + type NodeSpawnInvocation, + type NodeSpawnPort, + type NodeSpawnReceipt, + type NodeSpawnRequest, + type NodeSpawnTarget, +} from './node-spawn.js'; + +interface RelayAgent { id: string; name: string } +interface RelayNode { + id?: string; nodeId?: string; name: string; status: string; live?: boolean; + handlersLive?: boolean; capabilities: Array<{ name: string }>; + activeAgents?: number; maxAgents?: number; +} +interface RelayInvocation { + invocationId: string; actionName: string; status: string; handlerNodeId?: string | null; + dispatchedNodeId?: string | null; output?: Record | null; error?: string | null; +} +interface PlacementAck extends RelayInvocation { + node?: RelayNode; + placement: { node?: string; confirmed: boolean; state: string }; + confirmation?: RelayInvocation; +} + +export interface RemoteNodeSpawnClient { + agents: { get(name: string): Promise }; + nodes: { list(options?: Record): Promise }; + placement: { spawn(input: Record): Promise }; + commands: { getInvocation(name: string, invocationId: string): Promise }; +} + +/** Fleet node-spawn adapter backed only by the supported Agent Relay SDK. */ +export class RemoteNodeSpawnPort implements NodeSpawnPort { + constructor(private readonly client: RemoteNodeSpawnClient) {} + + async targets(): Promise { + return (await this.client.nodes.list()).map(node => ({ + id: node.nodeId ?? node.id, + name: node.name, + live: node.live ?? node.status === 'online', + handlersLive: node.handlersLive ?? false, + capabilities: node.capabilities.map(capability => capability.name), + activeAgents: node.activeAgents, + maxAgents: node.maxAgents, + })); + } + + async spawn(request: NodeSpawnRequest): Promise { + const capability = `spawn:${request.harness}`; + const ack = await this.client.placement.spawn({ + capability, + ...(request.node === undefined ? {} : { node: request.node }), + ...(request.repo === undefined ? {} : { repo: request.repo }), + ...(request.sandboxOnly === undefined ? {} : { sandboxOnly: request.sandboxOnly }), + failFast: true, + confirm: true, + verifyReady: true, + confirmTimeoutMs: request.timeoutMs ?? 360_000, + input: { + name: request.name, + task: request.task, + ...(request.model === undefined ? {} : { model: request.model }), + ...(request.directory === undefined ? {} : { worker_cwd: request.directory }), + }, + }); + const confirmation = ack.confirmation; + const output = confirmation?.output; + if (!confirmation || confirmation.status.toLowerCase() !== 'completed' + || output?.spawned !== true || output.ready !== true) { + throw new NodeSpawnError('The fleet node did not prove a ready spawned agent', 'spawn_unconfirmed'); + } + if (output.warning === 'spawn_task_unconfirmed' + || (isRecord(output.warning) && output.warning.code === 'spawn_task_unconfirmed')) { + throw new NodeSpawnError('The fleet node could not confirm initial task delivery', 'spawn_task_unconfirmed'); + } + const reportedName = nonempty(output.name) ? output.name : request.name; + if (reportedName !== request.name) throw new NodeSpawnError('The spawn result named a different agent', 'spawn_identity_mismatch'); + const agent = await this.resolveAgent(request.name); + if (agent.name !== request.name || !nonempty(agent.id) + || (nonempty(output.agent_id) && output.agent_id !== agent.id)) { + throw new NodeSpawnError('The spawned agent identity could not be resolved exactly', 'spawn_identity_mismatch'); + } + const nodeName = ack.node?.name ?? ack.placement.node; + if (!nodeName) throw new NodeSpawnError('The spawn result did not identify its node', 'spawn_node_unconfirmed'); + const nodeId = ack.node?.nodeId ?? ack.node?.id + ?? confirmation.dispatchedNodeId ?? confirmation.handlerNodeId ?? undefined; + return { + id: confirmation.invocationId, + action: 'spawn', + status: 'completed', + nodeId, + nodeName, + name: request.name, + agentId: agent.id, + spawned: true, + ready: true, + taskUnconfirmed: false, + }; + } + + async status(action: NodeSpawnAction, invocationId: string): Promise { + return mapInvocation(action, await this.client.commands.getInvocation(action, invocationId)); + } + + async release(request: NodeReleaseRequest): Promise { + const receipt = request.receipt; + const ack = await this.client.placement.spawn({ + capability: 'release', + actionName: 'release', + node: receipt.nodeName, + failFast: true, + confirm: false, + input: { + name: receipt.name, + expected_agent_id: receipt.agentId, + reason: request.reason ?? 'Relayflows attempt ended', + }, + }); + const deadline = Date.now() + 60_000; + let invocation = mapInvocation('release', ack); + while (!terminalStatus(invocation.status) && Date.now() < deadline) { + await delay(500); + invocation = await this.status('release', ack.invocationId); + } + if (invocation.status !== 'completed' || invocation.released !== true) { + throw new NodeSpawnError('The fleet node did not confirm exact agent release', invocation.errorCode ?? 'release_unconfirmed'); + } + return invocation; + } + + private async resolveAgent(name: string): Promise { + let last: unknown; + for (let attempt = 0; attempt < 10; attempt += 1) { + try { return await this.client.agents.get(name); } + catch (error) { last = error; await delay(250); } + } + throw new NodeSpawnError('The ready agent was not visible in the Relay roster', 'spawn_identity_unavailable', last); + } +} + +function mapInvocation(action: NodeSpawnAction, value: RelayInvocation): NodeSpawnInvocation { + const output = value.output; + const status = value.status.toLowerCase(); + return { + id: value.invocationId, + action, + status: normalizeStatus(status), + nodeId: value.dispatchedNodeId ?? value.handlerNodeId ?? undefined, + name: nonempty(output?.name) ? output.name : undefined, + spawned: output?.spawned === true, + ready: output?.ready === true, + released: output?.released === true, + taskUnconfirmed: output?.warning === 'spawn_task_unconfirmed', + errorCode: value.error?.split(':', 1)[0]?.trim() || undefined, + }; +} + +function normalizeStatus(status: string): NodeSpawnInvocation['status'] { + if (status === 'completed') return 'completed'; + if (terminalStatus(status)) return 'failed'; + if (status === 'running') return 'running'; + if (status === 'dispatched') return 'dispatched'; + return 'pending'; +} diff --git a/packages/sdk/src/node-spawn-result.ts b/packages/sdk/src/node-spawn-result.ts new file mode 100644 index 000000000..069185f40 --- /dev/null +++ b/packages/sdk/src/node-spawn-result.ts @@ -0,0 +1,114 @@ +import { canonicalize } from './canonical.js'; +import { isRecord, nonempty } from './node-spawn.js'; + +export const NODE_RESULT_PREFIX = 'RELAYFLOWS_RESULT_V1 '; + +export interface NodeResultEnvelope { + version: 1; + run_id: string; + step_id: string; + dispatch_id: string; + status: 'completed' | 'failed'; + output?: unknown; + error?: string; + accounting?: { tokens_input?: number; tokens_output?: number }; +} + +export interface NodeResultCorrelation { + runId: string; + stepId: string; + dispatchId: string; +} + +interface InboxItem { + id: string; + message: { id: string; text: string; from: { id?: string; name?: string } }; +} + +export interface NodeResultInboxClient { + inbox: { + subscribe(input?: { signal?: AbortSignal }): AsyncIterable; + ack(input: { inboxItemId: string; state?: 'delivered' | 'read'; metadata?: Record }): Promise; + fail(input: { inboxItemId: string; error: string; retry?: boolean; metadata?: Record }): Promise; + }; +} + +export interface NodeResultDelivery { + inboxItemId: string; + messageId: string; + envelope: NodeResultEnvelope; + /** Called only after the result has been durably appended to the run journal. */ + ack(): Promise; +} + +export function nodeResultInstruction(correlation: NodeResultCorrelation): string { + const template = { + version: 1, + run_id: correlation.runId, + step_id: correlation.stepId, + dispatch_id: correlation.dispatchId, + status: 'completed', + output: { summary: 'replace with the final task result' }, + }; + return `When the task is finished, use the Agent Relay report command supplied in your spawn instructions exactly once. ` + + `The entire report body must be this prefix followed by one compact JSON object: ${NODE_RESULT_PREFIX}${canonicalize(template)}. ` + + 'Keep version, run_id, step_id, and dispatch_id unchanged. Set status to completed and output to the final JSON value, ' + + 'or set status to failed and error to a concise reason. Wait for the report command to succeed before ending the turn.'; +} + +export async function waitForNodeResult( + client: NodeResultInboxClient, + expected: { agentId: string; agentName: string } & NodeResultCorrelation, + signal?: AbortSignal, +): Promise { + for await (const item of client.inbox.subscribe({ signal })) { + signal?.throwIfAborted(); + if (item.message.from.id !== expected.agentId || item.message.from.name !== expected.agentName) continue; + if (!item.message.text.startsWith(NODE_RESULT_PREFIX)) continue; + let raw: unknown; + try { raw = JSON.parse(item.message.text.slice(NODE_RESULT_PREFIX.length)); } + catch { raw = undefined; } + let envelope: NodeResultEnvelope; + try { envelope = readNodeResult(raw, expected); } + catch (error) { + const reason = error instanceof Error ? error.message : 'Malformed Relayflows result'; + return { + inboxItemId: item.id, + messageId: item.message.id, + envelope: { version: 1, run_id: expected.runId, step_id: expected.stepId, + dispatch_id: expected.dispatchId, status: 'failed', error: reason }, + ack: async () => { await client.inbox.fail({ inboxItemId: item.id, error: reason, retry: false, + metadata: { relayflows_result: 'rejected' } }); }, + }; + } + return { + inboxItemId: item.id, + messageId: item.message.id, + envelope, + ack: async () => { await client.inbox.ack({ inboxItemId: item.id, state: 'read', + metadata: { relayflows_result: 'journaled' } }); }, + }; + } + signal?.throwIfAborted(); + throw new Error('Agent Relay result inbox ended before the spawned agent reported completion'); +} + +export function readNodeResult(value: unknown, expected: NodeResultCorrelation): NodeResultEnvelope { + if (!isRecord(value) || value.version !== 1 || value.run_id !== expected.runId + || value.step_id !== expected.stepId || value.dispatch_id !== expected.dispatchId + || !['completed', 'failed'].includes(String(value.status))) { + throw new Error('Spawned agent result has invalid attempt correlation'); + } + if (value.status === 'completed' && !Object.hasOwn(value, 'output')) { + throw new Error('Spawned agent completion omitted its output'); + } + if (value.status === 'failed' && !nonempty(value.error)) { + throw new Error('Spawned agent failure omitted its reason'); + } + if (value.accounting !== undefined && (!isRecord(value.accounting) + || Object.entries(value.accounting).some(([key, amount]) => !['tokens_input', 'tokens_output'].includes(key) + || typeof amount !== 'number' || !Number.isFinite(amount) || amount < 0))) { + throw new Error('Spawned agent result has invalid accounting'); + } + return value as unknown as NodeResultEnvelope; +} diff --git a/packages/sdk/src/node-spawn-runtime.ts b/packages/sdk/src/node-spawn-runtime.ts new file mode 100644 index 000000000..8c3345b28 --- /dev/null +++ b/packages/sdk/src/node-spawn-runtime.ts @@ -0,0 +1,67 @@ +import { isIP } from 'node:net'; +import type { NodeResultInboxClient } from './node-spawn-result.js'; +import { NodeSpawnError } from './node-spawn.js'; +import { RemoteNodeSpawnPort, type RemoteNodeSpawnClient } from './node-spawn-remote.js'; +import type { NodeSpawnPort } from './node-spawn.js'; + +export interface NodeSpawnPublisher { id: string; name: string } +export interface NodeSpawnRuntime { + port: NodeSpawnPort; + inbox: NodeResultInboxClient; + publisher: NodeSpawnPublisher; + baseUrl: string; + node?: string; + repo?: string; + sandboxOnly?: boolean; +} + +interface RelayRuntimeClient extends RemoteNodeSpawnClient, NodeResultInboxClient { + agents: RemoteNodeSpawnClient['agents'] & { me(): Promise }; +} + +export async function createRemoteNodeSpawnRuntime(env: NodeJS.ProcessEnv = process.env): Promise { + const agentToken = env.RELAY_AGENT_TOKEN?.trim(); + if (!agentToken) throw new NodeSpawnError( + 'Node-spawn transport requires a dedicated publisher RELAY_AGENT_TOKEN.', 'publisher_token_required'); + const baseUrl = validateBaseUrl(env.RELAY_BASE_URL?.trim() || 'https://cast.agentrelay.com'); + let RelaycastMessagingClient: new (options: Record) => RelayRuntimeClient; + try { + ({ RelaycastMessagingClient } = await import('@agent-relay/sdk') as unknown as { + RelaycastMessagingClient: new (options: Record) => RelayRuntimeClient; + }); + } catch (error) { + throw new NodeSpawnError('Node-spawn transport requires optional @agent-relay/sdk >=17.2.1 <18.', + 'relay_sdk_unavailable', error); + } + const sandbox = env.RELAYFLOW_NODE_SANDBOX_ONLY?.trim(); + if (sandbox !== undefined && sandbox !== '' && sandbox !== '0' && sandbox !== '1') { + throw new NodeSpawnError('RELAYFLOW_NODE_SANDBOX_ONLY must be 0 or 1', 'spawn_invalid_input'); + } + const client = new RelaycastMessagingClient({ baseUrl, agentToken }); + const publisher = await client.agents.me(); + if (!publisher.id?.trim() || !publisher.name?.trim()) { + throw new NodeSpawnError('The node-spawn publisher token has no exact Relay identity', 'publisher_identity_missing'); + } + return { + port: new RemoteNodeSpawnPort(client), + inbox: client, + publisher, + baseUrl, + ...(env.RELAYFLOW_NODE_NAME?.trim() ? { node: env.RELAYFLOW_NODE_NAME.trim() } : {}), + ...(env.RELAYFLOW_NODE_REPO?.trim() ? { repo: env.RELAYFLOW_NODE_REPO.trim() } : {}), + ...(sandbox === '1' ? { sandboxOnly: true } : {}), + }; +} + +function validateBaseUrl(raw: string): string { + const url = new URL(raw); + if (url.username || url.password || url.search || url.hash || url.pathname !== '/' + || !['https:', 'http:'].includes(url.protocol)) { + throw new NodeSpawnError('Invalid Agent Relay base URL', 'spawn_invalid_input'); + } + const loopback = url.hostname === '[::1]' || (isIP(url.hostname) === 4 && url.hostname.startsWith('127.')); + if (url.protocol !== 'https:' && !loopback) { + throw new NodeSpawnError('Agent Relay credentials require HTTPS outside literal loopback addresses', 'spawn_invalid_input'); + } + return url.origin; +} diff --git a/packages/sdk/src/node-spawn-state.ts b/packages/sdk/src/node-spawn-state.ts new file mode 100644 index 000000000..f7af9fb8a --- /dev/null +++ b/packages/sdk/src/node-spawn-state.ts @@ -0,0 +1,67 @@ +import { createHash, randomUUID } from 'node:crypto'; +import { mkdir, open, readFile, rename } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import { canonicalize } from './canonical.js'; +import type { NodeSpawnReceipt } from './node-spawn.js'; + +export interface NodeSpawnClaim { + version: 2; + baseUrl: string; + publisherId: string; + publisherName: string; + runId: string; + stepId: string; + idempotencyKey: string; + agentName: string; + request: Record; + startedAt: number; + spawn?: NodeSpawnReceipt; + release?: { invocationId: string; status: string }; +} + +export async function claimNodeSpawn(dataDir: string, proposed: NodeSpawnClaim): Promise<{ + claim: NodeSpawnClaim; created: boolean; path: string; +}> { + const directory = join(dataDir, 'node-spawns'); + await mkdir(directory, { recursive: true }); + await syncDirectory(dataDir); + const digest = createHash('sha256') + .update(canonicalize([proposed.runId, proposed.stepId, proposed.idempotencyKey])) + .digest('hex'); + const path = join(directory, `${digest}.json`); + let file; + try { file = await open(path, 'wx', 0o600); } + catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + const prior = JSON.parse(await readFile(path, 'utf8')) as unknown; + if (!sameClaim(prior, proposed)) throw new Error('Node-spawn dispatch conflicts with its durable identity or input'); + return { claim: prior as NodeSpawnClaim, created: false, path }; + } + try { await file.writeFile(canonicalize(proposed)); await file.sync(); } + finally { await file.close(); } + await syncDirectory(directory); + return { claim: proposed, created: true, path }; +} + +export async function recordNodeSpawn(path: string, claim: NodeSpawnClaim): Promise { + const temporary = `${path}.${randomUUID()}.tmp`; + const file = await open(temporary, 'wx', 0o600); + try { await file.writeFile(canonicalize(claim)); await file.sync(); } + finally { await file.close(); } + await rename(temporary, path); + await syncDirectory(dirname(path)); +} + +function sameClaim(value: unknown, proposed: NodeSpawnClaim): boolean { + if (typeof value !== 'object' || value === null || !('startedAt' in value) + || typeof value.startedAt !== 'number' || !Number.isSafeInteger(value.startedAt)) return false; + const prior = value as NodeSpawnClaim; + const { spawn: _priorSpawn, release: _priorRelease, ...priorIdentity } = prior; + const { spawn: _spawn, release: _release, ...proposedIdentity } = proposed; + return canonicalize({ ...priorIdentity, startedAt: proposed.startedAt }) === canonicalize(proposedIdentity); +} + +async function syncDirectory(path: string): Promise { + const directory = await open(path, 'r'); + try { await directory.sync(); } finally { await directory.close(); } +} diff --git a/packages/sdk/src/node-spawn.ts b/packages/sdk/src/node-spawn.ts new file mode 100644 index 000000000..6370b13eb --- /dev/null +++ b/packages/sdk/src/node-spawn.ts @@ -0,0 +1,92 @@ +/** Native Agent Relay node-spawn contract used by Flows. */ +export interface NodeSpawnTarget { + id?: string; + name: string; + live: boolean; + handlersLive: boolean; + stale?: boolean; + capabilities: string[]; + activeAgents?: number; + reservedAgents?: number; + maxAgents?: number; +} + +export interface NodeSpawnRequest { + name: string; + harness: string; + task: string; + model?: string; + directory?: string; + idempotencyKey: string; + node?: string; + repo?: string; + timeoutMs?: number; + sandboxOnly?: boolean; +} + +export type NodeSpawnAction = 'spawn' | 'release'; +export type NodeSpawnStatus = 'pending' | 'dispatched' | 'running' | 'completed' | 'failed'; + +export interface NodeSpawnInvocation { + id: string; + action: NodeSpawnAction; + status: NodeSpawnStatus; + nodeId?: string; + nodeName?: string; + name?: string; + agentId?: string; + spawned: boolean; + ready: boolean; + released?: boolean; + taskUnconfirmed?: boolean; + errorCode?: string; +} + +export interface NodeSpawnReceipt extends NodeSpawnInvocation { + action: 'spawn'; + name: string; + agentId: string; + nodeName: string; + spawned: true; + ready: true; +} + +export interface NodeReleaseRequest { + receipt: NodeSpawnReceipt; + idempotencyKey: string; + reason?: string; +} + +/** Credentials and transport stay inside an implementation of this port. */ +export interface NodeSpawnPort { + targets(): Promise; + spawn(request: NodeSpawnRequest): Promise; + status(action: NodeSpawnAction, invocationId: string): Promise; + release(request: NodeReleaseRequest): Promise; +} + +export class NodeSpawnError extends Error { + constructor(message: string, public readonly code = 'node_spawn_failed', public readonly cause?: unknown) { + super(message); + this.name = 'NodeSpawnError'; + } +} + +export const isRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value); + +export const nonempty = (value: unknown): value is string => + typeof value === 'string' && value.trim().length > 0; + +export function validSegment(value: unknown): value is string { + return typeof value === 'string' && value.length > 0 && value.length <= 200 + && /^[A-Za-z0-9_-]+$/.test(value); +} + +export function validIdempotencyKey(value: unknown): value is string { + return validSegment(value) && value.length >= 16; +} + +export function terminalStatus(status: string): boolean { + return ['completed', 'failed', 'denied', 'cancelled', 'canceled', 'timed_out', 'timeout'].includes(status); +} diff --git a/packages/sdk/src/relayflowd-path.ts b/packages/sdk/src/relayflowd-path.ts index f5d4ec5f4..696ca3885 100644 --- a/packages/sdk/src/relayflowd-path.ts +++ b/packages/sdk/src/relayflowd-path.ts @@ -4,7 +4,7 @@ // with a minimal PATH, and that is exactly the case where a `which` lookup // fails to see the binary the installer placed next to its own launcher. The // multi-anchor order below is ported from ../relay's -// packages/harness-driver/src/broker-path.ts for that reason. +// Agent Relay's launcher resolution for that reason. import { createRequire } from 'node:module'; import { accessSync, constants, existsSync, realpathSync, statSync } from 'node:fs'; diff --git a/packages/sdk/src/worker-cli.ts b/packages/sdk/src/worker-cli.ts index 05e0ac044..75853c09c 100644 --- a/packages/sdk/src/worker-cli.ts +++ b/packages/sdk/src/worker-cli.ts @@ -47,6 +47,7 @@ import { import { pinCliAlias } from './cli/pinned-cli-alias.js'; import { prepareDevinRuntime } from './devin-runtime.js'; import { promptOperand } from './adapters/base.js'; +import type { NodeSpawnRuntime } from './node-spawn-runtime.js'; export type { CliTransportCause, CliTransportEvidence, CliTransportPhase } from './cli-transport-evidence.js'; export { agentCompletionReason } from './cli-transport-evidence.js'; @@ -68,6 +69,8 @@ export function cliInvocationArgv0(cli: string, cliIdentity?: string): string { export interface WorkerCliResult { relay_task?: import('./agent-relay-receipt.js').RelayTaskReceipt; + /** Settled only after the journal accepts stepComplete; never journaled. */ + relay_delivery?: { ack(): Promise }; tokens_input?: number; tokens_output?: number; exit_code: number | null; @@ -121,6 +124,7 @@ export interface AgentRelayContext { idempotencyKey: string; dataDir?: string; resultSchema?: unknown; + nodeSpawnRuntime?: NodeSpawnRuntime; } export async function runAgentCli( @@ -390,17 +394,18 @@ async function runViaAgentRelay( try { if (kind === 'relayflows-wrapper-v1') throw new Error('Relay task transport does not support same-process wrappers.'); if (!context?.dataDir) throw new Error('Relay task transport requires a durable data directory and journal dispatch identity.'); - const task = instruction + (wakeContext === undefined ? '' : `\n\nWake context (journaled):\n${JSON.stringify(wakeContext)}`) - + '\n\nReport the final task output with the injected agent_result tool and final=true. Wait for its successful durable acknowledgment before exiting.'; + const task = instruction + (wakeContext === undefined ? '' : `\n\nWake context (journaled):\n${JSON.stringify(wakeContext)}`); const received = await runAgentRelayTask({ cli: kind, task, model, worker_cwd, result_schema: context.resultSchema, runId: context.runId, stepId: context.stepId, idempotencyKey: context.idempotencyKey, dataDir: context.dataDir, - }, { signal }); - const receipt = { ...received, error: received.error === null ? null : redactRelayError(received.error) }; + }, { signal, runtime: context.nodeSpawnRuntime }); + const receipt = { ...received.receipt, + error: received.receipt.error === null ? null : redactRelayError(received.receipt.error) }; const accounting = receipt.task_execution.accounting; const result: WorkerCliResult = { - relay_task: receipt, exit_code: receipt.status === 'completed' ? 0 : 1, + relay_task: receipt, relay_delivery: received.delivery, + exit_code: receipt.status === 'completed' ? 0 : 1, stdout_tail: receipt.status === 'completed' ? JSON.stringify(receipt.output) : '', stderr_tail: receipt.status === 'failed' ? `Relay task failed: ${receipt.error}` : '', ...(accounting?.tokens_input === undefined ? {} : { tokens_input: accounting.tokens_input }), diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index 3d82c1b3d..78395dd27 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -1,5 +1,4 @@ import { brokerInstruction } from './communication/broker-spec.js'; -import { BrokerSessions } from './communication/broker-worker.js'; import { communicationInstruction } from './communication/spec.js'; import { workerSpend } from './worker-spend.js'; import { reportedCost } from './reported-cost.js'; @@ -23,6 +22,7 @@ import { redactText, type TranscriptDigest, } from './agent-transcript.js'; +import type { NodeSpawnRuntime } from './node-spawn-runtime.js'; export { MODEL_ENV, WAKE_CONTEXT_ENV } from './worker-cli.js'; @@ -43,6 +43,8 @@ export interface AgentWorkerOptions { runRoot?: string; /** Source environment; each provider CLI receives only its allowlist and step grants. */ environment?: NodeJS.ProcessEnv; + /** Optional approved node-spawn runtime; otherwise relay steps use the agent-token fleet client. */ + nodeSpawnRuntime?: NodeSpawnRuntime; } /** @@ -62,7 +64,6 @@ export interface AgentWorkerOptions { * shutdown. */ export class AgentWorker extends EventEmitter { - private readonly brokerSessions: BrokerSessions; private attached = false; private closing = false; private readonly inFlight: Set> = new Set(); @@ -72,7 +73,6 @@ export class AgentWorker extends EventEmitter { private readonly options: AgentWorkerOptions, ) { super(); - this.brokerSessions = new BrokerSessions(options.runRoot, options.environment); } async attach(): Promise { @@ -111,7 +111,6 @@ export class AgentWorker extends EventEmitter { if (pending.length > 0) { await Promise.allSettled(pending); } - await this.brokerSessions.close(); this.attached = false; } @@ -135,16 +134,14 @@ export class AgentWorker extends EventEmitter { } const broker = brokerInstruction(spec.instruction); if (broker) { - if (!this.options.dataDir) throw new Error('Broker agents require a worker data directory'); - await this.brokerSessions.complete(this.client, dispatch, broker, this.options.dataDir); + await this.retiredNodeSession(dispatch, + 'transport: broker was removed with the Relay broker. Use transport: relay for one-shot fleet execution; retained local sessions require the pending Desktop service-caller contract.'); return; } const communication = communicationInstruction(spec.instruction); if (communication) { - if (!this.options.dataDir) throw new Error('Agent communication requires a worker data directory'); - const { completeCommunicationDispatch } = await import('./communication/worker.js'); - await completeCommunicationDispatch(this.client, dispatch, communication, this.options.dataDir, - this.options.runRoot, this.options.environment); + await this.retiredNodeSession(dispatch, + 'Agent communication is unavailable without the retired broker/PTy path. Native node-spawn communication remains fail-closed until an authorized local service caller exists.'); return; } let humanIntervention = false; @@ -166,7 +163,8 @@ export class AgentWorker extends EventEmitter { }, cwd.directory ?? (spec.transport === 'relay' ? undefined : this.options.runRoot), spec.transport === 'relay' ? 'relay' : 'direct', { runId: dispatch.run_id, stepId: dispatch.step_id, idempotencyKey: dispatch.idempotency_key, - dataDir: this.options.dataDir, resultSchema: spec.verification?.json_schema }, + dataDir: this.options.dataDir, resultSchema: spec.verification?.json_schema, + nodeSpawnRuntime: this.options.nodeSpawnRuntime }, this.options.environment, spec.cli_identity, spec.timeout_ms, spec.resume, spec.secrets, this.client.supportsFeature?.('step_secret_scoping') ?? true) : Promise.resolve({ exit_code: null, stdout_tail: '', stderr_tail: 'agent step has no declared CLI' })); @@ -200,7 +198,7 @@ export class AgentWorker extends EventEmitter { // `transcript` is not part of the wrapper either: it is evidence about the // attempt, journaled in `trajectory_tail` below on success and failure // alike, where the kernel already accepts and bounds it (16 KiB). - const { transcript, transport, ...rawWrapper } = result; + const { transcript, transport, relay_delivery: relayDelivery, ...rawWrapper } = result; // `stderr_tail` is evidence, not authored output. Redact and bound it // before it reaches either the worker-failure render or the journal -- // against the environment the CLI ran with as well as this host's, since a @@ -239,6 +237,21 @@ export class AgentWorker extends EventEmitter { : dispatch.pins, }, ); + // Relay delivery is transport state, not execution truth. Only settle it + // after the journal has accepted the attempt's terminal fact. + await relayDelivery?.ack(); + } + + private async retiredNodeSession(dispatch: StepDispatchEvent, error: string): Promise { + await this.client.stepComplete( + dispatch.run_id, dispatch.step_id, dispatch.attempt, dispatch.idempotency_key, + 'worker_error', { + output: { error }, + usage: { tokens_in: 0, tokens_out: 0, dollars_unmetered: true }, + started_pins: dispatch.pins, + end_pins: dispatch.pins, + }, + ); } } diff --git a/packages/sdk/tests/agent-relay-hardening.test.ts b/packages/sdk/tests/agent-relay-hardening.test.ts index 58c3e1869..8860ce917 100644 --- a/packages/sdk/tests/agent-relay-hardening.test.ts +++ b/packages/sdk/tests/agent-relay-hardening.test.ts @@ -1,91 +1,44 @@ -import { readFile } from "node:fs/promises"; -import { describe, expect, it, vi } from "vitest"; -import { runAgentRelayTask } from "../src/agent-relay-transport.js"; -import { readTaskReceipt } from "../src/agent-relay-receipt.js"; +import { describe, expect, it, vi } from 'vitest'; +import { readNodeResult, waitForNodeResult, NODE_RESULT_PREFIX } from '../src/node-spawn-result.js'; -const wire = JSON.parse( - await readFile( - new URL("./fixtures/relay-task/relaycast-436.json", import.meta.url), - "utf8", - ), -); -const id = wire.ack.data.invocation_id; -const request = { - cli: "claude", - task: "fixture", - runId: "run", - stepId: "step", - idempotencyKey: "key", - dataDir: "/unused", -}; +const correlation = { runId: 'run_1', stepId: 'step_1', dispatchId: 'dispatch_1234567890' }; -describe("Relay credential origin", () => { - for (const baseUrl of [ - "http://cast.agentrelay.com", - "http://192.168.1.1", - "http://localhost.example", - "http://127.0.0.1.example", - ]) { - it(`refuses bearer credentials at ${baseUrl} before any request`, async () => { - const fetchMock = vi.fn(); - await expect( - runAgentRelayTask(request, { - baseUrl, - agentToken: "fixture-token", - fetch: fetchMock, - }), - ).rejects.toThrow(/require HTTPS/); - expect(fetchMock).not.toHaveBeenCalled(); - }); - } - for (const baseUrl of [ - "https://cast.agentrelay.com", - "http://127.0.0.1:1234", - "http://127.9.8.7:1234", - "http://[::1]:1234", - ]) { - it(`allows ${baseUrl} to reach the authenticated identity check`, async () => { - const fetchMock = vi.fn(async () => new Response("", { status: 401 })); - await expect( - runAgentRelayTask(request, { - baseUrl, - agentToken: "fixture-token", - fetch: fetchMock, - }), - ).rejects.toThrow("HTTP 401"); - expect(fetchMock).toHaveBeenCalledOnce(); - }); - } -}); +describe('node-spawn result hardening', () => { + it('requires immutable attempt correlation and terminal evidence', () => { + expect(() => readNodeResult({ version: 1, run_id: 'other', step_id: 'step_1', + dispatch_id: correlation.dispatchId, status: 'completed', output: {} }, correlation)).toThrow('correlation'); + expect(() => readNodeResult({ version: 1, run_id: 'run_1', step_id: 'step_1', + dispatch_id: correlation.dispatchId, status: 'failed' }, correlation)).toThrow('reason'); + }); -describe("Relay acceptance identity", () => { - it("allows generation and acceptance time to appear once on the same execution", () => { - const pending = readTaskReceipt(wire.dispatched.data, id, wire.input); - expect( - readTaskReceipt(wire.running.data, id, wire.input, pending).status, - ).toBe("running"); + it('ignores a forged name match without the exact immutable sender id', async () => { + const ack = vi.fn(); + const delivery = await waitForNodeResult({ + inbox: { + subscribe: async function* () { + for (const [id, sender] of [['forged', 'attacker'], ['real', 'agent_1']] as const) yield { + id, message: { id: `message_${id}`, from: { id: sender, name: 'worker' }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_1', step_id: 'step_1', + dispatch_id: correlation.dispatchId, status: 'completed', output: { id } }) }, + }; + }, + ack: async input => { ack(input); }, fail: async () => {}, + }, + }, { agentId: 'agent_1', agentName: 'worker', ...correlation }); + expect(delivery.envelope.output).toEqual({ id: 'real' }); + expect(ack).not.toHaveBeenCalled(); }); - it("refuses an execution change before the generation exists", () => { - const pending = readTaskReceipt(wire.dispatched.data, id, wire.input); - const changed = structuredClone(wire.running.data); - changed.task_execution.execution_id += "/replacement"; - expect(() => readTaskReceipt(changed, id, wire.input, pending)).toThrow( - /changed its execution/, - ); + + it('dead-letters a malformed exact-sender completion only after journal settlement', async () => { + const fail = vi.fn(async () => ({})); + const delivery = await waitForNodeResult({ inbox: { + subscribe: async function* () { yield { id: 'delivery_bad', message: { id: 'message_bad', + from: { id: 'agent_1', name: 'worker' }, text: NODE_RESULT_PREFIX + '{bad' } }; }, + ack: async () => ({}), fail, + } }, { agentId: 'agent_1', agentName: 'worker', ...correlation }); + expect(delivery.envelope.status).toBe('failed'); + expect(fail).not.toHaveBeenCalled(); + await delivery.ack(); + expect(fail).toHaveBeenCalledWith(expect.objectContaining({ inboxItemId: 'delivery_bad', retry: false })); }); - for (const field of ["worker_generation", "accepted_at"]) { - it(`refuses changed or removed ${field} after acceptance`, () => { - const running = readTaskReceipt(wire.running.data, id, wire.input); - const changed = structuredClone(wire.completed.data); - changed.task_execution[field] = - field === "accepted_at" ? "2026-01-01T00:00:00.000Z" : "replacement"; - expect(() => readTaskReceipt(changed, id, wire.input, running)).toThrow( - /changed its execution/, - ); - delete changed.task_execution[field]; - expect(() => readTaskReceipt(changed, id, wire.input, running)).toThrow( - /acceptance/, - ); - }); - } }); diff --git a/packages/sdk/tests/agent-relay-transport.test.ts b/packages/sdk/tests/agent-relay-transport.test.ts index 52c7a3469..75ca403a6 100644 --- a/packages/sdk/tests/agent-relay-transport.test.ts +++ b/packages/sdk/tests/agent-relay-transport.test.ts @@ -1,479 +1,149 @@ -import { createHash } from "node:crypto"; -import { mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { afterEach, describe, expect, it, vi } from "vitest"; -import { - runAgentRelayTask, - readAgentRelayEnv, - type AgentRelayTaskRequest, -} from "../src/agent-relay-transport.js"; +import { mkdtemp } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { describe, expect, it, vi } from 'vitest'; +import { runAgentRelayTask, type AgentRelayTaskRequest } from '../src/agent-relay-transport.js'; +import { NODE_RESULT_PREFIX } from '../src/node-spawn-result.js'; +import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; +import type { NodeSpawnPort, NodeSpawnReceipt } from '../src/node-spawn.js'; +import { EventEmitter } from 'node:events'; +import { AgentWorker } from '../src/worker.js'; +import type { JournalClient } from '../src/journal-client.js'; +import type { StepDispatchEvent } from '../src/protocol.js'; -const directories: string[] = []; -afterEach(async () => { - vi.restoreAllMocks(); - vi.unstubAllEnvs(); - vi.unstubAllGlobals(); - await Promise.all( - directories - .splice(0) - .map((path) => rm(path, { recursive: true, force: true })), - ); +const request = (dataDir: string): AgentRelayTaskRequest => ({ + cli: 'codex', task: 'Implement the change', runId: 'run_1', stepId: 'edit', + idempotencyKey: 'dispatch_1234567890', dataDir, timeoutMs: 10_000, }); -const ok = (data: unknown) => - new Response(JSON.stringify({ ok: true, data }), { - status: 200, - headers: { "content-type": "application/json" }, - }); -async function fixture() { - const dataDir = await mkdtemp(join(tmpdir(), "flows-relay-task-")); - directories.push(dataDir); - const request: AgentRelayTaskRequest = { - cli: "claude", - task: "return output", - runId: "run", - stepId: "step", - idempotencyKey: "journal-key", - dataDir, +function fixture(): { runtime: NodeSpawnRuntime; spawn: ReturnType; + release: ReturnType; status: ReturnType; ack: ReturnType } { + const receipt: NodeSpawnReceipt = { + id: 'inv_spawn_1', action: 'spawn', status: 'completed', nodeId: 'node_1', nodeName: 'desktop-1', + name: 'placeholder', agentId: 'agent_1', spawned: true, ready: true, }; - const invocationId = - "inv_idem_" + - createHash("sha256") - .update( - [ - "action-invoke-v1", - "workspace", - "caller", - "task.run", - "journal-key", - ].join("\0"), - ) - .digest("hex"); - let input: Record; - let status = "completed"; - let caller = "caller"; - let result: unknown = { answer: 42 }; - let generation = "generation-1"; - let losePost = false; - const deadline = new Date(Date.now() + 86400000).toISOString(); - let reads = 0; - const frames: { url: string; init?: RequestInit }[] = []; - const fetchMock = vi.fn( - async (url: string | URL | Request, init?: RequestInit) => { - const address = String(url); - frames.push({ url: address, init }); - expect((init?.headers as Record).authorization).toBe( - "Bearer fixture-agent-token", - ); - expect(init?.redirect).toBe("error"); - if (address.endsWith("/v1/agent")) - return ok({ - id: caller, - workspace_id: "workspace", - name: "caller-name", - }); - if (address.endsWith("/v1/workspace")) return ok({ id: "workspace" }); - if (address.endsWith("/invoke")) { - expect(init?.method).toBe("POST"); - expect( - (init?.headers as Record)["Idempotency-Key"], - ).toBe("journal-key"); - input = JSON.parse(String(init?.body)).input; - if (losePost) throw new Error("response lost after engine commit"); - return ok({ - invocation_id: invocationId, - action_name: "task.run", - input, - status: "completed", - }); - } - expect(address).toBe( - `https://cast.agentrelay.com/v1/actions/task.run/invocations/${invocationId}`, - ); - reads++; - return ok({ - invocation_id: invocationId, - action_name: "task.run", - caller_id: "caller", - input, - status, - task_execution: { - execution_id: `${invocationId}/1`, - run_id: "run", - step_id: "step", - dispatch_id: "journal-key", - deadline, - ...(status === "pending" - ? {} - : { - worker_generation: generation, - accepted_at: "2026-01-01T00:00:00.000Z", - }), - accounting: { tokens_input: 7, tokens_output: 3 }, - }, - output: status === "completed" ? result : null, - error: status === "failed" ? "task_deadline_exceeded" : null, - completed_at: ["completed", "failed"].includes(status) - ? "2026-01-01T00:00:01.000Z" - : null, - }); - }, - ); - return { - request, - fetchMock, - frames, - invocationId, - run: (signal?: AbortSignal) => - runAgentRelayTask(request, { - agentToken: "fixture-agent-token", - fetch: fetchMock as typeof fetch, - pollMs: 1, - signal, - }), - setStatus: (value: string) => { - status = value; - }, - setCaller: (value: string) => { - caller = value; - }, - setOutput: (value: unknown) => { - result = value; - }, - setGeneration: (value: string) => { - generation = value; - }, - losePost: () => { - losePost = true; - }, - get reads() { - return reads; - }, - posts: () => frames.filter((frame) => frame.init?.method === "POST"), + const spawn = vi.fn(async (input: { name: string }) => ({ ...receipt, name: input.name })); + const release = vi.fn(async () => ({ id: 'inv_release_1', action: 'release' as const, + status: 'completed' as const, spawned: false, ready: false, released: true })); + const status = vi.fn(async () => ({ ...receipt, name: receipt.name })); + const ack = vi.fn(async () => ({})); + const port: NodeSpawnPort = { + targets: async () => [], spawn, status, release, }; -} - -describe("durable Relay task transport", () => { - it("requires an agent token; a workspace key is not an invocation credential", () => { - expect(() => - readAgentRelayEnv({ RELAY_API_KEY: "fixture-workspace-key" }), - ).toThrow(/RELAY_AGENT_TOKEN/); - expect(readAgentRelayEnv({ RELAY_AGENT_TOKEN: " token " })).toEqual({ - agentToken: "token", - baseUrl: "https://cast.agentrelay.com", - }); - }); - it("uses supported routes and snake_case data envelopes, returning only final GET output", async () => { - const f = await fixture(); - const receipt = await f.run(); - expect(receipt.output).toEqual({ answer: 42 }); - expect(receipt.status).toBe("completed"); - expect(f.posts()).toHaveLength(1); - expect(f.reads).toBe(1); - expect( - JSON.parse(String(f.posts()[0]!.init!.body)).input.task_context, - ).toEqual({ - run_id: "run", - step_id: "step", - dispatch_id: "journal-key", - timeout_ms: 86400000, - }); - }); - it("keeps readiness/interim states pending even when the POST acknowledgment says completed", async () => { - const f = await fixture(); - f.setStatus("running"); - let settled = false; - const running = f.run().then((value) => { - settled = true; - return value; - }); - await vi.waitFor(() => expect(f.reads).toBeGreaterThan(1)); - expect(settled).toBe(false); - f.setStatus("completed"); - expect((await running).output).toEqual({ answer: 42 }); - }); - it("reconciles an ambiguous POST exclusively by GET without issuing another POST", async () => { - const f = await fixture(); - f.losePost(); - expect((await f.run()).status).toBe("completed"); - expect(f.posts()).toHaveLength(1); - expect(f.reads).toBe(1); - }); - it("reopens the durable dispatch after interruption and reads the original task", async () => { - const f = await fixture(); - f.setStatus("running"); - const abort = new AbortController(); - const running = f.run(abort.signal); - const rejection = expect(running).rejects.toThrow(); - await vi.waitFor(() => expect(f.reads).toBeGreaterThan(0)); - abort.abort(new Error("lease lost")); - await rejection; - f.setStatus("completed"); - expect((await f.run()).status).toBe("completed"); - expect(f.posts()).toHaveLength(1); - }); - it("refuses changed caller, canonical endpoint, or input before another invocation", async () => { - const f = await fixture(); - await f.run(); - f.setCaller("replacement"); - await expect(f.run()).rejects.toThrow(/durable caller/); - f.setCaller("caller"); - f.request.task = "changed"; - await expect(f.run()).rejects.toThrow(/durable caller/); - f.request.task = "return output"; - await expect( - runAgentRelayTask(f.request, { - baseUrl: "https://other.example", - agentToken: "fixture-agent-token", - fetch: f.fetchMock as typeof fetch, - }), - ).rejects.toThrow(/durable caller/); - expect(f.posts()).toHaveLength(1); - }); - it("recovers an old terminal receipt without redispatch after its reconciliation deadline", async () => { - const f = await fixture(); - await f.run(); - const directory = join(f.request.dataDir, "relay-tasks"); - const [name] = await readdir(directory); - const path = join(directory, name!); - const claim = JSON.parse(await readFile(path, "utf8")); - claim.startedAt = Date.now() - 2 * 86400000; - await writeFile(path, JSON.stringify(claim)); - expect((await f.run()).status).toBe("completed"); - expect(f.posts()).toHaveLength(1); - }); - it("fails on corrupt durable state rather than discarding it and replaying", async () => { - const f = await fixture(); - await f.run(); - const directory = join(f.request.dataDir, "relay-tasks"); - const [name] = await readdir(directory); - await writeFile(join(directory, name!), "{"); - await expect(f.run()).rejects.toThrow(); - expect(f.posts()).toHaveLength(1); - }); - it("refuses before POST when the durable dispatch cannot be written", async () => { - const f = await fixture(); - const file = join(f.request.dataDir, "not-a-directory"); - await writeFile(file, "fixture"); - f.request.dataDir = file; - await expect(f.run()).rejects.toThrow(); - expect(f.posts()).toHaveLength(0); - }); - it("preserves explicit terminal failure and scalar/array/null output", async () => { - const f = await fixture(); - f.setStatus("failed"); - expect((await f.run()).error).toBe("task_deadline_exceeded"); - for (const output of [null, 42, "text", [1, 2]]) { - const next = await fixture(); - next.setOutput(output); - expect((await next.run()).output).toEqual(output); - } - }); - it("rejects a changed accepted generation instead of accepting a stale terminal receipt", async () => { - const f = await fixture(); - f.setStatus("running"); - const running = f.run(); - const rejection = expect(running).rejects.toThrow(/generation/); - await vi.waitFor(() => expect(f.reads).toBeGreaterThan(0)); - f.setGeneration("replacement"); - f.setStatus("completed"); - await rejection; - }); - it("rejects wrong correlation and masks arbitrary HTTP error bodies", async () => { - const f = await fixture(); - await expect( - runAgentRelayTask(f.request, { - agentToken: "fixture-token", - fetch: vi.fn( - async () => new Response("secret-response-body", { status: 401 }), - ) as typeof fetch, - }), - ).rejects.toThrow("HTTP 401"); - const changed = vi.fn( - async (url: string | URL | Request, init?: RequestInit) => { - const response = await f.fetchMock(url, init); - const body = (await response.json()) as { - data: { task_execution: { dispatch_id: string } }; + const runtime: NodeSpawnRuntime = { + port, + publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + inbox: { inbox: { + subscribe: async function* () { + const name = spawn.mock.results[0]?.value === undefined + ? receipt.name : (await spawn.mock.results[0]!.value).name; + yield { + id: 'delivery_1', + message: { + id: 'message_1', + from: { id: 'agent_1', name }, + text: NODE_RESULT_PREFIX + JSON.stringify({ + version: 1, run_id: 'run_1', step_id: 'edit', dispatch_id: 'dispatch_1234567890', + status: 'completed', output: { summary: 'done' }, + accounting: { tokens_input: 10, tokens_output: 5 }, + }), + }, }; - if (String(url).includes("/invocations/")) - body.data.task_execution.dispatch_id = "wrong"; - return new Response(JSON.stringify(body), { status: 200 }); }, - ); - await expect( - runAgentRelayTask(f.request, { - agentToken: "fixture-agent-token", - fetch: changed as typeof fetch, - }), - ).rejects.toThrow(/correlation/); - }); -}); - -import { EventEmitter } from "node:events"; -import type { JournalClient } from "../src/journal-client.js"; -import type { StepDispatchEvent } from "../src/protocol.js"; -import { AgentWorker } from "../src/worker.js"; - -async function workerFixture(f: Awaited>, runRoot?: string) { - vi.stubEnv("RELAY_AGENT_TOKEN", "fixture-agent-token"); - vi.stubEnv("RELAY_BASE_URL", "https://cast.agentrelay.com"); - vi.stubGlobal("fetch", f.fetchMock); - const client = Object.assign(new EventEmitter(), { - workerAttach: vi.fn(async () => ({})), - stepHeartbeat: vi.fn(async () => ({ - lease_deadline_ms: Date.now() + 30000, - })), - stepComplete: vi.fn(async (..._args: unknown[]) => ({})), - }); - const worker = new AgentWorker(client as unknown as JournalClient, { - workerId: "fixture-worker", - pins: {}, - dataDir: f.request.dataDir, - ...(runRoot === undefined ? {} : { runRoot }), - }); - const errors: unknown[] = []; - worker.on("error", (error) => errors.push(error)); - await worker.attach(); - const dispatch: StepDispatchEvent = { - run_id: "run", - step_id: "step", - attempt: 1, - step_type: "agent", - spec: { - cli: "claude", - instruction: "return output", - transport: "relay", - model: "codex-medium", - verification: { json_schema: { type: "array" } }, - }, - idempotency_key: "journal-key", - lease_id: "lease", - lease_deadline_ms: Date.now() + 30000, - pins: {}, - wake_context: { triggering_event: { type: "fixture" } }, + ack, + fail: vi.fn(async () => ({})), + } }, }; - return { client, worker, errors, dispatch }; + return { runtime, spawn, release, status, ack }; } -describe("Relay completion at the journal boundary", () => { - it("does not complete at readiness and journals exact output, receipt, and priced accounting", async () => { - const f = await fixture(); - f.setStatus("running"); - f.setOutput([1, 2]); - const w = await workerFixture(f); - w.client.emit("step.dispatch", w.dispatch); - await vi.waitFor(() => expect(f.reads).toBe(1)); - expect(w.client.stepComplete).not.toHaveBeenCalled(); - f.setStatus("completed"); - await w.worker.close(); - expect(w.errors).toEqual([]); - expect(w.client.stepComplete).toHaveBeenCalledTimes(1); - expect(w.client.stepComplete.mock.calls[0]).toEqual([ - "run", - "step", - 1, - "journal-key", - "success", - expect.objectContaining({ - output: [1, 2], - usage: { tokens_in: 7, tokens_out: 3, dollars: "0.000038" }, - trajectory_tail: { - relay_task: expect.objectContaining({ - invocation_id: f.invocationId, - status: "completed", - }), - }, - }), - ]); - const input = JSON.parse(String(f.posts()[0]!.init!.body)).input; - expect(input.task).toContain("Wake context (journaled)"); - expect(input.task).toContain("final=true"); - expect(input.result_schema).toEqual({ type: "array" }); +describe('native node-spawn task transport', () => { + it('spawns once, captures exact-sender inbox evidence, and leaves ack to the journal caller', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-spawn-')); + const f = fixture(); + const result = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.spawn.mock.calls[0]![0]).toMatchObject({ harness: 'codex', idempotencyKey: 'dispatch_1234567890' }); + expect(f.release).toHaveBeenCalledWith(expect.objectContaining({ + receipt: expect.objectContaining({ agentId: 'agent_1', nodeName: 'desktop-1' }), + })); + expect(result.receipt).toMatchObject({ + action_name: 'spawn', status: 'completed', output: { summary: 'done' }, + task_execution: { agent_id: 'agent_1', delivery_id: 'delivery_1', release_invocation_id: 'inv_release_1' }, + }); + expect(f.ack).not.toHaveBeenCalled(); + await result.delivery.ack(); + expect(f.ack).toHaveBeenCalledWith(expect.objectContaining({ inboxItemId: 'delivery_1', state: 'read' })); }); - it("does not forward the local run root as worker_cwd to a remote relay worker", async () => { - const f = await fixture(); - const runRoot = await mkdtemp(join(tmpdir(), "flows-relay-runroot-")); - directories.push(runRoot); - const w = await workerFixture(f, runRoot); - w.client.emit("step.dispatch", w.dispatch); - await vi.waitFor(() => expect(f.posts()).toHaveLength(1)); - await w.worker.close(); - expect(w.errors).toEqual([]); - const input = JSON.parse(String(f.posts()[0]!.init!.body)).input; - expect(input.worker_cwd).toBeUndefined(); + + it('recovers a recorded spawn without issuing a second spawn or release', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-recover-')); + const f = fixture(); + await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + const recovered = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + expect(f.spawn).toHaveBeenCalledTimes(1); + expect(f.release).toHaveBeenCalledTimes(1); + expect(f.status).toHaveBeenCalledWith('spawn', 'inv_spawn_1'); + expect(recovered.receipt.status).toBe('completed'); }); - it("journals terminal task failure as worker_error with its explicit reason", async () => { - const f = await fixture(); - f.setStatus("failed"); - const w = await workerFixture(f); - w.client.emit("step.dispatch", w.dispatch); - await w.worker.close(); - expect(w.errors).toEqual([]); - expect(w.client.stepComplete.mock.calls[0]![4]).toBe("worker_error"); - expect(JSON.stringify(w.client.stepComplete.mock.calls[0])).toContain( - "task_deadline_exceeded", - ); + + it('fails closed across an ambiguous pre-ack crash instead of spawning twice', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-ambiguous-')); + const f = fixture(); + f.spawn.mockRejectedValueOnce(new Error('connection dropped')); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })).rejects.toThrow('connection dropped'); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })).rejects.toThrow('ambiguous pre-ack'); + expect(f.spawn).toHaveBeenCalledTimes(1); }); - it("aborts polling on rejected renewal and never writes a stale completion", async () => { - const f = await fixture(); - f.setStatus("running"); - const w = await workerFixture(f); - w.client.stepHeartbeat - // 3s lease, so worker-lease.ts renews at ~1s: after the durable claim's - // three fsyncs (agent-relay-state.ts `claimRelayTask`) and the POST, - // while `runAgentRelayTask` sits in its 1s poll delay. The old 150ms - // lease renewed at ~50ms and raced those fsyncs, which a loaded CI - // runner loses. `mockImplementationOnce` also reads the clock at the - // heartbeat rather than at mock setup, so the window is the stated one. - .mockImplementationOnce(async () => ({ lease_deadline_ms: Date.now() + 3_000 })) - .mockRejectedValueOnce(new Error("lease replaced")); - w.client.emit("step.dispatch", w.dispatch); - // The POST is the precondition for the abort being mid-poll, not a - // side effect of the timing: wait for it rather than assuming it landed. - await vi.waitFor(() => { expect(f.posts()).toHaveLength(1); }); - await w.worker.close(); - expect(w.client.stepComplete).not.toHaveBeenCalled(); - expect(String(w.errors[0])).toContain("lease replaced"); - expect(f.posts()).toHaveLength(1); + + it('releases the exact confirmed agent before propagating cancellation', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-cancel-')); + const f = fixture(); + const controller = new AbortController(); + f.spawn.mockImplementationOnce(async (input: { name: string }) => { + queueMicrotask(() => controller.abort(new Error('lease lost'))); + return { id: 'inv_spawn_1', action: 'spawn' as const, status: 'completed' as const, + nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, agentId: 'agent_1', + spawned: true as const, ready: true as const }; + }); + await expect(runAgentRelayTask(request(dataDir), { + runtime: f.runtime, signal: controller.signal, + })).rejects.toThrow('lease lost'); + expect(f.release).toHaveBeenCalledWith(expect.objectContaining({ + receipt: expect.objectContaining({ agentId: 'agent_1', nodeName: 'desktop-1' }), + })); }); -}); -import { readFile } from "node:fs/promises"; -it("matches exact HTTP fixtures generated by Relaycast #436, including its deterministic invocation ID", async () => { - const wire = JSON.parse( - await readFile( - new URL("./fixtures/relay-task/relaycast-436.json", import.meta.url), - "utf8", - ), - ); - const f = await fixture(); - let reads = 0; - const fetchMock = vi.fn( - async (url: string | URL | Request, init?: RequestInit) => { - if (String(url).endsWith("/v1/agent")) - return new Response(JSON.stringify(wire.identity), { status: 200 }); - if (String(url).endsWith("/invoke")) { - expect(JSON.parse(String(init?.body))).toEqual({ input: wire.input }); - return new Response(JSON.stringify(wire.ack), { status: 201 }); - } - expect(String(url)).toBe( - `https://cast.agentrelay.com/v1/actions/task.run/invocations/${wire.ack.data.invocation_id}`, - ); - return new Response( - JSON.stringify( - [wire.dispatched, wire.running, wire.completed][reads++], - ), - { status: 200 }, - ); - }, - ); - const result = await runAgentRelayTask(f.request, { - agentToken: "fixture-token", - fetch: fetchMock as typeof fetch, - pollMs: 1, + it('acks the inbox delivery only after stepComplete has entered the journal', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-journal-')); + const f = fixture(); + const order: string[] = []; + f.ack.mockImplementation(async () => { order.push('ack'); return {}; }); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), + stepComplete: vi.fn(async () => { expect(f.ack).not.toHaveBeenCalled(); order.push('journal'); return {}; }), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_1', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + const errors: unknown[] = []; + worker.on('error', error => errors.push(error)); + await worker.attach(); + const dispatch: StepDispatchEvent = { + run_id: 'run_1', step_id: 'edit', attempt: 1, step_type: 'agent', + spec: { cli: 'codex', instruction: 'Implement the change', transport: 'relay', model: 'codex-medium' }, + idempotency_key: 'dispatch_1234567890', lease_id: 'lease_1', + lease_deadline_ms: Date.now() + 30_000, pins: {}, + }; + client.emit('step.dispatch', dispatch); + await worker.close(); + expect(errors).toEqual([]); + expect(order).toEqual(['journal', 'ack']); + expect(client.stepComplete).toHaveBeenCalledWith('run_1', 'edit', 1, 'dispatch_1234567890', 'success', + expect.objectContaining({ output: { summary: 'done' }, trajectory_tail: { + relay_task: expect.objectContaining({ invocation_id: 'inv_spawn_1', status: 'completed' }), + } })); }); - expect(result.output).toEqual(wire.completed.data.output); - expect(result.task_execution).toEqual(wire.completed.data.task_execution); - expect(reads).toBe(3); }); diff --git a/packages/sdk/tests/authored-broker.test.ts b/packages/sdk/tests/authored-broker.test.ts deleted file mode 100644 index 3a41aa623..000000000 --- a/packages/sdk/tests/authored-broker.test.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { expect, it, vi } from 'vitest'; -import { authoredBroker } from '../src/authored-broker.js'; -import { brokerAgentName, brokerInstruction } from '../src/communication/broker-spec.js'; -import { toKernelSpec } from '../src/compile.js'; -import type { StepSpec } from '../src/spec.js'; -function setup(owner = 'root') { - const run = vi.fn(async (step: StepSpec) => { - const kernel = toKernelSpec({ version: '0.1.0', name: 'demo', steps: [step] }); - expect(kernel.steps[0]?.type).toBe('agent'); - expect(kernel.steps[0]).not.toHaveProperty('transport'); - const broker = step.type === 'agent' ? brokerInstruction(step.instruction)! : undefined!; - return { completionReason: 'success', summary: '{"ok":true}', artifacts: [], agentName: broker.agentName }; - }); - return { run, broker: authoredBroker({ name: 'My flow!' }, owner, 'local-worker', run) }; -} -it('lowers broker options into an agent instruction and preserves the JSON summary', async () => { - const { broker, run } = setup(); - const result = await broker.agent('agent-0', 'Implement #42', { transport: 'broker', task: 'Implement', cli: 'codex', - cwd: '/external/worktree', keepAlive: true, skills: ['review-fix-signoff-loop'] }); - expect(result.summary).toBe('{"ok":true}'); - const step = run.mock.calls[0]![0]; - expect(step).toMatchObject({ type: 'agent', timeoutMs: 3_600_000, transportRetries: 0, - surfaces: { streams: [{ stream: 'local-worker' }] } }); - expect(brokerInstruction((step as { instruction: string }).instruction)).toMatchObject({ - cwd: '/external/worktree', channel: 'flow-root', owner: 'root', flow: 'My flow!', step: 'Implement #42', - }); - await broker.message('agent-1', result.agentName!, 'Fix', { timeout: '5s' }); - expect(run.mock.calls[1]![0]).toMatchObject({ timeoutMs: 5000 }); - await broker.release('agent-2', result.agentName!); - await broker.release('agent-3', result.agentName!); - expect(run).toHaveBeenCalledTimes(4); -}); -it('names are sanitized, stable on replay, distinct across runs and duplicate display names', () => { - const name = brokerAgentName('Flow / 🚀', 'Step 42', 'root', 'agent-1'); - expect(name).toMatch(/^[a-z0-9][a-z0-9_-]{0,63}$/); - expect(brokerAgentName('Flow / 🚀', 'Step 42', 'root', 'agent-1')).toBe(name); - expect(brokerAgentName('Flow / 🚀', 'Step 42', 'other', 'agent-1')).not.toBe(name); - expect(brokerAgentName('Flow / 🚀', 'Step 42', 'root', 'agent-2')).not.toBe(name); -}); -it('only accepts retained agents created by this flow', async () => { - const { broker, run } = setup(); - await expect(broker.message('agent-1', 'someone-else', 'Fix')).rejects.toThrow('No retained'); - await expect(broker.release('agent-1', 'someone-else')).rejects.toThrow('No retained'); - expect(run).not.toHaveBeenCalled(); -}); -it('forwards a named gate on a retained message', async () => { - const { broker, run } = setup(); - const result = await broker.agent('agent-0', 'implement', { transport: 'broker', task: 'Implement', cli: 'codex', keepAlive: true }); - const gate = { type: 'artifact_exists' as const, path: 'result.json' }; - await broker.message('agent-1', result.agentName!, 'Fix', undefined, gate); - expect(run.mock.calls[1]![0]).toMatchObject({ verification: gate }); -}); -it('carries declared secrets onto spawn, message, and release', async () => { - const { broker, run } = setup(); - const result = await broker.agent('agent-0', 'implement', { transport: 'broker', task: 'Implement', cli: 'codex', - keepAlive: true, secrets: ['GH_TOKEN'] }, undefined, ['GH_TOKEN']); - expect(run.mock.calls[0]![0]).toMatchObject({ secrets: ['GH_TOKEN'] }); - await broker.message('agent-1', result.agentName!, 'Fix'); - expect(run.mock.calls[1]![0]).toMatchObject({ secrets: ['GH_TOKEN'] }); - await broker.release('agent-2', result.agentName!); - expect(run.mock.calls[2]![0]).toMatchObject({ secrets: ['GH_TOKEN'] }); -}); -it('refuses retained sessions without a local worker pin', async () => { - const run = vi.fn(); - const broker = authoredBroker({ name: 'demo' }, 'root', undefined, run); - await expect(broker.agent('agent-0', 'worker', { transport: 'broker', task: 'Work', keepAlive: true })) - .rejects.toThrow('local agent worker stream'); - expect(run).not.toHaveBeenCalled(); -}); -it.each([ - { skills: ['../unsafe'] }, { keepAlive: 'yes' }, { channel: 'bad channel' }, - { resume: 'previous' }, { maxIterations: 2 }, { transportRetries: 1 }, { cwd: '' }, -])('refuses unsupported or malformed options before admission: %j', async options => { - const { broker, run } = setup(); - await expect(broker.agent('agent-0', 'worker', { transport: 'broker', task: 'Work', ...options } as any)).rejects.toThrow(); - expect(run).not.toHaveBeenCalled(); -}); diff --git a/packages/sdk/tests/broker-environment.test.ts b/packages/sdk/tests/broker-environment.test.ts deleted file mode 100644 index c6b88cef3..000000000 --- a/packages/sdk/tests/broker-environment.test.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { beforeEach, expect, it, vi } from 'vitest'; -const fake = vi.hoisted(() => ({ resolve: vi.fn(), spawn: vi.fn() })); -vi.mock('node:module', () => ({ createRequire: () => ({ resolve: fake.resolve }) })); -vi.mock('node:child_process', () => ({ spawnSync: fake.spawn })); -import { checkBrokerEnvironment } from '../src/communication/broker-preflight.js'; -beforeEach(() => vi.resetAllMocks()); -it('refuses missing packages before attempting a connection', () => { - fake.resolve.mockImplementation(() => { throw new Error('MODULE_NOT_FOUND'); }); - expect(() => checkBrokerEnvironment('/run')).toThrow('optional @agent-relay'); - expect(fake.spawn).not.toHaveBeenCalled(); -}); -it.each([1, null])('refuses unavailable or timed-out local broker probes (%s)', status => { - fake.resolve.mockReturnValue('/installed/driver.js'); fake.spawn.mockReturnValue({ status }); - expect(() => checkBrokerEnvironment('/run')).toThrow('reachable local Relay broker'); -}); -it('uses SDK connect and healthCheck without spawning a broker or agent', () => { - fake.resolve.mockReturnValue('/installed/driver.js'); fake.spawn.mockReturnValue({ status: 0 }); - expect(() => checkBrokerEnvironment('/run')).not.toThrow(); - const args = fake.spawn.mock.calls[0]![1] as string[]; - expect(args[2]).toContain('HarnessDriverClient.connect'); - expect(args[2]).toContain('client.healthCheck'); - expect(args[2]).not.toContain('.spawn'); - expect(args.slice(-2)).toEqual(['file:///installed/driver.js', '/run']); - expect(fake.spawn.mock.calls[0]![2]).toMatchObject({ timeout: 10_000 }); -}); diff --git a/packages/sdk/tests/broker-flow.test.ts b/packages/sdk/tests/broker-flow.test.ts deleted file mode 100644 index a33cebfa0..000000000 --- a/packages/sdk/tests/broker-flow.test.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { afterEach, expect, it, vi } from 'vitest'; -import { flow } from '@relayflows/surface'; -import { executeAuthoredFlow } from '../src/authored-flow-executor.js'; -import { attachLocalAgent } from '../src/local-agent.js'; -import { chainFixture } from './flow-chain-fixture.js'; -const fake = vi.hoisted(() => ({ - invoke: undefined as undefined | ((request: { operation: string; values: string[] }) => Promise), - release: vi.fn(async () => {}), shutdown: vi.fn(async () => {}), spawn: vi.fn(), send: vi.fn(), -})); -vi.mock('../src/authored-preflight.js', () => ({ authoredPreflight: () => async (flow: unknown) => ({ - flow, report: { ok: true, diagnostics: [], resolutions: [], gates: [] }, -}) })); -vi.mock('../src/communication/relay.js', () => ({ connectLocalBroker: async () => ({ - spawnPty: fake.spawn, sendMessage: fake.send, shutdown: fake.shutdown, - onEvent: () => () => {}, connectEvents: () => {}, -}) })); -vi.mock('../src/communication/skills.js', () => ({ installBrokerSkills: async () => {} })); -vi.mock('../src/communication/tools.js', () => ({ openCommunicationTools: async (invoke: typeof fake.invoke) => { - fake.invoke = invoke; - return { path: '/tmp/fake.sock', token: 'fake-token', helperPath: '/tmp/fake.mjs', close: async () => {} }; -} })); -const cleanups: Array<() => Promise> = []; -afterEach(async () => { for (const close of cleanups.splice(0).reverse()) await close(); vi.clearAllMocks(); }); -it('journals spawn, message, and release through the authored surface and worker', async () => { - const fixture = chainFixture(); cleanups.push(() => fixture.close()); - const client = await fixture.connect(); - const worker = await attachLocalAgent(client, fixture.data, undefined, undefined, 2, fixture.root); - cleanups.push(() => worker.close()); - const complete = (text: string, json: string) => { - const turn = /complete '([^']+)'/.exec(text)![1]!; - setTimeout(() => void fake.invoke!({ operation: 'complete', values: [turn, json] }), 1); - }; - fake.spawn.mockImplementation(async input => { - complete(input.task, '{"pr":42}'); - return { name: input.name, release: fake.release, waitForReady: async () => ({ reason: 'ready' }) }; - }); - fake.send.mockImplementation(async input => { - complete(input.text, '{"fixed":true}'); return { event_id: 'event', targets: [input.to] }; - }); - const definition = flow('broker-flow', async f => { - const agent = await f.agent('implement', { task: 'Open PR', transport: 'broker', cli: 'codex', keepAlive: true }); - expect(JSON.parse(agent.summary)).toEqual({ pr: 42 }); - const reply = await f.message(agent.agentName!, 'Fix', { timeout: '5s' }); - expect(JSON.parse(reply.summary)).toEqual({ fixed: true }); - await f.release(agent.agentName!); - expect(fake.release).toHaveBeenCalledTimes(1); - f.done('success'); - }); - const report = await executeAuthoredFlow(definition, client, undefined, { - flowPath: fixture.flowPath, localAgentStream: worker.stream, dataDir: fixture.data, workerCapacity: 2, - }); - expect(report.completionReason).toBe('success'); - const agents = report.journalSteps.filter(step => step.id.startsWith('agent-')); - expect(agents).toHaveLength(3); - for (const step of agents) { - const entries = (await client.journalRead(step.runId, 1)).entries; - expect(entries.some(entry => entry.entry_type === 'step.completed' && entry.payload.completionReason === 'success')).toBe(true); - } - await worker.close(); expect(fake.release).toHaveBeenCalledTimes(1); -}); - -it('runs a named gate on f.message against the broker result', async () => { - const fixture = chainFixture(); cleanups.push(() => fixture.close()); - const client = await fixture.connect(); - const worker = await attachLocalAgent(client, fixture.data, undefined, undefined, 2, fixture.root); - cleanups.push(() => worker.close()); - const complete = (text: string, json: string) => { - const turn = /complete '([^']+)'/.exec(text)![1]!; - setTimeout(() => void fake.invoke!({ operation: 'complete', values: [turn, json] }), 1); - }; - fake.spawn.mockImplementation(async input => { - complete(input.task, '{"pr":42}'); - return { name: input.name, release: fake.release, waitForReady: async () => ({ reason: 'ready' }) }; - }); - fake.send.mockImplementation(async input => { - complete(input.text, '{"fixed":true}'); return { event_id: 'event', targets: [input.to] }; - }); - const definition = flow('broker-message-gate', async f => { - const agent = await f.agent('implement', { task: 'Open PR', transport: 'broker', cli: 'codex', keepAlive: true }); - await f.message(agent.agentName!, 'Fix').gate({ type: 'artifact_exists', path: 'result.json' }); - f.done('success'); - }); - await expect(executeAuthoredFlow(definition, client, undefined, { - flowPath: fixture.flowPath, localAgentStream: worker.stream, dataDir: fixture.data, workerCapacity: 2, - })).rejects.toThrow(/agent-\d+\.gate/); - expect(fake.spawn).toHaveBeenCalledTimes(1); - expect(fake.send).toHaveBeenCalledTimes(1); -}); - -it('releases a retained session after a budgeted named-gate failure', async () => { - const fixture = chainFixture(); cleanups.push(() => fixture.close()); - const client = await fixture.connect(); - const worker = await attachLocalAgent(client, fixture.data, undefined, undefined, 2, fixture.root); - cleanups.push(() => worker.close()); - const complete = (text: string, json: string) => { - const turn = /complete '([^']+)'/.exec(text)![1]!; - setTimeout(() => void fake.invoke!({ operation: 'complete', values: [turn, json] }), 1); - }; - fake.spawn.mockImplementation(async input => { - complete(input.task, '{"pr":42}'); - return { name: input.name, release: fake.release, waitForReady: async () => ({ reason: 'ready' }) }; - }); - fake.send.mockImplementation(async input => { - complete(input.text, '{"fixed":true}'); return { event_id: 'event', targets: [input.to] }; - }); - const definition = flow('broker-release-after-failure', { budget: '$1/run' }, async f => { - const agent = await f.agent('implement', { task: 'Open PR', transport: 'broker', cli: 'codex', keepAlive: true }); - try { - await f.message(agent.agentName!, 'Fix').gate({ type: 'artifact_exists', path: 'result.json' }); - } finally { - await f.release(agent.agentName!); - } - f.done('success'); - }); - await expect(executeAuthoredFlow(definition, client, undefined, { - flowPath: fixture.flowPath, localAgentStream: worker.stream, dataDir: fixture.data, workerCapacity: 2, - })).rejects.toThrow(/agent-\d+\.gate/); - expect(fake.release).toHaveBeenCalled(); -}); - -it.each(['named', 'predicate'] as const)('refuses a retained spawn with a %s gate before allocating a PTY', async gate => { - const fixture = chainFixture(); cleanups.push(() => fixture.close()); - const client = await fixture.connect(); - const definition = flow('broker-gated', async f => { - const agent = f.agent('implement', { task: 'Open PR', transport: 'broker', cli: 'codex', keepAlive: true }); - if (gate === 'predicate') await agent.gate(() => false); - else await agent.gate({ type: 'artifact_exists', path: 'result.json' }); - f.done('success'); - }); - await expect(executeAuthoredFlow(definition, client, undefined, { - flowPath: fixture.flowPath, localAgentStream: 'fake-local-worker', dataDir: fixture.data, - })).rejects.toThrow('Retained broker agents do not support gates'); - expect(fake.spawn).not.toHaveBeenCalled(); -}); diff --git a/packages/sdk/tests/broker-preflight.test.ts b/packages/sdk/tests/broker-preflight.test.ts index c197d81a4..afbfe8cac 100644 --- a/packages/sdk/tests/broker-preflight.test.ts +++ b/packages/sdk/tests/broker-preflight.test.ts @@ -1,51 +1,19 @@ -import { afterEach, expect, it, vi } from 'vitest'; -import { checkAuthoredFlow } from '../src/cli/check.js'; +import { expect, it } from 'vitest'; import { budgetDiagnostics } from '../src/budget-preflight.js'; import { compileSpec } from '../src/compile.js'; -import { cliProbeKey, type CliProbeOutcome } from '../src/preflight.js'; import { BROKER_TYPE } from '../src/communication/broker-spec.js'; -const environment = vi.hoisted(() => vi.fn()); -vi.mock('../src/communication/broker-preflight.js', () => ({ checkBrokerEnvironment: environment })); -afterEach(() => vi.resetAllMocks()); -function spec(operation: 'spawn' | 'message' | 'release' = 'spawn') { +import { checkBrokerEnvironment } from '../src/communication/broker-preflight.js'; + +function spec() { return { version: '0.1.0', budget: '$1/run', steps: [{ id: 'agent', type: 'agent', cli: 'claude', - model: 'claude-sonnet-4-6', instruction: JSON.stringify({ type: BROKER_TYPE, operation, + model: 'claude-sonnet-4-6', instruction: JSON.stringify({ type: BROKER_TYPE, operation: 'spawn', owner: 'root', flow: 'test', step: 'implement', agentName: 'test-agent', task: 'Work', skills: [], channel: 'flow-root', keepAlive: false, timeoutMs: 1000 }) }] }; } -it('warns that even a priced broker model cannot be bounded by token or dollar ceilings', () => { - const warnings = budgetDiagnostics(compileSpec(spec())); - expect(warnings).toEqual([expect.objectContaining({ kind: 'budget_unmetered', stepId: 'agent', - message: expect.stringContaining('Token and dollar ceilings cannot bound its spend') })]); -}); -it.each([undefined, { tokens: 100 }])('warns about unmetered sessions even without a dollar budget', budget => { - expect(budgetDiagnostics(compileSpec({ ...spec(), budget }))).toEqual([ + +it('keeps legacy broker instructions unmetered while refusing their runtime', () => { + expect(budgetDiagnostics(compileSpec(spec() as any))).toEqual([ expect.objectContaining({ kind: 'budget_unmetered', stepId: 'agent' }), ]); -}); -it.each([false, true])('checks broker prerequisites before returning an admissible flow (available=%s)', available => { - if (!available) environment.mockImplementation(() => { throw new Error('missing local broker'); }); - const cache = new Map([[cliProbeKey({ stepId: 'agent', cli: 'claude', source: 'step', - model: 'claude-sonnet-4-6' }, true), { result: { exists: true, authenticated: true, modelAvailable: true, executable: '/usr/bin/true' } }]]); - const checked = checkAuthoredFlow(spec() as any, '/tmp/test.flow.ts', { - projectConfig: { directory: '/tmp', models: [], executors: [] }, probeCache: cache, - }); - expect(environment).toHaveBeenCalledTimes(1); - expect(checked.report.ok).toBe(available); - if (!available) { - expect(checked.flow).toBeUndefined(); - expect(checked.report.diagnostics).toContainEqual(expect.objectContaining({ severity: 'refusal', kind: 'probe_failed', message: 'missing local broker' })); - } - expect(checked.report.diagnostics).toContainEqual(expect.objectContaining({ kind: 'budget_unmetered' })); -}); -it.each(['message', 'release'] as const)('leaves broker %s steps to the worker instead of probing again', operation => { - environment.mockImplementation(() => { throw new Error('missing local broker'); }); - const cache = new Map([[cliProbeKey({ stepId: 'agent', cli: 'claude', source: 'step', - model: 'claude-sonnet-4-6' }, true), { result: { exists: true, authenticated: true, modelAvailable: true, executable: '/usr/bin/true' } }]]); - const checked = checkAuthoredFlow(spec(operation) as any, '/tmp/test.flow.ts', { - projectConfig: { directory: '/tmp', models: [], executors: [] }, probeCache: cache, - }); - expect(environment).not.toHaveBeenCalled(); - expect(checked.report.ok).toBe(true); - expect(checked.flow).toBeDefined(); + expect(() => checkBrokerEnvironment()).toThrow('transport: broker was removed'); }); diff --git a/packages/sdk/tests/broker-retirement.test.ts b/packages/sdk/tests/broker-retirement.test.ts new file mode 100644 index 000000000..2d4e80e8d --- /dev/null +++ b/packages/sdk/tests/broker-retirement.test.ts @@ -0,0 +1,31 @@ +import { readFile } from 'node:fs/promises'; +import { expect, it, vi } from 'vitest'; +import { authoredBroker } from '../src/authored-broker.js'; +import { checkBrokerEnvironment } from '../src/communication/broker-preflight.js'; +import { checkCommunicationEnvironment } from '../src/communication/preflight.js'; + +it('removes harness-driver from the published dependency contract', async () => { + const manifest = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8')); + expect(JSON.stringify({ dependencies: manifest.dependencies, peerDependencies: manifest.peerDependencies, + optionalDependencies: manifest.optionalDependencies })).not.toContain('@agent-relay/harness-driver'); + expect(manifest.peerDependencies['@agent-relay/sdk']).toBe('>=17.2.1 <18'); +}); + +it('fails legacy retained sessions before creating a journal child run', async () => { + const run = vi.fn(); + const broker = authoredBroker({ name: 'demo' }, 'root', 'local-worker', run); + await expect(broker.agent('step', 'worker', { transport: 'broker', task: 'work' })) + .rejects.toThrow('transport: broker was removed'); + await expect(broker.message('step', 'worker', 'continue')).rejects.toThrow('transport: broker was removed'); + expect(run).not.toHaveBeenCalled(); +}); + +it('has no health-check fallback that can restart or reconnect to a broker', () => { + expect(() => checkBrokerEnvironment('/workspace')).toThrow('transport: broker was removed'); +}); + +it('fails communication admission until the native local caller contract exists', () => { + expect(() => checkCommunicationEnvironment({ version: '0.1.0', steps: [{ id: 'a', type: 'agent', cli: 'codex', + instruction: JSON.stringify({ type: 'relayflows.communication.v1', instruction: 'talk', incoming: [], + outgoing: ['b'], timeoutMs: 1000 }) }] })).toThrow('Native node-spawn communication'); +}); diff --git a/packages/sdk/tests/broker-worker.test.ts b/packages/sdk/tests/broker-worker.test.ts deleted file mode 100644 index 6b20dda97..000000000 --- a/packages/sdk/tests/broker-worker.test.ts +++ /dev/null @@ -1,226 +0,0 @@ -import { afterEach, beforeEach, expect, it, vi } from 'vitest'; -import type { JournalClient } from '../src/journal-client.js'; -import type { StepDispatchEvent } from '../src/protocol.js'; -import { BrokerSessions } from '../src/communication/broker-worker.js'; -import { BROKER_TYPE, type BrokerInstruction } from '../src/communication/broker-spec.js'; -const fake = vi.hoisted(() => ({ - event: undefined as undefined | ((event: { kind: string; name: string }) => void), - invoke: undefined as undefined | ((request: { operation: string; values: string[] }) => Promise), - release: vi.fn(async () => {}), shutdown: vi.fn(async () => {}), close: vi.fn(async () => {}), - ready: vi.fn(async () => ({ reason: 'ready' })), spawn: vi.fn(), send: vi.fn(), install: vi.fn(), -})); -vi.mock('../src/communication/relay.js', () => ({ connectLocalBroker: async () => ({ - spawnPty: fake.spawn, sendMessage: fake.send, shutdown: fake.shutdown, connectEvents: () => {}, onEvent: (callback: typeof fake.event) => { fake.event = callback; return () => {}; }, -}) })); -vi.mock('../src/communication/skills.js', () => ({ installBrokerSkills: fake.install })); -vi.mock('../src/communication/tools.js', () => ({ openCommunicationTools: async (invoke: typeof fake.invoke) => { - fake.invoke = invoke; - return { path: '/tmp/fake.sock', token: 'fake-token', helperPath: '/tmp/fake.mjs', close: fake.close }; -} })); -let sessions: BrokerSessions; -beforeEach(() => { - vi.clearAllMocks(); - sessions = new BrokerSessions(process.cwd()); - fake.ready.mockResolvedValue({ reason: 'ready' }); - const complete = (text: string) => { - const turn = /complete '([^']+)'/.exec(text)![1]!; - setTimeout(() => void fake.invoke!({ operation: 'complete', values: [turn, '{"ok":true}'] }), 1); - }; - fake.spawn.mockImplementation(async input => { - complete(input.task); - return { name: input.name, release: fake.release, waitForReady: fake.ready }; - }); - fake.send.mockImplementation(async input => { complete(input.text); return { event_id: 'event', targets: [input.to] }; }); -}); -afterEach(async () => sessions.close()); -const instruction: BrokerInstruction = { type: BROKER_TYPE, operation: 'spawn', owner: 'root', flow: 'demo', step: 'implement', - agentName: 'demo-implement-unique', task: 'Implement', skills: ['review-fix-signoff-loop'], channel: 'flow-root', keepAlive: true, timeoutMs: 2000 }; -function fixture() { - const client = { stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), stepComplete: vi.fn(async (..._args: unknown[]) => ({})) }; - let number = 0; - return { client, execute: (change: Partial = {}) => sessions.complete(client as unknown as JournalClient, { - run_id: `child-${++number}`, step_id: 'agent', attempt: 1, idempotency_key: `key-${number}`, pins: {}, - lease_id: 'lease', lease_deadline_ms: Date.now() + 30_000, - spec: { type: 'agent', cli: 'codex', model: 'gpt-5.4', instruction: '' }, - } as StepDispatchEvent, { ...instruction, ...change }, '/tmp/flows-broker-tests') }; -} -it('spawns with channel/team/metadata/skills, then reuses the retained session for a message', async () => { - const f = fixture(); await f.execute(); - expect(fake.install).toHaveBeenCalledWith(instruction.skills, 'codex', process.cwd(), expect.any(AbortSignal)); - expect(fake.spawn).toHaveBeenCalledWith(expect.objectContaining({ name: instruction.agentName, - skipRelayPrompt: false, channels: ['flow-root'], team: 'flow-root', cwd: process.cwd(), model: 'gpt-5.4', - harnessConfig: expect.objectContaining({ env: expect.objectContaining({ RELAYFLOW_RUN_ID: 'root', RELAYFLOW_FLOW_NAME: 'demo', RELAYFLOW_STEP_NAME: 'implement' }) }), - })); - expect(fake.release).not.toHaveBeenCalled(); - await f.execute({ operation: 'message', task: 'Fix review comments' }); - expect(fake.spawn).toHaveBeenCalledTimes(1); - expect(fake.send).toHaveBeenCalledWith(expect.objectContaining({ to: instruction.agentName, text: expect.stringContaining('Fix review comments') })); - expect(f.client.stepComplete).toHaveBeenLastCalledWith('child-2', 'agent', 1, 'key-2', 'success', expect.objectContaining({ - output: { completionReason: 'success', summary: '{"ok":true}', artifacts: [], agentName: instruction.agentName }, - })); - await sessions.close(); expect(fake.release).toHaveBeenCalledTimes(1); expect(fake.shutdown).toHaveBeenCalledTimes(1); -}); -it('passes a declared secret into the broker PTY and withholds other delivered secrets', async () => { - await sessions.close(); - sessions = new BrokerSessions(process.cwd(), { - OPENAI_API_KEY: 'codex-key', GH_TOKEN: 'repository-secret', OTHER_SECRET: 'granted', - RELAY_SECRET_ENV_NAMES: '["OPENAI_API_KEY","GH_TOKEN","OTHER_SECRET"]', RELAY_API_KEY: 'broker-key', - }); - const client = { supportsFeature: () => true, stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), - stepComplete: vi.fn(async () => ({})) }; - await sessions.complete(client as unknown as JournalClient, { - run_id: 'child-secret', step_id: 'agent', attempt: 1, idempotency_key: 'key-secret', pins: {}, - lease_id: 'lease', lease_deadline_ms: Date.now() + 30_000, - spec: { type: 'agent', cli: 'codex', instruction: '', secrets: ['OTHER_SECRET'] }, - } as StepDispatchEvent, instruction, '/tmp/flows-broker-tests'); - const env = fake.spawn.mock.calls[0]![0].harnessConfig.env as Record; - expect(env.OTHER_SECRET).toBe('granted'); - expect(env.OPENAI_API_KEY).toBe('codex-key'); - expect(env).not.toHaveProperty('GH_TOKEN'); - expect(env.RELAY_API_KEY).toBe(''); -}); -it('explicit release is idempotent and later messages fail closed', async () => { - const f = fixture(); await f.execute(); await f.execute({ operation: 'release' }); await f.execute({ operation: 'release' }); - expect(fake.release).toHaveBeenCalledTimes(1); - await f.execute({ operation: 'message' }); - expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('worker_error'); expect(fake.send).not.toHaveBeenCalled(); -}); -it('releases a non-retained agent before reporting success', async () => { - const f = fixture(); await f.execute({ keepAlive: false }); - expect(fake.release).toHaveBeenCalledTimes(1); - expect(fake.release.mock.invocationCallOrder[0]).toBeLessThan(f.client.stepComplete.mock.invocationCallOrder[0]!); -}); -it('rejects stale, duplicate, and non-JSON completions', async () => { - const f = fixture(); await f.execute(); - await expect(fake.invoke!({ operation: 'complete', values: ['child-1:agent:1', '{}'] })).rejects.toThrow('stale'); - fake.send.mockImplementationOnce(async input => { - await expect(fake.invoke!({ operation: 'complete', values: ['child-1:agent:1', '{}'] })).rejects.toThrow('stale'); - await expect(fake.invoke!({ operation: 'complete', values: ['child-2:agent:1', 'not json'] })).rejects.toThrow(); - await fake.invoke!({ operation: 'complete', values: ['child-2:agent:1', '[]'] }); - return { event_id: 'event', targets: [input.to] }; - }); - await f.execute({ operation: 'message' }); expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('success'); -}); -it('times out bounded messages and releases the session', async () => { - const f = fixture(); await f.execute(); fake.send.mockResolvedValue({ event_id: 'event', targets: [instruction.agentName] }); - await f.execute({ operation: 'message', timeoutMs: 10 }); - expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('timeout'); expect(fake.release).toHaveBeenCalledTimes(1); -}); -it('refuses missing delivery targets', async () => { - const f = fixture(); await f.execute(); fake.send.mockResolvedValue({ event_id: 'unsupported_operation', targets: [] }); - await f.execute({ operation: 'message' }); - expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('worker_error'); expect(fake.release).toHaveBeenCalledTimes(1); -}); -it('releases retained resources when the journal write fails', async () => { - const f = fixture(); f.client.stepComplete.mockRejectedValueOnce(new Error('journal unavailable')); - await expect(f.execute()).rejects.toThrow('journal unavailable'); - expect(fake.release).toHaveBeenCalledTimes(1); expect(fake.close).toHaveBeenCalledTimes(1); -}); -it('refuses invalid cwd and failed skills before spawn', async () => { - const f = fixture(); await f.execute({ cwd: '/this-directory-does-not-exist' }); expect(fake.spawn).not.toHaveBeenCalled(); - fake.install.mockRejectedValueOnce(new Error('unknown canonical skill')); - await f.execute(); expect(fake.spawn).not.toHaveBeenCalled(); expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('worker_error'); -}); -it('releases on readiness failure', async () => { - fake.spawn.mockImplementationOnce(async input => ({ name: input.name, release: fake.release, waitForReady: fake.ready })); - fake.ready.mockResolvedValue({ reason: 'timeout' }); - const f = fixture(); await f.execute(); expect(fake.release).toHaveBeenCalledTimes(1); - expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('worker_error'); -}); -it('times out a spawn that never returns and releases a PTY that arrives later', async () => { - let resolveSpawn: (value: unknown) => void = () => {}; - fake.spawn.mockImplementationOnce(() => new Promise(resolve => { resolveSpawn = resolve; })); - const f = fixture(); - await f.execute({ timeoutMs: 100 }); - expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('timeout'); - const lateRelease = vi.fn(async () => {}); - resolveSpawn({ name: 'late', release: lateRelease, waitForReady: fake.ready }); - await new Promise(resolve => setTimeout(resolve, 20)); - expect(lateRelease).toHaveBeenCalledWith('Flow broker session ended', { deleteIdentity: true }); - expect(fake.release).not.toHaveBeenCalled(); -}); -it('a late release failure stays inside the timed-out spawn', async () => { - let resolveSpawn: (value: unknown) => void = () => {}; - fake.spawn.mockImplementationOnce(() => new Promise(resolve => { resolveSpawn = resolve; })); - const rejections: unknown[] = []; - const onRejection = (error: unknown) => { rejections.push(error); }; - process.on('unhandledRejection', onRejection); - try { - const f = fixture(); - await f.execute({ timeoutMs: 100 }); - const lateRelease = vi.fn(async () => { throw new Error('late release failed'); }); - resolveSpawn({ name: 'late', release: lateRelease, waitForReady: fake.ready }); - await new Promise(resolve => setTimeout(resolve, 30)); - expect(lateRelease).toHaveBeenCalledTimes(1); - expect(rejections).toEqual([]); - expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('timeout'); - } finally { - process.off('unhandledRejection', onRejection); - } -}); -it('times out even when message submission never responds', async () => { - const f = fixture(); await f.execute(); fake.send.mockImplementationOnce(() => new Promise(() => {})); - await f.execute({ operation: 'message', timeoutMs: 10 }); - expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('timeout'); expect(fake.release).toHaveBeenCalledTimes(1); -}); -it('retains a failed release for retry instead of acknowledging it as removed', async () => { - const f = fixture(); await f.execute(); fake.release.mockRejectedValueOnce(new Error('broker unavailable')); - await f.execute({ operation: 'release' }); - expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('worker_error'); - expect(fake.close).not.toHaveBeenCalled(); - await f.execute({ operation: 'release' }); - expect(fake.release).toHaveBeenCalledTimes(2); expect(fake.close).toHaveBeenCalledTimes(1); -}); -it('a replacement worker refuses messages to lost sessions without spawning a replacement', async () => { - const f = fixture(); await f.execute({ operation: 'message' }); - expect(fake.spawn).not.toHaveBeenCalled(); expect(fake.send).not.toHaveBeenCalled(); - expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('worker_error'); -}); -it('releases on lease loss without writing a worker-error completion', async () => { - const { JournalRequestTimeoutError } = await import('../src/journal-client.js'); - const { isLeaseLost } = await import('../src/worker-lease.js'); - fake.spawn.mockImplementationOnce(async input => ({ name: input.name, release: fake.release, waitForReady: fake.ready })); - const client = { stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 100, lease_ttl_ms: 100 })), stepComplete: vi.fn() }; - client.stepHeartbeat.mockImplementationOnce(async () => ({ lease_deadline_ms: Date.now() + 100, lease_ttl_ms: 100 })); - client.stepHeartbeat.mockImplementation(async () => { throw new JournalRequestTimeoutError('step.heartbeat', 10); }); - const error = await sessions.complete(client as unknown as JournalClient, { - run_id: 'child', step_id: 'agent', attempt: 1, idempotency_key: 'key', pins: {}, - lease_id: 'lease', lease_deadline_ms: Date.now() + 100, lease_ttl_ms: 100, - spec: { type: 'agent', cli: 'codex', instruction: '' }, - } as StepDispatchEvent, instruction, '/tmp/flows-broker-tests').catch(error => error); - expect(isLeaseLost(error)).toBe(true); expect(client.stepComplete).not.toHaveBeenCalled(); - expect(fake.release).toHaveBeenCalledTimes(1); -}); - -it('accepts completion even when the PTY exits before the helper response flushes', async () => { - fake.spawn.mockImplementationOnce(async input => { - const turn = /complete '([^']+)'/.exec(input.task)![1]!; - await fake.invoke!({ operation: 'complete', values: [turn, '{}'] }); - fake.event!({ kind: 'agent_exit', name: input.name }); - return { name: input.name, release: fake.release, waitForReady: fake.ready }; - }); - const f = fixture(); await f.execute({ keepAlive: false }); - expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('success'); -}); - -it('retires an idle exited session before a later message can send', async () => { - const f = fixture(); await f.execute(); - fake.event!({ kind: 'agent_exit', name: instruction.agentName }); - await f.execute({ operation: 'message' }); - expect(fake.send).not.toHaveBeenCalled(); - expect(fake.release).toHaveBeenCalledTimes(1); - expect(fake.close).toHaveBeenCalledTimes(1); - expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('worker_error'); -}); - -it('retries failed idle-exit cleanup without sending to the dead PTY', async () => { - const f = fixture(); await f.execute(); - fake.release.mockRejectedValueOnce(new Error('broker unavailable')); - fake.event!({ kind: 'agent_exit', name: instruction.agentName }); - await new Promise(resolve => setTimeout(resolve, 0)); - await f.execute({ operation: 'message' }); - expect(fake.send).not.toHaveBeenCalled(); - expect(fake.release).toHaveBeenCalledTimes(2); - expect(fake.close).toHaveBeenCalledTimes(1); - expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('worker_error'); -}); diff --git a/packages/sdk/tests/check-helper-communication.test.ts b/packages/sdk/tests/check-helper-communication.test.ts index 36f0ad069..bc33713dd 100644 --- a/packages/sdk/tests/check-helper-communication.test.ts +++ b/packages/sdk/tests/check-helper-communication.test.ts @@ -53,7 +53,7 @@ it.each([false, true])('keeps the communication refusal and budget warning wheth const report = communicatingYaml('slack', mounted)(true); expect(report.ok).toBe(false); expect(report.diagnostics).toEqual(expect.arrayContaining([ - expect.objectContaining({ severity: 'refusal', kind: 'probe_failed', message: expect.stringContaining('RELAY_API_KEY') }), + expect.objectContaining({ severity: 'refusal', kind: 'probe_failed', message: expect.stringContaining('Native node-spawn communication') }), expect.objectContaining({ severity: 'warning', kind: 'budget_unmetered' }), ])); expect(report.diagnostics.some(d => d.kind === 'helper_credential_unresolved')).toBe(!mounted); diff --git a/packages/sdk/tests/communication-environment-preflight.test.ts b/packages/sdk/tests/communication-environment-preflight.test.ts index d59a956e8..54fbaea7b 100644 --- a/packages/sdk/tests/communication-environment-preflight.test.ts +++ b/packages/sdk/tests/communication-environment-preflight.test.ts @@ -14,19 +14,10 @@ it('ordinary flows never inspect Relay credentials, packages, or broker', () => checkCommunicationEnvironment({ version: '0.1.0', steps: [{ id: 'plain', type: 'agent', instruction: 'hello' }] }); expect(mocks.resolve).not.toHaveBeenCalled(); expect(mocks.spawn).not.toHaveBeenCalled(); }); -it('refuses missing credentials before package or daemon access', () => { - vi.stubEnv('RELAY_API_KEY', undefined); - expect(() => checkCommunicationEnvironment(spec)).toThrow('requires RELAY_API_KEY'); +it.each([undefined, '', 'rk_live_test'])('always refuses retired communication without probing the broker (credential=%s)', credential => { + vi.stubEnv('RELAY_API_KEY', credential); + expect(() => checkCommunicationEnvironment(spec)).toThrow('broker/PTy was removed'); + expect(mocks.resolve).not.toHaveBeenCalled(); expect(mocks.spawn).not.toHaveBeenCalled(); -}); -it('refuses missing optional packages', () => { - vi.stubEnv('RELAY_API_KEY', 'rk_live_test'); mocks.resolve.mockImplementation(() => { throw Error('MODULE_NOT_FOUND'); }); - expect(() => checkCommunicationEnvironment(spec)).toThrow('optional @agent-relay'); -}); -it.each(['missing', 'not-executable', 'ready'])('checks the broker executable: %s', state => { - vi.stubEnv('RELAY_API_KEY', 'rk_live_test'); mocks.resolve.mockReturnValue('/installed/broker-path.js'); - mocks.spawn.mockReturnValue({ status: state === 'missing' ? 1 : 0, stdout: '/installed/broker' }); - if (state === 'not-executable') mocks.access.mockImplementation(() => { throw Error('EACCES'); }); - if (state === 'ready') expect(() => checkCommunicationEnvironment(spec)).not.toThrow(); - else expect(() => checkCommunicationEnvironment(spec)).toThrow('executable Relay broker'); + expect(mocks.access).not.toHaveBeenCalled(); }); diff --git a/packages/sdk/tests/communication-feature-secret-scoping.test.ts b/packages/sdk/tests/communication-feature-secret-scoping.test.ts deleted file mode 100644 index a0ac18688..000000000 --- a/packages/sdk/tests/communication-feature-secret-scoping.test.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { EventEmitter } from 'node:events'; -import { mkdtempSync, rmSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { expect, it, vi } from 'vitest'; -import { completeCommunicationDispatch } from '../src/communication/worker.js'; -import type { JournalClient } from '../src/journal-client.js'; -import type { StepDispatchEvent } from '../src/protocol.js'; - -const state = vi.hoisted(() => ({ environments: [] as Array> })); -vi.mock('../src/communication/relay.js', () => ({ acquireRelayRuntime: async () => ({ - prefix: 'flow-test', - broker: { - onEvent: () => () => {}, - spawnPty: async (input: { harnessConfig: { env: Record } }) => { - state.environments.push(input.harnessConfig.env); - return { waitForReady: async () => ({ reason: 'failed' }), release: async () => {} }; - }, - }, - close: async () => {}, -}) })); - -it('communication worker uses the negotiated capability to scope its broker child', async () => { - const root = mkdtempSync(join(tmpdir(), 'flows-communication-capability-')); - state.environments.length = 0; - try { - for (const scoped of [true, false]) { - const client = new EventEmitter() as EventEmitter & Record; - client.supportsFeature = (name: string) => name === 'step_secret_scoping' && scoped; - client.stepHeartbeat = async () => ({ lease_deadline_ms: Date.now() + 30_000 }); - client.stepComplete = async () => ({}); - const dispatch = { - run_id: 'run', step_id: 'agent', step_type: 'agent', attempt: 1, - idempotency_key: `key-${scoped}`, lease_id: 'lease', lease_deadline_ms: Date.now() + 30_000, - pins: { workspace: [], streams: [] }, - spec: { cli: 'claude', instruction: 'task' }, - } satisfies StepDispatchEvent; - await completeCommunicationDispatch(client as unknown as JournalClient, dispatch, - { type: 'relayflows.communication.v1', instruction: 'task', incoming: [], outgoing: [], timeoutMs: 1000 }, - root, root, { - ANTHROPIC_API_KEY: 'delivered-provider-key', GH_TOKEN: 'user-repository-secret', - RELAY_SECRET_ENV_NAMES: '["ANTHROPIC_API_KEY","GH_TOKEN"]', - RELAY_API_KEY: 'broker-key', RELAYFLOW_V2_SIGNAL_READ_TOKEN: 'private-signal-token', - }); - const env = state.environments.at(-1)!; - expect(env.ANTHROPIC_API_KEY).toBe('delivered-provider-key'); - expect(env).not.toHaveProperty('GH_TOKEN'); - expect(env.RELAY_API_KEY).toBe(''); - expect(env).not.toHaveProperty('RELAYFLOW_V2_SIGNAL_READ_TOKEN'); - } - expect(state.environments).toHaveLength(2); - const client = new EventEmitter() as EventEmitter & Record; - client.supportsFeature = (name: string) => name === 'step_secret_scoping'; - client.stepHeartbeat = async () => ({ lease_deadline_ms: Date.now() + 30_000 }); - client.stepComplete = async () => ({}); - const dispatch = { - run_id: 'run', step_id: 'agent', step_type: 'agent', attempt: 1, - idempotency_key: 'key-granted', lease_id: 'lease', lease_deadline_ms: Date.now() + 30_000, - pins: { workspace: [], streams: [] }, - spec: { cli: 'claude', instruction: 'task', secrets: ['GH_TOKEN'] }, - } satisfies StepDispatchEvent; - await completeCommunicationDispatch(client as unknown as JournalClient, dispatch, - { type: 'relayflows.communication.v1', instruction: 'task', incoming: [], outgoing: [], timeoutMs: 1000 }, - root, root, { - ANTHROPIC_API_KEY: 'delivered-provider-key', GH_TOKEN: 'user-repository-secret', - RELAY_SECRET_ENV_NAMES: '["ANTHROPIC_API_KEY","GH_TOKEN"]', - RELAY_API_KEY: 'broker-key', - }); - expect(state.environments.at(-1)!.GH_TOKEN).toBe('user-repository-secret'); - expect(state.environments.at(-1)!.ANTHROPIC_API_KEY).toBe('delivered-provider-key'); - } finally { rmSync(root, { recursive: true, force: true }); } -}, 10_000); diff --git a/packages/sdk/tests/communication-mixed-resume.test.ts b/packages/sdk/tests/communication-mixed-resume.test.ts deleted file mode 100644 index 8133bd5c8..000000000 --- a/packages/sdk/tests/communication-mixed-resume.test.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; -import { join } from 'node:path'; -import { tmpdir } from 'node:os'; -import { expect, it, vi } from 'vitest'; -import { compileSpec, toKernelSpec } from '../src/compile.js'; -import { JournalClient } from '../src/journal-client.js'; -import { ensureDaemon } from '../src/daemon-lifecycle.js'; -import { resumeFlow, socketFor } from '../src/cli/run.js'; -import type { StepDispatchEvent } from '../src/protocol.js'; - -const state = vi.hoisted(() => ({ started: new Set(), ready: Promise.resolve(), release: () => {} })); -vi.mock('../src/communication/preflight.js', () => ({ checkCommunicationEnvironment: () => {}, CommunicationEnvironmentError: class extends Error {} })); -vi.mock('../src/communication/relay.js', () => ({ loadRelayModules: async () => ({}) })); -vi.mock('../src/worker-cli.js', () => ({ runAgentCli: async () => { - // Ordinary work cannot finish until BOTH conversation peers have capacity. - await state.ready; - return { exit_code: 0, stdout_tail: 'ordinary completed', stderr_tail: '' }; -} })); -vi.mock('../src/communication/worker.js', () => ({ requireCommunicationCli: () => {}, - completeCommunicationDispatch: async (client: JournalClient, dispatch: StepDispatchEvent) => { - state.started.add(dispatch.step_id); - if (state.started.size === 2) state.release(); - await state.ready; - await client.stepComplete(dispatch.run_id, dispatch.step_id, dispatch.attempt, dispatch.idempotency_key, - 'success', { output: { summary: 'peer started concurrently' }, started_pins: dispatch.pins, end_pins: dispatch.pins }); - }, -})); - -it('resumes mixed ordinary and linked agents through the real daemon without stealing peer capacity', async () => { - const root = mkdtempSync(join(tmpdir(), 'communication-resume-')); - const dataDir = join(root, 'data'); - state.started.clear(); - let reject!: (error: Error) => void; - state.ready = new Promise((yes, no) => { state.release = yes; reject = no; }); - void state.ready.catch(() => {}); - const timeout = setTimeout(() => reject(new Error('ordinary work occupied a conversation worker')), 5000); - const client = new JournalClient(socketFor(dataDir)); - try { - expect((await ensureDaemon(dataDir)).kind).toBe('attached'); - await client.connect(); await client.hello('mixed-resume-test'); - const spec = toKernelSpec(compileSpec({ version: '0.1.0', cli: 'test-cli', - communication: { links: [{ from: 'a', to: 'b' }, { from: 'b', to: 'a' }] }, - steps: ['ordinary1', 'ordinary2', 'a', 'b'].map(id => ({ id, type: 'agent', cli: 'test-cli', instruction: id })), - })); - const parked = await client.runStart(spec); - expect(parked.status).toBe('parked'); - const resumed = await resumeFlow(parked.run_id, dataDir, { localAgent: true, signal: AbortSignal.timeout(8000) }); - expect(resumed.report, JSON.stringify(resumed.report)).toMatchObject({ status: 'completed', completionReason: 'success', completedSteps: 4 }); - expect(state.started).toEqual(new Set(['a', 'b'])); - const entries = (await client.journalRead(parked.run_id, 1)).entries; - expect(entries.filter(entry => entry.entry_type === 'step.completed')).toHaveLength(4); - } finally { - clearTimeout(timeout); state.release(); client.close(); - try { process.kill(JSON.parse(readFileSync(join(dataDir, 'connection.json'), 'utf8')).pid, 'SIGTERM'); } - finally { rmSync(root, { recursive: true, force: true }); } - } -}, 15_000); diff --git a/packages/sdk/tests/communication-refusal.test.ts b/packages/sdk/tests/communication-refusal.test.ts index 9f328b259..f4a4ffde7 100644 --- a/packages/sdk/tests/communication-refusal.test.ts +++ b/packages/sdk/tests/communication-refusal.test.ts @@ -5,7 +5,7 @@ import { expect, it, vi } from 'vitest'; import { checkFlow } from '../src/cli/check.js'; import { runFlow } from '../src/cli/run.js'; -it('check and run refuse missing Relay prerequisites before creating a daemon or journal', async () => { +it('check and run refuse retired broker communication before creating a daemon or journal', async () => { const root = mkdtempSync(join(tmpdir(), 'communication-refusal-')); vi.stubEnv('RELAY_API_KEY', undefined); try { @@ -19,11 +19,11 @@ it('check and run refuse missing Relay prerequisites before creating a daemon or const checked = checkFlow(path); expect(checked.report.ok).toBe(false); expect(checked.flow).toBeUndefined(); - expect(checked.report.diagnostics).toContainEqual(expect.objectContaining({ severity: 'refusal', kind: 'probe_failed', message: expect.stringContaining('RELAY_API_KEY') })); + expect(checked.report.diagnostics).toContainEqual(expect.objectContaining({ severity: 'refusal', kind: 'probe_failed', message: expect.stringContaining('broker/PTy was removed') })); const dataDir = join(root, 'data'); const run = await runFlow(path, dataDir, { localAgent: true, daemon: { spawn: false } }); expect(run.exitCode).toBe(2); - expect(run.report.diagnostics).toContainEqual(expect.objectContaining({ kind: 'probe_failed', message: expect.stringContaining('RELAY_API_KEY') })); + expect(run.report.diagnostics).toContainEqual(expect.objectContaining({ kind: 'probe_failed', message: expect.stringContaining('broker/PTy was removed') })); expect(run.report.diagnostics.some(d => d.kind === 'protocol_error' || d.kind === 'daemon_unreachable')).toBe(false); expect(existsSync(dataDir)).toBe(false); } finally { vi.unstubAllEnvs(); rmSync(root, { recursive: true, force: true }); } diff --git a/packages/sdk/tests/communication-review.test.ts b/packages/sdk/tests/communication-review.test.ts index 798babe30..411c88240 100644 --- a/packages/sdk/tests/communication-review.test.ts +++ b/packages/sdk/tests/communication-review.test.ts @@ -70,7 +70,7 @@ it('communication agents drop delivered secrets by default on a scoped runtime', expect(agentEnvironment('claude', source, undefined, false).ANTHROPIC_API_KEY).toBe('delivered-provider-key'); expect(() => agentEnvironment('claude', { ...source, RELAY_SECRET_ENV_NAMES: JSON.stringify(['RELAY_API_KEY']), - }, ['RELAY_API_KEY'])).toThrow('Relay broker credentials'); + }, ['RELAY_API_KEY'])).toThrow('Agent Relay control credentials'); }); it('legacy communication agents still strip delivered repository names', () => { diff --git a/packages/sdk/tests/communication-worker.test.ts b/packages/sdk/tests/communication-worker.test.ts deleted file mode 100644 index 8d69f5556..000000000 --- a/packages/sdk/tests/communication-worker.test.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { it, expect, vi, beforeEach } from 'vitest'; -import { readlinkSync } from 'node:fs'; -import type { JournalClient } from '../src/journal-client.js'; -import type { StepDispatchEvent } from '../src/protocol.js'; -import { completeCommunicationDispatch } from '../src/communication/worker.js'; -const mocks = vi.hoisted(() => ({ - invoke: undefined as undefined | ((request: { operation: string; values: string[] }) => Promise), - release: vi.fn(async () => {}), close: vi.fn(async () => {}), toolClose: vi.fn(async () => {}), - ready: vi.fn(async () => ({ reason: 'ready' })), spawn: vi.fn(), -})); -vi.mock('../src/communication/relay.js', () => ({ acquireRelayRuntime: async () => ({ - prefix: 'managed', channel: 'wf-run', close: mocks.close, - broker: { onEvent: () => () => {}, spawnPty: mocks.spawn }, messaging: {}, -}) })); -vi.mock('../src/communication/tools.js', () => ({ openCommunicationTools: async (invoke: typeof mocks.invoke) => { - mocks.invoke = invoke; - return { path: '/tmp/test.sock', token: 'test-session-token', helperPath: '/tmp/tool.mjs', close: mocks.toolClose }; -} })); -beforeEach(() => { - vi.clearAllMocks(); - mocks.ready.mockResolvedValue({ reason: 'ready' }); - mocks.spawn.mockImplementation(async () => { - setTimeout(() => void mocks.invoke!({ operation: 'complete', values: ['done'] }), 5); - return { name: 'managed-agent', generation: 'generation', release: mocks.release, waitForReady: mocks.ready }; - }); -}); -function fixture(cli = 'claude', environment?: NodeJS.ProcessEnv, cliIdentity?: string) { - const client = { stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30000 })), - stepComplete: vi.fn(async () => ({})), channelReceive: vi.fn(async () => null) }; - const dispatch = { run_id: 'run', step_id: 'agent', attempt: 1, idempotency_key: 'key', pins: {}, - lease_id: 'lease', lease_deadline_ms: Date.now() + 30000, - spec: { type: 'agent', cli, ...(cliIdentity === undefined ? {} : { cli_identity: cliIdentity }), instruction: '' } } as StepDispatchEvent; - return { client, execute: () => completeCommunicationDispatch(client as unknown as JournalClient, dispatch, - { type: 'relayflows.communication.v1', instruction: 'test', incoming: ['peer'], outgoing: [], timeoutMs: 1000 }, - '/tmp/data', undefined, environment) }; -} -it('releases the exact owned identity and runtime before successful completion; never fabricates measured usage', async () => { - const f = fixture(); await f.execute(); - expect(mocks.release).toHaveBeenCalledWith('Flow communication attempt ended', { deleteIdentity: true }); - expect(mocks.close).toHaveBeenCalledTimes(1); - expect(mocks.toolClose).toHaveBeenCalledTimes(1); - expect(f.client.stepComplete).toHaveBeenCalledWith('run', 'agent', 1, 'key', 'success', expect.objectContaining({ - usage: { tokens_in: 0, tokens_out: 0, dollars_unmetered: true }, output: { summary: 'done' }, - })); -}); -it('journals readiness failure and still releases all acquired resources', async () => { - mocks.ready.mockResolvedValue({ reason: 'timeout' }); - const f = fixture(); await f.execute(); - expect(f.client.stepComplete).toHaveBeenCalledWith('run', 'agent', 1, 'key', 'worker_error', expect.objectContaining({ output: { error: 'Communication agent did not become ready: timeout' } })); - expect(mocks.release).toHaveBeenCalledTimes(1); - expect(mocks.close).toHaveBeenCalledTimes(1); -}); - -it.each(['claude', 'codex', 'gemini', 'cursor-agent', 'droid', 'opencode', 'aider', 'goose', 'grok', 'devin', 'pi', 'deepagents', '/opt/custom/tool'])('delegates %s launch and injection to Relay without Claude-only flags', async cli => { - const f = fixture(cli); await f.execute(); - expect(mocks.spawn).toHaveBeenCalledWith(expect.objectContaining({ - cli: cli.split('/').at(-1), - harnessConfig: expect.objectContaining({ command: `'${cli}'`, args: [], runtime: 'pty', - env: expect.objectContaining({ RELAYFLOW_COMMUNICATION_SOCKET: '/tmp/test.sock', RELAYFLOW_COMMUNICATION_TOKEN: 'test-session-token' }), - delivery: { mode: 'pty-injection', format: 'relay-block' } }), - })); - expect(f.client.stepComplete).toHaveBeenCalledWith('run', 'agent', 1, 'key', 'success', expect.anything()); -}); - -it('quotes an executable path as one command without losing spaces or apostrophes', async () => { - const f = fixture("/opt/agent tools/tool's cli"); await f.execute(); - expect(mocks.spawn.mock.calls[0]![0].harnessConfig.command).toBe("'/opt/agent tools/tool'\"'\"'s cli'"); -}); - -it('launches communication agents with the isolated provider credential', async () => { - const f = fixture('codex', { ...process.env, OPENAI_API_KEY: 'house-key' }); - await f.execute(); - expect(mocks.spawn.mock.calls[0]![0].harnessConfig.env).toMatchObject({ OPENAI_API_KEY: 'house-key' }); -}); - -it('uses the proved identity for a canonical generic communication executable', async () => { - let linkedTarget: string | undefined; - mocks.spawn.mockImplementationOnce(async input => { - const quotedCommand = (input as { harnessConfig: { command: string } }).harnessConfig.command; - linkedTarget = readlinkSync(quotedCommand.slice(1, -1)); - setTimeout(() => void mocks.invoke!({ operation: 'complete', values: ['done'] }), 5); - return { name: 'managed-agent', generation: 'generation', release: mocks.release, waitForReady: mocks.ready }; - }); - const f = fixture('/store/provider-cli.js', { ...process.env, OPENAI_API_KEY: 'house-key' }, 'codex'); - await f.execute(); - expect(linkedTarget).toBe('/store/provider-cli.js'); - expect(mocks.spawn).toHaveBeenCalledWith(expect.objectContaining({ - cli: 'codex', - harnessConfig: expect.objectContaining({ - command: expect.stringMatching(/^'\/tmp\/data\/communication\/cli-[^/]+\/codex'$/u), - env: expect.objectContaining({ OPENAI_API_KEY: 'house-key' }), - }), - })); -}); -it('leaves a heartbeat-lost attempt to the kernel instead of completing it as worker_error', async () => { - const { JournalRequestTimeoutError } = await import('../src/journal-client.js'); - const { isLeaseLost } = await import('../src/worker-lease.js'); - mocks.spawn.mockImplementation(async () => ({ name: 'managed-agent', generation: 'generation', release: mocks.release, waitForReady: mocks.ready })); - const f = fixture(); - f.client.stepHeartbeat.mockImplementation(async () => { throw new JournalRequestTimeoutError('step.heartbeat', 10); }); - (f as unknown as { client: { stepHeartbeat: unknown } }).client.stepHeartbeat = f.client.stepHeartbeat; - const dispatchDeadline = Date.now() + 200; - const error = await completeCommunicationDispatch(f.client as unknown as JournalClient, { - run_id: 'run', step_id: 'agent', attempt: 1, idempotency_key: 'key', pins: {}, lease_id: 'lease', - lease_deadline_ms: dispatchDeadline, spec: { type: 'agent', cli: 'claude', instruction: '' }, - } as StepDispatchEvent, { type: 'relayflows.communication.v1', instruction: 'test', incoming: ['peer'], outgoing: [], timeoutMs: 60_000 }, - '/tmp/data').catch(caught => caught); - expect(isLeaseLost(error)).toBe(true); - expect(f.client.stepComplete).not.toHaveBeenCalled(); -}); -it('leaves a retried attempt whose history read was interrupted to the kernel', async () => { - const { JournalRequestTimeoutError } = await import('../src/journal-client.js'); - const { isReadInterruptionError } = await import('../src/journal-read-policy.js'); - const f = fixture(); - (f.client as unknown as { journalRead: () => Promise }).journalRead = async () => { - throw new JournalRequestTimeoutError('journal.read', 10, 3, 300_000, 300_000); - }; - const error = await completeCommunicationDispatch(f.client as unknown as JournalClient, { - run_id: 'run', step_id: 'agent', attempt: 2, idempotency_key: 'key', pins: {}, lease_id: 'lease', - lease_deadline_ms: Date.now() + 30_000, spec: { type: 'agent', cli: 'claude', instruction: '' }, - } as StepDispatchEvent, { type: 'relayflows.communication.v1', instruction: 'test', incoming: ['peer'], outgoing: [], timeoutMs: 1000 }, - '/tmp/data').catch(caught => caught); - expect(isReadInterruptionError(error)).toBe(true); - expect(f.client.stepComplete).not.toHaveBeenCalled(); -}); -it('scopes the attempt history reads to its lease and releases them after', async () => { - const f = fixture(); - const release = vi.fn(); - const scopeReads = vi.fn((_signal: AbortSignal) => release); - (f.client as unknown as { scopeReads: typeof scopeReads }).scopeReads = scopeReads; - await f.execute(); - expect(scopeReads).toHaveBeenCalledTimes(1); - expect(scopeReads.mock.calls[0]![0]).toBeInstanceOf(AbortSignal); - expect(release).toHaveBeenCalledTimes(1); -}); diff --git a/packages/sdk/tests/node-spawn-ports.test.ts b/packages/sdk/tests/node-spawn-ports.test.ts new file mode 100644 index 000000000..3bfbd968c --- /dev/null +++ b/packages/sdk/tests/node-spawn-ports.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it, vi } from 'vitest'; +import { LocalNodeSpawnPort, type CoreClient } from '../src/node-spawn-local.js'; +import { RemoteNodeSpawnPort } from '../src/node-spawn-remote.js'; + +describe('local node-spawn port', () => { + it('uses the bridge v2 spawn methods and exact release identity', async () => { + const calls: Array<[string, Record | undefined]> = []; + const core: CoreClient = { + close() {}, + async call(method: string, params?: Record): Promise { + calls.push([method, params]); + if (method === 'spawn.targets') return { targets: [{ id: 'node_1', name: 'desktop-1', live: true, + handlers_live: true, stale: false, capabilities: ['spawn:codex'], active_agents: 0, + reserved_agents: 0, max_agents: 2 }] } as T; + if (method === 'spawn.invoke' || (method === 'spawn.status' && params?.action === 'spawn')) return { id: 'inv_1', status: 'completed', node_id: 'node_1', + agent_id: 'agent_1', name: 'worker_1', spawned: true, ready: true, task_unconfirmed: false } as T; + return { id: 'rel_1', status: 'completed', name: 'worker_1', spawned: false, + ready: false, released: true, task_unconfirmed: false } as T; + }, + }; + const port = new LocalNodeSpawnPort(core, 'authorized-caller'); + const receipt = await port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', + directory: '/workspace', idempotencyKey: 'dispatch_1234567890' }); + await port.release({ receipt, idempotencyKey: 'release_1234567890' }); + expect(calls[1]).toEqual(['spawn.invoke', expect.objectContaining({ caller_name: 'authorized-caller', + node_id: 'node_1', name: 'worker_1' })]); + expect(calls[3]).toEqual(['spawn.release', expect.objectContaining({ + expected_agent_id: 'agent_1', caller_name: 'authorized-caller' })]); + }); + + it('does not dispatch to an explicitly named node that is not live and handler-ready', async () => { + const calls = vi.fn(); + const core: CoreClient = { close() {}, async call(method: string): Promise { + calls(method); + if (method === 'spawn.targets') return { targets: [{ id: 'node_1', name: 'desktop-1', live: false, + handlers_live: true, stale: false, capabilities: ['spawn:codex'] }] } as T; + throw new Error('spawn must not be invoked'); + } }; + const port = new LocalNodeSpawnPort(core, 'authorized-caller'); + await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', node: 'desktop-1', + directory: '/workspace', idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ + code: 'no_eligible_node', + }); + expect(calls).toHaveBeenCalledTimes(1); + }); +}); + +describe('remote node-spawn port', () => { + it('confirms readiness, resolves exact identity, and targets release to the same node', async () => { + const placement = vi.fn() + .mockResolvedValueOnce({ invocationId: 'ack_1', actionName: 'spawn', status: 'pending', + node: { id: 'node_1', name: 'desktop-1', status: 'online', capabilities: [] }, + placement: { node: 'desktop-1', confirmed: true, state: 'ready' }, + confirmation: { invocationId: 'inv_1', actionName: 'spawn', status: 'completed', + output: { spawned: true, ready: true, name: 'worker_1' } } }) + .mockResolvedValueOnce({ invocationId: 'rel_1', actionName: 'release', status: 'pending', + placement: { node: 'desktop-1', confirmed: false, state: 'accepted' } }); + const getInvocation = vi.fn().mockResolvedValue({ invocationId: 'rel_1', actionName: 'release', + status: 'completed', output: { released: true, name: 'worker_1' } }); + const port = new RemoteNodeSpawnPort({ + agents: { get: async name => ({ id: 'agent_1', name }) }, + nodes: { list: async () => [] }, placement: { spawn: placement }, commands: { getInvocation }, + }); + const receipt = await port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', + idempotencyKey: 'dispatch_1234567890' }); + await port.release({ receipt, idempotencyKey: 'release_1234567890' }); + expect(receipt).toMatchObject({ agentId: 'agent_1', nodeName: 'desktop-1', ready: true }); + expect(placement.mock.calls[1]![0]).toMatchObject({ capability: 'release', actionName: 'release', + node: 'desktop-1', input: { name: 'worker_1', expected_agent_id: 'agent_1' } }); + expect(getInvocation).toHaveBeenCalledWith('release', 'rel_1'); + }); +}); diff --git a/packages/sdk/tsconfig.tests.json b/packages/sdk/tsconfig.tests.json index 9af6213f8..285efc99c 100644 --- a/packages/sdk/tsconfig.tests.json +++ b/packages/sdk/tsconfig.tests.json @@ -73,6 +73,7 @@ "tests/worker-cli-cwd.test.ts", "tests/agent-relay-transport.test.ts", "tests/agent-relay-hardening.test.ts", + "tests/node-spawn-ports.test.ts", "tests/webhook-hardening.test.ts", "tests/adapters/claude.test.ts", "tests/adapters/codex.test.ts", diff --git a/packages/surface/AUTHORING.md b/packages/surface/AUTHORING.md index f44957ecc..fac96948a 100644 --- a/packages/surface/AUTHORING.md +++ b/packages/surface/AUTHORING.md @@ -208,9 +208,7 @@ export interface AgentOptions { cli?: string; model?: string; /** - * Broker transport accepts absolute or run-root-relative directories, including - * external worktrees. For direct transport: - * Directory this agent's CLI is spawned in, relative to the run root — the + * Directory this direct agent's CLI is spawned in, relative to the run root — the * flow-runner's working directory, and where the CLI runs when this is * absent. One flow can therefore drive agents in sibling checkouts. The * path must be relative and free of `.`, `..` and empty components, and @@ -232,16 +230,16 @@ export interface AgentOptions { */ resume?: string; /** - * Dispatch transport (flows#385). `'direct'` (default) spawns the CLI as a - * local subprocess. `'broker'` opens an interactive local Relay session. `'relay'` posts to agent-relay so the agent registers - * as a first-class workspace participant that DMs can steer. + * Dispatch transport. `'direct'` (default) spawns a local subprocess. + * `'relay'` uses native Agent Relay node spawn. `'broker'` is retained only + * for source compatibility and is refused before admission. */ transport?: 'direct' | 'relay' | 'broker'; - /** Canonical AgentWorkforce/skills names installed for this broker agent. */ + /** Legacy broker option; currently refused. */ skills?: string[]; - /** Keep the broker session available for f.message until the flow ends. */ + /** Legacy broker option; currently refused. */ keepAlive?: boolean; - /** Broker session channel; defaults to flow-. */ + /** Legacy broker option; currently refused. */ channel?: string; /** Semantic executions allowed when verification rejects output. Default 1. */ maxIterations?: number; diff --git a/packages/surface/src/context.ts b/packages/surface/src/context.ts index 8bae41805..d0319ec58 100644 --- a/packages/surface/src/context.ts +++ b/packages/surface/src/context.ts @@ -49,9 +49,7 @@ export interface AgentOptions { cli?: string; model?: string; /** - * Broker transport accepts absolute or run-root-relative directories, including - * external worktrees. For direct transport: - * Directory this agent's CLI is spawned in, relative to the run root — the + * Directory this direct agent's CLI is spawned in, relative to the run root — the * flow-runner's working directory, and where the CLI runs when this is * absent. One flow can therefore drive agents in sibling checkouts. The * path must be relative and free of `.`, `..` and empty components, and @@ -73,16 +71,16 @@ export interface AgentOptions { */ resume?: string; /** - * Dispatch transport (flows#385). `'direct'` (default) spawns the CLI as a - * local subprocess. `'broker'` opens an interactive local Relay session. `'relay'` posts to agent-relay so the agent registers - * as a first-class workspace participant that DMs can steer. + * Dispatch transport. `'direct'` (default) spawns a local subprocess. + * `'relay'` uses native Agent Relay node spawn. `'broker'` is retained only + * for source compatibility and is refused before admission. */ transport?: 'direct' | 'relay' | 'broker'; - /** Canonical AgentWorkforce/skills names installed for this broker agent. */ + /** Legacy broker option; currently refused. */ skills?: string[]; - /** Keep the broker session available for f.message until the flow ends. */ + /** Legacy broker option; currently refused. */ keepAlive?: boolean; - /** Broker session channel; defaults to flow-. */ + /** Legacy broker option; currently refused. */ channel?: string; /** Semantic executions allowed when verification rejects output. Default 1. */ maxIterations?: number; From dad61da61b9ac5c1f0689c752e14e742923c99e2 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Sat, 7 Nov 2026 16:32:09 -0800 Subject: [PATCH 02/75] docs: record approved node-spawn follow-ups --- docs/MIGRATION-NODE-SPAWN.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/MIGRATION-NODE-SPAWN.md b/docs/MIGRATION-NODE-SPAWN.md index 182c4c928..d7d315df7 100644 --- a/docs/MIGRATION-NODE-SPAWN.md +++ b/docs/MIGRATION-NODE-SPAWN.md @@ -26,21 +26,23 @@ harness. Use `RELAYFLOW_NODE_NAME` for an exact node or broker health checks are no longer read. There is deliberately no compatibility fallback to broker 17.2.1. Retained -local sessions and linked communication remain unavailable until Desktop has a -supported headless Flows caller/delegation surface. An operator-supplied or -borrowed coding-session name is not a substitute. +local sessions and linked communication remain unavailable until Desktop ships +the approved core-bridge client package and authorized headless Flows caller. +An operator-supplied or borrowed coding-session name is not a substitute. ## Hosted and self-hosted execution Hosted workers use the agent-token fleet path. Self-hosted workers use the same path for remote nodes. Selecting the local Desktop bridge is withheld -until the service-caller contract exists. +until the approved client and service-caller contract land in a Desktop +release; the bridge adapter remains fail-closed in the meantime. Self-hosted Flows licensing is owned by relaycontrol's signed offline licence design. This SDK does not invent an entitlement store; deployments must fail closed until a signed `flows` entitlement attestation is available. Scheduled self-hosted submissions remain Kubernetes CronJobs, not `relaycron`. -Relay CLI migration is a separate follow-up: `relay flows` already consumes -the v2 SDK, while legacy `node workflow run` and the cloud worker must not move -until the Relay and Cloud owners approve that change. +Relay CLI migration remains a separate, approved follow-up: `relay flows` +already consumes the v2 SDK, while legacy `node workflow run` and the cloud +worker move together in their own stacked draft after the broker-retirement +change. This package is not published as part of either draft. From 70ba4763ef38659ad37a2bb4f3516e02a25873dd Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Sat, 7 Nov 2026 17:35:00 -0800 Subject: [PATCH 03/75] fix(sdk): require Desktop service caller secret --- docs/MIGRATION-NODE-SPAWN.md | 4 ++- docs/SURFACE.md | 3 +- packages/sdk/src/index.ts | 8 ++++- packages/sdk/src/node-spawn-local.ts | 34 ++++++++++++++++----- packages/sdk/tests/node-spawn-ports.test.ts | 23 +++++++++++--- 5 files changed, 58 insertions(+), 14 deletions(-) diff --git a/docs/MIGRATION-NODE-SPAWN.md b/docs/MIGRATION-NODE-SPAWN.md index d7d315df7..a7239e8b9 100644 --- a/docs/MIGRATION-NODE-SPAWN.md +++ b/docs/MIGRATION-NODE-SPAWN.md @@ -28,7 +28,9 @@ harness. Use `RELAYFLOW_NODE_NAME` for an exact node or There is deliberately no compatibility fallback to broker 17.2.1. Retained local sessions and linked communication remain unavailable until Desktop ships the approved core-bridge client package and authorized headless Flows caller. -An operator-supplied or borrowed coding-session name is not a substitute. +The local adapter requires both the Desktop-issued caller name and secret on +invoke, status, and release; an operator-supplied or borrowed coding-session +name is not a substitute. ## Hosted and self-hosted execution diff --git a/docs/SURFACE.md b/docs/SURFACE.md index 63e1bfccd..bc0dcfc7d 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -424,7 +424,8 @@ The retired `transport: "broker"` retained-session surface (`keepAlive`, `f.message`, and `f.release`) is no longer admitted. It has no PTY or local broker fallback. Native retained sessions remain fail-closed until Agent Relay Desktop exposes an authorized headless Flows caller/delegation contract; Flows -will not borrow a live coding session's registration. See +will not borrow a live coding session's registration. The local adapter carries +the Desktop-issued caller name and secret on invoke, status, and release. See [the migration note](MIGRATION-NODE-SPAWN.md). ### Agent retry and recovery controls diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 515cd8d01..139c61c16 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -251,7 +251,13 @@ export { type NodeSpawnRequest, type NodeSpawnTarget, } from './node-spawn.js'; -export { LocalNodeSpawnPort, connectCore, coreSocketPath, type CoreClient } from './node-spawn-local.js'; +export { + LocalNodeSpawnPort, + connectCore, + coreSocketPath, + type CoreClient, + type LocalNodeSpawnCaller, +} from './node-spawn-local.js'; export { RemoteNodeSpawnPort, type RemoteNodeSpawnClient } from './node-spawn-remote.js'; export { createRemoteNodeSpawnRuntime, diff --git a/packages/sdk/src/node-spawn-local.ts b/packages/sdk/src/node-spawn-local.ts index d47744bb5..6723edc59 100644 --- a/packages/sdk/src/node-spawn-local.ts +++ b/packages/sdk/src/node-spawn-local.ts @@ -134,13 +134,30 @@ interface CoreInvocation { error_code?: string; } +/** Desktop-issued identity for the authorized headless Flows service caller. */ +export interface LocalNodeSpawnCaller { + name: string; + secret: string; +} + /** - * Local native-spawn adapter. The caller name is deliberately explicit: this - * class never borrows a Desktop registration or provisions a service caller. + * Local native-spawn adapter. The caller identity is deliberately explicit: + * this class never borrows a Desktop registration or provisions a service + * caller. A core without the spawn-callers plugin refuses the secret. */ export class LocalNodeSpawnPort implements NodeSpawnPort { - constructor(private readonly core: CoreClient, private readonly callerName: string) { - if (!callerName.trim()) throw new NodeSpawnError('Local node spawn requires an explicitly authorized caller name', 'caller_required'); + private readonly callerName: string; + private readonly callerSecret: string; + + constructor(private readonly core: CoreClient, caller: LocalNodeSpawnCaller) { + if (!caller.name.trim() || !caller.secret.trim()) { + throw new NodeSpawnError( + 'Local node spawn requires an explicitly authorized service caller', + 'caller_required', + ); + } + this.callerName = caller.name; + this.callerSecret = caller.secret; } async targets(): Promise { @@ -164,7 +181,8 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { : targets.find(candidate => candidate.name === request.node); if (!target?.id) throw new NodeSpawnError('No eligible local Agent Relay spawn target', 'no_eligible_node'); const raw = await this.core.call('spawn.invoke', { - caller_name: this.callerName, node_id: target.id, node_name: target.name, + caller_name: this.callerName, caller_secret: this.callerSecret, + node_id: target.id, node_name: target.name, name: request.name, harness: request.harness, task: request.task, directory: request.directory, idempotency_key: request.idempotencyKey, }); @@ -180,7 +198,8 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { async status(action: NodeSpawnAction, invocationId: string): Promise { const raw = await this.core.call('spawn.status', { - caller_name: this.callerName, action, invocation_id: invocationId, + caller_name: this.callerName, caller_secret: this.callerSecret, + action, invocation_id: invocationId, }); return mapCoreInvocation(action, raw); } @@ -190,7 +209,8 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { const receipt = request.receipt; if (!receipt.nodeId) throw new NodeSpawnError('Local release requires the exact spawn node id', 'release_identity_mismatch'); const raw = await this.core.call('spawn.release', { - caller_name: this.callerName, node_id: receipt.nodeId, node_name: receipt.nodeName, + caller_name: this.callerName, caller_secret: this.callerSecret, + node_id: receipt.nodeId, node_name: receipt.nodeName, name: receipt.name, expected_agent_id: receipt.agentId, idempotency_key: request.idempotencyKey, }); const settled = raw.status === 'completed' ? mapCoreInvocation('release', raw) : await this.wait('release', raw.id); diff --git a/packages/sdk/tests/node-spawn-ports.test.ts b/packages/sdk/tests/node-spawn-ports.test.ts index 3bfbd968c..4828a4bd2 100644 --- a/packages/sdk/tests/node-spawn-ports.test.ts +++ b/packages/sdk/tests/node-spawn-ports.test.ts @@ -18,14 +18,26 @@ describe('local node-spawn port', () => { ready: false, released: true, task_unconfirmed: false } as T; }, }; - const port = new LocalNodeSpawnPort(core, 'authorized-caller'); + const port = new LocalNodeSpawnPort(core, { + name: 'authorized-caller', + secret: 'desktop-service-secret', + }); const receipt = await port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', directory: '/workspace', idempotencyKey: 'dispatch_1234567890' }); await port.release({ receipt, idempotencyKey: 'release_1234567890' }); expect(calls[1]).toEqual(['spawn.invoke', expect.objectContaining({ caller_name: 'authorized-caller', - node_id: 'node_1', name: 'worker_1' })]); + caller_secret: 'desktop-service-secret', node_id: 'node_1', name: 'worker_1' })]); + expect(calls[2]).toEqual(['spawn.status', expect.objectContaining({ caller_name: 'authorized-caller', + caller_secret: 'desktop-service-secret', invocation_id: 'inv_1' })]); expect(calls[3]).toEqual(['spawn.release', expect.objectContaining({ - expected_agent_id: 'agent_1', caller_name: 'authorized-caller' })]); + expected_agent_id: 'agent_1', caller_name: 'authorized-caller', + caller_secret: 'desktop-service-secret' })]); + }); + + it('refuses a local service caller without its Desktop-issued secret', () => { + const core: CoreClient = { close() {}, async call(): Promise { throw new Error('unused'); } }; + expect(() => new LocalNodeSpawnPort(core, { name: 'authorized-caller', secret: ' ' })) + .toThrow(expect.objectContaining({ code: 'caller_required' })); }); it('does not dispatch to an explicitly named node that is not live and handler-ready', async () => { @@ -36,7 +48,10 @@ describe('local node-spawn port', () => { handlers_live: true, stale: false, capabilities: ['spawn:codex'] }] } as T; throw new Error('spawn must not be invoked'); } }; - const port = new LocalNodeSpawnPort(core, 'authorized-caller'); + const port = new LocalNodeSpawnPort(core, { + name: 'authorized-caller', + secret: 'desktop-service-secret', + }); await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', node: 'desktop-1', directory: '/workspace', idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ code: 'no_eligible_node', From 8c2e44a32ed16a6c713585b5826834ac4ea88d6d Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Sat, 7 Nov 2026 18:32:01 -0800 Subject: [PATCH 04/75] fix(sdk): harden node-spawn recovery --- docs/AGENT-RELAY-TRANSPORT.md | 35 +-- docs/MIGRATION-NODE-SPAWN.md | 3 +- docs/SURFACE.md | 5 +- docs/agent-communication.md | 7 +- packages/sdk/src/agent-relay-transport.ts | 200 +++++++++++++++--- packages/sdk/src/node-spawn-local.ts | 65 ++++-- packages/sdk/src/node-spawn-remote.ts | 90 +++++--- packages/sdk/src/node-spawn-result.ts | 29 ++- packages/sdk/src/node-spawn-runtime.ts | 6 +- packages/sdk/src/node-spawn-state.ts | 35 ++- packages/sdk/src/node-spawn.ts | 34 ++- packages/sdk/src/worker.ts | 1 + .../sdk/tests/agent-relay-hardening.test.ts | 20 ++ .../sdk/tests/agent-relay-transport.test.ts | 148 +++++++++++-- packages/sdk/tests/broker-retirement.test.ts | 33 ++- packages/sdk/tests/node-spawn-ports.test.ts | 120 ++++++++++- packages/sdk/tsconfig.tests.json | 1 + packages/surface/AUTHORING.md | 10 +- packages/surface/src/context.ts | 10 +- 19 files changed, 706 insertions(+), 146 deletions(-) diff --git a/docs/AGENT-RELAY-TRANSPORT.md b/docs/AGENT-RELAY-TRANSPORT.md index 759bd626d..822aa35ed 100644 --- a/docs/AGENT-RELAY-TRANSPORT.md +++ b/docs/AGENT-RELAY-TRANSPORT.md @@ -41,23 +41,34 @@ no recorded acknowledgement is the ambiguous crash window and fails closed. It is never answered by a blind second spawn. The stable worker name provides additional collision protection at the node. +The claim also fixes the task's original start time. Resume never grants a new +deadline. Once a correlated result was already durably observed, a restart has +at most 30 seconds beyond that original deadline to recover the same unacked +delivery; it cannot use that window to spawn or extend remote execution. + Lease loss or timeout aborts the inbox wait and attempts exact release. A -failed cleanup remains auditable in the durable claim. A result delivery stays -unacked until its terminal journal append succeeds, so a worker crash cannot -turn Relay acknowledgement into fabricated execution truth. +post-spawn verification failure carries any exact identity already proven into +the same cleanup path. A failed cleanup is recorded as pending in the durable +claim and the result remains unacked for an idempotent retry. A result delivery +stays unacked until its terminal journal append succeeds, so a worker crash +cannot turn Relay acknowledgement into fabricated execution truth. ## Local Desktop bridge The SDK contains a local `NodeSpawnPort` adapter for the Desktop core's bridge v2 methods: `spawn.targets`, `spawn.invoke`, `spawn.status`, and -`spawn.release`. It connects to the private same-user `core.sock`, validates -socket ownership/mode, performs the `hello` handshake, passes no credentials, -and requires an explicit caller name. - -It is not selected by the CLI yet. Desktop currently authorizes only an -existing live coding-session registration; there is no headless Flows service -caller or delegation contract. Flows does not borrow another session's name. -Local admission therefore stays fail-closed until Desktop supplies that -contract. +`spawn.release`. On Unix it connects to `core.sock`, validates socket +ownership/mode, performs the `hello` handshake, and requires an explicit +Desktop-issued caller name and secret on invoke, status, and release. The +secret is kept in module-private storage rather than on the exported port +instance. Local node spawn is refused on Windows until Desktop exposes an +authenticated named-pipe server contract; a bridge-version hello alone is not +server authentication. +The current bridge does not accept model, repository, or sandbox placement +overrides, so the local adapter refuses those fields instead of dropping them. + +It is not selected by the CLI yet. Flows does not borrow another session's +name and does not invent or provision its caller secret. Local admission stays +fail-closed until Desktop supplies the service credential path and file shape. See [the migration note](MIGRATION-NODE-SPAWN.md) for removed broker surfaces. diff --git a/docs/MIGRATION-NODE-SPAWN.md b/docs/MIGRATION-NODE-SPAWN.md index a7239e8b9..f210b325a 100644 --- a/docs/MIGRATION-NODE-SPAWN.md +++ b/docs/MIGRATION-NODE-SPAWN.md @@ -19,7 +19,8 @@ harness. Use `RELAYFLOW_NODE_NAME` for an exact node or - `transport: 'broker'`, `keepAlive`, `f.message`, and `f.release` for retained sessions fail before child-run admission. -- YAML `communication.links` fails preflight. +- YAML with non-empty `communication.links` fails preflight. An empty list + declares no linked session and remains a no-op. - `examples/issue-swarm` is retained only as a labelled migration reference; it is not runnable on this release. - `RELAYFLOW_RELAY_BROKER_BIN`, broker connection files, helper sockets, and diff --git a/docs/SURFACE.md b/docs/SURFACE.md index bc0dcfc7d..820e23eb4 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -276,8 +276,9 @@ No process runs between events: the handler wakes, executes to its next await, p one, while a worker that dies stops renewing and the lease expires. The run's wallclock budget is separate and stops *new* work, draining whatever is already running rather than cancelling a step mid-flight. A wrapper step and - a native step therefore get the same duration by default. Agent steps may - declare the per-step `timeout` described below, enforced on both paths. + a native step therefore get the same duration by default. Direct agent steps + may declare the per-step `timeout` described below; relay transport refuses + that authored option because the worker cannot stop a remote process. `flows check` resolves the binary (a path is relative to the declaring flow or project config; a bare name resolves via `PATH`) and caches each resolved diff --git a/docs/agent-communication.md b/docs/agent-communication.md index e3eb5fe5b..7d2d7ab3f 100644 --- a/docs/agent-communication.md +++ b/docs/agent-communication.md @@ -4,9 +4,10 @@ The former communication runtime launched interactive broker-managed PTYs and injected a local helper socket. That path was removed with the Relay broker. Flows does not fall back to it. -Existing YAML `communication.links` declarations are still parsed so old -bundles receive a precise admission error. Preflight refuses them before a run -is admitted. Native node-spawn communication will require two contracts that +Existing YAML declarations with non-empty `communication.links` are still +parsed so old bundles receive a precise admission error. Preflight refuses +them before a run is admitted; an empty list declares no links and remains a +no-op. Native node-spawn communication will require two contracts that do not yet exist together: - a supported headless local service caller/delegation contract from Agent diff --git a/packages/sdk/src/agent-relay-transport.ts b/packages/sdk/src/agent-relay-transport.ts index cdc8ead7f..92f381d9b 100644 --- a/packages/sdk/src/agent-relay-transport.ts +++ b/packages/sdk/src/agent-relay-transport.ts @@ -2,7 +2,8 @@ import { createHash } from 'node:crypto'; import { claimNodeSpawn, recordNodeSpawn, type NodeSpawnClaim } from './node-spawn-state.js'; import { nodeResultInstruction, waitForNodeResult } from './node-spawn-result.js'; import { createRemoteNodeSpawnRuntime, type NodeSpawnRuntime } from './node-spawn-runtime.js'; -import { NodeSpawnError, nonempty, validIdempotencyKey } from './node-spawn.js'; +import { NodeSpawnError, nonempty, validIdempotencyKey, + type NodeSpawnIdentity, type NodeSpawnReceipt } from './node-spawn.js'; import type { RelayTaskResult } from './agent-relay-receipt.js'; export type AgentTransport = 'direct' | 'relay'; @@ -33,6 +34,8 @@ export class AgentRelayTransportError extends Error { } } +const RESULT_RECOVERY_WINDOW_MS = 30_000; + export function readAgentRelayEnv(env: NodeJS.ProcessEnv = process.env): Required> { const agentToken = env.RELAY_AGENT_TOKEN?.trim(); if (!agentToken) throw new AgentRelayTransportError( @@ -49,6 +52,7 @@ export async function runAgentRelayTask( request: AgentRelayTaskRequest, options: AgentRelayEnv & { runtime?: NodeSpawnRuntime; signal?: AbortSignal } = {}, ): Promise { + const taskStartedAt = Date.now(); options.signal?.throwIfAborted(); const timeoutMs = request.timeoutMs ?? 86_400_000; if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 86_400_000 @@ -88,37 +92,73 @@ export async function runAgentRelayTask( ...(runtime.repo === undefined ? {} : { repo: runtime.repo }), ...(runtime.sandboxOnly === undefined ? {} : { sandbox_only: runtime.sandboxOnly }), }, - startedAt: Date.now(), + startedAt: taskStartedAt, }; const claimed = await claimNodeSpawn(request.dataDir, proposed); let claim = claimed.claim; - if (!claimed.created && claim.spawn === undefined) { - throw new AgentRelayTransportError( - 'A prior node-spawn dispatch ended in the ambiguous pre-ack window; refusing a second spawn'); - } + let releaseAttempted = false; + const deadline = claim.startedAt + timeoutMs; try { + if (!claimed.created && claim.spawn === undefined) { + if (claim.cleanup === undefined) { + throw new AgentRelayTransportError( + 'A prior node-spawn dispatch ended in the ambiguous pre-ack window; refusing a second spawn'); + } + if (claim.release === undefined) { + releaseAttempted = true; + const cleanup = await tryRelease(runtime, claimed.path, claim, request.idempotencyKey); + claim = cleanup.claim; + if (cleanup.error !== undefined) throw cleanup.error; + } + throw new NodeSpawnError(claim.failure?.message ?? 'The prior node spawn failed after launch', + claim.failure?.code ?? 'spawn_unconfirmed'); + } + assertWithinDeadline(claim.result === undefined ? deadline : deadline + RESULT_RECOVERY_WINDOW_MS); + options.signal?.throwIfAborted(); if (claim.spawn === undefined) { - const spawn = await runtime.port.spawn({ + // This is deliberately adjacent to the irreversible dispatch. Runtime + // setup and durable claiming can take time or lose the worker lease. + options.signal?.throwIfAborted(); + const remaining = remainingDeadline(deadline); + const pendingSpawn = runtime.port.spawn({ name: agentName, harness: request.cli, task, - model: request.model, - directory: request.worker_cwd, + ...(request.model === undefined ? {} : { model: request.model }), + ...(request.worker_cwd === undefined ? {} : { directory: request.worker_cwd }), idempotencyKey: request.idempotencyKey, - node: runtime.node, - repo: runtime.repo, - sandboxOnly: runtime.sandboxOnly, - timeoutMs: Math.min(timeoutMs + 30_000, 86_430_000), + ...(runtime.node === undefined ? {} : { node: runtime.node }), + ...(runtime.repo === undefined ? {} : { repo: runtime.repo }), + ...(runtime.sandboxOnly === undefined ? {} : { sandboxOnly: runtime.sandboxOnly }), + timeoutMs: remaining, + }); + const spawnTimeout = AbortSignal.timeout(remaining); + const spawnSignal = options.signal === undefined + ? spawnTimeout : AbortSignal.any([options.signal, spawnTimeout]); + const spawn = await waitForSpawn(pendingSpawn, spawnSignal, outcome => { + void settleLateSpawn(outcome).catch(() => undefined); }); claim = { ...claim, spawn }; await recordNodeSpawn(claimed.path, claim); + options.signal?.throwIfAborted(); } else { const status = await runtime.port.status('spawn', claim.spawn.id); - if (status.status === 'failed') throw new NodeSpawnError('The recorded node spawn failed', status.errorCode); + if (status.status !== 'completed') { + throw new NodeSpawnError('The recorded node spawn is no longer completed', + status.errorCode ?? `spawn_${status.status}`); + } } const spawn = claim.spawn; if (spawn === undefined) throw new AgentRelayTransportError('Node-spawn claim lost its confirmed receipt'); - const timeout = AbortSignal.timeout(timeoutMs); + if (claim.result !== undefined && claim.release === undefined) { + releaseAttempted = true; + const cleanup = await tryRelease(runtime, claimed.path, claim, request.idempotencyKey); + claim = cleanup.claim; + if (cleanup.error !== undefined) throw cleanup.error; + } + const waitDeadline = claim.result === undefined ? deadline : deadline + RESULT_RECOVERY_WINDOW_MS; + assertWithinDeadline(waitDeadline); + const timeout = AbortSignal.timeout(remainingDeadline(waitDeadline)); const signal = options.signal === undefined ? timeout : AbortSignal.any([options.signal, timeout]); const delivery = await waitForNodeResult(runtime.inbox, { agentId: spawn.agentId, @@ -126,20 +166,25 @@ export async function runAgentRelayTask( ...correlation, }, signal); signal.throwIfAborted(); - let releaseError: unknown; + if (claim.result !== undefined + && (claim.result.inboxItemId !== delivery.inboxItemId || claim.result.messageId !== delivery.messageId)) { + throw new NodeSpawnError('The redelivered result did not match the durable delivery identity', + 'result_identity_mismatch'); + } + if (claim.result === undefined) { + claim = { ...claim, result: { inboxItemId: delivery.inboxItemId, + messageId: delivery.messageId, status: delivery.envelope.status, receivedAt: Date.now() } }; + await recordNodeSpawn(claimed.path, claim); + } if (claim.release === undefined) { - try { - const released = await runtime.port.release({ receipt: spawn, - idempotencyKey: releaseKey(request.idempotencyKey) }); - claim = { ...claim, release: { invocationId: released.id, status: released.status } }; - await recordNodeSpawn(claimed.path, claim); - } catch (error) { releaseError = error; } + releaseAttempted = true; + const cleanup = await tryRelease(runtime, claimed.path, claim, request.idempotencyKey); + claim = cleanup.claim; + if (cleanup.error !== undefined) throw cleanup.error; } signal.throwIfAborted(); const envelope = delivery.envelope; - const error = releaseError === undefined - ? envelope.status === 'failed' ? envelope.error! : null - : `Spawned agent release was not confirmed: ${safeError(releaseError)}`; + const error = envelope.status === 'failed' ? envelope.error! : null; const status = error === null ? 'completed' : 'failed'; return { spawn, @@ -170,19 +215,108 @@ export async function runAgentRelayTask( }, }; } catch (error) { - if (claim.spawn !== undefined && claim.release === undefined) { - try { - const released = await runtime.port.release({ receipt: claim.spawn, - idempotencyKey: releaseKey(request.idempotencyKey) }); - claim = { ...claim, release: { invocationId: released.id, status: released.status } }; - await recordNodeSpawn(claimed.path, claim); - } catch { /* Durable claim retains exact cleanup identity for audit/recovery. */ } + let cleanupError: unknown; + if (error instanceof NodeSpawnError && error.cleanup !== undefined && claim.spawn === undefined) { + claim = { ...claim, cleanup: error.cleanup, + failure: { code: error.code, message: error.message } }; + try { await recordNodeSpawn(claimed.path, claim); } + catch (stateError) { cleanupError = stateError; } + } + if ((claim.spawn !== undefined || claim.cleanup !== undefined) + && claim.release === undefined && !releaseAttempted) { + releaseAttempted = true; + const cleanup = await tryRelease(runtime, claimed.path, claim, request.idempotencyKey); + claim = cleanup.claim; + cleanupError ??= cleanup.error; } options.signal?.throwIfAborted(); - throw new AgentRelayTransportError(safeError(error), error); + const detail = cleanupError === undefined + ? safeError(error) + : `${safeError(error)}; exact cleanup remains pending: ${safeError(cleanupError)}`; + throw new AgentRelayTransportError(detail, error); + } + + async function settleLateSpawn(outcome: { spawn?: NodeSpawnReceipt; error?: unknown }): Promise { + if (outcome.spawn !== undefined) { + claim = { ...claim, spawn: outcome.spawn }; + } else if (outcome.error instanceof NodeSpawnError && outcome.error.cleanup !== undefined) { + claim = { ...claim, cleanup: outcome.error.cleanup, + failure: { code: outcome.error.code, message: outcome.error.message } }; + } else { + return; + } + try { await recordNodeSpawn(claimed.path, claim); } + catch { /* Exact in-memory identity is still sufficient for this cleanup attempt. */ } + const cleanup = await tryRelease(runtime, claimed.path, claim, request.idempotencyKey); + claim = cleanup.claim; + } +} + +async function waitForSpawn( + pending: Promise, + signal: AbortSignal, + onLate: (outcome: { spawn?: NodeSpawnReceipt; error?: unknown }) => void, +): Promise { + signal.throwIfAborted(); + let onAbort: (() => void) | undefined; + const aborted = new Promise((_resolve, reject) => { + onAbort = () => reject(signal.reason ?? new DOMException('The operation was aborted', 'AbortError')); + signal.addEventListener('abort', onAbort, { once: true }); + }); + try { return await Promise.race([pending, aborted]); } + catch (error) { + if (signal.aborted) { + void pending.then(spawn => onLate({ spawn }), lateError => onLate({ error: lateError })); + } + throw error; + } finally { + if (onAbort !== undefined) signal.removeEventListener('abort', onAbort); } } +async function tryRelease( + runtime: NodeSpawnRuntime, + path: string, + claim: NodeSpawnClaim, + idempotencyKey: string, +): Promise<{ claim: NodeSpawnClaim; error?: unknown }> { + const receipt: NodeSpawnIdentity | undefined = claim.spawn ?? claim.cleanup; + if (receipt === undefined || claim.release !== undefined) return { claim }; + try { + const released = await runtime.port.release({ receipt, + idempotencyKey: releaseKey(idempotencyKey) }); + const { cleanupPending: _cleanupPending, ...withoutPending } = claim; + const updated: NodeSpawnClaim = { ...withoutPending, + release: { invocationId: released.id, status: released.status } }; + await recordNodeSpawn(path, updated); + return { claim: updated }; + } catch (error) { + const updated: NodeSpawnClaim = { ...claim, cleanupPending: { + attempts: (claim.cleanupPending?.attempts ?? 0) + 1, + lastError: safeError(error), + updatedAt: Date.now(), + } }; + try { await recordNodeSpawn(path, updated); } + catch (stateError) { + return { claim: updated, + error: new AgentRelayTransportError(`${safeError(error)}; cleanup state was not durable: ${safeError(stateError)}`, error) }; + } + return { claim: updated, error }; + } +} + +function remainingDeadline(deadline: number): number { + const remaining = deadline - Date.now(); + if (!Number.isSafeInteger(remaining) || remaining < 1) { + throw new NodeSpawnError('The node-spawn task deadline elapsed', 'task_timeout'); + } + return remaining; +} + +function assertWithinDeadline(deadline: number): void { + remainingDeadline(deadline); +} + function stableAgentName(request: AgentRelayTaskRequest): string { const digest = createHash('sha256') .update(JSON.stringify([request.runId, request.stepId, request.idempotencyKey])) diff --git a/packages/sdk/src/node-spawn-local.ts b/packages/sdk/src/node-spawn-local.ts index 6723edc59..132c694d8 100644 --- a/packages/sdk/src/node-spawn-local.ts +++ b/packages/sdk/src/node-spawn-local.ts @@ -1,10 +1,11 @@ -import { createHash } from 'node:crypto'; import { existsSync, lstatSync, statSync } from 'node:fs'; import { homedir } from 'node:os'; import { dirname, isAbsolute, join } from 'node:path'; import { createConnection, type Socket } from 'node:net'; import { NodeSpawnError, + normalizeNodeSpawnStatus, + terminalStatus, type NodeReleaseRequest, type NodeSpawnAction, type NodeSpawnInvocation, @@ -30,15 +31,8 @@ export function coreSocketPath(env: NodeJS.ProcessEnv = process.env, home = home return isAbsolute(configured) ? configured : join(dirname(fallback), configured); } -function pipeName(socket: string): string { - let key = socket.replace(/\//g, '\\').toLowerCase(); - if (key.startsWith('\\\\?\\')) key = key.slice(4); - while (key.length > 1 && key.endsWith('\\')) key = key.slice(0, -1); - return `\\\\.\\pipe\\agent-relay-${createHash('sha256').update(key).digest('hex').slice(0, 16)}`; -} - function verifySocket(socket: string): void { - if (process.platform === 'win32' || typeof process.getuid !== 'function') return; + if (typeof process.getuid !== 'function') return; const directory = dirname(socket); const entry = lstatSync(directory); const target = statSync(directory); @@ -54,11 +48,15 @@ function verifySocket(socket: string): void { /** Connects to Desktop's credential-holding core bridge; never starts a daemon. */ export async function connectCore(socket = coreSocketPath(), timeoutMs = DEFAULT_TIMEOUT_MS): Promise { - if (process.platform !== 'win32') { - if (!existsSync(socket)) return undefined; - verifySocket(socket); + if (process.platform === 'win32') { + throw new NodeSpawnError( + 'Local node spawn is unavailable on Windows until the Desktop pipe server can be authenticated', + 'untrusted_core_socket', + ); } - const stream = await openSocket(process.platform === 'win32' ? pipeName(socket) : socket, timeoutMs); + if (!existsSync(socket)) return undefined; + verifySocket(socket); + const stream = await openSocket(socket, timeoutMs); if (!stream) return undefined; let nextId = 1; let buffer = ''; @@ -140,6 +138,8 @@ export interface LocalNodeSpawnCaller { secret: string; } +const callerSecrets = new WeakMap(); + /** * Local native-spawn adapter. The caller identity is deliberately explicit: * this class never borrows a Desktop registration or provisions a service @@ -147,7 +147,6 @@ export interface LocalNodeSpawnCaller { */ export class LocalNodeSpawnPort implements NodeSpawnPort { private readonly callerName: string; - private readonly callerSecret: string; constructor(private readonly core: CoreClient, caller: LocalNodeSpawnCaller) { if (!caller.name.trim() || !caller.secret.trim()) { @@ -157,7 +156,7 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { ); } this.callerName = caller.name; - this.callerSecret = caller.secret; + callerSecrets.set(this, caller.secret); } async targets(): Promise { @@ -174,6 +173,12 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { async spawn(request: NodeSpawnRequest): Promise { if (!request.directory || !isAbsolute(request.directory)) throw new NodeSpawnError('Local node spawn requires an absolute directory', 'spawn_invalid_input'); if (!validIdempotencyKey(request.idempotencyKey)) throw new NodeSpawnError('Invalid node-spawn idempotency key', 'spawn_invalid_input'); + if (request.model !== undefined || request.repo !== undefined || request.sandboxOnly !== undefined) { + throw new NodeSpawnError( + 'The Desktop core bridge does not accept model, repository, or sandbox placement overrides', + 'spawn_invalid_input', + ); + } const targets = (await this.targets()).filter(candidate => candidate.live && candidate.handlersLive && !candidate.stale && candidate.capabilities.includes(`spawn:${request.harness}`)); const target = request.node === undefined @@ -181,7 +186,7 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { : targets.find(candidate => candidate.name === request.node); if (!target?.id) throw new NodeSpawnError('No eligible local Agent Relay spawn target', 'no_eligible_node'); const raw = await this.core.call('spawn.invoke', { - caller_name: this.callerName, caller_secret: this.callerSecret, + caller_name: this.callerName, caller_secret: callerSecret(this), node_id: target.id, node_name: target.name, name: request.name, harness: request.harness, task: request.task, directory: request.directory, idempotency_key: request.idempotencyKey, @@ -189,8 +194,18 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { // spawn.invoke is the action receipt; spawn.status is also the boundary // that enriches it from the core's exact same-workspace registration. const settled = await this.wait('spawn', raw.id, request.timeoutMs); - if (settled.status !== 'completed' || !settled.spawned || !settled.ready || !settled.agentId || !settled.name) { - throw new NodeSpawnError('The local node did not prove a ready spawned agent', settled.errorCode); + const cleanup = settled.agentId && settled.name && settled.nodeId === target.id + ? { id: settled.id, nodeId: settled.nodeId, nodeName: target.name, + name: settled.name, agentId: settled.agentId } + : undefined; + if (settled.status !== 'completed' || !settled.spawned || !settled.ready + || settled.taskUnconfirmed || !settled.agentId || !settled.name) { + throw new NodeSpawnError('The local node did not prove a ready spawned agent', + settled.taskUnconfirmed ? 'spawn_task_unconfirmed' : settled.errorCode, undefined, cleanup); + } + if (settled.name !== request.name || settled.nodeId !== target.id) { + throw new NodeSpawnError('The local spawn result did not preserve its requested identity', + 'spawn_identity_mismatch', undefined, cleanup); } return { ...settled, action: 'spawn', nodeName: target.name, name: settled.name, agentId: settled.agentId, spawned: true, ready: true }; @@ -198,7 +213,7 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { async status(action: NodeSpawnAction, invocationId: string): Promise { const raw = await this.core.call('spawn.status', { - caller_name: this.callerName, caller_secret: this.callerSecret, + caller_name: this.callerName, caller_secret: callerSecret(this), action, invocation_id: invocationId, }); return mapCoreInvocation(action, raw); @@ -209,7 +224,7 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { const receipt = request.receipt; if (!receipt.nodeId) throw new NodeSpawnError('Local release requires the exact spawn node id', 'release_identity_mismatch'); const raw = await this.core.call('spawn.release', { - caller_name: this.callerName, caller_secret: this.callerSecret, + caller_name: this.callerName, caller_secret: callerSecret(this), node_id: receipt.nodeId, node_name: receipt.nodeName, name: receipt.name, expected_agent_id: receipt.agentId, idempotency_key: request.idempotencyKey, }); @@ -224,16 +239,22 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { const value = await this.status(action, id); - if (value.status === 'completed' || value.status === 'failed') return value; + if (terminalStatus(value.status)) return value; await new Promise(resolve => setTimeout(resolve, 500)); } throw new NodeSpawnError(`Timed out waiting for local ${action} confirmation`, `${action}_unconfirmed`); } } +function callerSecret(port: LocalNodeSpawnPort): string { + const secret = callerSecrets.get(port); + if (secret === undefined) throw new NodeSpawnError('Local node spawn caller authority was lost', 'caller_required'); + return secret; +} + function mapCoreInvocation(action: NodeSpawnAction, value: CoreInvocation): NodeSpawnInvocation { return { - id: value.id, action, status: value.status as NodeSpawnInvocation['status'], + id: value.id, action, status: normalizeNodeSpawnStatus(value.status), nodeId: value.node_id, name: value.name, agentId: value.agent_id, spawned: value.spawned, ready: value.ready, taskUnconfirmed: value.task_unconfirmed, released: value.released, diff --git a/packages/sdk/src/node-spawn-remote.ts b/packages/sdk/src/node-spawn-remote.ts index 0965e89cd..7959a6470 100644 --- a/packages/sdk/src/node-spawn-remote.ts +++ b/packages/sdk/src/node-spawn-remote.ts @@ -2,10 +2,12 @@ import { setTimeout as delay } from 'node:timers/promises'; import { NodeSpawnError, isRecord, + normalizeNodeSpawnStatus, nonempty, terminalStatus, type NodeReleaseRequest, type NodeSpawnAction, + type NodeSpawnIdentity, type NodeSpawnInvocation, type NodeSpawnPort, type NodeSpawnReceipt, @@ -72,33 +74,58 @@ export class RemoteNodeSpawnPort implements NodeSpawnPort { }); const confirmation = ack.confirmation; const output = confirmation?.output; + const reportedName = nonempty(output?.name) ? output.name : request.name; + const nodeId = ack.node?.nodeId ?? ack.node?.id + ?? confirmation?.dispatchedNodeId ?? confirmation?.handlerNodeId ?? undefined; + let nodeName = ack.node?.name ?? ack.placement.node ?? request.node; + if (!nodeName && nodeId) { + nodeName = await this.resolveNodeName(nodeId); + } + const taskUnconfirmed = output?.warning === 'spawn_task_unconfirmed' + || (isRecord(output?.warning) && output.warning.code === 'spawn_task_unconfirmed'); + let cleanup: NodeSpawnIdentity | undefined; + if (confirmation && output?.spawned === true && nodeName) { + const agent = await this.resolveAgent(reportedName); + const exactAgentName = nonempty(agent.name) ? agent.name : reportedName; + if (!nonempty(agent.id)) { + throw new NodeSpawnError('The spawned agent identity could not be resolved exactly', 'spawn_identity_mismatch'); + } + cleanup = { + id: confirmation.invocationId, + nodeId, + nodeName, + name: exactAgentName, + agentId: agent.id, + }; + if (exactAgentName !== reportedName || (nonempty(output.agent_id) && output.agent_id !== agent.id)) { + throw new NodeSpawnError('The spawned agent identity could not be resolved exactly', + 'spawn_identity_mismatch', undefined, cleanup); + } + } if (!confirmation || confirmation.status.toLowerCase() !== 'completed' || output?.spawned !== true || output.ready !== true) { - throw new NodeSpawnError('The fleet node did not prove a ready spawned agent', 'spawn_unconfirmed'); + throw new NodeSpawnError('The fleet node did not prove a ready spawned agent', + 'spawn_unconfirmed', undefined, cleanup); } - if (output.warning === 'spawn_task_unconfirmed' - || (isRecord(output.warning) && output.warning.code === 'spawn_task_unconfirmed')) { - throw new NodeSpawnError('The fleet node could not confirm initial task delivery', 'spawn_task_unconfirmed'); + if (!cleanup) { + throw new NodeSpawnError('The spawn result did not identify its exact agent and node', 'spawn_identity_unavailable'); } - const reportedName = nonempty(output.name) ? output.name : request.name; - if (reportedName !== request.name) throw new NodeSpawnError('The spawn result named a different agent', 'spawn_identity_mismatch'); - const agent = await this.resolveAgent(request.name); - if (agent.name !== request.name || !nonempty(agent.id) - || (nonempty(output.agent_id) && output.agent_id !== agent.id)) { - throw new NodeSpawnError('The spawned agent identity could not be resolved exactly', 'spawn_identity_mismatch'); + if (ack.placement.confirmed !== true) { + throw new NodeSpawnError('The fleet placement was not confirmed', + 'spawn_placement_unconfirmed', undefined, cleanup); + } + if (taskUnconfirmed) { + throw new NodeSpawnError('The fleet node could not confirm initial task delivery', + 'spawn_task_unconfirmed', undefined, cleanup); + } + if (reportedName !== request.name) { + throw new NodeSpawnError('The spawn result named a different agent', + 'spawn_identity_mismatch', undefined, cleanup); } - const nodeName = ack.node?.name ?? ack.placement.node; - if (!nodeName) throw new NodeSpawnError('The spawn result did not identify its node', 'spawn_node_unconfirmed'); - const nodeId = ack.node?.nodeId ?? ack.node?.id - ?? confirmation.dispatchedNodeId ?? confirmation.handlerNodeId ?? undefined; return { - id: confirmation.invocationId, + ...cleanup, action: 'spawn', status: 'completed', - nodeId, - nodeName, - name: request.name, - agentId: agent.id, spawned: true, ready: true, taskUnconfirmed: false, @@ -143,6 +170,18 @@ export class RemoteNodeSpawnPort implements NodeSpawnPort { } throw new NodeSpawnError('The ready agent was not visible in the Relay roster', 'spawn_identity_unavailable', last); } + + private async resolveNodeName(id: string): Promise { + for (let attempt = 0; attempt < 10; attempt += 1) { + try { + const nodes = await this.client.nodes.list(); + const name = nodes.find(node => (node.nodeId ?? node.id) === id)?.name; + if (name) return name; + } catch { /* Roster convergence is retried within the confirmation budget. */ } + if (attempt < 9) await delay(250); + } + return undefined; + } } function mapInvocation(action: NodeSpawnAction, value: RelayInvocation): NodeSpawnInvocation { @@ -151,21 +190,14 @@ function mapInvocation(action: NodeSpawnAction, value: RelayInvocation): NodeSpa return { id: value.invocationId, action, - status: normalizeStatus(status), + status: normalizeNodeSpawnStatus(status), nodeId: value.dispatchedNodeId ?? value.handlerNodeId ?? undefined, name: nonempty(output?.name) ? output.name : undefined, spawned: output?.spawned === true, ready: output?.ready === true, released: output?.released === true, - taskUnconfirmed: output?.warning === 'spawn_task_unconfirmed', + taskUnconfirmed: output?.warning === 'spawn_task_unconfirmed' + || (isRecord(output?.warning) && output.warning.code === 'spawn_task_unconfirmed'), errorCode: value.error?.split(':', 1)[0]?.trim() || undefined, }; } - -function normalizeStatus(status: string): NodeSpawnInvocation['status'] { - if (status === 'completed') return 'completed'; - if (terminalStatus(status)) return 'failed'; - if (status === 'running') return 'running'; - if (status === 'dispatched') return 'dispatched'; - return 'pending'; -} diff --git a/packages/sdk/src/node-spawn-result.ts b/packages/sdk/src/node-spawn-result.ts index 069185f40..a2edfebb3 100644 --- a/packages/sdk/src/node-spawn-result.ts +++ b/packages/sdk/src/node-spawn-result.ts @@ -61,7 +61,11 @@ export async function waitForNodeResult( expected: { agentId: string; agentName: string } & NodeResultCorrelation, signal?: AbortSignal, ): Promise { - for await (const item of client.inbox.subscribe({ signal })) { + const iterator = client.inbox.subscribe({ signal })[Symbol.asyncIterator](); + for (;;) { + const next = await abortableNext(iterator, signal); + if (next.done) break; + const item = next.value; signal?.throwIfAborted(); if (item.message.from.id !== expected.agentId || item.message.from.name !== expected.agentName) continue; if (!item.message.text.startsWith(NODE_RESULT_PREFIX)) continue; @@ -93,6 +97,27 @@ export async function waitForNodeResult( throw new Error('Agent Relay result inbox ended before the spawned agent reported completion'); } +async function abortableNext(iterator: AsyncIterator, signal?: AbortSignal): Promise> { + signal?.throwIfAborted(); + const pending = iterator.next(); + if (signal === undefined) return pending; + let onAbort: (() => void) | undefined; + const aborted = new Promise((_resolve, reject) => { + onAbort = () => reject(signal.reason ?? new DOMException('The operation was aborted', 'AbortError')); + signal.addEventListener('abort', onAbort, { once: true }); + }); + try { return await Promise.race([pending, aborted]); } + catch (error) { + if (signal.aborted) { + void pending.catch(() => undefined); + void iterator.return?.(); + } + throw error; + } finally { + if (onAbort !== undefined) signal.removeEventListener('abort', onAbort); + } +} + export function readNodeResult(value: unknown, expected: NodeResultCorrelation): NodeResultEnvelope { if (!isRecord(value) || value.version !== 1 || value.run_id !== expected.runId || value.step_id !== expected.stepId || value.dispatch_id !== expected.dispatchId @@ -107,7 +132,7 @@ export function readNodeResult(value: unknown, expected: NodeResultCorrelation): } if (value.accounting !== undefined && (!isRecord(value.accounting) || Object.entries(value.accounting).some(([key, amount]) => !['tokens_input', 'tokens_output'].includes(key) - || typeof amount !== 'number' || !Number.isFinite(amount) || amount < 0))) { + || typeof amount !== 'number' || !Number.isSafeInteger(amount) || amount < 0))) { throw new Error('Spawned agent result has invalid accounting'); } return value as unknown as NodeResultEnvelope; diff --git a/packages/sdk/src/node-spawn-runtime.ts b/packages/sdk/src/node-spawn-runtime.ts index 8c3345b28..bb3bdf36e 100644 --- a/packages/sdk/src/node-spawn-runtime.ts +++ b/packages/sdk/src/node-spawn-runtime.ts @@ -54,7 +54,11 @@ export async function createRemoteNodeSpawnRuntime(env: NodeJS.ProcessEnv = proc } function validateBaseUrl(raw: string): string { - const url = new URL(raw); + let url: URL; + try { url = new URL(raw); } + catch (error) { + throw new NodeSpawnError('Invalid Agent Relay base URL', 'spawn_invalid_input', error); + } if (url.username || url.password || url.search || url.hash || url.pathname !== '/' || !['https:', 'http:'].includes(url.protocol)) { throw new NodeSpawnError('Invalid Agent Relay base URL', 'spawn_invalid_input'); diff --git a/packages/sdk/src/node-spawn-state.ts b/packages/sdk/src/node-spawn-state.ts index f7af9fb8a..7d0d328d7 100644 --- a/packages/sdk/src/node-spawn-state.ts +++ b/packages/sdk/src/node-spawn-state.ts @@ -1,8 +1,8 @@ import { createHash, randomUUID } from 'node:crypto'; -import { mkdir, open, readFile, rename } from 'node:fs/promises'; +import { link, mkdir, open, readFile, rename, unlink } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import { canonicalize } from './canonical.js'; -import type { NodeSpawnReceipt } from './node-spawn.js'; +import type { NodeSpawnIdentity, NodeSpawnReceipt } from './node-spawn.js'; export interface NodeSpawnClaim { version: 2; @@ -16,6 +16,10 @@ export interface NodeSpawnClaim { request: Record; startedAt: number; spawn?: NodeSpawnReceipt; + cleanup?: NodeSpawnIdentity; + failure?: { code: string; message: string }; + result?: { inboxItemId: string; messageId: string; status: 'completed' | 'failed'; receivedAt: number }; + cleanupPending?: { attempts: number; lastError: string; updatedAt: number }; release?: { invocationId: string; status: string }; } @@ -29,18 +33,24 @@ export async function claimNodeSpawn(dataDir: string, proposed: NodeSpawnClaim): .update(canonicalize([proposed.runId, proposed.stepId, proposed.idempotencyKey])) .digest('hex'); const path = join(directory, `${digest}.json`); - let file; - try { file = await open(path, 'wx', 0o600); } - catch (error) { + const temporary = join(directory, `.${digest}.${randomUUID()}.tmp`); + const file = await open(temporary, 'wx', 0o600); + try { await file.writeFile(canonicalize(proposed)); await file.sync(); } + finally { await file.close(); } + try { + // A hard link publishes the already-complete inode atomically and refuses + // to replace a claim won by another worker. + await link(temporary, path); + await syncDirectory(directory); + return { claim: proposed, created: true, path }; + } catch (error) { if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; const prior = JSON.parse(await readFile(path, 'utf8')) as unknown; if (!sameClaim(prior, proposed)) throw new Error('Node-spawn dispatch conflicts with its durable identity or input'); return { claim: prior as NodeSpawnClaim, created: false, path }; + } finally { + await unlink(temporary).catch(() => undefined); } - try { await file.writeFile(canonicalize(proposed)); await file.sync(); } - finally { await file.close(); } - await syncDirectory(directory); - return { claim: proposed, created: true, path }; } export async function recordNodeSpawn(path: string, claim: NodeSpawnClaim): Promise { @@ -56,8 +66,11 @@ function sameClaim(value: unknown, proposed: NodeSpawnClaim): boolean { if (typeof value !== 'object' || value === null || !('startedAt' in value) || typeof value.startedAt !== 'number' || !Number.isSafeInteger(value.startedAt)) return false; const prior = value as NodeSpawnClaim; - const { spawn: _priorSpawn, release: _priorRelease, ...priorIdentity } = prior; - const { spawn: _spawn, release: _release, ...proposedIdentity } = proposed; + const { spawn: _priorSpawn, cleanup: _priorCleanup, failure: _priorFailure, + result: _priorResult, cleanupPending: _priorCleanupPending, + release: _priorRelease, ...priorIdentity } = prior; + const { spawn: _spawn, cleanup: _cleanup, failure: _failure, result: _result, + cleanupPending: _cleanupPending, release: _release, ...proposedIdentity } = proposed; return canonicalize({ ...priorIdentity, startedAt: proposed.startedAt }) === canonicalize(proposedIdentity); } diff --git a/packages/sdk/src/node-spawn.ts b/packages/sdk/src/node-spawn.ts index 6370b13eb..9cbdc91de 100644 --- a/packages/sdk/src/node-spawn.ts +++ b/packages/sdk/src/node-spawn.ts @@ -25,7 +25,8 @@ export interface NodeSpawnRequest { } export type NodeSpawnAction = 'spawn' | 'release'; -export type NodeSpawnStatus = 'pending' | 'dispatched' | 'running' | 'completed' | 'failed'; +export type NodeSpawnStatus = 'pending' | 'dispatched' | 'running' | 'completed' | 'failed' + | 'denied' | 'cancelled' | 'timed_out'; export interface NodeSpawnInvocation { id: string; @@ -51,8 +52,17 @@ export interface NodeSpawnReceipt extends NodeSpawnInvocation { ready: true; } +/** Exact identity sufficient for a safe release, even when readiness failed. */ +export interface NodeSpawnIdentity { + id: string; + nodeId?: string; + nodeName: string; + name: string; + agentId: string; +} + export interface NodeReleaseRequest { - receipt: NodeSpawnReceipt; + receipt: NodeSpawnIdentity; idempotencyKey: string; reason?: string; } @@ -66,7 +76,13 @@ export interface NodeSpawnPort { } export class NodeSpawnError extends Error { - constructor(message: string, public readonly code = 'node_spawn_failed', public readonly cause?: unknown) { + constructor( + message: string, + public readonly code = 'node_spawn_failed', + public readonly cause?: unknown, + /** Exact identity to release when spawn happened but later verification failed. */ + public readonly cleanup?: NodeSpawnIdentity, + ) { super(message); this.name = 'NodeSpawnError'; } @@ -90,3 +106,15 @@ export function validIdempotencyKey(value: unknown): value is string { export function terminalStatus(status: string): boolean { return ['completed', 'failed', 'denied', 'cancelled', 'canceled', 'timed_out', 'timeout'].includes(status); } + +export function normalizeNodeSpawnStatus(status: string): NodeSpawnStatus { + const normalized = status.toLowerCase(); + if (normalized === 'completed') return 'completed'; + if (normalized === 'denied') return 'denied'; + if (normalized === 'cancelled' || normalized === 'canceled') return 'cancelled'; + if (normalized === 'timed_out' || normalized === 'timeout') return 'timed_out'; + if (normalized === 'failed') return 'failed'; + if (normalized === 'running') return 'running'; + if (normalized === 'dispatched') return 'dispatched'; + return 'pending'; +} diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index 78395dd27..1ca0ec618 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -243,6 +243,7 @@ export class AgentWorker extends EventEmitter { } private async retiredNodeSession(dispatch: StepDispatchEvent, error: string): Promise { + await withWorkerLease(this.client, dispatch, async signal => { signal.throwIfAborted(); }); await this.client.stepComplete( dispatch.run_id, dispatch.step_id, dispatch.attempt, dispatch.idempotency_key, 'worker_error', { diff --git a/packages/sdk/tests/agent-relay-hardening.test.ts b/packages/sdk/tests/agent-relay-hardening.test.ts index 8860ce917..42c157c89 100644 --- a/packages/sdk/tests/agent-relay-hardening.test.ts +++ b/packages/sdk/tests/agent-relay-hardening.test.ts @@ -9,6 +9,11 @@ describe('node-spawn result hardening', () => { dispatch_id: correlation.dispatchId, status: 'completed', output: {} }, correlation)).toThrow('correlation'); expect(() => readNodeResult({ version: 1, run_id: 'run_1', step_id: 'step_1', dispatch_id: correlation.dispatchId, status: 'failed' }, correlation)).toThrow('reason'); + for (const amount of [0.5, Number.MAX_SAFE_INTEGER + 1]) { + expect(() => readNodeResult({ version: 1, run_id: 'run_1', step_id: 'step_1', + dispatch_id: correlation.dispatchId, status: 'completed', output: {}, + accounting: { tokens_input: amount } }, correlation)).toThrow('accounting'); + } }); it('ignores a forged name match without the exact immutable sender id', async () => { @@ -41,4 +46,19 @@ describe('node-spawn result hardening', () => { await delivery.ack(); expect(fail).toHaveBeenCalledWith(expect.objectContaining({ inboxItemId: 'delivery_bad', retry: false })); }); + + it('cancels an initial inbox read even when the client ignores its signal option', async () => { + const returned = vi.fn(); + const controller = new AbortController(); + const waiting = waitForNodeResult({ inbox: { + subscribe: () => ({ [Symbol.asyncIterator]: () => ({ + next: () => new Promise>(() => {}), + return: async () => { returned(); return { done: true, value: undefined }; }, + }) }), + ack: async () => ({}), fail: async () => ({}), + } }, { agentId: 'agent_1', agentName: 'worker', ...correlation }, controller.signal); + controller.abort(new Error('lease lost')); + await expect(waiting).rejects.toThrow('lease lost'); + expect(returned).toHaveBeenCalledOnce(); + }); }); diff --git a/packages/sdk/tests/agent-relay-transport.test.ts b/packages/sdk/tests/agent-relay-transport.test.ts index 75ca403a6..8c0ff51c1 100644 --- a/packages/sdk/tests/agent-relay-transport.test.ts +++ b/packages/sdk/tests/agent-relay-transport.test.ts @@ -1,11 +1,11 @@ -import { mkdtemp } from 'node:fs/promises'; +import { mkdtemp, rm } from 'node:fs/promises'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; -import { describe, expect, it, vi } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { runAgentRelayTask, type AgentRelayTaskRequest } from '../src/agent-relay-transport.js'; import { NODE_RESULT_PREFIX } from '../src/node-spawn-result.js'; import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; -import type { NodeSpawnPort, NodeSpawnReceipt } from '../src/node-spawn.js'; +import { NodeSpawnError, type NodeSpawnPort, type NodeSpawnReceipt } from '../src/node-spawn.js'; import { EventEmitter } from 'node:events'; import { AgentWorker } from '../src/worker.js'; import type { JournalClient } from '../src/journal-client.js'; @@ -16,7 +16,15 @@ const request = (dataDir: string): AgentRelayTaskRequest => ({ idempotencyKey: 'dispatch_1234567890', dataDir, timeoutMs: 10_000, }); -function fixture(): { runtime: NodeSpawnRuntime; spawn: ReturnType; +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +async function dataDirectory(label: string): Promise { + const directory = await mkdtemp(join(tmpdir(), label)); + roots.push(directory); + return directory; +} + +function fixture(result: 'completed' | 'failed' = 'completed'): { runtime: NodeSpawnRuntime; spawn: ReturnType; release: ReturnType; status: ReturnType; ack: ReturnType } { const receipt: NodeSpawnReceipt = { id: 'inv_spawn_1', action: 'spawn', status: 'completed', nodeId: 'node_1', nodeName: 'desktop-1', @@ -45,7 +53,8 @@ function fixture(): { runtime: NodeSpawnRuntime; spawn: ReturnType from: { id: 'agent_1', name }, text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_1', step_id: 'edit', dispatch_id: 'dispatch_1234567890', - status: 'completed', output: { summary: 'done' }, + status: result, + ...(result === 'completed' ? { output: { summary: 'done' } } : { error: 'agent reported failure' }), accounting: { tokens_input: 10, tokens_output: 5 }, }), }, @@ -60,11 +69,12 @@ function fixture(): { runtime: NodeSpawnRuntime; spawn: ReturnType describe('native node-spawn task transport', () => { it('spawns once, captures exact-sender inbox evidence, and leaves ack to the journal caller', async () => { - const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-spawn-')); + const dataDir = await dataDirectory('flows-node-spawn-'); const f = fixture(); const result = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); expect(f.spawn).toHaveBeenCalledOnce(); expect(f.spawn.mock.calls[0]![0]).toMatchObject({ harness: 'codex', idempotencyKey: 'dispatch_1234567890' }); + expect(f.spawn.mock.calls[0]![0]).not.toHaveProperty('directory'); expect(f.release).toHaveBeenCalledWith(expect.objectContaining({ receipt: expect.objectContaining({ agentId: 'agent_1', nodeName: 'desktop-1' }), })); @@ -78,7 +88,7 @@ describe('native node-spawn task transport', () => { }); it('recovers a recorded spawn without issuing a second spawn or release', async () => { - const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-recover-')); + const dataDir = await dataDirectory('flows-node-recover-'); const f = fixture(); await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); const recovered = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); @@ -89,7 +99,7 @@ describe('native node-spawn task transport', () => { }); it('fails closed across an ambiguous pre-ack crash instead of spawning twice', async () => { - const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-ambiguous-')); + const dataDir = await dataDirectory('flows-node-ambiguous-'); const f = fixture(); f.spawn.mockRejectedValueOnce(new Error('connection dropped')); await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })).rejects.toThrow('connection dropped'); @@ -97,8 +107,27 @@ describe('native node-spawn task transport', () => { expect(f.spawn).toHaveBeenCalledTimes(1); }); + it('durably releases exact identity attached to a post-spawn verification failure', async () => { + const dataDir = await dataDirectory('flows-node-postspawn-failure-'); + const f = fixture(); + f.spawn.mockRejectedValueOnce(new NodeSpawnError('task delivery unconfirmed', + 'spawn_task_unconfirmed', undefined, { + id: 'inv_spawn_1', nodeId: 'node_1', nodeName: 'desktop-1', + name: 'worker_1', agentId: 'agent_1', + })); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('task delivery unconfirmed'); + expect(f.release).toHaveBeenCalledWith(expect.objectContaining({ + receipt: expect.objectContaining({ agentId: 'agent_1', nodeName: 'desktop-1' }), + })); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('task delivery unconfirmed'); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).toHaveBeenCalledOnce(); + }); + it('releases the exact confirmed agent before propagating cancellation', async () => { - const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-cancel-')); + const dataDir = await dataDirectory('flows-node-cancel-'); const f = fixture(); const controller = new AbortController(); f.spawn.mockImplementationOnce(async (input: { name: string }) => { @@ -110,13 +139,43 @@ describe('native node-spawn task transport', () => { await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime, signal: controller.signal, })).rejects.toThrow('lease lost'); - expect(f.release).toHaveBeenCalledWith(expect.objectContaining({ - receipt: expect.objectContaining({ agentId: 'agent_1', nodeName: 'desktop-1' }), + await vi.waitFor(() => { + expect(f.release).toHaveBeenCalledWith(expect.objectContaining({ + receipt: expect.objectContaining({ agentId: 'agent_1', nodeName: 'desktop-1' }), + })); + }); + }); + + it('returns promptly from a cancelled confirmation and cleans up its late exact receipt', async () => { + const dataDir = await dataDirectory('flows-node-confirm-cancel-'); + const f = fixture(); + const controller = new AbortController(); + let confirm!: (receipt: NodeSpawnReceipt) => void; + f.spawn.mockImplementationOnce((input: { name: string }) => new Promise(resolve => { + confirm = receipt => resolve({ ...receipt, name: input.name }); })); + const pending = runAgentRelayTask(request(dataDir), { runtime: f.runtime, signal: controller.signal }); + await vi.waitFor(() => expect(f.spawn).toHaveBeenCalledOnce()); + controller.abort(new Error('lease lost during confirmation')); + await expect(pending).rejects.toThrow('lease lost during confirmation'); + expect(f.release).not.toHaveBeenCalled(); + confirm({ id: 'inv_spawn_1', action: 'spawn', status: 'completed', nodeId: 'node_1', + nodeName: 'desktop-1', name: 'placeholder', agentId: 'agent_1', spawned: true, ready: true }); + await vi.waitFor(() => expect(f.release).toHaveBeenCalledOnce()); + }); + + it('rechecks cancellation after durable claiming immediately before spawn dispatch', async () => { + const dataDir = await dataDirectory('flows-node-prespawn-cancel-'); + const f = fixture(); + const controller = new AbortController(); + const pending = runAgentRelayTask(request(dataDir), { runtime: f.runtime, signal: controller.signal }); + queueMicrotask(() => controller.abort(new Error('lease lost before dispatch'))); + await expect(pending).rejects.toThrow('lease lost before dispatch'); + expect(f.spawn).not.toHaveBeenCalled(); }); it('acks the inbox delivery only after stepComplete has entered the journal', async () => { - const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-journal-')); + const dataDir = await dataDirectory('flows-node-journal-'); const f = fixture(); const order: string[] = []; f.ack.mockImplementation(async () => { order.push('ack'); return {}; }); @@ -146,4 +205,69 @@ describe('native node-spawn task transport', () => { relay_task: expect.objectContaining({ invocation_id: 'inv_spawn_1', status: 'completed' }), } })); }); + + it('keeps a result unacked and durably retries exact cleanup when release is unconfirmed', async () => { + const dataDir = await dataDirectory('flows-node-cleanup-'); + const f = fixture(); + f.release.mockRejectedValueOnce(new Error('release response lost')); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('release response lost'); + expect(f.ack).not.toHaveBeenCalled(); + const recovered = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).toHaveBeenCalledTimes(2); + expect(recovered.receipt.status).toBe('completed'); + }); + + it('keeps the original deadline across restart and bounds completed-result recovery', async () => { + vi.useFakeTimers({ toFake: ['Date'] }); + try { + const started = new Date('2026-11-07T00:00:00.000Z'); + vi.setSystemTime(started); + const dataDir = await dataDirectory('flows-node-deadline-'); + const f = fixture(); + const bounded = { ...request(dataDir), timeoutMs: 1_000 }; + await runAgentRelayTask(bounded, { runtime: f.runtime }); + vi.setSystemTime(started.getTime() + 1_001); + await expect(runAgentRelayTask(bounded, { runtime: f.runtime })).resolves.toBeDefined(); + expect(f.status).toHaveBeenCalledTimes(1); + vi.setSystemTime(started.getTime() + 31_001); + await expect(runAgentRelayTask(bounded, { runtime: f.runtime })).rejects.toThrow('deadline elapsed'); + expect(f.status).toHaveBeenCalledTimes(1); + expect(f.spawn).toHaveBeenCalledOnce(); + } finally { + vi.useRealTimers(); + } + }); + + it('journals a failed agent envelope before acknowledging its delivery', async () => { + const dataDir = await dataDirectory('flows-node-failed-result-'); + const f = fixture('failed'); + const order: string[] = []; + f.ack.mockImplementation(async () => { order.push('ack'); return {}; }); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), + stepComplete: vi.fn(async () => { expect(f.ack).not.toHaveBeenCalled(); order.push('journal'); return {}; }), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_1', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + const errors: unknown[] = []; + worker.on('error', error => errors.push(error)); + await worker.attach(); + client.emit('step.dispatch', { + run_id: 'run_1', step_id: 'edit', attempt: 1, step_type: 'agent', + spec: { cli: 'codex', instruction: 'Implement the change', transport: 'relay' }, + idempotency_key: 'dispatch_1234567890', lease_id: 'lease_1', + lease_deadline_ms: Date.now() + 30_000, pins: {}, + } satisfies StepDispatchEvent); + await worker.close(); + expect(errors).toEqual([]); + expect(order).toEqual(['journal', 'ack']); + expect(client.stepComplete).toHaveBeenCalledWith('run_1', 'edit', 1, 'dispatch_1234567890', + expect.not.stringMatching(/^success$/), expect.objectContaining({ + trajectory_tail: { relay_task: expect.objectContaining({ status: 'failed', error: 'agent reported failure' }) }, + })); + }); }); diff --git a/packages/sdk/tests/broker-retirement.test.ts b/packages/sdk/tests/broker-retirement.test.ts index 2d4e80e8d..7f190134c 100644 --- a/packages/sdk/tests/broker-retirement.test.ts +++ b/packages/sdk/tests/broker-retirement.test.ts @@ -1,8 +1,16 @@ +import { chmodSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { readFile } from 'node:fs/promises'; -import { expect, it, vi } from 'vitest'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, expect, it, vi } from 'vitest'; import { authoredBroker } from '../src/authored-broker.js'; +import { checkAuthoredFlow } from '../src/cli/check.js'; import { checkBrokerEnvironment } from '../src/communication/broker-preflight.js'; import { checkCommunicationEnvironment } from '../src/communication/preflight.js'; +import type { FlowSpec } from '../src/spec.js'; + +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); it('removes harness-driver from the published dependency contract', async () => { const manifest = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8')); @@ -24,6 +32,29 @@ it('has no health-check fallback that can restart or reconnect to a broker', () expect(() => checkBrokerEnvironment('/workspace')).toThrow('transport: broker was removed'); }); +it('refuses a compiled broker spawn through the public check path', () => { + const root = mkdtempSync(join(tmpdir(), 'broker-retirement-')); + roots.push(root); + const cli = join(root, 'codex'); + writeFileSync(cli, '#!/bin/sh\nexit 0\n'); + chmodSync(cli, 0o755); + const instruction = JSON.stringify({ type: 'relayflows.broker.v1', operation: 'spawn', + owner: 'owner', flow: 'flow', step: 'step', agentName: 'agent', task: 'work', + channel: 'default', keepAlive: false, timeoutMs: 1000, skills: [] }); + const flow: FlowSpec = { version: '0.1.0', cli, steps: [ + { id: 'step', type: 'agent', instruction }, + ] }; + const checked = checkAuthoredFlow(flow, join(root, 'flow.yaml'), { + projectConfig: { directory: root, models: [], executors: [] }, + }); + expect(checked.report.ok).toBe(false); + expect(checked.report.diagnostics).toEqual(expect.arrayContaining([ + expect.objectContaining({ severity: 'refusal', kind: 'probe_failed', + message: expect.stringContaining('transport: broker was removed') }), + ])); + expect(checked.flow).toBeUndefined(); +}); + it('fails communication admission until the native local caller contract exists', () => { expect(() => checkCommunicationEnvironment({ version: '0.1.0', steps: [{ id: 'a', type: 'agent', cli: 'codex', instruction: JSON.stringify({ type: 'relayflows.communication.v1', instruction: 'talk', incoming: [], diff --git a/packages/sdk/tests/node-spawn-ports.test.ts b/packages/sdk/tests/node-spawn-ports.test.ts index 4828a4bd2..8a427113a 100644 --- a/packages/sdk/tests/node-spawn-ports.test.ts +++ b/packages/sdk/tests/node-spawn-ports.test.ts @@ -1,8 +1,17 @@ import { describe, expect, it, vi } from 'vitest'; -import { LocalNodeSpawnPort, type CoreClient } from '../src/node-spawn-local.js'; +import { connectCore, LocalNodeSpawnPort, type CoreClient } from '../src/node-spawn-local.js'; import { RemoteNodeSpawnPort } from '../src/node-spawn-remote.js'; describe('local node-spawn port', () => { + it('refuses Windows before opening an unauthenticated named pipe', async () => { + const platform = vi.spyOn(process, 'platform', 'get').mockReturnValue('win32'); + try { + await expect(connectCore()).rejects.toMatchObject({ code: 'untrusted_core_socket' }); + } finally { + platform.mockRestore(); + } + }); + it('uses the bridge v2 spawn methods and exact release identity', async () => { const calls: Array<[string, Record | undefined]> = []; const core: CoreClient = { @@ -22,6 +31,8 @@ describe('local node-spawn port', () => { name: 'authorized-caller', secret: 'desktop-service-secret', }); + expect(JSON.stringify(port)).not.toContain('desktop-service-secret'); + expect(Object.values(port)).not.toContain('desktop-service-secret'); const receipt = await port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', directory: '/workspace', idempotencyKey: 'dispatch_1234567890' }); await port.release({ receipt, idempotencyKey: 'release_1234567890' }); @@ -40,12 +51,26 @@ describe('local node-spawn port', () => { .toThrow(expect.objectContaining({ code: 'caller_required' })); }); - it('does not dispatch to an explicitly named node that is not live and handler-ready', async () => { + it('refuses local-only fields the Desktop core bridge cannot preserve', async () => { + const calls = vi.fn(); + const core: CoreClient = { close() {}, async call(): Promise { calls(); throw new Error('unused'); } }; + const port = new LocalNodeSpawnPort(core, { name: 'authorized-caller', secret: 'desktop-service-secret' }); + await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', directory: '/workspace', + model: 'codex-medium', idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ + code: 'spawn_invalid_input', + }); + expect(calls).not.toHaveBeenCalled(); + }); + + it.each([ + { live: false, handlers_live: true }, + { live: true, handlers_live: false }, + ])('does not dispatch to an explicitly named node unless both liveness checks pass (%j)', async readiness => { const calls = vi.fn(); const core: CoreClient = { close() {}, async call(method: string): Promise { calls(method); - if (method === 'spawn.targets') return { targets: [{ id: 'node_1', name: 'desktop-1', live: false, - handlers_live: true, stale: false, capabilities: ['spawn:codex'] }] } as T; + if (method === 'spawn.targets') return { targets: [{ id: 'node_1', name: 'desktop-1', + ...readiness, stale: false, capabilities: ['spawn:codex'] }] } as T; throw new Error('spawn must not be invoked'); } }; const port = new LocalNodeSpawnPort(core, { @@ -58,6 +83,37 @@ describe('local node-spawn port', () => { }); expect(calls).toHaveBeenCalledTimes(1); }); + + it('stops polling on a denied terminal status', async () => { + const status = vi.fn(); + const core: CoreClient = { close() {}, async call(method: string): Promise { + if (method === 'spawn.targets') return { targets: [{ id: 'node_1', name: 'desktop-1', live: true, + handlers_live: true, stale: false, capabilities: ['spawn:codex'] }] } as T; + if (method === 'spawn.invoke') return { id: 'inv_1', status: 'pending', spawned: false, + ready: false, task_unconfirmed: false } as T; + status(); + return { id: 'inv_1', status: 'denied', spawned: false, ready: false, + task_unconfirmed: false, error_code: 'outside_scope' } as T; + } }; + const port = new LocalNodeSpawnPort(core, { name: 'authorized-caller', secret: 'desktop-service-secret' }); + await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', directory: '/workspace', + idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ code: 'outside_scope' }); + expect(status).toHaveBeenCalledOnce(); + }); + + it('refuses a completed local spawn whose returned node identity changed', async () => { + const core: CoreClient = { close() {}, async call(method: string): Promise { + if (method === 'spawn.targets') return { targets: [{ id: 'node_1', name: 'desktop-1', live: true, + handlers_live: true, stale: false, capabilities: ['spawn:codex'] }] } as T; + if (method === 'spawn.invoke') return { id: 'inv_1', status: 'pending', spawned: false, + ready: false, task_unconfirmed: false } as T; + return { id: 'inv_1', status: 'completed', node_id: 'node_other', agent_id: 'agent_1', + name: 'worker_1', spawned: true, ready: true, task_unconfirmed: false } as T; + } }; + const port = new LocalNodeSpawnPort(core, { name: 'authorized-caller', secret: 'desktop-service-secret' }); + await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', directory: '/workspace', + idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ code: 'spawn_identity_mismatch' }); + }); }); describe('remote node-spawn port', () => { @@ -80,8 +136,64 @@ describe('remote node-spawn port', () => { idempotencyKey: 'dispatch_1234567890' }); await port.release({ receipt, idempotencyKey: 'release_1234567890' }); expect(receipt).toMatchObject({ agentId: 'agent_1', nodeName: 'desktop-1', ready: true }); + expect(placement.mock.calls[0]![0]).toMatchObject({ confirm: true, verifyReady: true }); expect(placement.mock.calls[1]![0]).toMatchObject({ capability: 'release', actionName: 'release', node: 'desktop-1', input: { name: 'worker_1', expected_agent_id: 'agent_1' } }); expect(getInvocation).toHaveBeenCalledWith('release', 'rel_1'); }); + + it('returns exact cleanup identity when task delivery verification fails after spawn', async () => { + const port = new RemoteNodeSpawnPort({ + agents: { get: async name => ({ id: 'agent_1', name }) }, + nodes: { list: async () => [] }, + placement: { spawn: async () => ({ invocationId: 'ack_1', actionName: 'spawn', status: 'pending', + node: { id: 'node_1', name: 'desktop-1', status: 'online', capabilities: [] }, + placement: { node: 'desktop-1', confirmed: true, state: 'ready' }, + confirmation: { invocationId: 'inv_1', actionName: 'spawn', status: 'completed', + output: { spawned: true, ready: true, name: 'worker_1', warning: 'spawn_task_unconfirmed' } } }) }, + commands: { getInvocation: vi.fn() }, + }); + await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', + idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ + code: 'spawn_task_unconfirmed', + cleanup: { id: 'inv_1', nodeId: 'node_1', nodeName: 'desktop-1', + name: 'worker_1', agentId: 'agent_1' }, + }); + }); + + it('recovers the selected node name from its dispatch id when placement metadata lags', async () => { + const list = vi.fn(async () => [{ id: 'node_1', name: 'desktop-1', status: 'online', capabilities: [] }]); + const port = new RemoteNodeSpawnPort({ + agents: { get: async name => ({ id: 'agent_1', name }) }, + nodes: { list }, + placement: { spawn: async () => ({ invocationId: 'ack_1', actionName: 'spawn', status: 'pending', + placement: { confirmed: true, state: 'ready' }, + confirmation: { invocationId: 'inv_1', actionName: 'spawn', status: 'completed', + dispatchedNodeId: 'node_1', output: { spawned: true, ready: true, name: 'worker_1' } } }) }, + commands: { getInvocation: vi.fn() }, + }); + await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', + idempotencyKey: 'dispatch_1234567890' })).resolves.toMatchObject({ + nodeId: 'node_1', nodeName: 'desktop-1', agentId: 'agent_1', + }); + expect(list).toHaveBeenCalledOnce(); + }); + + it('refuses an otherwise ready spawn when placement was not confirmed', async () => { + const port = new RemoteNodeSpawnPort({ + agents: { get: async name => ({ id: 'agent_1', name }) }, + nodes: { list: async () => [] }, + placement: { spawn: async () => ({ invocationId: 'ack_1', actionName: 'spawn', status: 'pending', + node: { id: 'node_1', name: 'desktop-1', status: 'online', capabilities: [] }, + placement: { node: 'desktop-1', confirmed: false, state: 'accepted' }, + confirmation: { invocationId: 'inv_1', actionName: 'spawn', status: 'completed', + output: { spawned: true, ready: true, name: 'worker_1' } } }) }, + commands: { getInvocation: vi.fn() }, + }); + await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', + idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ + code: 'spawn_placement_unconfirmed', + cleanup: expect.objectContaining({ agentId: 'agent_1', nodeName: 'desktop-1' }), + }); + }); }); diff --git a/packages/sdk/tsconfig.tests.json b/packages/sdk/tsconfig.tests.json index 285efc99c..2bb784394 100644 --- a/packages/sdk/tsconfig.tests.json +++ b/packages/sdk/tsconfig.tests.json @@ -72,6 +72,7 @@ "tests/shipped-source-worker-invocations.test.ts", "tests/worker-cli-cwd.test.ts", "tests/agent-relay-transport.test.ts", + "tests/broker-retirement.test.ts", "tests/agent-relay-hardening.test.ts", "tests/node-spawn-ports.test.ts", "tests/webhook-hardening.test.ts", diff --git a/packages/surface/AUTHORING.md b/packages/surface/AUTHORING.md index fac96948a..f12b88750 100644 --- a/packages/surface/AUTHORING.md +++ b/packages/surface/AUTHORING.md @@ -94,9 +94,9 @@ export interface Ctx extends Helpers { /** Classify evidence with Jev; the journal replays the recorded answer on resume. */ classify>(options: ClassifyOptions): Step>; agent(name: string, options: AgentOptions): Step; - /** Send work to a retained broker agent and await its next JSON completion. */ + /** Retired retained-session method; calling it is refused before admission. */ message(agentName: string, text: string, options?: { timeout?: string | number }): Step; - /** Release a retained broker agent before cleaning up its working directory. */ + /** Retired retained-session method; calling it is refused before admission. */ release(agentName: string): Step; /** Open a durable, bounded event subscription for this running body. */ on(source: TriggerSource, options: ActivityOptions): Activity; @@ -196,8 +196,8 @@ export interface RunResult { ```ts export interface AgentOptions { - /** CLI time limit: milliseconds or ms/s/m duration, at most 60m. - * Resolves with completionReason: timeout. No default for direct; broker defaults to 60m. Requires maxIterations 1. */ + /** Direct CLI time limit: milliseconds or ms/s/m duration, at most 60m. + * Resolves with completionReason: timeout. No default. Relay transport refuses this option. Requires maxIterations 1. */ timeout?: string | number; task: string; /** Run environment secrets this agent may receive by name. Default: none. */ @@ -283,7 +283,7 @@ export interface AgentResult { * continue the same conversation. */ sessionId?: string; - /** Broker session name; pass to f.message while keepAlive is enabled. */ + /** Retired broker session identifier; no native node-spawn step populates it. */ agentName?: string; } ``` diff --git a/packages/surface/src/context.ts b/packages/surface/src/context.ts index d0319ec58..3549a1b43 100644 --- a/packages/surface/src/context.ts +++ b/packages/surface/src/context.ts @@ -23,7 +23,7 @@ export interface AgentResult { * continue the same conversation. */ sessionId?: string; - /** Broker session name; pass to f.message while keepAlive is enabled. */ + /** Retired broker session identifier; no native node-spawn step populates it. */ agentName?: string; } @@ -37,8 +37,8 @@ export interface PermissionsSpec { } export interface AgentOptions { - /** CLI time limit: milliseconds or ms/s/m duration, at most 60m. - * Resolves with completionReason: timeout. No default for direct; broker defaults to 60m. Requires maxIterations 1. */ + /** Direct CLI time limit: milliseconds or ms/s/m duration, at most 60m. + * Resolves with completionReason: timeout. No default. Relay transport refuses this option. Requires maxIterations 1. */ timeout?: string | number; task: string; /** Run environment secrets this agent may receive by name. Default: none. */ @@ -212,9 +212,9 @@ export interface Ctx extends Helpers { /** Classify evidence with Jev; the journal replays the recorded answer on resume. */ classify>(options: ClassifyOptions): Step>; agent(name: string, options: AgentOptions): Step; - /** Send work to a retained broker agent and await its next JSON completion. */ + /** Retired retained-session method; calling it is refused before admission. */ message(agentName: string, text: string, options?: { timeout?: string | number }): Step; - /** Release a retained broker agent before cleaning up its working directory. */ + /** Retired retained-session method; calling it is refused before admission. */ release(agentName: string): Step; /** Open a durable, bounded event subscription for this running body. */ on(source: TriggerSource, options: ActivityOptions): Activity; From 2aa4b8ce5e6740df21f21e11c3e3ff8632a762d3 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sat, 7 Nov 2026 19:03:18 -0800 Subject: [PATCH 05/75] fix(sdk): keep node cleanup recoverable --- docs/AGENT-RELAY-TRANSPORT.md | 3 +- packages/sdk/src/agent-relay-transport.ts | 12 +++- packages/sdk/src/node-spawn-result.ts | 65 +++++++++++-------- packages/sdk/src/worker-cli.ts | 4 ++ .../sdk/tests/agent-relay-hardening.test.ts | 27 ++++++++ .../sdk/tests/agent-relay-transport.test.ts | 34 ++++++++++ 6 files changed, 114 insertions(+), 31 deletions(-) diff --git a/docs/AGENT-RELAY-TRANSPORT.md b/docs/AGENT-RELAY-TRANSPORT.md index 822aa35ed..9a104072c 100644 --- a/docs/AGENT-RELAY-TRANSPORT.md +++ b/docs/AGENT-RELAY-TRANSPORT.md @@ -49,7 +49,8 @@ delivery; it cannot use that window to spawn or extend remote execution. Lease loss or timeout aborts the inbox wait and attempts exact release. A post-spawn verification failure carries any exact identity already proven into the same cleanup path. A failed cleanup is recorded as pending in the durable -claim and the result remains unacked for an idempotent retry. A result delivery +claim, the attempt remains incomplete for kernel redispatch, and the result +remains unacked for an idempotent retry. A result delivery stays unacked until its terminal journal append succeeds, so a worker crash cannot turn Relay acknowledgement into fabricated execution truth. diff --git a/packages/sdk/src/agent-relay-transport.ts b/packages/sdk/src/agent-relay-transport.ts index 92f381d9b..9c3aa3e64 100644 --- a/packages/sdk/src/agent-relay-transport.ts +++ b/packages/sdk/src/agent-relay-transport.ts @@ -28,7 +28,12 @@ export interface AgentRelayEnv { sandboxOnly?: boolean; } export class AgentRelayTransportError extends Error { - constructor(message: string, public readonly cause?: unknown) { + constructor( + message: string, + public readonly cause?: unknown, + /** Exact cleanup is durable and must be retried by a replacement attempt. */ + public readonly cleanupPending = false, + ) { super(message); this.name = 'AgentRelayTransportError'; } @@ -233,7 +238,10 @@ export async function runAgentRelayTask( const detail = cleanupError === undefined ? safeError(error) : `${safeError(error)}; exact cleanup remains pending: ${safeError(cleanupError)}`; - throw new AgentRelayTransportError(detail, error); + const exactCleanupPending = claim.release === undefined + && claim.cleanupPending !== undefined + && (claim.spawn !== undefined || claim.cleanup !== undefined); + throw new AgentRelayTransportError(detail, error, exactCleanupPending); } async function settleLateSpawn(outcome: { spawn?: NodeSpawnReceipt; error?: unknown }): Promise { diff --git a/packages/sdk/src/node-spawn-result.ts b/packages/sdk/src/node-spawn-result.ts index a2edfebb3..4f612354c 100644 --- a/packages/sdk/src/node-spawn-result.ts +++ b/packages/sdk/src/node-spawn-result.ts @@ -62,39 +62,49 @@ export async function waitForNodeResult( signal?: AbortSignal, ): Promise { const iterator = client.inbox.subscribe({ signal })[Symbol.asyncIterator](); - for (;;) { - const next = await abortableNext(iterator, signal); - if (next.done) break; - const item = next.value; - signal?.throwIfAborted(); - if (item.message.from.id !== expected.agentId || item.message.from.name !== expected.agentName) continue; - if (!item.message.text.startsWith(NODE_RESULT_PREFIX)) continue; - let raw: unknown; - try { raw = JSON.parse(item.message.text.slice(NODE_RESULT_PREFIX.length)); } - catch { raw = undefined; } - let envelope: NodeResultEnvelope; - try { envelope = readNodeResult(raw, expected); } - catch (error) { - const reason = error instanceof Error ? error.message : 'Malformed Relayflows result'; + try { + for (;;) { + const next = await abortableNext(iterator, signal); + if (next.done) break; + const item = next.value; + signal?.throwIfAborted(); + if (item.message.from.id !== expected.agentId || item.message.from.name !== expected.agentName) continue; + if (!item.message.text.startsWith(NODE_RESULT_PREFIX)) continue; + let raw: unknown; + try { raw = JSON.parse(item.message.text.slice(NODE_RESULT_PREFIX.length)); } + catch { raw = undefined; } + let envelope: NodeResultEnvelope; + try { envelope = readNodeResult(raw, expected); } + catch (error) { + const reason = error instanceof Error ? error.message : 'Malformed Relayflows result'; + return { + inboxItemId: item.id, + messageId: item.message.id, + envelope: { version: 1, run_id: expected.runId, step_id: expected.stepId, + dispatch_id: expected.dispatchId, status: 'failed', error: reason }, + ack: async () => { await client.inbox.fail({ inboxItemId: item.id, error: reason, retry: false, + metadata: { relayflows_result: 'rejected' } }); }, + }; + } return { inboxItemId: item.id, messageId: item.message.id, - envelope: { version: 1, run_id: expected.runId, step_id: expected.stepId, - dispatch_id: expected.dispatchId, status: 'failed', error: reason }, - ack: async () => { await client.inbox.fail({ inboxItemId: item.id, error: reason, retry: false, - metadata: { relayflows_result: 'rejected' } }); }, + envelope, + ack: async () => { await client.inbox.ack({ inboxItemId: item.id, state: 'read', + metadata: { relayflows_result: 'journaled' } }); }, }; } - return { - inboxItemId: item.id, - messageId: item.message.id, - envelope, - ack: async () => { await client.inbox.ack({ inboxItemId: item.id, state: 'read', - metadata: { relayflows_result: 'journaled' } }); }, - }; + signal?.throwIfAborted(); + throw new Error('Agent Relay result inbox ended before the spawned agent reported completion'); + } finally { + const closing = iterator.return?.(); + // A client that ignores cancellation may leave next() pending forever. + // Preserve prompt lease loss while still requesting resource cleanup. + if (closing !== undefined) { + if (signal?.aborted) void closing.catch(() => undefined); + else await closing; + } } - signal?.throwIfAborted(); - throw new Error('Agent Relay result inbox ended before the spawned agent reported completion'); } async function abortableNext(iterator: AsyncIterator, signal?: AbortSignal): Promise> { @@ -110,7 +120,6 @@ async function abortableNext(iterator: AsyncIterator, signal?: AbortSignal catch (error) { if (signal.aborted) { void pending.catch(() => undefined); - void iterator.return?.(); } throw error; } finally { diff --git a/packages/sdk/src/worker-cli.ts b/packages/sdk/src/worker-cli.ts index 75853c09c..1793e8e54 100644 --- a/packages/sdk/src/worker-cli.ts +++ b/packages/sdk/src/worker-cli.ts @@ -414,6 +414,10 @@ async function runViaAgentRelay( return requirePricedUsage(result, model); } catch (error) { signal?.throwIfAborted(); + // A durable exact release remains pending. Do not journal worker_error: + // leaving this attempt incomplete lets the kernel redispatch it, and the + // saved node-spawn claim resumes cleanup without spawning another agent. + if (error instanceof AgentRelayTransportError && error.cleanupPending) throw error; const detail = error instanceof AgentRelayTransportError ? error.message : error instanceof Error ? error.message : 'Relay task transport failed'; return { exit_code: null, stdout_tail: '', stderr_tail: redactRelayError(detail) }; diff --git a/packages/sdk/tests/agent-relay-hardening.test.ts b/packages/sdk/tests/agent-relay-hardening.test.ts index 42c157c89..61e193ed8 100644 --- a/packages/sdk/tests/agent-relay-hardening.test.ts +++ b/packages/sdk/tests/agent-relay-hardening.test.ts @@ -61,4 +61,31 @@ describe('node-spawn result hardening', () => { await expect(waiting).rejects.toThrow('lease lost'); expect(returned).toHaveBeenCalledOnce(); }); + + it('closes a completed inbox reader without acknowledging the delivery', async () => { + const returned = vi.fn(); + const ack = vi.fn(); + const delivery = await waitForNodeResult({ inbox: { + subscribe: () => ({ [Symbol.asyncIterator]: () => { + let delivered = false; + return { + next: async () => { + if (delivered) return { done: true, value: undefined } as const; + delivered = true; + return { done: false, value: { id: 'delivery_1', message: { id: 'message_1', + from: { id: 'agent_1', name: 'worker' }, text: NODE_RESULT_PREFIX + JSON.stringify({ + version: 1, run_id: 'run_1', step_id: 'step_1', dispatch_id: correlation.dispatchId, + status: 'completed', output: { ok: true }, + }) } } } as const; + }, + return: async () => { returned(); return { done: true, value: undefined } as const; }, + }; + } }), + ack: async input => { ack(input); }, fail: async () => ({}), + } }, { agentId: 'agent_1', agentName: 'worker', ...correlation }); + expect(returned).toHaveBeenCalledOnce(); + expect(ack).not.toHaveBeenCalled(); + await delivery.ack(); + expect(ack).toHaveBeenCalledOnce(); + }); }); diff --git a/packages/sdk/tests/agent-relay-transport.test.ts b/packages/sdk/tests/agent-relay-transport.test.ts index 8c0ff51c1..bd747a7a0 100644 --- a/packages/sdk/tests/agent-relay-transport.test.ts +++ b/packages/sdk/tests/agent-relay-transport.test.ts @@ -219,6 +219,40 @@ describe('native node-spawn task transport', () => { expect(recovered.receipt.status).toBe('completed'); }); + it('leaves failed cleanup nonterminal so a worker redispatch can finish exact release', async () => { + const dataDir = await dataDirectory('flows-node-worker-cleanup-'); + const f = fixture(); + f.release.mockRejectedValueOnce(new Error('release response lost')); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), + stepComplete: vi.fn(async () => ({})), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_1', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + const errors: unknown[] = []; + worker.on('error', error => errors.push(error)); + await worker.attach(); + const dispatch: StepDispatchEvent = { + run_id: 'run_1', step_id: 'edit', attempt: 1, step_type: 'agent', + spec: { cli: 'codex', instruction: 'Implement the change', transport: 'relay' }, + idempotency_key: 'dispatch_1234567890', lease_id: 'lease_1', + lease_deadline_ms: Date.now() + 30_000, pins: {}, + }; + client.emit('step.dispatch', dispatch); + await vi.waitFor(() => expect(errors).toHaveLength(1)); + expect(client.stepComplete).not.toHaveBeenCalled(); + expect(f.ack).not.toHaveBeenCalled(); + + client.emit('step.dispatch', dispatch); + await vi.waitFor(() => expect(client.stepComplete).toHaveBeenCalledOnce()); + await worker.close(); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).toHaveBeenCalledTimes(2); + expect(f.ack).toHaveBeenCalledOnce(); + }); + it('keeps the original deadline across restart and bounds completed-result recovery', async () => { vi.useFakeTimers({ toFake: ['Date'] }); try { From cefb92856f68227ec162d874f8f143ca53d5c7e7 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Sat, 7 Nov 2026 19:05:01 -0800 Subject: [PATCH 06/75] fix(sdk): preserve node-spawn cleanup recovery --- docs/AGENT-RELAY-TRANSPORT.md | 13 ++- packages/sdk/src/agent-relay-transport.ts | 51 ++++++++++- packages/sdk/src/node-spawn-local.ts | 31 ++++--- packages/sdk/src/node-spawn-remote.ts | 28 ++++-- packages/sdk/src/node-spawn-result.ts | 72 ++++++++------- packages/sdk/src/worker-cli.ts | 2 + .../sdk/tests/agent-relay-hardening.test.ts | 40 +++++++++ .../sdk/tests/agent-relay-transport.test.ts | 87 +++++++++++++++++-- packages/sdk/tests/node-spawn-ports.test.ts | 46 ++++++++-- 9 files changed, 307 insertions(+), 63 deletions(-) diff --git a/docs/AGENT-RELAY-TRANSPORT.md b/docs/AGENT-RELAY-TRANSPORT.md index 822aa35ed..46b3f2a64 100644 --- a/docs/AGENT-RELAY-TRANSPORT.md +++ b/docs/AGENT-RELAY-TRANSPORT.md @@ -34,8 +34,10 @@ Before dispatch, Flows creates and fsyncs a private claim under `/node-spawns/`. It binds publisher identity, Relay origin, journal identity, stable worker name, and full spawn input. Tokens are never stored. -The current fleet placement SDK has no caller-provided idempotency key. Flows -therefore sends at most one spawn for a newly created claim. A recorded +The current fleet placement SDK has no caller-provided idempotency key for the +placement invocation itself. Flows forwards the durable key to the selected +node in the spawn action input, and still sends at most one placement spawn +for a newly created claim. A recorded invocation is reconciled with `commands.getInvocation`; an existing claim with no recorded acknowledgement is the ambiguous crash window and fails closed. It is never answered by a blind second spawn. The stable worker name provides @@ -48,8 +50,11 @@ delivery; it cannot use that window to spawn or extend remote execution. Lease loss or timeout aborts the inbox wait and attempts exact release. A post-spawn verification failure carries any exact identity already proven into -the same cleanup path. A failed cleanup is recorded as pending in the durable -claim and the result remains unacked for an idempotent retry. A result delivery +the same cleanup path. Spawn and release action inputs carry their stable +idempotency keys. A failed cleanup is recorded as pending in the durable claim, +the result remains unacked, and the worker refuses to journal a terminal step +fact. Kernel redelivery therefore reopens the same claim and retries exact +release before it can accept a result or issue another spawn. A result delivery stays unacked until its terminal journal append succeeds, so a worker crash cannot turn Relay acknowledgement into fabricated execution truth. diff --git a/packages/sdk/src/agent-relay-transport.ts b/packages/sdk/src/agent-relay-transport.ts index 92f381d9b..f306b26b9 100644 --- a/packages/sdk/src/agent-relay-transport.ts +++ b/packages/sdk/src/agent-relay-transport.ts @@ -34,6 +34,19 @@ export class AgentRelayTransportError extends Error { } } +/** + * Exact cleanup is durable but incomplete. The worker must not journal a + * terminal step fact: leaving the lease unresolved lets the kernel dispatch + * the same durable claim again, where cleanup is retried before any result is + * accepted or any second spawn can occur. + */ +export class AgentRelayCleanupPendingError extends AgentRelayTransportError { + constructor(message: string, cause?: unknown) { + super(message, cause); + this.name = 'AgentRelayCleanupPendingError'; + } +} + const RESULT_RECOVERY_WINDOW_MS = 30_000; export function readAgentRelayEnv(env: NodeJS.ProcessEnv = process.env): Required> { @@ -142,6 +155,23 @@ export async function runAgentRelayTask( await recordNodeSpawn(claimed.path, claim); options.signal?.throwIfAborted(); } else { + if (claim.cleanupPending !== undefined && claim.result === undefined + && claim.release === undefined) { + releaseAttempted = true; + const cleanup = await tryRelease(runtime, claimed.path, claim, request.idempotencyKey); + claim = cleanup.claim; + if (cleanup.error !== undefined) throw cleanup.error; + throw new NodeSpawnError( + claim.failure?.message ?? 'The prior node spawn failed after launch', + claim.failure?.code ?? 'spawn_unconfirmed', + ); + } + if (claim.release !== undefined && claim.result === undefined) { + throw new NodeSpawnError( + claim.failure?.message ?? 'The recorded node spawn was released before it reported a result', + claim.failure?.code ?? 'spawn_released', + ); + } const status = await runtime.port.status('spawn', claim.spawn.id); if (status.status !== 'completed') { throw new NodeSpawnError('The recorded node spawn is no longer completed', @@ -216,6 +246,12 @@ export async function runAgentRelayTask( }; } catch (error) { let cleanupError: unknown; + if ((claim.spawn !== undefined || claim.cleanup !== undefined) + && claim.release === undefined && claim.failure === undefined) { + claim = { ...claim, failure: failureFrom(error) }; + try { await recordNodeSpawn(claimed.path, claim); } + catch (stateError) { cleanupError = stateError; } + } if (error instanceof NodeSpawnError && error.cleanup !== undefined && claim.spawn === undefined) { claim = { ...claim, cleanup: error.cleanup, failure: { code: error.code, message: error.message } }; @@ -233,12 +269,18 @@ export async function runAgentRelayTask( const detail = cleanupError === undefined ? safeError(error) : `${safeError(error)}; exact cleanup remains pending: ${safeError(cleanupError)}`; + if (claim.cleanupPending !== undefined && claim.release === undefined) { + throw new AgentRelayCleanupPendingError(detail, error); + } throw new AgentRelayTransportError(detail, error); } async function settleLateSpawn(outcome: { spawn?: NodeSpawnReceipt; error?: unknown }): Promise { if (outcome.spawn !== undefined) { - claim = { ...claim, spawn: outcome.spawn }; + claim = { ...claim, spawn: outcome.spawn, failure: { + code: 'spawn_cancelled', + message: 'The node spawn confirmed after its caller stopped waiting', + } }; } else if (outcome.error instanceof NodeSpawnError && outcome.error.cleanup !== undefined) { claim = { ...claim, cleanup: outcome.error.cleanup, failure: { code: outcome.error.code, message: outcome.error.message } }; @@ -332,3 +374,10 @@ function releaseKey(key: string): string { function safeError(error: unknown): string { return error instanceof Error && error.message ? error.message : 'Node-spawn task failed'; } + +function failureFrom(error: unknown): { code: string; message: string } { + return { + code: error instanceof NodeSpawnError ? error.code : 'node_spawn_failed', + message: safeError(error), + }; +} diff --git a/packages/sdk/src/node-spawn-local.ts b/packages/sdk/src/node-spawn-local.ts index 132c694d8..fdce3c03d 100644 --- a/packages/sdk/src/node-spawn-local.ts +++ b/packages/sdk/src/node-spawn-local.ts @@ -4,6 +4,8 @@ import { dirname, isAbsolute, join } from 'node:path'; import { createConnection, type Socket } from 'node:net'; import { NodeSpawnError, + isRecord, + nonempty, normalizeNodeSpawnStatus, terminalStatus, type NodeReleaseRequest, @@ -161,13 +163,20 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { async targets(): Promise { const result = await this.core.call<{ targets: Array> }>('spawn.targets'); - if (!Array.isArray(result.targets)) throw new NodeSpawnError('The core returned invalid spawn targets'); - return result.targets.map(value => ({ - id: String(value.id), name: String(value.name), live: value.live === true, - handlersLive: value.handlers_live === true, stale: value.stale === true, - capabilities: Array.isArray(value.capabilities) ? value.capabilities.filter(v => typeof v === 'string') : [], - activeAgents: Number(value.active_agents), reservedAgents: Number(value.reserved_agents), maxAgents: Number(value.max_agents), - })); + if (!isRecord(result) || !Array.isArray(result.targets)) { + throw new NodeSpawnError('The core returned invalid spawn targets', 'invalid_core_response'); + } + return result.targets.map(value => { + if (!isRecord(value) || !nonempty(value.id) || !nonempty(value.name)) { + throw new NodeSpawnError('The core returned a spawn target without an exact identity', 'invalid_core_response'); + } + return { + id: value.id, name: value.name, live: value.live === true, + handlersLive: value.handlers_live === true, stale: value.stale === true, + capabilities: Array.isArray(value.capabilities) ? value.capabilities.filter(nonempty) : [], + activeAgents: Number(value.active_agents), reservedAgents: Number(value.reserved_agents), maxAgents: Number(value.max_agents), + }; + }); } async spawn(request: NodeSpawnRequest): Promise { @@ -179,7 +188,8 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { 'spawn_invalid_input', ); } - const targets = (await this.targets()).filter(candidate => candidate.live && candidate.handlersLive + const allTargets = await this.targets(); + const targets = allTargets.filter(candidate => candidate.live && candidate.handlersLive && !candidate.stale && candidate.capabilities.includes(`spawn:${request.harness}`)); const target = request.node === undefined ? targets[0] @@ -194,8 +204,9 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { // spawn.invoke is the action receipt; spawn.status is also the boundary // that enriches it from the core's exact same-workspace registration. const settled = await this.wait('spawn', raw.id, request.timeoutMs); - const cleanup = settled.agentId && settled.name && settled.nodeId === target.id - ? { id: settled.id, nodeId: settled.nodeId, nodeName: target.name, + const settledTarget = allTargets.find(candidate => candidate.id === settled.nodeId); + const cleanup = settled.agentId && settled.name && settledTarget?.id + ? { id: settled.id, nodeId: settledTarget.id, nodeName: settledTarget.name, name: settled.name, agentId: settled.agentId } : undefined; if (settled.status !== 'completed' || !settled.spawned || !settled.ready diff --git a/packages/sdk/src/node-spawn-remote.ts b/packages/sdk/src/node-spawn-remote.ts index 7959a6470..7f669da34 100644 --- a/packages/sdk/src/node-spawn-remote.ts +++ b/packages/sdk/src/node-spawn-remote.ts @@ -5,6 +5,7 @@ import { normalizeNodeSpawnStatus, nonempty, terminalStatus, + validIdempotencyKey, type NodeReleaseRequest, type NodeSpawnAction, type NodeSpawnIdentity, @@ -55,6 +56,9 @@ export class RemoteNodeSpawnPort implements NodeSpawnPort { } async spawn(request: NodeSpawnRequest): Promise { + if (!validIdempotencyKey(request.idempotencyKey)) { + throw new NodeSpawnError('Invalid node-spawn idempotency key', 'spawn_invalid_input'); + } const capability = `spawn:${request.harness}`; const ack = await this.client.placement.spawn({ capability, @@ -68,6 +72,7 @@ export class RemoteNodeSpawnPort implements NodeSpawnPort { input: { name: request.name, task: request.task, + idempotency_key: request.idempotencyKey, ...(request.model === undefined ? {} : { model: request.model }), ...(request.directory === undefined ? {} : { worker_cwd: request.directory }), }, @@ -85,10 +90,23 @@ export class RemoteNodeSpawnPort implements NodeSpawnPort { || (isRecord(output?.warning) && output.warning.code === 'spawn_task_unconfirmed'); let cleanup: NodeSpawnIdentity | undefined; if (confirmation && output?.spawned === true && nodeName) { + const outputAgentId = nonempty(output.agent_id) ? output.agent_id : undefined; + const outputCleanup: NodeSpawnIdentity | undefined = outputAgentId === undefined ? undefined : { + id: confirmation.invocationId, + nodeId, + nodeName, + name: reportedName, + agentId: outputAgentId, + }; const agent = await this.resolveAgent(reportedName); const exactAgentName = nonempty(agent.name) ? agent.name : reportedName; if (!nonempty(agent.id)) { - throw new NodeSpawnError('The spawned agent identity could not be resolved exactly', 'spawn_identity_mismatch'); + throw new NodeSpawnError('The spawned agent identity could not be resolved exactly', + 'spawn_identity_mismatch', undefined, outputCleanup); + } + if (exactAgentName !== reportedName || (outputAgentId !== undefined && outputAgentId !== agent.id)) { + throw new NodeSpawnError('The spawned agent identity could not be resolved exactly', + 'spawn_identity_mismatch', undefined, outputCleanup); } cleanup = { id: confirmation.invocationId, @@ -97,10 +115,6 @@ export class RemoteNodeSpawnPort implements NodeSpawnPort { name: exactAgentName, agentId: agent.id, }; - if (exactAgentName !== reportedName || (nonempty(output.agent_id) && output.agent_id !== agent.id)) { - throw new NodeSpawnError('The spawned agent identity could not be resolved exactly', - 'spawn_identity_mismatch', undefined, cleanup); - } } if (!confirmation || confirmation.status.toLowerCase() !== 'completed' || output?.spawned !== true || output.ready !== true) { @@ -137,6 +151,9 @@ export class RemoteNodeSpawnPort implements NodeSpawnPort { } async release(request: NodeReleaseRequest): Promise { + if (!validIdempotencyKey(request.idempotencyKey)) { + throw new NodeSpawnError('Invalid node-release idempotency key', 'spawn_invalid_input'); + } const receipt = request.receipt; const ack = await this.client.placement.spawn({ capability: 'release', @@ -147,6 +164,7 @@ export class RemoteNodeSpawnPort implements NodeSpawnPort { input: { name: receipt.name, expected_agent_id: receipt.agentId, + idempotency_key: request.idempotencyKey, reason: request.reason ?? 'Relayflows attempt ended', }, }); diff --git a/packages/sdk/src/node-spawn-result.ts b/packages/sdk/src/node-spawn-result.ts index a2edfebb3..38f743146 100644 --- a/packages/sdk/src/node-spawn-result.ts +++ b/packages/sdk/src/node-spawn-result.ts @@ -62,42 +62,56 @@ export async function waitForNodeResult( signal?: AbortSignal, ): Promise { const iterator = client.inbox.subscribe({ signal })[Symbol.asyncIterator](); - for (;;) { - const next = await abortableNext(iterator, signal); - if (next.done) break; - const item = next.value; - signal?.throwIfAborted(); - if (item.message.from.id !== expected.agentId || item.message.from.name !== expected.agentName) continue; - if (!item.message.text.startsWith(NODE_RESULT_PREFIX)) continue; - let raw: unknown; - try { raw = JSON.parse(item.message.text.slice(NODE_RESULT_PREFIX.length)); } - catch { raw = undefined; } - let envelope: NodeResultEnvelope; - try { envelope = readNodeResult(raw, expected); } - catch (error) { - const reason = error instanceof Error ? error.message : 'Malformed Relayflows result'; + let closing: Promise | undefined; + const close = (): Promise => { + closing ??= Promise.resolve(iterator.return?.()).then(() => undefined); + return closing; + }; + try { + for (;;) { + const next = await abortableNext(iterator, signal, close); + if (next.done) break; + const item = next.value; + signal?.throwIfAborted(); + if (item.message.from.id !== expected.agentId || item.message.from.name !== expected.agentName) continue; + if (!item.message.text.startsWith(NODE_RESULT_PREFIX)) continue; + let raw: unknown; + try { raw = JSON.parse(item.message.text.slice(NODE_RESULT_PREFIX.length)); } + catch { raw = undefined; } + let envelope: NodeResultEnvelope; + try { envelope = readNodeResult(raw, expected); } + catch (error) { + const reason = error instanceof Error ? error.message : 'Malformed Relayflows result'; + return { + inboxItemId: item.id, + messageId: item.message.id, + envelope: { version: 1, run_id: expected.runId, step_id: expected.stepId, + dispatch_id: expected.dispatchId, status: 'failed', error: reason }, + ack: async () => { await client.inbox.fail({ inboxItemId: item.id, error: reason, retry: false, + metadata: { relayflows_result: 'rejected' } }); }, + }; + } return { inboxItemId: item.id, messageId: item.message.id, - envelope: { version: 1, run_id: expected.runId, step_id: expected.stepId, - dispatch_id: expected.dispatchId, status: 'failed', error: reason }, - ack: async () => { await client.inbox.fail({ inboxItemId: item.id, error: reason, retry: false, - metadata: { relayflows_result: 'rejected' } }); }, + envelope, + ack: async () => { await client.inbox.ack({ inboxItemId: item.id, state: 'read', + metadata: { relayflows_result: 'journaled' } }); }, }; } - return { - inboxItemId: item.id, - messageId: item.message.id, - envelope, - ack: async () => { await client.inbox.ack({ inboxItemId: item.id, state: 'read', - metadata: { relayflows_result: 'journaled' } }); }, - }; + signal?.throwIfAborted(); + throw new Error('Agent Relay result inbox ended before the spawned agent reported completion'); + } finally { + if (signal?.aborted) void close(); + else await close(); } - signal?.throwIfAborted(); - throw new Error('Agent Relay result inbox ended before the spawned agent reported completion'); } -async function abortableNext(iterator: AsyncIterator, signal?: AbortSignal): Promise> { +async function abortableNext( + iterator: AsyncIterator, + signal: AbortSignal | undefined, + close: () => Promise, +): Promise> { signal?.throwIfAborted(); const pending = iterator.next(); if (signal === undefined) return pending; @@ -110,7 +124,7 @@ async function abortableNext(iterator: AsyncIterator, signal?: AbortSignal catch (error) { if (signal.aborted) { void pending.catch(() => undefined); - void iterator.return?.(); + void close(); } throw error; } finally { diff --git a/packages/sdk/src/worker-cli.ts b/packages/sdk/src/worker-cli.ts index 75853c09c..bd0f4c946 100644 --- a/packages/sdk/src/worker-cli.ts +++ b/packages/sdk/src/worker-cli.ts @@ -41,6 +41,7 @@ import { } from './cli-transport-evidence.js'; import { runAgentRelayTask, + AgentRelayCleanupPendingError, AgentRelayTransportError, type AgentTransport, } from './agent-relay-transport.js'; @@ -414,6 +415,7 @@ async function runViaAgentRelay( return requirePricedUsage(result, model); } catch (error) { signal?.throwIfAborted(); + if (error instanceof AgentRelayCleanupPendingError) throw error; const detail = error instanceof AgentRelayTransportError ? error.message : error instanceof Error ? error.message : 'Relay task transport failed'; return { exit_code: null, stdout_tail: '', stderr_tail: redactRelayError(detail) }; diff --git a/packages/sdk/tests/agent-relay-hardening.test.ts b/packages/sdk/tests/agent-relay-hardening.test.ts index 42c157c89..ed944f44e 100644 --- a/packages/sdk/tests/agent-relay-hardening.test.ts +++ b/packages/sdk/tests/agent-relay-hardening.test.ts @@ -47,6 +47,31 @@ describe('node-spawn result hardening', () => { expect(fail).toHaveBeenCalledWith(expect.objectContaining({ inboxItemId: 'delivery_bad', retry: false })); }); + it('closes a normal inbox subscription without acknowledging the delivery', async () => { + const returned = vi.fn(async () => ({ done: true as const, value: undefined })); + let delivered = false; + const ack = vi.fn(async () => ({})); + const delivery = await waitForNodeResult({ inbox: { + subscribe: () => ({ [Symbol.asyncIterator]: () => ({ + next: async () => { + if (delivered) return new Promise>(() => {}); + delivered = true; + return { done: false as const, value: { id: 'delivery_1', message: { id: 'message_1', + from: { id: 'agent_1', name: 'worker' }, text: NODE_RESULT_PREFIX + JSON.stringify({ + version: 1, run_id: 'run_1', step_id: 'step_1', dispatch_id: correlation.dispatchId, + status: 'completed', output: { ok: true }, + }) } } }; + }, + return: returned, + }) }), + ack, fail: async () => ({}), + } }, { agentId: 'agent_1', agentName: 'worker', ...correlation }); + expect(returned).toHaveBeenCalledOnce(); + expect(ack).not.toHaveBeenCalled(); + await delivery.ack(); + expect(ack).toHaveBeenCalledOnce(); + }); + it('cancels an initial inbox read even when the client ignores its signal option', async () => { const returned = vi.fn(); const controller = new AbortController(); @@ -61,4 +86,19 @@ describe('node-spawn result hardening', () => { await expect(waiting).rejects.toThrow('lease lost'); expect(returned).toHaveBeenCalledOnce(); }); + + it('does not wait for an uncooperative iterator close after cancellation', async () => { + const returned = vi.fn(() => new Promise>(() => {})); + const controller = new AbortController(); + const waiting = waitForNodeResult({ inbox: { + subscribe: () => ({ [Symbol.asyncIterator]: () => ({ + next: () => new Promise>(() => {}), + return: returned, + }) }), + ack: async () => ({}), fail: async () => ({}), + } }, { agentId: 'agent_1', agentName: 'worker', ...correlation }, controller.signal); + controller.abort(new Error('lease lost')); + await expect(waiting).rejects.toThrow('lease lost'); + expect(returned).toHaveBeenCalledOnce(); + }); }); diff --git a/packages/sdk/tests/agent-relay-transport.test.ts b/packages/sdk/tests/agent-relay-transport.test.ts index 8c0ff51c1..aa12c7597 100644 --- a/packages/sdk/tests/agent-relay-transport.test.ts +++ b/packages/sdk/tests/agent-relay-transport.test.ts @@ -2,7 +2,8 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import { runAgentRelayTask, type AgentRelayTaskRequest } from '../src/agent-relay-transport.js'; +import { AgentRelayCleanupPendingError, runAgentRelayTask, + type AgentRelayTaskRequest } from '../src/agent-relay-transport.js'; import { NODE_RESULT_PREFIX } from '../src/node-spawn-result.js'; import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; import { NodeSpawnError, type NodeSpawnPort, type NodeSpawnReceipt } from '../src/node-spawn.js'; @@ -136,14 +137,23 @@ describe('native node-spawn task transport', () => { nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, agentId: 'agent_1', spawned: true as const, ready: true as const }; }); - await expect(runAgentRelayTask(request(dataDir), { - runtime: f.runtime, signal: controller.signal, - })).rejects.toThrow('lease lost'); - await vi.waitFor(() => { - expect(f.release).toHaveBeenCalledWith(expect.objectContaining({ - receipt: expect.objectContaining({ agentId: 'agent_1', nodeName: 'desktop-1' }), - })); + const order: string[] = []; + f.release.mockImplementationOnce(async () => { + order.push('release'); + return { id: 'inv_release_1', action: 'release' as const, + status: 'completed' as const, spawned: false, ready: false, released: true }; }); + try { + await runAgentRelayTask(request(dataDir), { runtime: f.runtime, signal: controller.signal }); + throw new Error('expected cancellation'); + } catch (error) { + expect(error).toMatchObject({ message: 'lease lost' }); + order.push('rejected'); + } + expect(order).toEqual(['release', 'rejected']); + expect(f.release).toHaveBeenCalledWith(expect.objectContaining({ + receipt: expect.objectContaining({ agentId: 'agent_1', nodeName: 'desktop-1' }), + })); }); it('returns promptly from a cancelled confirmation and cleans up its late exact receipt', async () => { @@ -211,7 +221,7 @@ describe('native node-spawn task transport', () => { const f = fixture(); f.release.mockRejectedValueOnce(new Error('release response lost')); await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) - .rejects.toThrow('release response lost'); + .rejects.toBeInstanceOf(AgentRelayCleanupPendingError); expect(f.ack).not.toHaveBeenCalled(); const recovered = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); expect(f.spawn).toHaveBeenCalledOnce(); @@ -219,6 +229,65 @@ describe('native node-spawn task transport', () => { expect(recovered.receipt.status).toBe('completed'); }); + it('does not terminally journal a cleanup failure and recovers through the worker path', async () => { + const dataDir = await dataDirectory('flows-node-worker-cleanup-'); + const f = fixture(); + f.release.mockRejectedValueOnce(new Error('release response lost')); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), + stepComplete: vi.fn(async () => ({})), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_1', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + await worker.attach(); + const dispatch = (attempt: number): StepDispatchEvent => ({ + run_id: 'run_1', step_id: 'edit', attempt, step_type: 'agent', + spec: { cli: 'codex', instruction: 'Implement the change', transport: 'relay' }, + idempotency_key: 'dispatch_1234567890', lease_id: `lease_${attempt}`, + lease_deadline_ms: Date.now() + 30_000, pins: {}, + }); + const failed = new Promise(resolve => worker.once('error', resolve)); + client.emit('step.dispatch', dispatch(1)); + await expect(failed).resolves.toBeInstanceOf(AgentRelayCleanupPendingError); + expect(client.stepComplete).not.toHaveBeenCalled(); + client.emit('step.dispatch', dispatch(2)); + await vi.waitFor(() => expect(client.stepComplete).toHaveBeenCalledOnce()); + await worker.close(); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).toHaveBeenCalledTimes(2); + expect(client.stepComplete).toHaveBeenCalledWith('run_1', 'edit', 2, + 'dispatch_1234567890', 'success', expect.any(Object)); + }); + + it('fails immediately when a prior error already released an agent without a result', async () => { + const dataDir = await dataDirectory('flows-node-released-without-result-'); + const f = fixture(); + f.runtime.inbox.inbox.subscribe = async function* () {}; + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('inbox ended'); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('inbox ended'); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).toHaveBeenCalledOnce(); + expect(f.status).not.toHaveBeenCalled(); + }); + + it('retries pending cleanup before failing a result-less spawn recovery', async () => { + const dataDir = await dataDirectory('flows-node-resultless-cleanup-'); + const f = fixture(); + f.runtime.inbox.inbox.subscribe = async function* () {}; + f.release.mockRejectedValueOnce(new Error('release response lost')); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toBeInstanceOf(AgentRelayCleanupPendingError); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('inbox ended'); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).toHaveBeenCalledTimes(2); + expect(f.status).not.toHaveBeenCalled(); + }); + it('keeps the original deadline across restart and bounds completed-result recovery', async () => { vi.useFakeTimers({ toFake: ['Date'] }); try { diff --git a/packages/sdk/tests/node-spawn-ports.test.ts b/packages/sdk/tests/node-spawn-ports.test.ts index 8a427113a..e0c65b0c2 100644 --- a/packages/sdk/tests/node-spawn-ports.test.ts +++ b/packages/sdk/tests/node-spawn-ports.test.ts @@ -62,6 +62,15 @@ describe('local node-spawn port', () => { expect(calls).not.toHaveBeenCalled(); }); + it('refuses local targets without an exact string id instead of fabricating one', async () => { + const core: CoreClient = { close() {}, async call(): Promise { + return { targets: [{ id: undefined, name: 'desktop-1', live: true, + handlers_live: true, capabilities: ['spawn:codex'] }] } as T; + } }; + const port = new LocalNodeSpawnPort(core, { name: 'authorized-caller', secret: 'desktop-service-secret' }); + await expect(port.targets()).rejects.toMatchObject({ code: 'invalid_core_response' }); + }); + it.each([ { live: false, handlers_live: true }, { live: true, handlers_live: false }, @@ -103,8 +112,12 @@ describe('local node-spawn port', () => { it('refuses a completed local spawn whose returned node identity changed', async () => { const core: CoreClient = { close() {}, async call(method: string): Promise { - if (method === 'spawn.targets') return { targets: [{ id: 'node_1', name: 'desktop-1', live: true, - handlers_live: true, stale: false, capabilities: ['spawn:codex'] }] } as T; + if (method === 'spawn.targets') return { targets: [ + { id: 'node_1', name: 'desktop-1', live: true, + handlers_live: true, stale: false, capabilities: ['spawn:codex'] }, + { id: 'node_other', name: 'desktop-2', live: true, + handlers_live: true, stale: false, capabilities: ['spawn:codex'] }, + ] } as T; if (method === 'spawn.invoke') return { id: 'inv_1', status: 'pending', spawned: false, ready: false, task_unconfirmed: false } as T; return { id: 'inv_1', status: 'completed', node_id: 'node_other', agent_id: 'agent_1', @@ -112,7 +125,10 @@ describe('local node-spawn port', () => { } }; const port = new LocalNodeSpawnPort(core, { name: 'authorized-caller', secret: 'desktop-service-secret' }); await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', directory: '/workspace', - idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ code: 'spawn_identity_mismatch' }); + idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ + code: 'spawn_identity_mismatch', + cleanup: { nodeId: 'node_other', nodeName: 'desktop-2', name: 'worker_1', agentId: 'agent_1' }, + }); }); }); @@ -136,9 +152,11 @@ describe('remote node-spawn port', () => { idempotencyKey: 'dispatch_1234567890' }); await port.release({ receipt, idempotencyKey: 'release_1234567890' }); expect(receipt).toMatchObject({ agentId: 'agent_1', nodeName: 'desktop-1', ready: true }); - expect(placement.mock.calls[0]![0]).toMatchObject({ confirm: true, verifyReady: true }); + expect(placement.mock.calls[0]![0]).toMatchObject({ confirm: true, verifyReady: true, + input: { idempotency_key: 'dispatch_1234567890' } }); expect(placement.mock.calls[1]![0]).toMatchObject({ capability: 'release', actionName: 'release', - node: 'desktop-1', input: { name: 'worker_1', expected_agent_id: 'agent_1' } }); + node: 'desktop-1', input: { name: 'worker_1', expected_agent_id: 'agent_1', + idempotency_key: 'release_1234567890' } }); expect(getInvocation).toHaveBeenCalledWith('release', 'rel_1'); }); @@ -161,6 +179,24 @@ describe('remote node-spawn port', () => { }); }); + it('never substitutes a conflicting roster id for the handler cleanup identity', async () => { + const port = new RemoteNodeSpawnPort({ + agents: { get: async name => ({ id: 'agent_roster_replacement', name }) }, + nodes: { list: async () => [] }, + placement: { spawn: async () => ({ invocationId: 'ack_1', actionName: 'spawn', status: 'pending', + node: { id: 'node_1', name: 'desktop-1', status: 'online', capabilities: [] }, + placement: { node: 'desktop-1', confirmed: true, state: 'ready' }, + confirmation: { invocationId: 'inv_1', actionName: 'spawn', status: 'completed', + output: { spawned: true, ready: true, name: 'worker_1', agent_id: 'agent_spawned' } } }) }, + commands: { getInvocation: vi.fn() }, + }); + await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', + idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ + code: 'spawn_identity_mismatch', + cleanup: { agentId: 'agent_spawned', name: 'worker_1', nodeName: 'desktop-1' }, + }); + }); + it('recovers the selected node name from its dispatch id when placement metadata lags', async () => { const list = vi.fn(async () => [{ id: 'node_1', name: 'desktop-1', status: 'online', capabilities: [] }]); const port = new RemoteNodeSpawnPort({ From 32e71b2946fe27f45263460ff0025b853786e3f9 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sat, 7 Nov 2026 19:34:18 -0800 Subject: [PATCH 07/75] fix(sdk): close node-spawn recovery gaps --- packages/sdk/src/agent-relay-transport.ts | 8 +++++--- packages/sdk/tests/agent-relay-transport.test.ts | 6 ++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/sdk/src/agent-relay-transport.ts b/packages/sdk/src/agent-relay-transport.ts index 311f1316f..205365bd2 100644 --- a/packages/sdk/src/agent-relay-transport.ts +++ b/packages/sdk/src/agent-relay-transport.ts @@ -240,7 +240,7 @@ export async function runAgentRelayTask( let cleanupError: unknown; if ((claim.spawn !== undefined || claim.cleanup !== undefined) && claim.release === undefined && claim.failure === undefined) { - claim = { ...claim, failure: failureFrom(error) }; + claim = { ...claim, failure: failureRecord(error, options.signal) }; try { await recordNodeSpawn(claimed.path, claim); } catch (stateError) { cleanupError = stateError; } } @@ -367,9 +367,11 @@ function safeError(error: unknown): string { return error instanceof Error && error.message ? error.message : 'Node-spawn task failed'; } -function failureFrom(error: unknown): { code: string; message: string } { +function failureRecord(error: unknown, signal?: AbortSignal): { code: string; message: string } { return { - code: error instanceof NodeSpawnError ? error.code : 'node_spawn_failed', + code: error instanceof NodeSpawnError + ? error.code + : signal?.aborted === true ? 'task_cancelled' : 'node_spawn_failed', message: safeError(error), }; } diff --git a/packages/sdk/tests/agent-relay-transport.test.ts b/packages/sdk/tests/agent-relay-transport.test.ts index d926a2325..e9daeb850 100644 --- a/packages/sdk/tests/agent-relay-transport.test.ts +++ b/packages/sdk/tests/agent-relay-transport.test.ts @@ -154,6 +154,12 @@ describe('native node-spawn task transport', () => { expect(f.release).toHaveBeenCalledWith(expect.objectContaining({ receipt: expect.objectContaining({ agentId: 'agent_1', nodeName: 'desktop-1' }), })); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })).rejects.toMatchObject({ + cause: expect.objectContaining({ code: 'task_cancelled' }), + }); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).toHaveBeenCalledOnce(); + expect(f.status).not.toHaveBeenCalled(); }); it('returns promptly from a cancelled confirmation and cleans up its late exact receipt', async () => { From 7465cf5903f5ebda1c3e00da6f1ca108fa6b4588 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sat, 7 Nov 2026 19:50:42 -0800 Subject: [PATCH 08/75] fix(sdk): retain cleanup identity across roster outage --- packages/sdk/src/node-spawn-remote.ts | 10 +++++++- packages/sdk/tests/node-spawn-ports.test.ts | 26 +++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/packages/sdk/src/node-spawn-remote.ts b/packages/sdk/src/node-spawn-remote.ts index 7f669da34..dc1f8f0c3 100644 --- a/packages/sdk/src/node-spawn-remote.ts +++ b/packages/sdk/src/node-spawn-remote.ts @@ -98,7 +98,15 @@ export class RemoteNodeSpawnPort implements NodeSpawnPort { name: reportedName, agentId: outputAgentId, }; - const agent = await this.resolveAgent(reportedName); + let agent: RelayAgent; + try { + agent = await this.resolveAgent(reportedName); + } catch (error) { + if (error instanceof NodeSpawnError) { + throw new NodeSpawnError(error.message, error.code, error.cause ?? error, outputCleanup); + } + throw error; + } const exactAgentName = nonempty(agent.name) ? agent.name : reportedName; if (!nonempty(agent.id)) { throw new NodeSpawnError('The spawned agent identity could not be resolved exactly', diff --git a/packages/sdk/tests/node-spawn-ports.test.ts b/packages/sdk/tests/node-spawn-ports.test.ts index e0c65b0c2..91015cd29 100644 --- a/packages/sdk/tests/node-spawn-ports.test.ts +++ b/packages/sdk/tests/node-spawn-ports.test.ts @@ -197,6 +197,32 @@ describe('remote node-spawn port', () => { }); }); + it('retains the handler cleanup identity when the ready-agent roster is unavailable', async () => { + vi.useFakeTimers(); + try { + const port = new RemoteNodeSpawnPort({ + agents: { get: async () => { throw new Error('roster unavailable'); } }, + nodes: { list: async () => [] }, + placement: { spawn: async () => ({ invocationId: 'ack_1', actionName: 'spawn', status: 'pending', + node: { id: 'node_1', name: 'desktop-1', status: 'online', capabilities: [] }, + placement: { node: 'desktop-1', confirmed: true, state: 'ready' }, + confirmation: { invocationId: 'inv_1', actionName: 'spawn', status: 'completed', + output: { spawned: true, ready: true, name: 'worker_1', agent_id: 'agent_spawned' } } }) }, + commands: { getInvocation: vi.fn() }, + }); + const pending = port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', + idempotencyKey: 'dispatch_1234567890' }); + await vi.runAllTimersAsync(); + await expect(pending).rejects.toMatchObject({ + code: 'spawn_identity_unavailable', + cleanup: { id: 'inv_1', nodeId: 'node_1', nodeName: 'desktop-1', + name: 'worker_1', agentId: 'agent_spawned' }, + }); + } finally { + vi.useRealTimers(); + } + }); + it('recovers the selected node name from its dispatch id when placement metadata lags', async () => { const list = vi.fn(async () => [{ id: 'node_1', name: 'desktop-1', status: 'online', capabilities: [] }]); const port = new RemoteNodeSpawnPort({ From 837a9311f550e4e5c870fb336cca716192bbbba5 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sat, 7 Nov 2026 20:03:26 -0800 Subject: [PATCH 09/75] fix(sdk): close node-spawn review gaps --- packages/sdk/src/agent-relay-transport.ts | 2 +- packages/sdk/src/node-spawn-local.ts | 4 +- packages/sdk/src/node-spawn-result.ts | 4 +- packages/sdk/src/node-spawn-runtime.ts | 33 +++++++- .../sdk/tests/agent-relay-hardening.test.ts | 16 ++++ packages/sdk/tests/node-spawn-ports.test.ts | 30 ++++++- packages/sdk/tests/node-spawn-runtime.test.ts | 79 +++++++++++++++++++ packages/sdk/tsconfig.tests.json | 1 + 8 files changed, 156 insertions(+), 13 deletions(-) create mode 100644 packages/sdk/tests/node-spawn-runtime.test.ts diff --git a/packages/sdk/src/agent-relay-transport.ts b/packages/sdk/src/agent-relay-transport.ts index 205365bd2..1fa606820 100644 --- a/packages/sdk/src/agent-relay-transport.ts +++ b/packages/sdk/src/agent-relay-transport.ts @@ -72,7 +72,7 @@ export async function runAgentRelayTask( ...(options.node === undefined ? {} : { RELAYFLOW_NODE_NAME: options.node }), ...(options.repo === undefined ? {} : { RELAYFLOW_NODE_REPO: options.repo }), ...(options.sandboxOnly === undefined ? {} : { RELAYFLOW_NODE_SANDBOX_ONLY: options.sandboxOnly ? '1' : '0' }), - }); + }, { signal: options.signal }); const correlation = { runId: request.runId, stepId: request.stepId, dispatchId: request.idempotencyKey }; const agentName = stableAgentName(request); const task = [ diff --git a/packages/sdk/src/node-spawn-local.ts b/packages/sdk/src/node-spawn-local.ts index fdce3c03d..7a46d0fd5 100644 --- a/packages/sdk/src/node-spawn-local.ts +++ b/packages/sdk/src/node-spawn-local.ts @@ -210,9 +210,9 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { name: settled.name, agentId: settled.agentId } : undefined; if (settled.status !== 'completed' || !settled.spawned || !settled.ready - || settled.taskUnconfirmed || !settled.agentId || !settled.name) { + || settled.taskUnconfirmed !== false || !settled.agentId || !settled.name) { throw new NodeSpawnError('The local node did not prove a ready spawned agent', - settled.taskUnconfirmed ? 'spawn_task_unconfirmed' : settled.errorCode, undefined, cleanup); + settled.taskUnconfirmed !== false ? 'spawn_task_unconfirmed' : settled.errorCode, undefined, cleanup); } if (settled.name !== request.name || settled.nodeId !== target.id) { throw new NodeSpawnError('The local spawn result did not preserve its requested identity', diff --git a/packages/sdk/src/node-spawn-result.ts b/packages/sdk/src/node-spawn-result.ts index 072aa07da..0acf065f9 100644 --- a/packages/sdk/src/node-spawn-result.ts +++ b/packages/sdk/src/node-spawn-result.ts @@ -104,7 +104,7 @@ export async function waitForNodeResult( } finally { // A client that ignores cancellation may leave next() pending forever. // Preserve prompt lease loss while still requesting resource cleanup. - if (signal?.aborted) void close(); + if (signal?.aborted) void close().catch(() => undefined); else await close(); } } @@ -126,7 +126,7 @@ async function abortableNext( catch (error) { if (signal.aborted) { void pending.catch(() => undefined); - void close(); + void close().catch(() => undefined); } throw error; } finally { diff --git a/packages/sdk/src/node-spawn-runtime.ts b/packages/sdk/src/node-spawn-runtime.ts index bb3bdf36e..9168c725d 100644 --- a/packages/sdk/src/node-spawn-runtime.ts +++ b/packages/sdk/src/node-spawn-runtime.ts @@ -17,13 +17,19 @@ export interface NodeSpawnRuntime { interface RelayRuntimeClient extends RemoteNodeSpawnClient, NodeResultInboxClient { agents: RemoteNodeSpawnClient['agents'] & { me(): Promise }; + events: { disconnect(): Promise }; } -export async function createRemoteNodeSpawnRuntime(env: NodeJS.ProcessEnv = process.env): Promise { +const PUBLISHER_DISCOVERY_TIMEOUT_MS = 15_000; + +export async function createRemoteNodeSpawnRuntime( + env: NodeJS.ProcessEnv = process.env, + options: { signal?: AbortSignal } = {}, +): Promise { const agentToken = env.RELAY_AGENT_TOKEN?.trim(); if (!agentToken) throw new NodeSpawnError( 'Node-spawn transport requires a dedicated publisher RELAY_AGENT_TOKEN.', 'publisher_token_required'); - const baseUrl = validateBaseUrl(env.RELAY_BASE_URL?.trim() || 'https://cast.agentrelay.com'); + const baseUrl = validateNodeSpawnBaseUrl(env.RELAY_BASE_URL?.trim() || 'https://cast.agentrelay.com'); let RelaycastMessagingClient: new (options: Record) => RelayRuntimeClient; try { ({ RelaycastMessagingClient } = await import('@agent-relay/sdk') as unknown as { @@ -38,7 +44,9 @@ export async function createRemoteNodeSpawnRuntime(env: NodeJS.ProcessEnv = proc throw new NodeSpawnError('RELAYFLOW_NODE_SANDBOX_ONLY must be 0 or 1', 'spawn_invalid_input'); } const client = new RelaycastMessagingClient({ baseUrl, agentToken }); - const publisher = await client.agents.me(); + const timeout = AbortSignal.timeout(PUBLISHER_DISCOVERY_TIMEOUT_MS); + const signal = options.signal === undefined ? timeout : AbortSignal.any([options.signal, timeout]); + const publisher = await discoverPublisher(client, signal); if (!publisher.id?.trim() || !publisher.name?.trim()) { throw new NodeSpawnError('The node-spawn publisher token has no exact Relay identity', 'publisher_identity_missing'); } @@ -53,7 +61,24 @@ export async function createRemoteNodeSpawnRuntime(env: NodeJS.ProcessEnv = proc }; } -function validateBaseUrl(raw: string): string { +async function discoverPublisher(client: RelayRuntimeClient, signal: AbortSignal): Promise { + let onAbort: (() => void) | undefined; + try { + signal.throwIfAborted(); + const aborted = new Promise((_resolve, reject) => { + onAbort = () => reject(signal.reason ?? new DOMException('The operation was aborted', 'AbortError')); + signal.addEventListener('abort', onAbort, { once: true }); + }); + return await Promise.race([client.agents.me(), aborted]); + } catch (error) { + if (signal.aborted) void client.events.disconnect().catch(() => undefined); + throw error; + } finally { + if (onAbort !== undefined) signal.removeEventListener('abort', onAbort); + } +} + +function validateNodeSpawnBaseUrl(raw: string): string { let url: URL; try { url = new URL(raw); } catch (error) { diff --git a/packages/sdk/tests/agent-relay-hardening.test.ts b/packages/sdk/tests/agent-relay-hardening.test.ts index ed944f44e..0db5bcc00 100644 --- a/packages/sdk/tests/agent-relay-hardening.test.ts +++ b/packages/sdk/tests/agent-relay-hardening.test.ts @@ -101,4 +101,20 @@ describe('node-spawn result hardening', () => { await expect(waiting).rejects.toThrow('lease lost'); expect(returned).toHaveBeenCalledOnce(); }); + + it('preserves cancellation when a detached iterator close rejects', async () => { + const returned = vi.fn(async () => { throw new Error('close failed'); }); + const controller = new AbortController(); + const waiting = waitForNodeResult({ inbox: { + subscribe: () => ({ [Symbol.asyncIterator]: () => ({ + next: () => new Promise>(() => {}), + return: returned, + }) }), + ack: async () => ({}), fail: async () => ({}), + } }, { agentId: 'agent_1', agentName: 'worker', ...correlation }, controller.signal); + controller.abort(new Error('lease lost')); + await expect(waiting).rejects.toThrow('lease lost'); + expect(returned).toHaveBeenCalledOnce(); + await new Promise(resolve => setImmediate(resolve)); + }); }); diff --git a/packages/sdk/tests/node-spawn-ports.test.ts b/packages/sdk/tests/node-spawn-ports.test.ts index 91015cd29..3e59a85a4 100644 --- a/packages/sdk/tests/node-spawn-ports.test.ts +++ b/packages/sdk/tests/node-spawn-ports.test.ts @@ -42,7 +42,8 @@ describe('local node-spawn port', () => { caller_secret: 'desktop-service-secret', invocation_id: 'inv_1' })]); expect(calls[3]).toEqual(['spawn.release', expect.objectContaining({ expected_agent_id: 'agent_1', caller_name: 'authorized-caller', - caller_secret: 'desktop-service-secret' })]); + caller_secret: 'desktop-service-secret', node_id: 'node_1', node_name: 'desktop-1', + name: 'worker_1', idempotency_key: 'release_1234567890' })]); }); it('refuses a local service caller without its Desktop-issued secret', () => { @@ -100,9 +101,12 @@ describe('local node-spawn port', () => { handlers_live: true, stale: false, capabilities: ['spawn:codex'] }] } as T; if (method === 'spawn.invoke') return { id: 'inv_1', status: 'pending', spawned: false, ready: false, task_unconfirmed: false } as T; - status(); - return { id: 'inv_1', status: 'denied', spawned: false, ready: false, - task_unconfirmed: false, error_code: 'outside_scope' } as T; + if (method === 'spawn.status') { + status(); + return { id: 'inv_1', status: 'denied', spawned: false, ready: false, + task_unconfirmed: false, error_code: 'outside_scope' } as T; + } + throw new Error(`unexpected core method: ${method}`); } }; const port = new LocalNodeSpawnPort(core, { name: 'authorized-caller', secret: 'desktop-service-secret' }); await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', directory: '/workspace', @@ -130,6 +134,24 @@ describe('local node-spawn port', () => { cleanup: { nodeId: 'node_other', nodeName: 'desktop-2', name: 'worker_1', agentId: 'agent_1' }, }); }); + + it('refuses a completed local spawn without explicit task delivery confirmation', async () => { + const core: CoreClient = { close() {}, async call(method: string): Promise { + if (method === 'spawn.targets') return { targets: [{ id: 'node_1', name: 'desktop-1', live: true, + handlers_live: true, stale: false, capabilities: ['spawn:codex'] }] } as T; + if (method === 'spawn.invoke') return { id: 'inv_1', status: 'pending', spawned: false, + ready: false, task_unconfirmed: false } as T; + if (method === 'spawn.status') return { id: 'inv_1', status: 'completed', node_id: 'node_1', + agent_id: 'agent_1', name: 'worker_1', spawned: true, ready: true } as T; + throw new Error(`unexpected core method: ${method}`); + } }; + const port = new LocalNodeSpawnPort(core, { name: 'authorized-caller', secret: 'desktop-service-secret' }); + await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', directory: '/workspace', + idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ + code: 'spawn_task_unconfirmed', + cleanup: { nodeId: 'node_1', nodeName: 'desktop-1', name: 'worker_1', agentId: 'agent_1' }, + }); + }); }); describe('remote node-spawn port', () => { diff --git a/packages/sdk/tests/node-spawn-runtime.test.ts b/packages/sdk/tests/node-spawn-runtime.test.ts new file mode 100644 index 000000000..a0877f56a --- /dev/null +++ b/packages/sdk/tests/node-spawn-runtime.test.ts @@ -0,0 +1,79 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const relay = vi.hoisted(() => ({ + constructorOptions: [] as Array>, + disconnect: vi.fn(async () => {}), + me: vi.fn<() => Promise<{ id: string; name: string }>>(), +})); + +vi.mock('@agent-relay/sdk', () => ({ + RelaycastMessagingClient: class { + readonly agents = { me: relay.me, get: vi.fn() }; + readonly commands = { getInvocation: vi.fn() }; + readonly events = { disconnect: relay.disconnect }; + readonly inbox = { subscribe: vi.fn(), ack: vi.fn(), fail: vi.fn() }; + readonly nodes = { list: vi.fn() }; + readonly placement = { spawn: vi.fn() }; + + constructor(options: Record) { + relay.constructorOptions.push(options); + } + }, +})); + +import { createRemoteNodeSpawnRuntime } from '../src/node-spawn-runtime.js'; +import { runAgentRelayTask } from '../src/agent-relay-transport.js'; + +describe('remote node-spawn runtime', () => { + beforeEach(() => { + relay.constructorOptions.length = 0; + relay.disconnect.mockClear(); + relay.me.mockReset(); + }); + + it('rejects a non-HTTPS credential origin before constructing a Relay client', async () => { + await expect(createRemoteNodeSpawnRuntime({ + RELAY_AGENT_TOKEN: 'agent-token', + RELAY_BASE_URL: 'http://relay.example', + })).rejects.toMatchObject({ code: 'spawn_invalid_input' }); + expect(relay.constructorOptions).toHaveLength(0); + }); + + it('allows a literal loopback HTTP credential origin', async () => { + relay.me.mockResolvedValue({ id: 'publisher_1', name: 'publisher' }); + await expect(createRemoteNodeSpawnRuntime({ + RELAY_AGENT_TOKEN: 'agent-token', + RELAY_BASE_URL: 'http://127.0.0.1:8787', + })).resolves.toMatchObject({ + baseUrl: 'http://127.0.0.1:8787', + publisher: { id: 'publisher_1', name: 'publisher' }, + }); + expect(relay.constructorOptions).toEqual([{ + agentToken: 'agent-token', + baseUrl: 'http://127.0.0.1:8787', + }]); + }); + + it('honors cancellation during publisher discovery and cleans up the client', async () => { + relay.me.mockImplementation(() => new Promise(() => {})); + const controller = new AbortController(); + const pending = createRemoteNodeSpawnRuntime({ RELAY_AGENT_TOKEN: 'agent-token' }, { + signal: controller.signal, + }); + controller.abort(new Error('lease lost')); + await expect(pending).rejects.toThrow('lease lost'); + expect(relay.disconnect).toHaveBeenCalledOnce(); + }); + + it('forwards the worker lease cancellation into publisher discovery', async () => { + relay.me.mockImplementation(() => new Promise(() => {})); + const controller = new AbortController(); + const pending = runAgentRelayTask({ + cli: 'codex', task: 'work', runId: 'run_1', stepId: 'step_1', + idempotencyKey: 'dispatch_1234567890', dataDir: '/tmp/unused', + }, { agentToken: 'agent-token', signal: controller.signal }); + controller.abort(new Error('lease lost')); + await expect(pending).rejects.toThrow('lease lost'); + expect(relay.disconnect).toHaveBeenCalledOnce(); + }); +}); diff --git a/packages/sdk/tsconfig.tests.json b/packages/sdk/tsconfig.tests.json index 2bb784394..70f3e886a 100644 --- a/packages/sdk/tsconfig.tests.json +++ b/packages/sdk/tsconfig.tests.json @@ -75,6 +75,7 @@ "tests/broker-retirement.test.ts", "tests/agent-relay-hardening.test.ts", "tests/node-spawn-ports.test.ts", + "tests/node-spawn-runtime.test.ts", "tests/webhook-hardening.test.ts", "tests/adapters/claude.test.ts", "tests/adapters/codex.test.ts", From 70dc70c0d12b776a76c639c875a990f44bed5c16 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sat, 7 Nov 2026 20:08:16 -0800 Subject: [PATCH 10/75] test(sdk): pin publisher discovery timeout --- packages/sdk/tests/node-spawn-runtime.test.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/sdk/tests/node-spawn-runtime.test.ts b/packages/sdk/tests/node-spawn-runtime.test.ts index a0877f56a..3f67ead7a 100644 --- a/packages/sdk/tests/node-spawn-runtime.test.ts +++ b/packages/sdk/tests/node-spawn-runtime.test.ts @@ -65,6 +65,13 @@ describe('remote node-spawn runtime', () => { expect(relay.disconnect).toHaveBeenCalledOnce(); }); + it('bounds publisher discovery and cleans up the client on timeout', async () => { + relay.me.mockImplementation(() => new Promise(() => {})); + const pending = createRemoteNodeSpawnRuntime({ RELAY_AGENT_TOKEN: 'agent-token' }); + await expect(pending).rejects.toMatchObject({ name: 'TimeoutError' }); + expect(relay.disconnect).toHaveBeenCalledOnce(); + }, 20_000); + it('forwards the worker lease cancellation into publisher discovery', async () => { relay.me.mockImplementation(() => new Promise(() => {})); const controller = new AbortController(); From bdb05fea1e407c9684fe4efeb91e2b38c406fdb3 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sat, 7 Nov 2026 20:14:06 -0800 Subject: [PATCH 11/75] fix(sdk): close node-spawn inbox transport --- packages/sdk/src/node-spawn-result.ts | 11 +++++++++-- packages/sdk/src/node-spawn-runtime.ts | 2 +- packages/sdk/tests/agent-relay-hardening.test.ts | 9 +++++++-- packages/sdk/tests/node-spawn-runtime.test.ts | 7 +++++-- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/packages/sdk/src/node-spawn-result.ts b/packages/sdk/src/node-spawn-result.ts index 0acf065f9..d6703e4e2 100644 --- a/packages/sdk/src/node-spawn-result.ts +++ b/packages/sdk/src/node-spawn-result.ts @@ -31,6 +31,8 @@ export interface NodeResultInboxClient { ack(input: { inboxItemId: string; state?: 'delivered' | 'read'; metadata?: Record }): Promise; fail(input: { inboxItemId: string; error: string; retry?: boolean; metadata?: Record }): Promise; }; + /** Closes the agent-scoped event transport after this subscription ends. */ + disconnect?(): Promise; } export interface NodeResultDelivery { @@ -104,8 +106,13 @@ export async function waitForNodeResult( } finally { // A client that ignores cancellation may leave next() pending forever. // Preserve prompt lease loss while still requesting resource cleanup. - if (signal?.aborted) void close().catch(() => undefined); - else await close(); + if (signal?.aborted) { + void close().catch(() => undefined); + void client.disconnect?.().catch(() => undefined); + } else { + try { await close(); } + finally { await client.disconnect?.(); } + } } } diff --git a/packages/sdk/src/node-spawn-runtime.ts b/packages/sdk/src/node-spawn-runtime.ts index 9168c725d..754ea30ba 100644 --- a/packages/sdk/src/node-spawn-runtime.ts +++ b/packages/sdk/src/node-spawn-runtime.ts @@ -52,7 +52,7 @@ export async function createRemoteNodeSpawnRuntime( } return { port: new RemoteNodeSpawnPort(client), - inbox: client, + inbox: { inbox: client.inbox, disconnect: () => client.events.disconnect() }, publisher, baseUrl, ...(env.RELAYFLOW_NODE_NAME?.trim() ? { node: env.RELAYFLOW_NODE_NAME.trim() } : {}), diff --git a/packages/sdk/tests/agent-relay-hardening.test.ts b/packages/sdk/tests/agent-relay-hardening.test.ts index 0db5bcc00..102859eed 100644 --- a/packages/sdk/tests/agent-relay-hardening.test.ts +++ b/packages/sdk/tests/agent-relay-hardening.test.ts @@ -49,9 +49,10 @@ describe('node-spawn result hardening', () => { it('closes a normal inbox subscription without acknowledging the delivery', async () => { const returned = vi.fn(async () => ({ done: true as const, value: undefined })); + const disconnect = vi.fn(async () => {}); let delivered = false; const ack = vi.fn(async () => ({})); - const delivery = await waitForNodeResult({ inbox: { + const delivery = await waitForNodeResult({ disconnect, inbox: { subscribe: () => ({ [Symbol.asyncIterator]: () => ({ next: async () => { if (delivered) return new Promise>(() => {}); @@ -67,6 +68,7 @@ describe('node-spawn result hardening', () => { ack, fail: async () => ({}), } }, { agentId: 'agent_1', agentName: 'worker', ...correlation }); expect(returned).toHaveBeenCalledOnce(); + expect(disconnect).toHaveBeenCalledOnce(); expect(ack).not.toHaveBeenCalled(); await delivery.ack(); expect(ack).toHaveBeenCalledOnce(); @@ -74,8 +76,9 @@ describe('node-spawn result hardening', () => { it('cancels an initial inbox read even when the client ignores its signal option', async () => { const returned = vi.fn(); + const disconnect = vi.fn(async () => {}); const controller = new AbortController(); - const waiting = waitForNodeResult({ inbox: { + const waiting = waitForNodeResult({ disconnect, inbox: { subscribe: () => ({ [Symbol.asyncIterator]: () => ({ next: () => new Promise>(() => {}), return: async () => { returned(); return { done: true, value: undefined }; }, @@ -85,6 +88,8 @@ describe('node-spawn result hardening', () => { controller.abort(new Error('lease lost')); await expect(waiting).rejects.toThrow('lease lost'); expect(returned).toHaveBeenCalledOnce(); + await new Promise(resolve => setImmediate(resolve)); + expect(disconnect).toHaveBeenCalledOnce(); }); it('does not wait for an uncooperative iterator close after cancellation', async () => { diff --git a/packages/sdk/tests/node-spawn-runtime.test.ts b/packages/sdk/tests/node-spawn-runtime.test.ts index 3f67ead7a..5a7694ca1 100644 --- a/packages/sdk/tests/node-spawn-runtime.test.ts +++ b/packages/sdk/tests/node-spawn-runtime.test.ts @@ -41,10 +41,11 @@ describe('remote node-spawn runtime', () => { it('allows a literal loopback HTTP credential origin', async () => { relay.me.mockResolvedValue({ id: 'publisher_1', name: 'publisher' }); - await expect(createRemoteNodeSpawnRuntime({ + const runtime = await createRemoteNodeSpawnRuntime({ RELAY_AGENT_TOKEN: 'agent-token', RELAY_BASE_URL: 'http://127.0.0.1:8787', - })).resolves.toMatchObject({ + }); + expect(runtime).toMatchObject({ baseUrl: 'http://127.0.0.1:8787', publisher: { id: 'publisher_1', name: 'publisher' }, }); @@ -52,6 +53,8 @@ describe('remote node-spawn runtime', () => { agentToken: 'agent-token', baseUrl: 'http://127.0.0.1:8787', }]); + await runtime.inbox.disconnect?.(); + expect(relay.disconnect).toHaveBeenCalledOnce(); }); it('honors cancellation during publisher discovery and cleans up the client', async () => { From e48df159312a3d3510a337464b6f208478088667 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sat, 7 Nov 2026 20:26:21 -0800 Subject: [PATCH 12/75] fix(sdk): preserve node results when teardown fails --- packages/sdk/src/node-spawn-result.ts | 2 +- packages/sdk/tests/agent-relay-hardening.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/sdk/src/node-spawn-result.ts b/packages/sdk/src/node-spawn-result.ts index d6703e4e2..39a104427 100644 --- a/packages/sdk/src/node-spawn-result.ts +++ b/packages/sdk/src/node-spawn-result.ts @@ -111,7 +111,7 @@ export async function waitForNodeResult( void client.disconnect?.().catch(() => undefined); } else { try { await close(); } - finally { await client.disconnect?.(); } + finally { await client.disconnect?.().catch(() => undefined); } } } } diff --git a/packages/sdk/tests/agent-relay-hardening.test.ts b/packages/sdk/tests/agent-relay-hardening.test.ts index 102859eed..6389ffaf0 100644 --- a/packages/sdk/tests/agent-relay-hardening.test.ts +++ b/packages/sdk/tests/agent-relay-hardening.test.ts @@ -49,7 +49,7 @@ describe('node-spawn result hardening', () => { it('closes a normal inbox subscription without acknowledging the delivery', async () => { const returned = vi.fn(async () => ({ done: true as const, value: undefined })); - const disconnect = vi.fn(async () => {}); + const disconnect = vi.fn(async () => { throw new Error('disconnect failed'); }); let delivered = false; const ack = vi.fn(async () => ({})); const delivery = await waitForNodeResult({ disconnect, inbox: { From eeb765a1ab1b7181cf19d16c42366e5610c3ca3a Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sat, 7 Nov 2026 21:14:05 -0800 Subject: [PATCH 13/75] fix(sdk): share relay publisher runtime across tasks --- docs/AGENT-RELAY-TRANSPORT.md | 4 + packages/sdk/src/agent-relay-transport.ts | 42 ++++--- packages/sdk/src/node-spawn-result.ts | 10 +- packages/sdk/src/node-spawn-runtime.ts | 67 ++++++++++- packages/sdk/src/worker-cli.ts | 3 +- packages/sdk/src/worker.ts | 9 +- .../sdk/tests/agent-relay-hardening.test.ts | 11 +- packages/sdk/tests/node-spawn-runtime.test.ts | 28 ++++- .../tests/worker-node-runtime-sharing.test.ts | 109 ++++++++++++++++++ 9 files changed, 246 insertions(+), 37 deletions(-) create mode 100644 packages/sdk/tests/worker-node-runtime-sharing.test.ts diff --git a/docs/AGENT-RELAY-TRANSPORT.md b/docs/AGENT-RELAY-TRANSPORT.md index 76985029f..0ecbdab20 100644 --- a/docs/AGENT-RELAY-TRANSPORT.md +++ b/docs/AGENT-RELAY-TRANSPORT.md @@ -25,6 +25,10 @@ The remote path uses `@agent-relay/sdk >=17.2.1 <18`: 5. Only after `stepComplete` is durably accepted by the journal does Flows ack the inbox delivery. Relay is transport; the journal is execution authority. +One worker owns one lazy publisher connection across all of its concurrent +relay steps. Each step closes only its inbox iterator; worker shutdown drains +every step before disconnecting the shared publisher identity. + Spawn readiness is not task completion. A result without the exact sender and attempt correlation cannot complete a step. diff --git a/packages/sdk/src/agent-relay-transport.ts b/packages/sdk/src/agent-relay-transport.ts index 1fa606820..de0ce5bea 100644 --- a/packages/sdk/src/agent-relay-transport.ts +++ b/packages/sdk/src/agent-relay-transport.ts @@ -55,7 +55,8 @@ export function readAgentRelayEnv(env: NodeJS.ProcessEnv = process.env): Require */ export async function runAgentRelayTask( request: AgentRelayTaskRequest, - options: AgentRelayEnv & { runtime?: NodeSpawnRuntime; signal?: AbortSignal } = {}, + options: AgentRelayEnv & { runtime?: NodeSpawnRuntime; + runtimeProvider?: (signal?: AbortSignal) => Promise; signal?: AbortSignal } = {}, ): Promise { const taskStartedAt = Date.now(); options.signal?.throwIfAborted(); @@ -65,7 +66,8 @@ export async function runAgentRelayTask( || !validIdempotencyKey(request.idempotencyKey)) { throw new AgentRelayTransportError('Node-spawn task requires durable dispatch identity and a valid deadline'); } - const runtime = options.runtime ?? await createRemoteNodeSpawnRuntime({ + const ownsRuntime = options.runtime === undefined && options.runtimeProvider === undefined; + const runtime = options.runtime ?? await options.runtimeProvider?.(options.signal) ?? await createRemoteNodeSpawnRuntime({ ...process.env, ...(options.baseUrl === undefined ? {} : { RELAY_BASE_URL: options.baseUrl }), ...(options.agentToken === undefined ? {} : { RELAY_AGENT_TOKEN: options.agentToken }), @@ -73,6 +75,20 @@ export async function runAgentRelayTask( ...(options.repo === undefined ? {} : { RELAYFLOW_NODE_REPO: options.repo }), ...(options.sandboxOnly === undefined ? {} : { RELAYFLOW_NODE_SANDBOX_ONLY: options.sandboxOnly ? '1' : '0' }), }, { signal: options.signal }); + try { + return await executeAgentRelayTask(request, runtime, options.signal, taskStartedAt, timeoutMs); + } finally { + if (ownsRuntime) await runtime.disconnect?.().catch(() => undefined); + } +} + +async function executeAgentRelayTask( + request: AgentRelayTaskRequest, + runtime: NodeSpawnRuntime, + signal: AbortSignal | undefined, + taskStartedAt: number, + timeoutMs: number, +): Promise { const correlation = { runId: request.runId, stepId: request.stepId, dispatchId: request.idempotencyKey }; const agentName = stableAgentName(request); const task = [ @@ -119,11 +135,11 @@ export async function runAgentRelayTask( claim.failure?.code ?? 'spawn_unconfirmed'); } assertWithinDeadline(claim.result === undefined ? deadline : deadline + RESULT_RECOVERY_WINDOW_MS); - options.signal?.throwIfAborted(); + signal?.throwIfAborted(); if (claim.spawn === undefined) { // This is deliberately adjacent to the irreversible dispatch. Runtime // setup and durable claiming can take time or lose the worker lease. - options.signal?.throwIfAborted(); + signal?.throwIfAborted(); const remaining = remainingDeadline(deadline); const pendingSpawn = runtime.port.spawn({ name: agentName, @@ -138,14 +154,14 @@ export async function runAgentRelayTask( timeoutMs: remaining, }); const spawnTimeout = AbortSignal.timeout(remaining); - const spawnSignal = options.signal === undefined - ? spawnTimeout : AbortSignal.any([options.signal, spawnTimeout]); + const spawnSignal = signal === undefined + ? spawnTimeout : AbortSignal.any([signal, spawnTimeout]); const spawn = await waitForSpawn(pendingSpawn, spawnSignal, outcome => { void settleLateSpawn(outcome).catch(() => undefined); }); claim = { ...claim, spawn }; await recordNodeSpawn(claimed.path, claim); - options.signal?.throwIfAborted(); + signal?.throwIfAborted(); } else { if (claim.cleanupPending !== undefined && claim.result === undefined && claim.release === undefined) { @@ -181,13 +197,13 @@ export async function runAgentRelayTask( const waitDeadline = claim.result === undefined ? deadline : deadline + RESULT_RECOVERY_WINDOW_MS; assertWithinDeadline(waitDeadline); const timeout = AbortSignal.timeout(remainingDeadline(waitDeadline)); - const signal = options.signal === undefined ? timeout : AbortSignal.any([options.signal, timeout]); + const resultSignal = signal === undefined ? timeout : AbortSignal.any([signal, timeout]); const delivery = await waitForNodeResult(runtime.inbox, { agentId: spawn.agentId, agentName: spawn.name, ...correlation, - }, signal); - signal.throwIfAborted(); + }, resultSignal); + resultSignal.throwIfAborted(); if (claim.result !== undefined && (claim.result.inboxItemId !== delivery.inboxItemId || claim.result.messageId !== delivery.messageId)) { throw new NodeSpawnError('The redelivered result did not match the durable delivery identity', @@ -204,7 +220,7 @@ export async function runAgentRelayTask( claim = cleanup.claim; if (cleanup.error !== undefined) throw cleanup.error; } - signal.throwIfAborted(); + resultSignal.throwIfAborted(); const envelope = delivery.envelope; const error = envelope.status === 'failed' ? envelope.error! : null; const status = error === null ? 'completed' : 'failed'; @@ -240,7 +256,7 @@ export async function runAgentRelayTask( let cleanupError: unknown; if ((claim.spawn !== undefined || claim.cleanup !== undefined) && claim.release === undefined && claim.failure === undefined) { - claim = { ...claim, failure: failureRecord(error, options.signal) }; + claim = { ...claim, failure: failureRecord(error, signal) }; try { await recordNodeSpawn(claimed.path, claim); } catch (stateError) { cleanupError = stateError; } } @@ -257,7 +273,7 @@ export async function runAgentRelayTask( claim = cleanup.claim; cleanupError ??= cleanup.error; } - options.signal?.throwIfAborted(); + signal?.throwIfAborted(); const detail = cleanupError === undefined ? safeError(error) : `${safeError(error)}; exact cleanup remains pending: ${safeError(cleanupError)}`; diff --git a/packages/sdk/src/node-spawn-result.ts b/packages/sdk/src/node-spawn-result.ts index 39a104427..9b68877fc 100644 --- a/packages/sdk/src/node-spawn-result.ts +++ b/packages/sdk/src/node-spawn-result.ts @@ -31,8 +31,6 @@ export interface NodeResultInboxClient { ack(input: { inboxItemId: string; state?: 'delivered' | 'read'; metadata?: Record }): Promise; fail(input: { inboxItemId: string; error: string; retry?: boolean; metadata?: Record }): Promise; }; - /** Closes the agent-scoped event transport after this subscription ends. */ - disconnect?(): Promise; } export interface NodeResultDelivery { @@ -105,13 +103,13 @@ export async function waitForNodeResult( throw new Error('Agent Relay result inbox ended before the spawned agent reported completion'); } finally { // A client that ignores cancellation may leave next() pending forever. - // Preserve prompt lease loss while still requesting resource cleanup. + // Preserve prompt lease loss while still closing only this subscription. + // The runtime owner closes the shared agent connection after every + // concurrent task has drained. if (signal?.aborted) { void close().catch(() => undefined); - void client.disconnect?.().catch(() => undefined); } else { - try { await close(); } - finally { await client.disconnect?.().catch(() => undefined); } + await close(); } } } diff --git a/packages/sdk/src/node-spawn-runtime.ts b/packages/sdk/src/node-spawn-runtime.ts index 754ea30ba..3fb0f88f1 100644 --- a/packages/sdk/src/node-spawn-runtime.ts +++ b/packages/sdk/src/node-spawn-runtime.ts @@ -13,6 +13,13 @@ export interface NodeSpawnRuntime { node?: string; repo?: string; sandboxOnly?: boolean; + /** Closes the publisher connection after every task using it has drained. */ + disconnect?(): Promise; +} + +export interface NodeSpawnRuntimeOwner { + get(signal?: AbortSignal): Promise; + close(): Promise; } interface RelayRuntimeClient extends RemoteNodeSpawnClient, NodeResultInboxClient { @@ -46,21 +53,72 @@ export async function createRemoteNodeSpawnRuntime( const client = new RelaycastMessagingClient({ baseUrl, agentToken }); const timeout = AbortSignal.timeout(PUBLISHER_DISCOVERY_TIMEOUT_MS); const signal = options.signal === undefined ? timeout : AbortSignal.any([options.signal, timeout]); - const publisher = await discoverPublisher(client, signal); - if (!publisher.id?.trim() || !publisher.name?.trim()) { - throw new NodeSpawnError('The node-spawn publisher token has no exact Relay identity', 'publisher_identity_missing'); + let publisher: NodeSpawnPublisher; + try { + publisher = await discoverPublisher(client, signal); + if (!publisher.id?.trim() || !publisher.name?.trim()) { + throw new NodeSpawnError('The node-spawn publisher token has no exact Relay identity', 'publisher_identity_missing'); + } + } catch (error) { + await client.events.disconnect().catch(() => undefined); + throw error; } return { port: new RemoteNodeSpawnPort(client), - inbox: { inbox: client.inbox, disconnect: () => client.events.disconnect() }, + inbox: { inbox: client.inbox }, publisher, baseUrl, + disconnect: () => client.events.disconnect(), ...(env.RELAYFLOW_NODE_NAME?.trim() ? { node: env.RELAYFLOW_NODE_NAME.trim() } : {}), ...(env.RELAYFLOW_NODE_REPO?.trim() ? { repo: env.RELAYFLOW_NODE_REPO.trim() } : {}), ...(sandbox === '1' ? { sandboxOnly: true } : {}), }; } +/** One lazy publisher connection shared by every relay task on a worker. */ +export function createNodeSpawnRuntimeOwner( + provided?: NodeSpawnRuntime, + factory: (signal: AbortSignal) => Promise = signal => + createRemoteNodeSpawnRuntime(process.env, { signal }), +): NodeSpawnRuntimeOwner { + const controller = new AbortController(); + let pending: Promise | undefined; + let closed = false; + const get = async (signal?: AbortSignal): Promise => { + signal?.throwIfAborted(); + if (closed) throw new NodeSpawnError('Node-spawn runtime owner is closed', 'task_cancelled'); + if (provided !== undefined) return provided; + if (pending === undefined) { + const created = factory(controller.signal); + pending = created; + void created.catch(() => { if (pending === created) pending = undefined; }); + } + return waitForRuntime(pending, signal); + }; + return { + get, + close: async () => { + if (closed) return; + closed = true; + controller.abort(new Error('Node-spawn runtime owner closed')); + if (provided !== undefined) return; + const runtime = await pending?.catch(() => undefined); + await runtime?.disconnect?.().catch(() => undefined); + }, + }; +} + +async function waitForRuntime(pending: Promise, signal?: AbortSignal): Promise { + if (signal === undefined) return pending; + let onAbort: (() => void) | undefined; + const aborted = new Promise((_resolve, reject) => { + onAbort = () => reject(signal.reason ?? new DOMException('The operation was aborted', 'AbortError')); + signal.addEventListener('abort', onAbort, { once: true }); + }); + try { return await Promise.race([pending, aborted]); } + finally { if (onAbort !== undefined) signal.removeEventListener('abort', onAbort); } +} + async function discoverPublisher(client: RelayRuntimeClient, signal: AbortSignal): Promise { let onAbort: (() => void) | undefined; try { @@ -71,7 +129,6 @@ async function discoverPublisher(client: RelayRuntimeClient, signal: AbortSignal }); return await Promise.race([client.agents.me(), aborted]); } catch (error) { - if (signal.aborted) void client.events.disconnect().catch(() => undefined); throw error; } finally { if (onAbort !== undefined) signal.removeEventListener('abort', onAbort); diff --git a/packages/sdk/src/worker-cli.ts b/packages/sdk/src/worker-cli.ts index 1793e8e54..8d745fcfd 100644 --- a/packages/sdk/src/worker-cli.ts +++ b/packages/sdk/src/worker-cli.ts @@ -125,6 +125,7 @@ export interface AgentRelayContext { dataDir?: string; resultSchema?: unknown; nodeSpawnRuntime?: NodeSpawnRuntime; + nodeSpawnRuntimeProvider?: (signal?: AbortSignal) => Promise; } export async function runAgentCli( @@ -399,7 +400,7 @@ async function runViaAgentRelay( cli: kind, task, model, worker_cwd, result_schema: context.resultSchema, runId: context.runId, stepId: context.stepId, idempotencyKey: context.idempotencyKey, dataDir: context.dataDir, - }, { signal, runtime: context.nodeSpawnRuntime }); + }, { signal, runtime: context.nodeSpawnRuntime, runtimeProvider: context.nodeSpawnRuntimeProvider }); const receipt = { ...received.receipt, error: received.receipt.error === null ? null : redactRelayError(received.receipt.error) }; const accounting = receipt.task_execution.accounting; diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index 1ca0ec618..318b4a939 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -22,7 +22,8 @@ import { redactText, type TranscriptDigest, } from './agent-transcript.js'; -import type { NodeSpawnRuntime } from './node-spawn-runtime.js'; +import { createNodeSpawnRuntimeOwner, type NodeSpawnRuntime, + type NodeSpawnRuntimeOwner } from './node-spawn-runtime.js'; export { MODEL_ENV, WAKE_CONTEXT_ENV } from './worker-cli.js'; @@ -67,12 +68,14 @@ export class AgentWorker extends EventEmitter { private attached = false; private closing = false; private readonly inFlight: Set> = new Set(); + private readonly nodeSpawnRuntimeOwner: NodeSpawnRuntimeOwner; constructor( private readonly client: JournalClient, private readonly options: AgentWorkerOptions, ) { super(); + this.nodeSpawnRuntimeOwner = createNodeSpawnRuntimeOwner(options.nodeSpawnRuntime); } async attach(): Promise { @@ -111,6 +114,7 @@ export class AgentWorker extends EventEmitter { if (pending.length > 0) { await Promise.allSettled(pending); } + await this.nodeSpawnRuntimeOwner.close(); this.attached = false; } @@ -164,7 +168,8 @@ export class AgentWorker extends EventEmitter { spec.transport === 'relay' ? 'relay' : 'direct', { runId: dispatch.run_id, stepId: dispatch.step_id, idempotencyKey: dispatch.idempotency_key, dataDir: this.options.dataDir, resultSchema: spec.verification?.json_schema, - nodeSpawnRuntime: this.options.nodeSpawnRuntime }, + nodeSpawnRuntime: this.options.nodeSpawnRuntime, + nodeSpawnRuntimeProvider: signal => this.nodeSpawnRuntimeOwner.get(signal) }, this.options.environment, spec.cli_identity, spec.timeout_ms, spec.resume, spec.secrets, this.client.supportsFeature?.('step_secret_scoping') ?? true) : Promise.resolve({ exit_code: null, stdout_tail: '', stderr_tail: 'agent step has no declared CLI' })); diff --git a/packages/sdk/tests/agent-relay-hardening.test.ts b/packages/sdk/tests/agent-relay-hardening.test.ts index 6389ffaf0..108349d05 100644 --- a/packages/sdk/tests/agent-relay-hardening.test.ts +++ b/packages/sdk/tests/agent-relay-hardening.test.ts @@ -47,12 +47,11 @@ describe('node-spawn result hardening', () => { expect(fail).toHaveBeenCalledWith(expect.objectContaining({ inboxItemId: 'delivery_bad', retry: false })); }); - it('closes a normal inbox subscription without acknowledging the delivery', async () => { + it('closes only the normal inbox subscription without acknowledging the delivery', async () => { const returned = vi.fn(async () => ({ done: true as const, value: undefined })); - const disconnect = vi.fn(async () => { throw new Error('disconnect failed'); }); let delivered = false; const ack = vi.fn(async () => ({})); - const delivery = await waitForNodeResult({ disconnect, inbox: { + const delivery = await waitForNodeResult({ inbox: { subscribe: () => ({ [Symbol.asyncIterator]: () => ({ next: async () => { if (delivered) return new Promise>(() => {}); @@ -68,7 +67,6 @@ describe('node-spawn result hardening', () => { ack, fail: async () => ({}), } }, { agentId: 'agent_1', agentName: 'worker', ...correlation }); expect(returned).toHaveBeenCalledOnce(); - expect(disconnect).toHaveBeenCalledOnce(); expect(ack).not.toHaveBeenCalled(); await delivery.ack(); expect(ack).toHaveBeenCalledOnce(); @@ -76,9 +74,8 @@ describe('node-spawn result hardening', () => { it('cancels an initial inbox read even when the client ignores its signal option', async () => { const returned = vi.fn(); - const disconnect = vi.fn(async () => {}); const controller = new AbortController(); - const waiting = waitForNodeResult({ disconnect, inbox: { + const waiting = waitForNodeResult({ inbox: { subscribe: () => ({ [Symbol.asyncIterator]: () => ({ next: () => new Promise>(() => {}), return: async () => { returned(); return { done: true, value: undefined }; }, @@ -88,8 +85,6 @@ describe('node-spawn result hardening', () => { controller.abort(new Error('lease lost')); await expect(waiting).rejects.toThrow('lease lost'); expect(returned).toHaveBeenCalledOnce(); - await new Promise(resolve => setImmediate(resolve)); - expect(disconnect).toHaveBeenCalledOnce(); }); it('does not wait for an uncooperative iterator close after cancellation', async () => { diff --git a/packages/sdk/tests/node-spawn-runtime.test.ts b/packages/sdk/tests/node-spawn-runtime.test.ts index 5a7694ca1..16a78d4be 100644 --- a/packages/sdk/tests/node-spawn-runtime.test.ts +++ b/packages/sdk/tests/node-spawn-runtime.test.ts @@ -21,7 +21,8 @@ vi.mock('@agent-relay/sdk', () => ({ }, })); -import { createRemoteNodeSpawnRuntime } from '../src/node-spawn-runtime.js'; +import { createNodeSpawnRuntimeOwner, createRemoteNodeSpawnRuntime, + type NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; import { runAgentRelayTask } from '../src/agent-relay-transport.js'; describe('remote node-spawn runtime', () => { @@ -53,10 +54,33 @@ describe('remote node-spawn runtime', () => { agentToken: 'agent-token', baseUrl: 'http://127.0.0.1:8787', }]); - await runtime.inbox.disconnect?.(); + await runtime.disconnect?.(); expect(relay.disconnect).toHaveBeenCalledOnce(); }); + it('shares one publisher runtime across concurrent tasks and closes it only with its owner', async () => { + let resolve!: (runtime: NodeSpawnRuntime) => void; + const disconnect = vi.fn(async () => { throw new Error('disconnect failed'); }); + const runtime = { port: {} as NodeSpawnRuntime['port'], inbox: {} as NodeSpawnRuntime['inbox'], + publisher: { id: 'publisher_1', name: 'publisher' }, baseUrl: 'https://cast.agentrelay.com', + disconnect } satisfies NodeSpawnRuntime; + const factory = vi.fn((_signal: AbortSignal) => new Promise(done => { resolve = done; })); + const owner = createNodeSpawnRuntimeOwner(undefined, factory); + const cancelled = new AbortController(); + const first = owner.get(cancelled.signal); + const second = owner.get(); + cancelled.abort(new Error('first task lost its lease')); + await expect(first).rejects.toThrow('first task lost its lease'); + expect(factory).toHaveBeenCalledOnce(); + expect(factory.mock.calls[0]![0].aborted).toBe(false); + resolve(runtime); + await expect(second).resolves.toBe(runtime); + await expect(owner.get()).resolves.toBe(runtime); + expect(disconnect).not.toHaveBeenCalled(); + await expect(owner.close()).resolves.toBeUndefined(); + expect(disconnect).toHaveBeenCalledOnce(); + }); + it('honors cancellation during publisher discovery and cleans up the client', async () => { relay.me.mockImplementation(() => new Promise(() => {})); const controller = new AbortController(); diff --git a/packages/sdk/tests/worker-node-runtime-sharing.test.ts b/packages/sdk/tests/worker-node-runtime-sharing.test.ts new file mode 100644 index 000000000..d1fe22335 --- /dev/null +++ b/packages/sdk/tests/worker-node-runtime-sharing.test.ts @@ -0,0 +1,109 @@ +import { EventEmitter } from 'node:events'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { NODE_RESULT_PREFIX } from '../src/node-spawn-result.js'; +import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; +import type { JournalClient } from '../src/journal-client.js'; +import type { StepDispatchEvent } from '../src/protocol.js'; + +const owner = vi.hoisted(() => ({ get: vi.fn(), close: vi.fn(async () => {}) })); +vi.mock('../src/node-spawn-runtime.js', async importOriginal => ({ + ...await importOriginal(), + createNodeSpawnRuntimeOwner: () => owner, +})); + +import { AgentWorker } from '../src/worker.js'; + +const roots: string[] = []; +afterEach(async () => { + owner.get.mockReset(); + owner.close.mockClear(); + await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); +}); + +describe('AgentWorker node-spawn runtime ownership', () => { + it('shares one publisher runtime across parallel relay steps and closes it after both journal writes', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-shared-node-runtime-')); + roots.push(dataDir); + const dispatches = [dispatch('edit-a', 'dispatch_aaaaaaaaaa'), dispatch('edit-b', 'dispatch_bbbbbbbbbb')]; + const spawned: Array<{ name: string; agentId: string; key: string }> = []; + let bothSpawned!: () => void; + const ready = new Promise(resolve => { bothSpawned = resolve; }); + const ack = vi.fn(async () => ({})); + const runtime: NodeSpawnRuntime = { + publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + port: { + targets: async () => [], + spawn: vi.fn(async input => { + const agentId = `agent_${spawned.length + 1}`; + spawned.push({ name: input.name, agentId, key: input.idempotencyKey }); + if (spawned.length === 2) bothSpawned(); + return { id: `spawn_${agentId}`, action: 'spawn', status: 'completed', + nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, agentId, + spawned: true, ready: true }; + }), + status: vi.fn(), + release: vi.fn(async input => ({ id: `release_${input.receipt.agentId}`, + action: 'release', status: 'completed', released: true })), + }, + inbox: { inbox: { + subscribe: async function* () { + await ready; + for (const item of spawned) { + const expected = dispatches.find(value => value.idempotency_key === item.key)!; + yield { id: `delivery_${item.agentId}`, message: { id: `message_${item.agentId}`, + from: { id: item.agentId, name: item.name }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: expected.run_id, + step_id: expected.step_id, dispatch_id: expected.idempotency_key, + status: 'completed', output: { step: expected.step_id }, + accounting: { tokens_input: 1, tokens_output: 1 } }) } }; + } + }, + ack, + fail: vi.fn(async () => ({})), + } }, + }; + owner.get.mockResolvedValue(runtime); + const order: string[] = []; + owner.close.mockImplementation(async () => { order.push('close'); }); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), + stepComplete: vi.fn(async () => { order.push('journal'); return {}; }), + supportsFeature: vi.fn(() => true), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_1', pins: {}, capacity: 2, dataDir, + }); + const errors: unknown[] = []; + worker.on('error', error => errors.push(error)); + await worker.attach(); + for (const value of dispatches) client.emit('step.dispatch', value); + await worker.close(); + + expect(errors).toEqual([]); + expect(owner.get).toHaveBeenCalledTimes(2); + expect(runtime.port.spawn).toHaveBeenCalledTimes(2); + expect(client.stepComplete).toHaveBeenCalledTimes(2); + expect(client.stepComplete.mock.calls.map(call => [call[1], call[4], call[5]?.output])) + .toEqual(expect.arrayContaining([ + ['edit-a', 'success', { step: 'edit-a' }], + ['edit-b', 'success', { step: 'edit-b' }], + ])); + expect(ack).toHaveBeenCalledTimes(2); + expect(owner.close).toHaveBeenCalledOnce(); + expect(order).toEqual(['journal', 'journal', 'close']); + }); +}); + +function dispatch(stepId: string, idempotencyKey: string): StepDispatchEvent { + return { + run_id: 'run_parallel', step_id: stepId, attempt: 1, step_type: 'agent', + spec: { cli: 'codex', model: 'codex-medium', instruction: `Implement ${stepId}`, transport: 'relay' }, + idempotency_key: idempotencyKey, lease_id: `lease_${stepId}`, + lease_deadline_ms: Date.now() + 30_000, pins: {}, + }; +} From fb6d2f98b1fcf2551b02cb49896cb81445fdb0a7 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sat, 7 Nov 2026 21:21:07 -0800 Subject: [PATCH 14/75] test(sdk): typecheck shared runtime regression --- docs/AGENT-RELAY-TRANSPORT.md | 7 ++-- .../tests/worker-node-runtime-sharing.test.ts | 37 ++++++++++++------- packages/sdk/tsconfig.tests.json | 1 + 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/docs/AGENT-RELAY-TRANSPORT.md b/docs/AGENT-RELAY-TRANSPORT.md index 0ecbdab20..05e5208a4 100644 --- a/docs/AGENT-RELAY-TRANSPORT.md +++ b/docs/AGENT-RELAY-TRANSPORT.md @@ -25,9 +25,10 @@ The remote path uses `@agent-relay/sdk >=17.2.1 <18`: 5. Only after `stepComplete` is durably accepted by the journal does Flows ack the inbox delivery. Relay is transport; the journal is execution authority. -One worker owns one lazy publisher connection across all of its concurrent -relay steps. Each step closes only its inbox iterator; worker shutdown drains -every step before disconnecting the shared publisher identity. +One worker uses one lazy publisher connection across all of its concurrent +relay steps. Each step closes only its inbox iterator. The default runtime is +disconnected after worker shutdown drains every step; cancellation teardown is +best-effort, and injected runtimes remain caller-owned. Spawn readiness is not task completion. A result without the exact sender and attempt correlation cannot complete a step. diff --git a/packages/sdk/tests/worker-node-runtime-sharing.test.ts b/packages/sdk/tests/worker-node-runtime-sharing.test.ts index d1fe22335..8abef899f 100644 --- a/packages/sdk/tests/worker-node-runtime-sharing.test.ts +++ b/packages/sdk/tests/worker-node-runtime-sharing.test.ts @@ -5,6 +5,7 @@ import { join } from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { NODE_RESULT_PREFIX } from '../src/node-spawn-result.js'; import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; +import type { NodeSpawnPort } from '../src/node-spawn.js'; import type { JournalClient } from '../src/journal-client.js'; import type { StepDispatchEvent } from '../src/protocol.js'; @@ -32,22 +33,26 @@ describe('AgentWorker node-spawn runtime ownership', () => { let bothSpawned!: () => void; const ready = new Promise(resolve => { bothSpawned = resolve; }); const ack = vi.fn(async () => ({})); + const spawn = vi.fn(async input => { + const agentId = `agent_${spawned.length + 1}`; + spawned.push({ name: input.name, agentId, key: input.idempotencyKey }); + if (spawned.length === 2) bothSpawned(); + return { id: `spawn_${agentId}`, action: 'spawn', status: 'completed', + nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, agentId, + spawned: true, ready: true }; + }); + const release = vi.fn(async input => ({ + id: `release_${input.receipt.agentId}`, action: 'release', status: 'completed', + spawned: false, ready: false, released: true, + })); const runtime: NodeSpawnRuntime = { publisher: { id: 'publisher_1', name: 'flows-publisher' }, baseUrl: 'https://cast.agentrelay.com', port: { targets: async () => [], - spawn: vi.fn(async input => { - const agentId = `agent_${spawned.length + 1}`; - spawned.push({ name: input.name, agentId, key: input.idempotencyKey }); - if (spawned.length === 2) bothSpawned(); - return { id: `spawn_${agentId}`, action: 'spawn', status: 'completed', - nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, agentId, - spawned: true, ready: true }; - }), + spawn, status: vi.fn(), - release: vi.fn(async input => ({ id: `release_${input.receipt.agentId}`, - action: 'release', status: 'completed', released: true })), + release, }, inbox: { inbox: { subscribe: async function* () { @@ -69,10 +74,14 @@ describe('AgentWorker node-spawn runtime ownership', () => { owner.get.mockResolvedValue(runtime); const order: string[] = []; owner.close.mockImplementation(async () => { order.push('close'); }); + const stepComplete = vi.fn(async () => { + order.push('journal'); + return {} as Awaited>; + }); const client = Object.assign(new EventEmitter(), { workerAttach: vi.fn(async () => ({})), stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), - stepComplete: vi.fn(async () => { order.push('journal'); return {}; }), + stepComplete, supportsFeature: vi.fn(() => true), }); const worker = new AgentWorker(client as unknown as JournalClient, { @@ -86,9 +95,9 @@ describe('AgentWorker node-spawn runtime ownership', () => { expect(errors).toEqual([]); expect(owner.get).toHaveBeenCalledTimes(2); - expect(runtime.port.spawn).toHaveBeenCalledTimes(2); - expect(client.stepComplete).toHaveBeenCalledTimes(2); - expect(client.stepComplete.mock.calls.map(call => [call[1], call[4], call[5]?.output])) + expect(spawn).toHaveBeenCalledTimes(2); + expect(stepComplete).toHaveBeenCalledTimes(2); + expect(stepComplete.mock.calls.map(call => [call[1], call[4], call[5]?.output])) .toEqual(expect.arrayContaining([ ['edit-a', 'success', { step: 'edit-a' }], ['edit-b', 'success', { step: 'edit-b' }], diff --git a/packages/sdk/tsconfig.tests.json b/packages/sdk/tsconfig.tests.json index 70f3e886a..58e7a151f 100644 --- a/packages/sdk/tsconfig.tests.json +++ b/packages/sdk/tsconfig.tests.json @@ -76,6 +76,7 @@ "tests/agent-relay-hardening.test.ts", "tests/node-spawn-ports.test.ts", "tests/node-spawn-runtime.test.ts", + "tests/worker-node-runtime-sharing.test.ts", "tests/webhook-hardening.test.ts", "tests/adapters/claude.test.ts", "tests/adapters/codex.test.ts", From 4bbd5670af44bd8f2b6da58cbae351b055d60042 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sat, 7 Nov 2026 21:24:05 -0800 Subject: [PATCH 15/75] fix(sdk): preserve results across iterator teardown --- docs/AGENT-RELAY-TRANSPORT.md | 3 ++- packages/sdk/src/node-spawn-result.ts | 4 +++- .../sdk/tests/agent-relay-hardening.test.ts | 19 ++++++++++++++++++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/docs/AGENT-RELAY-TRANSPORT.md b/docs/AGENT-RELAY-TRANSPORT.md index 05e5208a4..792867ba2 100644 --- a/docs/AGENT-RELAY-TRANSPORT.md +++ b/docs/AGENT-RELAY-TRANSPORT.md @@ -28,7 +28,8 @@ The remote path uses `@agent-relay/sdk >=17.2.1 <18`: One worker uses one lazy publisher connection across all of its concurrent relay steps. Each step closes only its inbox iterator. The default runtime is disconnected after worker shutdown drains every step; cancellation teardown is -best-effort, and injected runtimes remain caller-owned. +best-effort. Injected runtimes remain caller-owned: their caller must +disconnect them after every worker sharing that runtime has drained. Spawn readiness is not task completion. A result without the exact sender and attempt correlation cannot complete a step. diff --git a/packages/sdk/src/node-spawn-result.ts b/packages/sdk/src/node-spawn-result.ts index 9b68877fc..a78c9ceb8 100644 --- a/packages/sdk/src/node-spawn-result.ts +++ b/packages/sdk/src/node-spawn-result.ts @@ -109,7 +109,9 @@ export async function waitForNodeResult( if (signal?.aborted) { void close().catch(() => undefined); } else { - await close(); + // The outcome is already decided. Iterator teardown must not replace it + // or hold the worker lease forever when return() rejects or never settles. + void close().catch(() => undefined); } } } diff --git a/packages/sdk/tests/agent-relay-hardening.test.ts b/packages/sdk/tests/agent-relay-hardening.test.ts index 108349d05..888b20415 100644 --- a/packages/sdk/tests/agent-relay-hardening.test.ts +++ b/packages/sdk/tests/agent-relay-hardening.test.ts @@ -48,7 +48,7 @@ describe('node-spawn result hardening', () => { }); it('closes only the normal inbox subscription without acknowledging the delivery', async () => { - const returned = vi.fn(async () => ({ done: true as const, value: undefined })); + const returned = vi.fn(async () => { throw new Error('iterator close failed'); }); let delivered = false; const ack = vi.fn(async () => ({})); const delivery = await waitForNodeResult({ inbox: { @@ -72,6 +72,23 @@ describe('node-spawn result hardening', () => { expect(ack).toHaveBeenCalledOnce(); }); + it('does not wait for an uncooperative normal iterator close after receiving a result', async () => { + const returned = vi.fn(() => new Promise>(() => {})); + const delivery = await waitForNodeResult({ inbox: { + subscribe: () => ({ [Symbol.asyncIterator]: () => ({ + next: async () => ({ done: false as const, value: { id: 'delivery_1', message: { + id: 'message_1', from: { id: 'agent_1', name: 'worker' }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_1', step_id: 'step_1', + dispatch_id: correlation.dispatchId, status: 'completed', output: { ok: true } }), + } } }), + return: returned, + }) }), + ack: async () => ({}), fail: async () => ({}), + } }, { agentId: 'agent_1', agentName: 'worker', ...correlation }); + expect(delivery.envelope.output).toEqual({ ok: true }); + expect(returned).toHaveBeenCalledOnce(); + }); + it('cancels an initial inbox read even when the client ignores its signal option', async () => { const returned = vi.fn(); const controller = new AbortController(); From 87a1c99b1fc9393ebe4b0f33d4143f179ed0448e Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sat, 7 Nov 2026 21:45:52 -0800 Subject: [PATCH 16/75] test(sdk): prove node spawn through live journal --- packages/sdk/tests/node-spawn-live.test.ts | 289 +++++++++++++++++++++ packages/sdk/tsconfig.tests.json | 1 + 2 files changed, 290 insertions(+) create mode 100644 packages/sdk/tests/node-spawn-live.test.ts diff --git a/packages/sdk/tests/node-spawn-live.test.ts b/packages/sdk/tests/node-spawn-live.test.ts new file mode 100644 index 000000000..deaf769b8 --- /dev/null +++ b/packages/sdk/tests/node-spawn-live.test.ts @@ -0,0 +1,289 @@ +import { + accessSync, + constants, + existsSync, + lstatSync, + mkdtempSync, + readdirSync, + rmSync, + statSync, +} from 'node:fs'; +import { homedir, tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { spawn, type ChildProcess } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { afterEach, beforeAll, describe, expect, it } from 'vitest'; +import { compileYaml, toKernelSpec } from '../src/compile.js'; +import { socketPathFor } from '../src/daemon-connection.js'; +import { JournalClient } from '../src/journal-client.js'; +import { NODE_RESULT_PREFIX } from '../src/node-spawn-result.js'; +import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; +import type { NodeSpawnReceipt } from '../src/node-spawn.js'; +import { AgentWorker } from '../src/worker.js'; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..', '..'); +const TOOLCHAIN_TARGET = + process.env['CARGO_TARGET_DIR'] ?? + join( + process.env['RELAYFLOWS_TOOLCHAIN_HOME'] ?? + join(homedir(), '.relayflows-toolchain'), + 'target', + ); +const RELAYFLOWD = resolve(process.env['RELAYFLOWD_BIN'] ?? locateRelayflowd()); +const roots: string[] = []; +const daemons: ChildProcess[] = []; +const clients: JournalClient[] = []; + +beforeAll(() => accessSync(RELAYFLOWD, constants.X_OK)); + +afterEach(async () => { + for (const client of clients.splice(0)) client.close(); + for (const daemon of daemons.splice(0)) await stopDaemon(daemon); + for (const root of roots.splice(0)) + rmSync(root, { recursive: true, force: true }); +}); + +describe('node-spawn transport through the real journal', () => { + it('runs a relay agent step, releases exact identity, journals, then acknowledges', async () => { + const dataDir = mkdtempSync(join(tmpdir(), 'flows-node-spawn-live-')); + roots.push(dataDir); + await startDaemon(dataDir); + const client = await connectClient(dataDir); + await client.hello('node-spawn-live-test'); + + let runId = ''; + let allowSpawn!: () => void; + const runReady = new Promise((resolveReady) => { + allowSpawn = resolveReady; + }); + let resolveSpawn!: (receipt: NodeSpawnReceipt) => void; + const spawned = new Promise((resolveReceipt) => { + resolveSpawn = resolveReceipt; + }); + let dispatchId = ''; + const releases: Array<{ + receipt: NodeSpawnReceipt; + idempotencyKey: string; + }> = []; + const ackStates: string[] = []; + const runtime: NodeSpawnRuntime = { + baseUrl: 'https://cast.agentrelay.com', + publisher: { id: 'publisher_1', name: 'flows-publisher' }, + port: { + targets: async () => [], + spawn: async (input) => { + await runReady; + dispatchId = input.idempotencyKey; + const receipt: NodeSpawnReceipt = { + id: 'spawn_1', + action: 'spawn', + status: 'completed', + nodeId: 'node_1', + nodeName: 'desktop-1', + name: input.name, + agentId: 'agent_1', + spawned: true, + ready: true, + }; + resolveSpawn(receipt); + return receipt; + }, + status: async () => { + throw new Error('first-attempt run must not poll a prior spawn'); + }, + release: async (input) => { + releases.push({ + receipt: input.receipt as NodeSpawnReceipt, + idempotencyKey: input.idempotencyKey, + }); + return { + id: 'release_1', + action: 'release', + status: 'completed', + spawned: false, + ready: false, + released: true, + }; + }, + }, + inbox: { + inbox: { + subscribe: async function* () { + const receipt = await spawned; + yield { + id: 'delivery_1', + message: { + id: 'message_1', + from: { id: receipt.agentId, name: receipt.name }, + text: + NODE_RESULT_PREFIX + + JSON.stringify({ + version: 1, + run_id: runId, + step_id: 'edit', + dispatch_id: dispatchId, + status: 'completed', + output: { summary: 'node-spawn journal round trip' }, + accounting: { tokens_input: 3, tokens_output: 5 }, + }), + }, + }; + }, + ack: async () => { + const snapshot = await client.runGet(runId); + ackStates.push(snapshot.steps['edit']?.state ?? 'missing'); + return {}; + }, + fail: async () => { + throw new Error('valid result must not fail its delivery'); + }, + }, + }, + }; + const worker = new AgentWorker(client, { + workerId: 'node-spawn-live-worker', + pins: { + workspace: [{ surface: 'repo', revision_id: 'rev-a' }], + streams: [], + }, + dataDir, + nodeSpawnRuntime: runtime, + }); + const workerErrors: unknown[] = []; + worker.on('error', (error) => workerErrors.push(error)); + await worker.attach(); + + try { + const started = await client.runStart( + toKernelSpec( + compileYaml(` +version: '0.1.0' +steps: + - id: edit + type: agent + cli: codex + model: codex-medium + transport: relay + instruction: Perform the declared work. +`), + ), + ); + runId = started.run_id; + allowSpawn(); + const step = await waitForStep(client, runId, 'edit', 'done'); + const completed = (await client.journalRead(runId, 1, 100)).entries.find( + (entry) => + (entry as { entry_type?: string; step_id?: string }).entry_type === + 'step.completed' && + (entry as { step_id?: string }).step_id === 'edit', + ); + + expect(workerErrors).toEqual([]); + expect(step).toMatchObject({ state: 'done', type: 'agent' }); + expect(completed).toMatchObject({ + payload: { + completionReason: 'success', + output: { summary: 'node-spawn journal round trip' }, + }, + }); + expect(releases).toEqual([ + { + receipt: expect.objectContaining({ + id: 'spawn_1', + nodeId: 'node_1', + nodeName: 'desktop-1', + agentId: 'agent_1', + }), + idempotencyKey: expect.stringMatching(/^release_[a-f0-9]{32}$/), + }, + ]); + expect(ackStates).toEqual(['done']); + } finally { + allowSpawn(); + await worker.close(); + } + }, 20_000); +}); + +function locateRelayflowd(): string { + const direct = join(TOOLCHAIN_TARGET, 'debug', 'relayflowd'); + if (existsSync(direct)) return direct; + if (!existsSync(TOOLCHAIN_TARGET)) + return join(ROOT, 'kernel', 'target', 'debug', 'relayflowd'); + return ( + readdirSync(TOOLCHAIN_TARGET) + .map((entry) => join(TOOLCHAIN_TARGET, entry, 'debug', 'relayflowd')) + .filter((candidate) => existsSync(candidate)) + .sort((a, b) => statSync(b).mtimeMs - statSync(a).mtimeMs)[0] ?? + join(ROOT, 'kernel', 'target', 'debug', 'relayflowd') + ); +} + +async function startDaemon(dataDir: string): Promise { + const daemon = spawn(RELAYFLOWD, ['--data-dir', dataDir, 'serve'], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + daemons.push(daemon); + const stderr: Buffer[] = []; + daemon.stderr?.on('data', (chunk: Buffer) => stderr.push(chunk)); + const socket = socketPathFor(dataDir); + const deadline = Date.now() + 5_000; + while (Date.now() < deadline) { + if (existsSync(socket) && lstatSync(socket).isSocket()) { + const ready = new JournalClient(socket, { requestTimeoutMs: 1_000 }); + try { + await ready.connect(); + await ready.hello('node-spawn-live-readiness'); + return; + } catch { + /* Startup is still in progress. */ + } finally { + ready.close(); + } + } + if (daemon.exitCode !== null || daemon.signalCode !== null) { + throw new Error( + `relayflowd exited before readiness: ${Buffer.concat(stderr).toString('utf8')}`, + ); + } + await delay(20); + } + throw new Error(`relayflowd did not bind ${socket} within 5000ms`); +} + +async function connectClient(dataDir: string): Promise { + const client = new JournalClient(socketPathFor(dataDir), { + requestTimeoutMs: 5_000, + }); + clients.push(client); + await client.connect(); + return client; +} + +async function stopDaemon(daemon: ChildProcess): Promise { + if (daemon.exitCode !== null || daemon.signalCode !== null) return; + const exited = new Promise((resolveExit) => + daemon.once('exit', () => resolveExit()), + ); + daemon.kill('SIGTERM'); + await exited; +} + +async function waitForStep( + client: JournalClient, + runId: string, + stepId: string, + state: string, +) { + const deadline = Date.now() + 10_000; + while (Date.now() < deadline) { + const snapshot = await client.runGet(runId); + if (snapshot.steps[stepId]?.state === state) return snapshot.steps[stepId]; + await delay(20); + } + throw new Error(`step ${stepId} did not reach ${state} within 10000ms`); +} + +function delay(ms: number): Promise { + return new Promise((resolveDelay) => setTimeout(resolveDelay, ms)); +} diff --git a/packages/sdk/tsconfig.tests.json b/packages/sdk/tsconfig.tests.json index 58e7a151f..c231f7ca6 100644 --- a/packages/sdk/tsconfig.tests.json +++ b/packages/sdk/tsconfig.tests.json @@ -76,6 +76,7 @@ "tests/agent-relay-hardening.test.ts", "tests/node-spawn-ports.test.ts", "tests/node-spawn-runtime.test.ts", + "tests/node-spawn-live.test.ts", "tests/worker-node-runtime-sharing.test.ts", "tests/webhook-hardening.test.ts", "tests/adapters/claude.test.ts", From f7be47d0cd2e8d52b9d2f91b2b779056e9765c82 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sat, 7 Nov 2026 21:49:45 -0800 Subject: [PATCH 17/75] test(sdk): type exact cleanup identity --- packages/sdk/tests/node-spawn-live.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/sdk/tests/node-spawn-live.test.ts b/packages/sdk/tests/node-spawn-live.test.ts index deaf769b8..9060a68f0 100644 --- a/packages/sdk/tests/node-spawn-live.test.ts +++ b/packages/sdk/tests/node-spawn-live.test.ts @@ -18,7 +18,7 @@ import { socketPathFor } from '../src/daemon-connection.js'; import { JournalClient } from '../src/journal-client.js'; import { NODE_RESULT_PREFIX } from '../src/node-spawn-result.js'; import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; -import type { NodeSpawnReceipt } from '../src/node-spawn.js'; +import type { NodeSpawnIdentity, NodeSpawnReceipt } from '../src/node-spawn.js'; import { AgentWorker } from '../src/worker.js'; const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..', '..'); @@ -62,7 +62,7 @@ describe('node-spawn transport through the real journal', () => { }); let dispatchId = ''; const releases: Array<{ - receipt: NodeSpawnReceipt; + receipt: NodeSpawnIdentity; idempotencyKey: string; }> = []; const ackStates: string[] = []; @@ -93,7 +93,7 @@ describe('node-spawn transport through the real journal', () => { }, release: async (input) => { releases.push({ - receipt: input.receipt as NodeSpawnReceipt, + receipt: input.receipt, idempotencyKey: input.idempotencyKey, }); return { From e19dc92cadd9a92a1d932090a782079eb83c11f7 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sat, 7 Nov 2026 22:20:36 -0800 Subject: [PATCH 18/75] test(sdk): pin node runtime teardown order --- .../tests/worker-node-runtime-sharing.test.ts | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/packages/sdk/tests/worker-node-runtime-sharing.test.ts b/packages/sdk/tests/worker-node-runtime-sharing.test.ts index 8abef899f..684273212 100644 --- a/packages/sdk/tests/worker-node-runtime-sharing.test.ts +++ b/packages/sdk/tests/worker-node-runtime-sharing.test.ts @@ -30,9 +30,13 @@ describe('AgentWorker node-spawn runtime ownership', () => { roots.push(dataDir); const dispatches = [dispatch('edit-a', 'dispatch_aaaaaaaaaa'), dispatch('edit-b', 'dispatch_bbbbbbbbbb')]; const spawned: Array<{ name: string; agentId: string; key: string }> = []; + const order: string[] = []; let bothSpawned!: () => void; const ready = new Promise(resolve => { bothSpawned = resolve; }); - const ack = vi.fn(async () => ({})); + const ack = vi.fn(async ({ inboxItemId }: { inboxItemId: string }) => { + order.push(`ack:${inboxItemId}`); + return {}; + }); const spawn = vi.fn(async input => { const agentId = `agent_${spawned.length + 1}`; spawned.push({ name: input.name, agentId, key: input.idempotencyKey }); @@ -41,10 +45,11 @@ describe('AgentWorker node-spawn runtime ownership', () => { nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, agentId, spawned: true, ready: true }; }); - const release = vi.fn(async input => ({ - id: `release_${input.receipt.agentId}`, action: 'release', status: 'completed', - spawned: false, ready: false, released: true, - })); + const release = vi.fn(async input => { + order.push(`release:${input.receipt.agentId}`); + return { id: `release_${input.receipt.agentId}`, action: 'release', status: 'completed', + spawned: false, ready: false, released: true }; + }); const runtime: NodeSpawnRuntime = { publisher: { id: 'publisher_1', name: 'flows-publisher' }, baseUrl: 'https://cast.agentrelay.com', @@ -72,10 +77,9 @@ describe('AgentWorker node-spawn runtime ownership', () => { } }, }; owner.get.mockResolvedValue(runtime); - const order: string[] = []; owner.close.mockImplementation(async () => { order.push('close'); }); - const stepComplete = vi.fn(async () => { - order.push('journal'); + const stepComplete = vi.fn(async (_runId, stepId) => { + order.push(`journal:${stepId}`); return {} as Awaited>; }); const client = Object.assign(new EventEmitter(), { @@ -104,7 +108,18 @@ describe('AgentWorker node-spawn runtime ownership', () => { ])); expect(ack).toHaveBeenCalledTimes(2); expect(owner.close).toHaveBeenCalledOnce(); - expect(order).toEqual(['journal', 'journal', 'close']); + for (const item of spawned) { + const stepId = dispatches.find(value => value.idempotency_key === item.key)!.step_id; + const releaseIndex = order.indexOf(`release:${item.agentId}`); + const journalIndex = order.indexOf(`journal:${stepId}`); + const ackIndex = order.indexOf(`ack:delivery_${item.agentId}`); + const closeIndex = order.indexOf('close'); + expect([releaseIndex, journalIndex, ackIndex, closeIndex].every(index => index >= 0)).toBe(true); + expect(releaseIndex).toBeLessThan(journalIndex); + expect(journalIndex).toBeLessThan(ackIndex); + expect(ackIndex).toBeLessThan(closeIndex); + } + expect(order.at(-1)).toBe('close'); }); }); From caa33dc30c761d67929e5b3b2ea4343d2cb7b474 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sat, 7 Nov 2026 22:32:12 -0800 Subject: [PATCH 19/75] fix(sdk): contain synchronous inbox teardown errors --- packages/sdk/src/node-spawn-result.ts | 2 +- .../sdk/tests/agent-relay-hardening.test.ts | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/sdk/src/node-spawn-result.ts b/packages/sdk/src/node-spawn-result.ts index a78c9ceb8..48feb64e5 100644 --- a/packages/sdk/src/node-spawn-result.ts +++ b/packages/sdk/src/node-spawn-result.ts @@ -64,7 +64,7 @@ export async function waitForNodeResult( const iterator = client.inbox.subscribe({ signal })[Symbol.asyncIterator](); let closing: Promise | undefined; const close = (): Promise => { - closing ??= Promise.resolve(iterator.return?.()).then(() => undefined); + closing ??= Promise.resolve().then(() => iterator.return?.()).then(() => undefined); return closing; }; try { diff --git a/packages/sdk/tests/agent-relay-hardening.test.ts b/packages/sdk/tests/agent-relay-hardening.test.ts index 888b20415..7e212459a 100644 --- a/packages/sdk/tests/agent-relay-hardening.test.ts +++ b/packages/sdk/tests/agent-relay-hardening.test.ts @@ -72,6 +72,23 @@ describe('node-spawn result hardening', () => { expect(ack).toHaveBeenCalledOnce(); }); + it('preserves a received result when iterator close throws synchronously', async () => { + const returned = vi.fn(() => { throw new Error('synchronous iterator close failed'); }); + const delivery = await waitForNodeResult({ inbox: { + subscribe: () => ({ [Symbol.asyncIterator]: () => ({ + next: async () => ({ done: false as const, value: { id: 'delivery_1', message: { + id: 'message_1', from: { id: 'agent_1', name: 'worker' }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_1', step_id: 'step_1', + dispatch_id: correlation.dispatchId, status: 'completed', output: { ok: true } }), + } } }), + return: returned, + }) }), + ack: async () => ({}), fail: async () => ({}), + } }, { agentId: 'agent_1', agentName: 'worker', ...correlation }); + expect(delivery.envelope.output).toEqual({ ok: true }); + expect(returned).toHaveBeenCalledOnce(); + }); + it('does not wait for an uncooperative normal iterator close after receiving a result', async () => { const returned = vi.fn(() => new Promise>(() => {})); const delivery = await waitForNodeResult({ inbox: { From 03850e30e1d2069e3093b09496b4c54cd815fa5b Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Sun, 8 Nov 2026 02:30:13 -0800 Subject: [PATCH 20/75] fix(sdk): share concurrent worker shutdown --- packages/sdk/src/worker.ts | 12 ++++-- .../tests/worker-node-runtime-sharing.test.ts | 43 +++++++++++++++++-- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index 318b4a939..8590fdfce 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -67,6 +67,7 @@ export interface AgentWorkerOptions { export class AgentWorker extends EventEmitter { private attached = false; private closing = false; + private closePromise?: Promise; private readonly inFlight: Set> = new Set(); private readonly nodeSpawnRuntimeOwner: NodeSpawnRuntimeOwner; @@ -99,15 +100,20 @@ export class AgentWorker extends EventEmitter { /** * Async, drain-aware shutdown. Awaits every dispatch in-flight; ignores - * dispatches that arrive after close() begins. Idempotent. + * dispatches that arrive after close() begins. Concurrent and later callers + * receive the same promise, so none can observe shutdown before the drain. * * A prior synchronous close() did NOT drain, so a caller shutting * mid-dispatch could silently lose an in-flight step's stepComplete * journal write when the socket was immediately shut. This closes * that hole. */ - async close(): Promise { - if (this.closing) return; + close(): Promise { + this.closePromise ??= this.closeOnce(); + return this.closePromise; + } + + private async closeOnce(): Promise { this.closing = true; this.client.off('step.dispatch', this.onDispatch); const pending = Array.from(this.inFlight); diff --git a/packages/sdk/tests/worker-node-runtime-sharing.test.ts b/packages/sdk/tests/worker-node-runtime-sharing.test.ts index 684273212..dadba6ea2 100644 --- a/packages/sdk/tests/worker-node-runtime-sharing.test.ts +++ b/packages/sdk/tests/worker-node-runtime-sharing.test.ts @@ -25,7 +25,7 @@ afterEach(async () => { }); describe('AgentWorker node-spawn runtime ownership', () => { - it('shares one publisher runtime across parallel relay steps and closes it after both journal writes', async () => { + it('shares one runtime and one concurrent close promise through dispatch drain', async () => { const dataDir = await mkdtemp(join(tmpdir(), 'flows-shared-node-runtime-')); roots.push(dataDir); const dispatches = [dispatch('edit-a', 'dispatch_aaaaaaaaaa'), dispatch('edit-b', 'dispatch_bbbbbbbbbb')]; @@ -33,6 +33,14 @@ describe('AgentWorker node-spawn runtime ownership', () => { const order: string[] = []; let bothSpawned!: () => void; const ready = new Promise(resolve => { bothSpawned = resolve; }); + let bothJournalsStarted!: () => void; + const journalsStarted = new Promise(resolve => { bothJournalsStarted = resolve; }); + let allowJournals!: () => void; + const journalGate = new Promise(resolve => { allowJournals = resolve; }); + let ownerCloseStarted!: () => void; + const closeStarted = new Promise(resolve => { ownerCloseStarted = resolve; }); + let allowOwnerClose!: () => void; + const ownerCloseGate = new Promise(resolve => { allowOwnerClose = resolve; }); const ack = vi.fn(async ({ inboxItemId }: { inboxItemId: string }) => { order.push(`ack:${inboxItemId}`); return {}; @@ -77,9 +85,17 @@ describe('AgentWorker node-spawn runtime ownership', () => { } }, }; owner.get.mockResolvedValue(runtime); - owner.close.mockImplementation(async () => { order.push('close'); }); + owner.close.mockImplementation(async () => { + order.push('close'); + ownerCloseStarted(); + await ownerCloseGate; + }); + let journalCalls = 0; const stepComplete = vi.fn(async (_runId, stepId) => { order.push(`journal:${stepId}`); + journalCalls += 1; + if (journalCalls === dispatches.length) bothJournalsStarted(); + await journalGate; return {} as Awaited>; }); const client = Object.assign(new EventEmitter(), { @@ -95,7 +111,28 @@ describe('AgentWorker node-spawn runtime ownership', () => { worker.on('error', error => errors.push(error)); await worker.attach(); for (const value of dispatches) client.emit('step.dispatch', value); - await worker.close(); + const firstClose = worker.close(); + const secondClose = worker.close(); + expect(secondClose).toBe(firstClose); + let firstSettled = false; + let secondSettled = false; + void firstClose.then(() => { firstSettled = true; }); + void secondClose.then(() => { secondSettled = true; }); + + await journalsStarted; + await Promise.resolve(); + expect([firstSettled, secondSettled]).toEqual([false, false]); + expect(owner.close).not.toHaveBeenCalled(); + + allowJournals(); + await closeStarted; + await Promise.resolve(); + expect(owner.close).toHaveBeenCalledOnce(); + expect([firstSettled, secondSettled]).toEqual([false, false]); + + allowOwnerClose(); + await Promise.all([firstClose, secondClose]); + expect([firstSettled, secondSettled]).toEqual([true, true]); expect(errors).toEqual([]); expect(owner.get).toHaveBeenCalledTimes(2); From 2304c938f2a00a458f4592cb324dcaaf8841d346 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 8 Nov 2026 03:29:33 -0800 Subject: [PATCH 21/75] test(sdk): stress shared node runtime drain --- .../tests/worker-node-runtime-sharing.test.ts | 141 +++++++++++++++++- 1 file changed, 139 insertions(+), 2 deletions(-) diff --git a/packages/sdk/tests/worker-node-runtime-sharing.test.ts b/packages/sdk/tests/worker-node-runtime-sharing.test.ts index dadba6ea2..7ace31043 100644 --- a/packages/sdk/tests/worker-node-runtime-sharing.test.ts +++ b/packages/sdk/tests/worker-node-runtime-sharing.test.ts @@ -8,6 +8,7 @@ import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; import type { NodeSpawnPort } from '../src/node-spawn.js'; import type { JournalClient } from '../src/journal-client.js'; import type { StepDispatchEvent } from '../src/protocol.js'; +import { WorkerLeaseLostError } from '../src/worker-lease.js'; const owner = vi.hoisted(() => ({ get: vi.fn(), close: vi.fn(async () => {}) })); vi.mock('../src/node-spawn-runtime.js', async importOriginal => ({ @@ -158,13 +159,149 @@ describe('AgentWorker node-spawn runtime ownership', () => { } expect(order.at(-1)).toBe('close'); }); + + it('drains a six-task mixed-order fan with one lease cancellation before closing the shared runtime', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-shared-node-runtime-fan-')); + roots.push(dataDir); + const cancelledIndex = 2; + const dispatches = Array.from({ length: 6 }, (_, index) => dispatch( + `edit-${index}`, + `dispatch_${String(index).padStart(10, '0')}`, + index === cancelledIndex ? 300 : 30_000, + )); + const spawned: Array<{ name: string; agentId: string; key: string }> = []; + const order: string[] = []; + let allSpawned!: () => void; + const spawnedGate = new Promise(resolve => { allSpawned = resolve; }); + let allowResults!: () => void; + const resultsGate = new Promise(resolve => { allowResults = resolve; }); + const completionOrder = [5, 1, 4, 0, 3]; + const ack = vi.fn(async ({ inboxItemId }: { inboxItemId: string }) => { + order.push(`ack:${inboxItemId}`); + return {}; + }); + const spawn = vi.fn(async input => { + const agentId = `agent_${spawned.length}`; + spawned.push({ name: input.name, agentId, key: input.idempotencyKey }); + if (spawned.length === dispatches.length) allSpawned(); + return { id: `spawn_${agentId}`, action: 'spawn', status: 'completed', + nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, agentId, + spawned: true, ready: true }; + }); + const release = vi.fn(async input => { + order.push(`release:${input.receipt.agentId}`); + return { id: `release_${input.receipt.agentId}`, action: 'release', status: 'completed', + spawned: false, ready: false, released: true }; + }); + const runtime: NodeSpawnRuntime = { + publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + port: { targets: async () => [], spawn, status: vi.fn(), release }, + inbox: { inbox: { + subscribe: async function* () { + await spawnedGate; + await resultsGate; + for (const index of completionOrder) { + const expected = dispatches[index]!; + const item = spawned.find(value => value.key === expected.idempotency_key)!; + yield { id: `delivery_${item.agentId}`, message: { id: `message_${item.agentId}`, + from: { id: item.agentId, name: item.name }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: expected.run_id, + step_id: expected.step_id, dispatch_id: expected.idempotency_key, + status: 'completed', output: { step: expected.step_id }, + accounting: { tokens_input: 1, tokens_output: 1 } }) } }; + } + await new Promise(() => {}); + }, + ack, + fail: vi.fn(async () => ({})), + } }, + }; + owner.get.mockResolvedValue(runtime); + owner.close.mockImplementation(async () => { order.push('close'); }); + const heartbeatCalls = new Map(); + const stepHeartbeat = vi.fn(async (_runId, stepId) => { + const count = (heartbeatCalls.get(stepId) ?? 0) + 1; + heartbeatCalls.set(stepId, count); + if (stepId === dispatches[cancelledIndex]!.step_id && count > 1) { + throw new WorkerLeaseLostError('renewal_expired', 'review cancellation'); + } + const ttl = stepId === dispatches[cancelledIndex]!.step_id ? 300 : 30_000; + return { lease_deadline_ms: Date.now() + ttl, lease_ttl_ms: ttl }; + }); + const stepComplete = vi.fn(async (_runId, stepId) => { + order.push(`journal:${stepId}`); + return {} as Awaited>; + }); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat, + stepComplete, + supportsFeature: vi.fn(() => true), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_fan', pins: {}, capacity: dispatches.length, dataDir, + }); + const errors: Array<{ error: unknown; stepId: string }> = []; + worker.on('error', (error, value: StepDispatchEvent) => { + errors.push({ error, stepId: value.step_id }); + order.push(`error:${value.step_id}`); + }); + await worker.attach(); + for (const value of dispatches) client.emit('step.dispatch', value); + await spawnedGate; + + const firstClose = worker.close(); + const secondClose = worker.close(); + expect(secondClose).toBe(firstClose); + let closed = false; + void firstClose.then(() => { closed = true; }); + await Promise.resolve(); + expect(closed).toBe(false); + expect(owner.close).not.toHaveBeenCalled(); + + allowResults(); + await firstClose; + + expect(errors).toHaveLength(1); + expect(errors[0]).toMatchObject({ + error: { name: 'WorkerLeaseLostError', reason: 'renewal_expired' }, + stepId: dispatches[cancelledIndex]!.step_id, + }); + expect(owner.get).toHaveBeenCalledTimes(dispatches.length); + expect(spawn).toHaveBeenCalledTimes(dispatches.length); + expect(release).toHaveBeenCalledTimes(dispatches.length); + expect(stepComplete).toHaveBeenCalledTimes(dispatches.length - 1); + expect(ack).toHaveBeenCalledTimes(dispatches.length - 1); + expect(owner.close).toHaveBeenCalledOnce(); + for (const index of completionOrder) { + const expected = dispatches[index]!; + const item = spawned.find(value => value.key === expected.idempotency_key)!; + const stepId = expected.step_id; + const releaseIndex = order.indexOf(`release:${item.agentId}`); + const journalIndex = order.indexOf(`journal:${stepId}`); + const ackIndex = order.indexOf(`ack:delivery_${item.agentId}`); + const closeIndex = order.indexOf('close'); + expect(releaseIndex).toBeLessThan(journalIndex); + expect(journalIndex).toBeLessThan(ackIndex); + expect(ackIndex).toBeLessThan(closeIndex); + } + const cancelled = spawned.find(value => value.key === dispatches[cancelledIndex]!.idempotency_key)!; + expect(order.indexOf(`release:${cancelled.agentId}`)) + .toBeLessThan(order.indexOf(`error:${dispatches[cancelledIndex]!.step_id}`)); + expect(order.indexOf(`error:${dispatches[cancelledIndex]!.step_id}`)) + .toBeLessThan(order.indexOf('close')); + expect(order.at(-1)).toBe('close'); + }); }); -function dispatch(stepId: string, idempotencyKey: string): StepDispatchEvent { +function dispatch(stepId: string, idempotencyKey: string, leaseTtlMs?: number): StepDispatchEvent { return { run_id: 'run_parallel', step_id: stepId, attempt: 1, step_type: 'agent', spec: { cli: 'codex', model: 'codex-medium', instruction: `Implement ${stepId}`, transport: 'relay' }, idempotency_key: idempotencyKey, lease_id: `lease_${stepId}`, - lease_deadline_ms: Date.now() + 30_000, pins: {}, + lease_deadline_ms: Date.now() + (leaseTtlMs ?? 30_000), + ...(leaseTtlMs === undefined ? {} : { lease_ttl_ms: leaseTtlMs }), + pins: {}, }; } From 29fe24b8913b2f64da55dd6c96fe4bf1ab7f55a3 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 8 Nov 2026 13:02:23 -0800 Subject: [PATCH 22/75] fix(sdk): share node runtime owner shutdown --- packages/sdk/src/node-spawn-runtime.ts | 18 ++++++------- packages/sdk/tests/node-spawn-runtime.test.ts | 25 +++++++++++++++++++ 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/packages/sdk/src/node-spawn-runtime.ts b/packages/sdk/src/node-spawn-runtime.ts index 3fb0f88f1..2780baac8 100644 --- a/packages/sdk/src/node-spawn-runtime.ts +++ b/packages/sdk/src/node-spawn-runtime.ts @@ -84,6 +84,7 @@ export function createNodeSpawnRuntimeOwner( const controller = new AbortController(); let pending: Promise | undefined; let closed = false; + let closePromise: Promise | undefined; const get = async (signal?: AbortSignal): Promise => { signal?.throwIfAborted(); if (closed) throw new NodeSpawnError('Node-spawn runtime owner is closed', 'task_cancelled'); @@ -95,17 +96,14 @@ export function createNodeSpawnRuntimeOwner( } return waitForRuntime(pending, signal); }; - return { - get, - close: async () => { - if (closed) return; - closed = true; - controller.abort(new Error('Node-spawn runtime owner closed')); - if (provided !== undefined) return; - const runtime = await pending?.catch(() => undefined); - await runtime?.disconnect?.().catch(() => undefined); - }, + const closeOnce = async (): Promise => { + closed = true; + controller.abort(new Error('Node-spawn runtime owner closed')); + if (provided !== undefined) return; + const runtime = await pending?.catch(() => undefined); + await runtime?.disconnect?.().catch(() => undefined); }; + return { get, close: () => closePromise ??= closeOnce() }; } async function waitForRuntime(pending: Promise, signal?: AbortSignal): Promise { diff --git a/packages/sdk/tests/node-spawn-runtime.test.ts b/packages/sdk/tests/node-spawn-runtime.test.ts index 16a78d4be..3a8308b51 100644 --- a/packages/sdk/tests/node-spawn-runtime.test.ts +++ b/packages/sdk/tests/node-spawn-runtime.test.ts @@ -81,6 +81,31 @@ describe('remote node-spawn runtime', () => { expect(disconnect).toHaveBeenCalledOnce(); }); + it('shares one pending close across concurrent owner shutdown callers', async () => { + let releaseDisconnect!: () => void; + const disconnect = vi.fn(() => new Promise(resolve => { releaseDisconnect = resolve; })); + const runtime = { port: {} as NodeSpawnRuntime['port'], inbox: {} as NodeSpawnRuntime['inbox'], + publisher: { id: 'publisher_1', name: 'publisher' }, baseUrl: 'https://cast.agentrelay.com', + disconnect } satisfies NodeSpawnRuntime; + const owner = createNodeSpawnRuntimeOwner(undefined, async () => runtime); + await expect(owner.get()).resolves.toBe(runtime); + + let firstSettled = false; + let secondSettled = false; + const first = owner.close(); + const second = owner.close(); + void first.finally(() => { firstSettled = true; }); + void second.finally(() => { secondSettled = true; }); + + expect(first).toBe(second); + await vi.waitFor(() => expect(disconnect).toHaveBeenCalledOnce()); + expect(firstSettled).toBe(false); + expect(secondSettled).toBe(false); + + releaseDisconnect(); + await expect(Promise.all([first, second])).resolves.toEqual([undefined, undefined]); + }); + it('honors cancellation during publisher discovery and cleans up the client', async () => { relay.me.mockImplementation(() => new Promise(() => {})); const controller = new AbortController(); From b6ad369c691d82bbfac3a6904f963eda6c5bb757 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 8 Nov 2026 20:27:03 -0800 Subject: [PATCH 23/75] fix(sdk): harden relay node task recovery --- docs/AGENT-RELAY-TRANSPORT.md | 17 +- packages/sdk/src/agent-relay-transport.ts | 217 ++++++---- packages/sdk/src/node-spawn-deadline.ts | 43 ++ packages/sdk/src/node-spawn-remote.ts | 79 +++- packages/sdk/src/node-spawn-result.ts | 5 + packages/sdk/src/node-spawn-runtime.ts | 5 +- packages/sdk/src/node-spawn-settlement.ts | 172 ++++++++ packages/sdk/src/node-spawn-state.ts | 100 ++++- packages/sdk/src/node-spawn.ts | 14 + packages/sdk/src/worker-cli.ts | 8 +- packages/sdk/src/worker.ts | 97 ++++- .../sdk/tests/agent-relay-transport.test.ts | 146 ++++++- packages/sdk/tests/node-spawn-ports.test.ts | 22 + packages/sdk/tests/node-spawn-runtime.test.ts | 16 +- .../sdk/tests/node-spawn-settlement.test.ts | 397 ++++++++++++++++++ .../tests/worker-node-runtime-sharing.test.ts | 61 +++ 16 files changed, 1275 insertions(+), 124 deletions(-) create mode 100644 packages/sdk/src/node-spawn-deadline.ts create mode 100644 packages/sdk/src/node-spawn-settlement.ts create mode 100644 packages/sdk/tests/node-spawn-settlement.test.ts diff --git a/docs/AGENT-RELAY-TRANSPORT.md b/docs/AGENT-RELAY-TRANSPORT.md index 792867ba2..cfd621704 100644 --- a/docs/AGENT-RELAY-TRANSPORT.md +++ b/docs/AGENT-RELAY-TRANSPORT.md @@ -25,6 +25,13 @@ The remote path uses `@agent-relay/sdk >=17.2.1 <18`: 5. Only after `stepComplete` is durably accepted by the journal does Flows ack the inbox delivery. Relay is transport; the journal is execution authority. +Before the journal append, Flows records a settlement intent containing the +kernel attempt and expected completion reason. The journal's terminal fact +must contain that reason and the exact Relay delivery and message ids before a +restart can acknowledge the inbox item. Inbox acknowledgement and rejection +have independent deadlines; an unconfirmed operation remains in the durable +outbox and is retried on worker attach, after dispatch drain, and on shutdown. + One worker uses one lazy publisher connection across all of its concurrent relay steps. Each step closes only its inbox iterator. The default runtime is disconnected after worker shutdown drains every step; cancellation teardown is @@ -50,14 +57,16 @@ It is never answered by a blind second spawn. The stable worker name provides additional collision protection at the node. The claim also fixes the task's original start time. Resume never grants a new -deadline. Once a correlated result was already durably observed, a restart has -at most 30 seconds beyond that original deadline to recover the same unacked -delivery; it cannot use that window to spawn or extend remote execution. +execution deadline. Once a correlated result was durably observed before that +deadline, the validated envelope is part of the claim and can be replayed to +the journal after any restart delay. Recovery cannot use that fact to spawn or +extend remote execution. Lease loss or timeout aborts the inbox wait and attempts exact release. A post-spawn verification failure carries any exact identity already proven into the same cleanup path. Spawn and release action inputs carry their stable -idempotency keys. A failed cleanup is recorded as pending in the durable claim, +idempotency keys. Every release attempt has an independent deadline. A failed +or unconfirmed cleanup is recorded as pending in the durable claim, the attempt remains incomplete for kernel redispatch, and the result remains unacked. Kernel redelivery therefore reopens the same claim and retries exact release before it can accept a result or issue another spawn. A result delivery diff --git a/packages/sdk/src/agent-relay-transport.ts b/packages/sdk/src/agent-relay-transport.ts index de0ce5bea..dc9e7f714 100644 --- a/packages/sdk/src/agent-relay-transport.ts +++ b/packages/sdk/src/agent-relay-transport.ts @@ -1,8 +1,11 @@ import { createHash } from 'node:crypto'; import { claimNodeSpawn, recordNodeSpawn, type NodeSpawnClaim } from './node-spawn-state.js'; import { nodeResultInstruction, waitForNodeResult } from './node-spawn-result.js'; +import { durableNodeResultDelivery } from './node-spawn-settlement.js'; +import { waitForNodeOperation, waitForNodeSpawn } from './node-spawn-deadline.js'; import { createRemoteNodeSpawnRuntime, type NodeSpawnRuntime } from './node-spawn-runtime.js'; import { NodeSpawnError, nonempty, validIdempotencyKey, + validNodeSpawnIdentity, validNodeSpawnReceipt, type NodeSpawnIdentity, type NodeSpawnReceipt } from './node-spawn.js'; import type { RelayTaskResult } from './agent-relay-receipt.js'; @@ -16,6 +19,7 @@ export interface AgentRelayTaskRequest { runId: string; stepId: string; idempotencyKey: string; + attempt: number; dataDir: string; /** Engine task contract ceiling; bounded independently of the worker lease. */ timeoutMs?: number; @@ -39,7 +43,7 @@ export class AgentRelayTransportError extends Error { } } -const RESULT_RECOVERY_WINDOW_MS = 30_000; +const CLEANUP_ATTEMPT_TIMEOUT_MS = 15_000; export function readAgentRelayEnv(env: NodeJS.ProcessEnv = process.env): Required> { const agentToken = env.RELAY_AGENT_TOKEN?.trim(); @@ -62,19 +66,45 @@ export async function runAgentRelayTask( options.signal?.throwIfAborted(); const timeoutMs = request.timeoutMs ?? 86_400_000; if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 86_400_000 + || !Number.isSafeInteger(request.attempt) || request.attempt < 1 || ![request.runId, request.stepId, request.dataDir, request.cli].every(nonempty) || !validIdempotencyKey(request.idempotencyKey)) { throw new AgentRelayTransportError('Node-spawn task requires durable dispatch identity and a valid deadline'); } + const taskDeadline = taskStartedAt + timeoutMs; + let acquisitionRemaining: number; + try { acquisitionRemaining = remainingDeadline(taskDeadline); } + catch (error) { + throw new AgentRelayTransportError('The node-spawn task deadline elapsed', error); + } const ownsRuntime = options.runtime === undefined && options.runtimeProvider === undefined; - const runtime = options.runtime ?? await options.runtimeProvider?.(options.signal) ?? await createRemoteNodeSpawnRuntime({ - ...process.env, - ...(options.baseUrl === undefined ? {} : { RELAY_BASE_URL: options.baseUrl }), - ...(options.agentToken === undefined ? {} : { RELAY_AGENT_TOKEN: options.agentToken }), - ...(options.node === undefined ? {} : { RELAYFLOW_NODE_NAME: options.node }), - ...(options.repo === undefined ? {} : { RELAYFLOW_NODE_REPO: options.repo }), - ...(options.sandboxOnly === undefined ? {} : { RELAYFLOW_NODE_SANDBOX_ONLY: options.sandboxOnly ? '1' : '0' }), - }, { signal: options.signal }); + const acquisitionTimeout = AbortSignal.timeout(acquisitionRemaining); + const acquisitionSignal = options.signal === undefined + ? acquisitionTimeout : AbortSignal.any([options.signal, acquisitionTimeout]); + let runtime: NodeSpawnRuntime; + try { + if (options.runtime !== undefined) { + runtime = options.runtime; + } else if (options.runtimeProvider !== undefined) { + runtime = await waitForNodeOperation(options.runtimeProvider(acquisitionSignal), acquisitionSignal); + } else { + runtime = await createRemoteNodeSpawnRuntime({ + ...process.env, + ...(options.baseUrl === undefined ? {} : { RELAY_BASE_URL: options.baseUrl }), + ...(options.agentToken === undefined ? {} : { RELAY_AGENT_TOKEN: options.agentToken }), + ...(options.node === undefined ? {} : { RELAYFLOW_NODE_NAME: options.node }), + ...(options.repo === undefined ? {} : { RELAYFLOW_NODE_REPO: options.repo }), + ...(options.sandboxOnly === undefined + ? {} : { RELAYFLOW_NODE_SANDBOX_ONLY: options.sandboxOnly ? '1' : '0' }), + }, { signal: acquisitionSignal }); + } + } catch (error) { + if (options.signal?.aborted !== true && acquisitionTimeout.aborted) { + const timeout = new NodeSpawnError('The node-spawn task deadline elapsed', 'task_timeout', error); + throw new AgentRelayTransportError(timeout.message, timeout); + } + throw error; + } try { return await executeAgentRelayTask(request, runtime, options.signal, taskStartedAt, timeoutMs); } finally { @@ -106,6 +136,7 @@ async function executeAgentRelayTask( stepId: request.stepId, idempotencyKey: request.idempotencyKey, agentName, + timeoutMs, request: { cli: request.cli, task, ...(request.model === undefined ? {} : { model: request.model }), ...(request.worker_cwd === undefined ? {} : { worker_cwd: request.worker_cwd }), @@ -118,7 +149,7 @@ async function executeAgentRelayTask( const claimed = await claimNodeSpawn(request.dataDir, proposed); let claim = claimed.claim; let releaseAttempted = false; - const deadline = claim.startedAt + timeoutMs; + const deadline = claim.startedAt + claim.timeoutMs; try { if (!claimed.created && claim.spawn === undefined) { if (claim.cleanup === undefined) { @@ -134,7 +165,7 @@ async function executeAgentRelayTask( throw new NodeSpawnError(claim.failure?.message ?? 'The prior node spawn failed after launch', claim.failure?.code ?? 'spawn_unconfirmed'); } - assertWithinDeadline(claim.result === undefined ? deadline : deadline + RESULT_RECOVERY_WINDOW_MS); + if (claim.result === undefined) assertWithinDeadline(deadline); signal?.throwIfAborted(); if (claim.spawn === undefined) { // This is deliberately adjacent to the irreversible dispatch. Runtime @@ -156,34 +187,48 @@ async function executeAgentRelayTask( const spawnTimeout = AbortSignal.timeout(remaining); const spawnSignal = signal === undefined ? spawnTimeout : AbortSignal.any([signal, spawnTimeout]); - const spawn = await waitForSpawn(pendingSpawn, spawnSignal, outcome => { + const spawn = await waitForNodeSpawn(pendingSpawn, spawnSignal, outcome => { void settleLateSpawn(outcome).catch(() => undefined); }); + if (!validNodeSpawnReceipt(spawn) || spawn.name !== agentName) { + throw new NodeSpawnError('The node spawn did not return a valid ready identity', + 'spawn_identity_mismatch', undefined, + validNodeSpawnIdentity(spawn) ? spawn : undefined); + } claim = { ...claim, spawn }; await recordNodeSpawn(claimed.path, claim); signal?.throwIfAborted(); } else { - if (claim.cleanupPending !== undefined && claim.result === undefined - && claim.release === undefined) { - releaseAttempted = true; - const cleanup = await tryRelease(runtime, claimed.path, claim, request.idempotencyKey); - claim = cleanup.claim; - if (cleanup.error !== undefined) throw cleanup.error; - throw new NodeSpawnError( - claim.failure?.message ?? 'The prior node spawn failed after launch', - claim.failure?.code ?? 'spawn_unconfirmed', - ); - } - if (claim.release !== undefined && claim.result === undefined) { - throw new NodeSpawnError( - claim.failure?.message ?? 'The recorded node spawn was released before it reported a result', - claim.failure?.code ?? 'spawn_released', - ); - } - const status = await runtime.port.status('spawn', claim.spawn.id); - if (status.status !== 'completed') { - throw new NodeSpawnError('The recorded node spawn is no longer completed', - status.errorCode ?? `spawn_${status.status}`); + // A durable result is the authoritative completion record. Requiring a + // live placement lookup here would make crash recovery depend on a + // remote invocation record that may expire after exact cleanup. + if (claim.result === undefined) { + if ((claim.cleanupPending !== undefined || claim.failure !== undefined) + && claim.release === undefined) { + releaseAttempted = true; + const cleanup = await tryRelease(runtime, claimed.path, claim, request.idempotencyKey); + claim = cleanup.claim; + if (cleanup.error !== undefined) throw cleanup.error; + throw new NodeSpawnError( + claim.failure?.message ?? 'The prior node spawn failed after launch', + claim.failure?.code ?? 'spawn_unconfirmed', + ); + } + if (claim.release !== undefined) { + throw new NodeSpawnError( + claim.failure?.message ?? 'The recorded node spawn was released before it reported a result', + claim.failure?.code ?? 'spawn_released', + ); + } + const remaining = remainingDeadline(deadline); + const statusTimeout = AbortSignal.timeout(remaining); + const statusSignal = signal === undefined + ? statusTimeout : AbortSignal.any([signal, statusTimeout]); + const status = await waitForNodeOperation(runtime.port.status('spawn', claim.spawn.id), statusSignal); + if (status.status !== 'completed') { + throw new NodeSpawnError('The recorded node spawn is no longer completed', + status.errorCode ?? `spawn_${status.status}`); + } } } const spawn = claim.spawn; @@ -194,24 +239,31 @@ async function executeAgentRelayTask( claim = cleanup.claim; if (cleanup.error !== undefined) throw cleanup.error; } - const waitDeadline = claim.result === undefined ? deadline : deadline + RESULT_RECOVERY_WINDOW_MS; - assertWithinDeadline(waitDeadline); - const timeout = AbortSignal.timeout(remainingDeadline(waitDeadline)); - const resultSignal = signal === undefined ? timeout : AbortSignal.any([signal, timeout]); - const delivery = await waitForNodeResult(runtime.inbox, { - agentId: spawn.agentId, - agentName: spawn.name, - ...correlation, - }, resultSignal); - resultSignal.throwIfAborted(); + const resultSignal = claim.result === undefined + ? (signal === undefined + ? AbortSignal.timeout(remainingDeadline(deadline)) + : AbortSignal.any([signal, AbortSignal.timeout(remainingDeadline(deadline))])) + : signal; + const observed = claim.result === undefined + ? await waitForNodeResult(runtime.inbox, { + agentId: spawn.agentId, + agentName: spawn.name, + ...correlation, + }, resultSignal) + : undefined; + resultSignal?.throwIfAborted(); if (claim.result !== undefined - && (claim.result.inboxItemId !== delivery.inboxItemId || claim.result.messageId !== delivery.messageId)) { + && observed !== undefined + && (claim.result.inboxItemId !== observed.inboxItemId || claim.result.messageId !== observed.messageId)) { throw new NodeSpawnError('The redelivered result did not match the durable delivery identity', 'result_identity_mismatch'); } if (claim.result === undefined) { - claim = { ...claim, result: { inboxItemId: delivery.inboxItemId, - messageId: delivery.messageId, status: delivery.envelope.status, receivedAt: Date.now() } }; + if (observed === undefined) throw new NodeSpawnError('Node result disappeared before it became durable'); + claim = { ...claim, result: { inboxItemId: observed.inboxItemId, + messageId: observed.messageId, status: observed.envelope.status, + envelope: observed.envelope, settlementAction: observed.settlementAction, + receivedAt: Date.now() } }; await recordNodeSpawn(claimed.path, claim); } if (claim.release === undefined) { @@ -220,7 +272,11 @@ async function executeAgentRelayTask( claim = cleanup.claim; if (cleanup.error !== undefined) throw cleanup.error; } - resultSignal.throwIfAborted(); + // Construct the settlement wrapper from the post-release claim. Otherwise + // preparing its journal intent could rewrite the file from a stale copy + // and silently discard the exact release receipt. + const delivery = durableNodeResultDelivery(runtime, claimed.path, claim); + resultSignal?.throwIfAborted(); const envelope = delivery.envelope; const error = envelope.status === 'failed' ? envelope.error! : null; const status = error === null ? 'completed' : 'failed'; @@ -237,7 +293,7 @@ async function executeAgentRelayTask( run_id: request.runId, step_id: request.stepId, dispatch_id: request.idempotencyKey, - deadline: new Date(claim.startedAt + timeoutMs).toISOString(), + deadline: new Date(deadline).toISOString(), node_id: spawn.nodeId, node_name: spawn.nodeName, agent_id: spawn.agentId, @@ -260,7 +316,8 @@ async function executeAgentRelayTask( try { await recordNodeSpawn(claimed.path, claim); } catch (stateError) { cleanupError = stateError; } } - if (error instanceof NodeSpawnError && error.cleanup !== undefined && claim.spawn === undefined) { + if (error instanceof NodeSpawnError && validNodeSpawnIdentity(error.cleanup) + && claim.spawn === undefined) { claim = { ...claim, cleanup: error.cleanup, failure: { code: error.code, message: error.message } }; try { await recordNodeSpawn(claimed.path, claim); } @@ -285,11 +342,17 @@ async function executeAgentRelayTask( async function settleLateSpawn(outcome: { spawn?: NodeSpawnReceipt; error?: unknown }): Promise { if (outcome.spawn !== undefined) { - claim = { ...claim, spawn: outcome.spawn, failure: { + const identity = validNodeSpawnIdentity(outcome.spawn) ? outcome.spawn : undefined; + if (identity === undefined) return; + claim = { ...claim, + ...(validNodeSpawnReceipt(outcome.spawn) && outcome.spawn.name === agentName + ? { spawn: outcome.spawn } : { cleanup: identity }), + failure: { code: 'spawn_cancelled', message: 'The node spawn confirmed after its caller stopped waiting', } }; - } else if (outcome.error instanceof NodeSpawnError && outcome.error.cleanup !== undefined) { + } else if (outcome.error instanceof NodeSpawnError + && validNodeSpawnIdentity(outcome.error.cleanup)) { claim = { ...claim, cleanup: outcome.error.cleanup, failure: { code: outcome.error.code, message: outcome.error.message } }; } else { @@ -302,28 +365,6 @@ async function executeAgentRelayTask( } } -async function waitForSpawn( - pending: Promise, - signal: AbortSignal, - onLate: (outcome: { spawn?: NodeSpawnReceipt; error?: unknown }) => void, -): Promise { - signal.throwIfAborted(); - let onAbort: (() => void) | undefined; - const aborted = new Promise((_resolve, reject) => { - onAbort = () => reject(signal.reason ?? new DOMException('The operation was aborted', 'AbortError')); - signal.addEventListener('abort', onAbort, { once: true }); - }); - try { return await Promise.race([pending, aborted]); } - catch (error) { - if (signal.aborted) { - void pending.then(spawn => onLate({ spawn }), lateError => onLate({ error: lateError })); - } - throw error; - } finally { - if (onAbort !== undefined) signal.removeEventListener('abort', onAbort); - } -} - async function tryRelease( runtime: NodeSpawnRuntime, path: string, @@ -333,8 +374,13 @@ async function tryRelease( const receipt: NodeSpawnIdentity | undefined = claim.spawn ?? claim.cleanup; if (receipt === undefined || claim.release !== undefined) return { claim }; try { - const released = await runtime.port.release({ receipt, - idempotencyKey: releaseKey(idempotencyKey) }); + const released = await boundedCleanup(runtime.port.release({ receipt, + idempotencyKey: releaseKey(idempotencyKey) })); + if (released.action !== 'release' || released.status !== 'completed' + || released.released !== true || !nonempty(released.id)) { + throw new NodeSpawnError('The node did not confirm exact agent release', + released.errorCode ?? 'release_unconfirmed'); + } const { cleanupPending: _cleanupPending, ...withoutPending } = claim; const updated: NodeSpawnClaim = { ...withoutPending, release: { invocationId: released.id, status: released.status } }; @@ -355,6 +401,27 @@ async function tryRelease( } } +async function boundedCleanup(pending: Promise): Promise { + let timer: NodeJS.Timeout | undefined; + const deadline = new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new NodeSpawnError( + 'Exact node cleanup did not confirm within its independent deadline', 'release_unconfirmed')), + CLEANUP_ATTEMPT_TIMEOUT_MS); + }); + // A transport that ignores cancellation may settle after this attempt has + // durably recorded cleanupPending. Fold that late outcome into this promise + // so it cannot become an unhandled rejection; the next attempt reuses the + // same idempotency key and reconciles the exact release. + const outcome = pending.then(value => ({ ok: true as const, value }), error => ({ ok: false as const, error })); + try { + const settled = await Promise.race([outcome, deadline]); + if (!settled.ok) throw settled.error; + return settled.value; + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} + function remainingDeadline(deadline: number): number { const remaining = deadline - Date.now(); if (!Number.isSafeInteger(remaining) || remaining < 1) { diff --git a/packages/sdk/src/node-spawn-deadline.ts b/packages/sdk/src/node-spawn-deadline.ts new file mode 100644 index 000000000..77d74f139 --- /dev/null +++ b/packages/sdk/src/node-spawn-deadline.ts @@ -0,0 +1,43 @@ +import type { NodeSpawnReceipt } from './node-spawn.js'; + +/** Races a side-effect-free node read/runtime acquisition against its caller deadline. */ +export async function waitForNodeOperation(pending: Promise, signal: AbortSignal): Promise { + signal.throwIfAborted(); + let onAbort: (() => void) | undefined; + const aborted = new Promise((_resolve, reject) => { + onAbort = () => reject(signal.reason ?? new DOMException('The operation was aborted', 'AbortError')); + signal.addEventListener('abort', onAbort, { once: true }); + }); + // Fold a late rejection from a transport that ignored cancellation. + const outcome = pending.then(value => ({ ok: true as const, value }), error => ({ ok: false as const, error })); + try { + const settled = await Promise.race([outcome, aborted]); + if (!settled.ok) throw settled.error; + return settled.value; + } finally { + if (onAbort !== undefined) signal.removeEventListener('abort', onAbort); + } +} + +/** Preserves the exact receipt when spawn confirmation settles after cancellation. */ +export async function waitForNodeSpawn( + pending: Promise, + signal: AbortSignal, + onLate: (outcome: { spawn?: NodeSpawnReceipt; error?: unknown }) => void, +): Promise { + signal.throwIfAborted(); + let onAbort: (() => void) | undefined; + const aborted = new Promise((_resolve, reject) => { + onAbort = () => reject(signal.reason ?? new DOMException('The operation was aborted', 'AbortError')); + signal.addEventListener('abort', onAbort, { once: true }); + }); + try { return await Promise.race([pending, aborted]); } + catch (error) { + if (signal.aborted) { + void pending.then(spawn => onLate({ spawn }), lateError => onLate({ error: lateError })); + } + throw error; + } finally { + if (onAbort !== undefined) signal.removeEventListener('abort', onAbort); + } +} diff --git a/packages/sdk/src/node-spawn-remote.ts b/packages/sdk/src/node-spawn-remote.ts index dc1f8f0c3..6f9fe11b9 100644 --- a/packages/sdk/src/node-spawn-remote.ts +++ b/packages/sdk/src/node-spawn-remote.ts @@ -32,6 +32,13 @@ interface PlacementAck extends RelayInvocation { confirmation?: RelayInvocation; } +interface PlacementFailure extends Error { + code?: unknown; + node?: unknown; + invocationId?: unknown; + receipt?: unknown; +} + export interface RemoteNodeSpawnClient { agents: { get(name: string): Promise }; nodes: { list(options?: Record): Promise }; @@ -60,23 +67,43 @@ export class RemoteNodeSpawnPort implements NodeSpawnPort { throw new NodeSpawnError('Invalid node-spawn idempotency key', 'spawn_invalid_input'); } const capability = `spawn:${request.harness}`; - const ack = await this.client.placement.spawn({ - capability, - ...(request.node === undefined ? {} : { node: request.node }), - ...(request.repo === undefined ? {} : { repo: request.repo }), - ...(request.sandboxOnly === undefined ? {} : { sandboxOnly: request.sandboxOnly }), - failFast: true, - confirm: true, - verifyReady: true, - confirmTimeoutMs: request.timeoutMs ?? 360_000, - input: { - name: request.name, - task: request.task, - idempotency_key: request.idempotencyKey, - ...(request.model === undefined ? {} : { model: request.model }), - ...(request.directory === undefined ? {} : { worker_cwd: request.directory }), - }, - }); + let ack: PlacementAck; + try { + ack = await this.client.placement.spawn({ + capability, + ...(request.node === undefined ? {} : { node: request.node }), + ...(request.repo === undefined ? {} : { repo: request.repo }), + ...(request.sandboxOnly === undefined ? {} : { sandboxOnly: request.sandboxOnly }), + failFast: true, + confirm: true, + verifyReady: true, + confirmTimeoutMs: request.timeoutMs ?? 360_000, + input: { + name: request.name, + task: request.task, + idempotency_key: request.idempotencyKey, + ...(request.model === undefined ? {} : { model: request.model }), + ...(request.directory === undefined ? {} : { worker_cwd: request.directory }), + }, + }); + } catch (error) { + const cleanup = placementFailureCleanup(error, request); + if (cleanup !== undefined) { + const failure = error as PlacementFailure; + const output = isRecord(failure.receipt) && isRecord(failure.receipt.output) + ? failure.receipt.output : undefined; + const taskUnconfirmed = output?.warning === 'spawn_task_unconfirmed' + || (isRecord(output?.warning) && output.warning.code === 'spawn_task_unconfirmed'); + throw new NodeSpawnError( + error instanceof Error ? error.message : 'Fleet placement failed after dispatch', + taskUnconfirmed ? 'spawn_task_unconfirmed' + : nonempty(failure.code) ? failure.code : 'spawn_unconfirmed', + error, + cleanup, + ); + } + throw error; + } const confirmation = ack.confirmation; const output = confirmation?.output; const reportedName = nonempty(output?.name) ? output.name : request.name; @@ -210,6 +237,24 @@ export class RemoteNodeSpawnPort implements NodeSpawnPort { } } +function placementFailureCleanup(error: unknown, request: NodeSpawnRequest): NodeSpawnIdentity | undefined { + if (!(error instanceof Error)) return undefined; + const failure = error as PlacementFailure; + if (!isRecord(failure.receipt)) return undefined; + const receipt = failure.receipt; + const output = isRecord(receipt.output) ? receipt.output : undefined; + const invocationId = nonempty(failure.invocationId) ? failure.invocationId + : nonempty(receipt.invocationId) ? receipt.invocationId : undefined; + const nodeName = nonempty(failure.node) ? failure.node : request.node; + const nodeId = nonempty(receipt.dispatchedNodeId) ? receipt.dispatchedNodeId + : nonempty(receipt.handlerNodeId) ? receipt.handlerNodeId : undefined; + const agentId = nonempty(output?.agent_id) ? output.agent_id : undefined; + const name = nonempty(output?.name) ? output.name : request.name; + if (invocationId === undefined || nodeName === undefined || agentId === undefined + || output?.spawned !== true || name !== request.name) return undefined; + return { id: invocationId, nodeId, nodeName, name, agentId }; +} + function mapInvocation(action: NodeSpawnAction, value: RelayInvocation): NodeSpawnInvocation { const output = value.output; const status = value.status.toLowerCase(); diff --git a/packages/sdk/src/node-spawn-result.ts b/packages/sdk/src/node-spawn-result.ts index 48feb64e5..0312b1dfa 100644 --- a/packages/sdk/src/node-spawn-result.ts +++ b/packages/sdk/src/node-spawn-result.ts @@ -37,6 +37,9 @@ export interface NodeResultDelivery { inboxItemId: string; messageId: string; envelope: NodeResultEnvelope; + settlementAction: 'ack' | 'fail'; + /** Durably records the journal settlement that must precede inbox settlement. */ + prepareSettlement?(input: { attempt: number; completionReason: string }): Promise; /** Called only after the result has been durably appended to the run journal. */ ack(): Promise; } @@ -87,6 +90,7 @@ export async function waitForNodeResult( messageId: item.message.id, envelope: { version: 1, run_id: expected.runId, step_id: expected.stepId, dispatch_id: expected.dispatchId, status: 'failed', error: reason }, + settlementAction: 'fail', ack: async () => { await client.inbox.fail({ inboxItemId: item.id, error: reason, retry: false, metadata: { relayflows_result: 'rejected' } }); }, }; @@ -95,6 +99,7 @@ export async function waitForNodeResult( inboxItemId: item.id, messageId: item.message.id, envelope, + settlementAction: 'ack', ack: async () => { await client.inbox.ack({ inboxItemId: item.id, state: 'read', metadata: { relayflows_result: 'journaled' } }); }, }; diff --git a/packages/sdk/src/node-spawn-runtime.ts b/packages/sdk/src/node-spawn-runtime.ts index 2780baac8..4166c7401 100644 --- a/packages/sdk/src/node-spawn-runtime.ts +++ b/packages/sdk/src/node-spawn-runtime.ts @@ -39,10 +39,11 @@ export async function createRemoteNodeSpawnRuntime( const baseUrl = validateNodeSpawnBaseUrl(env.RELAY_BASE_URL?.trim() || 'https://cast.agentrelay.com'); let RelaycastMessagingClient: new (options: Record) => RelayRuntimeClient; try { - ({ RelaycastMessagingClient } = await import('@agent-relay/sdk') as unknown as { + ({ RelaycastMessagingClient } = await waitForRuntime(import('@agent-relay/sdk'), options.signal) as unknown as { RelaycastMessagingClient: new (options: Record) => RelayRuntimeClient; }); } catch (error) { + if (options.signal?.aborted === true) throw options.signal.reason ?? error; throw new NodeSpawnError('Node-spawn transport requires optional @agent-relay/sdk >=17.2.1 <18.', 'relay_sdk_unavailable', error); } @@ -106,7 +107,7 @@ export function createNodeSpawnRuntimeOwner( return { get, close: () => closePromise ??= closeOnce() }; } -async function waitForRuntime(pending: Promise, signal?: AbortSignal): Promise { +async function waitForRuntime(pending: Promise, signal?: AbortSignal): Promise { if (signal === undefined) return pending; let onAbort: (() => void) | undefined; const aborted = new Promise((_resolve, reject) => { diff --git a/packages/sdk/src/node-spawn-settlement.ts b/packages/sdk/src/node-spawn-settlement.ts new file mode 100644 index 000000000..c8a5447bc --- /dev/null +++ b/packages/sdk/src/node-spawn-settlement.ts @@ -0,0 +1,172 @@ +import type { JournalClient } from './journal-client.js'; +import type { JournalEvent } from './journal-reader.js'; +import type { NodeResultDelivery } from './node-spawn-result.js'; +import { readNodeSpawnClaims, recordNodeSpawn, type NodeSpawnClaim } from './node-spawn-state.js'; +import type { NodeSpawnRuntime } from './node-spawn-runtime.js'; +import { NodeSpawnError, isRecord } from './node-spawn.js'; + +const SETTLEMENT_ATTEMPT_TIMEOUT_MS = 10_000; + +export function durableNodeResultDelivery( + runtime: NodeSpawnRuntime, + path: string, + initial: NodeSpawnClaim, +): NodeResultDelivery { + const result = initial.result; + if (result === undefined) throw new NodeSpawnError('Node result settlement requires a durable result'); + let claim = initial; + let settling: Promise | undefined; + const settleOnce = async (): Promise => { + const current = claim.result; + if (current === undefined) throw new NodeSpawnError('Durable node result disappeared before settlement'); + if (current.settlement === undefined) { + throw new NodeSpawnError('Node result cannot settle before its journal intent is durable', + 'result_settlement_unprepared'); + } + await settleInbox(runtime, current.settlementAction, current.inboxItemId, + current.envelope.error ?? 'Relayflows rejected an invalid result'); + claim = { ...claim, result: { ...current, settlement: { + ...current.settlement, + settledAt: Date.now(), settledAs: current.settlementAction, + } } }; + await recordNodeSpawn(path, claim); + }; + const settle = (): Promise => { + if (settling !== undefined) return settling; + const attempt = settleOnce(); + settling = attempt; + void attempt.catch(() => { if (settling === attempt) settling = undefined; }); + return attempt; + }; + return { + inboxItemId: result.inboxItemId, + messageId: result.messageId, + envelope: result.envelope, + settlementAction: result.settlementAction, + prepareSettlement: async input => { + const current = claim.result; + if (current === undefined) throw new NodeSpawnError('Durable node result disappeared before settlement'); + const prior = current.settlement; + if (prior !== undefined && prior.completionReason !== input.completionReason) { + throw new NodeSpawnError('Node result settlement conflicts with its journal identity', + 'result_identity_mismatch'); + } + if (prior?.settledAt !== undefined && prior.attempt !== input.attempt) { + throw new NodeSpawnError('A settled node result cannot move to a different journal attempt', + 'result_identity_mismatch'); + } + if (prior !== undefined && input.attempt < prior.attempt) { + throw new NodeSpawnError('Node result settlement cannot move back to an older journal attempt', + 'result_identity_mismatch'); + } + claim = { ...claim, result: { ...current, settlement: + prior !== undefined && prior.attempt === input.attempt ? prior : { + attempt: input.attempt, + completionReason: input.completionReason, + preparedAt: Date.now(), + } } }; + await recordNodeSpawn(path, claim); + }, + ack: settle, + }; +} + +/** Reconciles journaled result settlements left by a crash or failed inbox call. */ +export async function reconcileNodeResultSettlements( + dataDir: string, + runtime: NodeSpawnRuntime, + journal: Pick, +): Promise { + let pending = 0; + for (const entry of await readNodeSpawnClaims(dataDir)) { + const result = entry.claim.result; + const settlement = result?.settlement; + if (result === undefined || settlement === undefined || settlement.settledAt !== undefined) continue; + if (entry.claim.baseUrl !== runtime.baseUrl + || entry.claim.publisherId !== runtime.publisher.id + || entry.claim.publisherName !== runtime.publisher.name) { + throw new NodeSpawnError('Pending node-result settlement belongs to a different publisher identity', + 'publisher_identity_mismatch'); + } + const terminal = await terminalEntry(journal, entry.claim.runId, entry.claim.stepId, settlement.attempt); + if (terminal === undefined) { pending += 1; continue; } + const exact = terminalMatchesResult(terminal, entry.claim, settlement, result); + const action = exact ? result.settlementAction : 'fail'; + await settleInbox(runtime, action, result.inboxItemId, + exact ? (result.envelope.error ?? 'Relayflows rejected an invalid result') + : 'The journal completed this attempt with a different terminal outcome'); + const updated: NodeSpawnClaim = { ...entry.claim, result: { ...result, settlement: { + ...settlement, settledAt: Date.now(), settledAs: action, + } } }; + await recordNodeSpawn(entry.path, updated); + } + return pending; +} + +function terminalMatchesResult( + terminal: JournalEvent, + claim: NodeSpawnClaim, + settlement: NonNullable['settlement']>, + result: NonNullable, +): boolean { + const payload = isRecord(terminal.payload) ? terminal.payload : {}; + const tail = isRecord(payload.trajectory_tail) ? payload.trajectory_tail : {}; + const relayTask = isRecord(tail.relay_task) ? tail.relay_task : {}; + const execution = isRecord(relayTask.task_execution) ? relayTask.task_execution : {}; + return payload.completionReason === settlement.completionReason + && execution.run_id === claim.runId + && execution.step_id === claim.stepId + && execution.dispatch_id === claim.idempotencyKey + && execution.delivery_id === result.inboxItemId + && execution.message_id === result.messageId; +} + +async function terminalEntry( + journal: Pick, + runId: string, + stepId: string, + attempt: number, +): Promise { + let fromSeq = 1; + while (true) { + const entries = (await journal.journalRead(runId, fromSeq, 256)).entries as JournalEvent[]; + const terminal = entries.find(entry => entry.entry_type === 'step.completed' + && entry.step_id === stepId && entry.attempt === attempt); + if (terminal !== undefined || entries.length < 256) return terminal; + const last = entries.at(-1); + if (last === undefined || !Number.isSafeInteger(last.seq) || last.seq < fromSeq) { + throw new NodeSpawnError('Journal settlement scan did not advance', 'result_settlement_unconfirmed'); + } + fromSeq = last.seq + 1; + } +} + +async function settleInbox( + runtime: NodeSpawnRuntime, + action: 'ack' | 'fail', + inboxItemId: string, + error: string, +): Promise { + const pending = action === 'ack' + ? runtime.inbox.inbox.ack({ inboxItemId, state: 'read', metadata: { relayflows_result: 'journaled' } }) + : runtime.inbox.inbox.fail({ inboxItemId, error, retry: false, + metadata: { relayflows_result: 'rejected' } }); + await bounded(pending); +} + +async function bounded(pending: Promise): Promise { + let timer: NodeJS.Timeout | undefined; + const deadline = new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new NodeSpawnError( + 'Node-result inbox settlement did not confirm within its independent deadline', + 'result_settlement_unconfirmed')), + SETTLEMENT_ATTEMPT_TIMEOUT_MS); + }); + const outcome = pending.then(() => ({ ok: true as const }), error => ({ ok: false as const, error })); + try { + const settled = await Promise.race([outcome, deadline]); + if (!settled.ok) throw settled.error; + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} diff --git a/packages/sdk/src/node-spawn-state.ts b/packages/sdk/src/node-spawn-state.ts index 7d0d328d7..da99fa9bb 100644 --- a/packages/sdk/src/node-spawn-state.ts +++ b/packages/sdk/src/node-spawn-state.ts @@ -1,8 +1,10 @@ import { createHash, randomUUID } from 'node:crypto'; -import { link, mkdir, open, readFile, rename, unlink } from 'node:fs/promises'; +import { link, mkdir, open, readFile, readdir, rename, unlink } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import { canonicalize } from './canonical.js'; -import type { NodeSpawnIdentity, NodeSpawnReceipt } from './node-spawn.js'; +import { readNodeResult, type NodeResultEnvelope } from './node-spawn-result.js'; +import { isRecord, nonempty, validIdempotencyKey, validNodeSpawnIdentity, validNodeSpawnReceipt, + type NodeSpawnIdentity, type NodeSpawnReceipt } from './node-spawn.js'; export interface NodeSpawnClaim { version: 2; @@ -13,12 +15,16 @@ export interface NodeSpawnClaim { stepId: string; idempotencyKey: string; agentName: string; + timeoutMs: number; request: Record; startedAt: number; spawn?: NodeSpawnReceipt; cleanup?: NodeSpawnIdentity; failure?: { code: string; message: string }; - result?: { inboxItemId: string; messageId: string; status: 'completed' | 'failed'; receivedAt: number }; + result?: { inboxItemId: string; messageId: string; status: 'completed' | 'failed'; + envelope: NodeResultEnvelope; settlementAction: 'ack' | 'fail'; receivedAt: number; + settlement?: { attempt: number; completionReason: string; preparedAt: number; + settledAt?: number; settledAs?: 'ack' | 'fail' } }; cleanupPending?: { attempts: number; lastError: string; updatedAt: number }; release?: { invocationId: string; status: string }; } @@ -45,9 +51,9 @@ export async function claimNodeSpawn(dataDir: string, proposed: NodeSpawnClaim): return { claim: proposed, created: true, path }; } catch (error) { if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; - const prior = JSON.parse(await readFile(path, 'utf8')) as unknown; + const prior = decodeNodeSpawnClaim(JSON.parse(await readFile(path, 'utf8')) as unknown, path); if (!sameClaim(prior, proposed)) throw new Error('Node-spawn dispatch conflicts with its durable identity or input'); - return { claim: prior as NodeSpawnClaim, created: false, path }; + return { claim: prior, created: false, path }; } finally { await unlink(temporary).catch(() => undefined); } @@ -62,10 +68,25 @@ export async function recordNodeSpawn(path: string, claim: NodeSpawnClaim): Prom await syncDirectory(dirname(path)); } -function sameClaim(value: unknown, proposed: NodeSpawnClaim): boolean { - if (typeof value !== 'object' || value === null || !('startedAt' in value) - || typeof value.startedAt !== 'number' || !Number.isSafeInteger(value.startedAt)) return false; - const prior = value as NodeSpawnClaim; +export async function readNodeSpawnClaims(dataDir: string): Promise> { + const directory = join(dataDir, 'node-spawns'); + let names: string[]; + try { names = await readdir(directory); } + catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; + throw error; + } + const claims: Array<{ path: string; claim: NodeSpawnClaim }> = []; + for (const name of names.sort()) { + if (!/^[a-f0-9]{64}\.json$/.test(name)) continue; + const path = join(directory, name); + const value = JSON.parse(await readFile(path, 'utf8')) as unknown; + claims.push({ path, claim: decodeNodeSpawnClaim(value, name) }); + } + return claims; +} + +function sameClaim(prior: NodeSpawnClaim, proposed: NodeSpawnClaim): boolean { const { spawn: _priorSpawn, cleanup: _priorCleanup, failure: _priorFailure, result: _priorResult, cleanupPending: _priorCleanupPending, release: _priorRelease, ...priorIdentity } = prior; @@ -74,6 +95,67 @@ function sameClaim(value: unknown, proposed: NodeSpawnClaim): boolean { return canonicalize({ ...priorIdentity, startedAt: proposed.startedAt }) === canonicalize(proposedIdentity); } +function decodeNodeSpawnClaim(value: unknown, label: string): NodeSpawnClaim { + const invalid = (): never => { throw new Error(`Invalid durable node-spawn claim: ${label}`); }; + const record = isRecord(value) ? value : invalid(); + if (record.version !== 2 + || ![record.baseUrl, record.publisherId, record.publisherName, record.runId, record.stepId, record.agentName] + .every(nonempty) + || !validIdempotencyKey(record.idempotencyKey) + || !positiveInteger(record.timeoutMs) || record.timeoutMs > 86_400_000 + || !isRecord(record.request) + || !safeTime(record.startedAt)) invalid(); + const claim = record as unknown as NodeSpawnClaim; + if (record.spawn !== undefined && !validNodeSpawnReceipt(record.spawn)) invalid(); + if (record.cleanup !== undefined && !validNodeSpawnIdentity(record.cleanup)) invalid(); + if (record.spawn !== undefined && record.cleanup !== undefined) invalid(); + if (record.failure !== undefined) { + const failure = isRecord(record.failure) ? record.failure : invalid(); + if (!nonempty(failure.code) || !nonempty(failure.message) + || (record.spawn === undefined && record.cleanup === undefined)) invalid(); + } + if (record.cleanupPending !== undefined) { + const pending = isRecord(record.cleanupPending) ? record.cleanupPending : invalid(); + if (!positiveInteger(pending.attempts) || !nonempty(pending.lastError) || !safeTime(pending.updatedAt) + || (record.spawn === undefined && record.cleanup === undefined) || record.release !== undefined) invalid(); + } + if (record.release !== undefined) { + const release = isRecord(record.release) ? record.release : invalid(); + if (!nonempty(release.invocationId) || release.status !== 'completed' + || (record.spawn === undefined && record.cleanup === undefined)) invalid(); + } + if (record.result !== undefined) { + const result = isRecord(record.result) ? record.result : invalid(); + if (!nonempty(result.inboxItemId) + || !nonempty(result.messageId) + || !['completed', 'failed'].includes(String(result.status)) + || !['ack', 'fail'].includes(String(result.settlementAction)) + || !safeTime(result.receivedAt)) invalid(); + let envelope: NodeResultEnvelope; + try { + envelope = readNodeResult(result.envelope, { + runId: claim.runId, stepId: claim.stepId, dispatchId: claim.idempotencyKey, + }); + } catch { invalid(); } + if (envelope!.status !== result.status || record.spawn === undefined) invalid(); + const settlement = result.settlement; + if (settlement !== undefined) { + if (!isRecord(settlement) || !positiveInteger(settlement.attempt) + || !nonempty(settlement.completionReason) || !safeTime(settlement.preparedAt) + || (settlement.settledAt !== undefined && !safeTime(settlement.settledAt)) + || (settlement.settledAs !== undefined && !['ack', 'fail'].includes(String(settlement.settledAs))) + || (settlement.settledAt === undefined) !== (settlement.settledAs === undefined)) invalid(); + } + } + return claim; +} + +const safeTime = (value: unknown): value is number => + typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; + +const positiveInteger = (value: unknown): value is number => + typeof value === 'number' && Number.isSafeInteger(value) && value > 0; + async function syncDirectory(path: string): Promise { const directory = await open(path, 'r'); try { await directory.sync(); } finally { await directory.close(); } diff --git a/packages/sdk/src/node-spawn.ts b/packages/sdk/src/node-spawn.ts index 9cbdc91de..c6568b777 100644 --- a/packages/sdk/src/node-spawn.ts +++ b/packages/sdk/src/node-spawn.ts @@ -94,6 +94,20 @@ export const isRecord = (value: unknown): value is Record => export const nonempty = (value: unknown): value is string => typeof value === 'string' && value.trim().length > 0; +export function validNodeSpawnIdentity(value: unknown): value is NodeSpawnIdentity { + if (!isRecord(value)) return false; + return [value.id, value.nodeName, value.name, value.agentId].every(nonempty) + && (value.nodeId === undefined || nonempty(value.nodeId)); +} + +export function validNodeSpawnReceipt(value: unknown): value is NodeSpawnReceipt { + return validNodeSpawnIdentity(value) + && (value as unknown as Record).action === 'spawn' + && (value as unknown as Record).status === 'completed' + && (value as unknown as Record).spawned === true + && (value as unknown as Record).ready === true; +} + export function validSegment(value: unknown): value is string { return typeof value === 'string' && value.length > 0 && value.length <= 200 && /^[A-Za-z0-9_-]+$/.test(value); diff --git a/packages/sdk/src/worker-cli.ts b/packages/sdk/src/worker-cli.ts index 8d745fcfd..b21e8e946 100644 --- a/packages/sdk/src/worker-cli.ts +++ b/packages/sdk/src/worker-cli.ts @@ -70,7 +70,10 @@ export function cliInvocationArgv0(cli: string, cliIdentity?: string): string { export interface WorkerCliResult { relay_task?: import('./agent-relay-receipt.js').RelayTaskReceipt; /** Settled only after the journal accepts stepComplete; never journaled. */ - relay_delivery?: { ack(): Promise }; + relay_delivery?: { + prepareSettlement?(input: { attempt: number; completionReason: string }): Promise; + ack(): Promise; + }; tokens_input?: number; tokens_output?: number; exit_code: number | null; @@ -122,6 +125,7 @@ export interface AgentRelayContext { runId: string; stepId: string; idempotencyKey: string; + attempt: number; dataDir?: string; resultSchema?: unknown; nodeSpawnRuntime?: NodeSpawnRuntime; @@ -399,7 +403,7 @@ async function runViaAgentRelay( const received = await runAgentRelayTask({ cli: kind, task, model, worker_cwd, result_schema: context.resultSchema, runId: context.runId, stepId: context.stepId, idempotencyKey: context.idempotencyKey, - dataDir: context.dataDir, + attempt: context.attempt, dataDir: context.dataDir, }, { signal, runtime: context.nodeSpawnRuntime, runtimeProvider: context.nodeSpawnRuntimeProvider }); const receipt = { ...received.receipt, error: received.receipt.error === null ? null : redactRelayError(received.receipt.error) }; diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index 8590fdfce..a788e92ac 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -24,6 +24,8 @@ import { } from './agent-transcript.js'; import { createNodeSpawnRuntimeOwner, type NodeSpawnRuntime, type NodeSpawnRuntimeOwner } from './node-spawn-runtime.js'; +import { reconcileNodeResultSettlements } from './node-spawn-settlement.js'; +import { readNodeSpawnClaims } from './node-spawn-state.js'; export { MODEL_ENV, WAKE_CONTEXT_ENV } from './worker-cli.js'; @@ -68,6 +70,8 @@ export class AgentWorker extends EventEmitter { private attached = false; private closing = false; private closePromise?: Promise; + private settlementTimer?: NodeJS.Timeout; + private settlementReconcile?: Promise; private readonly inFlight: Set> = new Set(); private readonly nodeSpawnRuntimeOwner: NodeSpawnRuntimeOwner; @@ -84,6 +88,7 @@ export class AgentWorker extends EventEmitter { if (this.closing) throw new Error('agent worker: cannot attach a closed worker (construct a new one)'); this.client.on('step.dispatch', this.onDispatch); try { + await this.reconcileRelaySettlements(); await this.client.workerAttach( this.options.workerId, ['agent'], @@ -94,6 +99,23 @@ export class AgentWorker extends EventEmitter { this.attached = true; } catch (error) { this.client.off('step.dispatch', this.onDispatch); + if (this.settlementTimer !== undefined) { + clearTimeout(this.settlementTimer); + this.settlementTimer = undefined; + } + // An attach attempt can create the lazy Relay runtime while reconciling + // a prior inbox settlement. Once attachment fails, this worker is not + // reusable: close that owned connection before the caller loses the + // worker instance. + this.closing = true; + try { + const close = this.nodeSpawnRuntimeOwner.close(); + this.closePromise = close; + await close; + } catch (closeError) { + throw new AggregateError([error, closeError], + 'Agent worker could not attach or close its node runtime'); + } throw error; } } @@ -120,18 +142,44 @@ export class AgentWorker extends EventEmitter { if (pending.length > 0) { await Promise.allSettled(pending); } - await this.nodeSpawnRuntimeOwner.close(); - this.attached = false; + if (this.settlementTimer !== undefined) { + clearTimeout(this.settlementTimer); + this.settlementTimer = undefined; + } + let settlementError: unknown; + try { + await this.reconcileRelaySettlements(); + } catch (error) { + settlementError = error; + } + try { + await this.nodeSpawnRuntimeOwner.close(); + } catch (closeError) { + if (settlementError !== undefined) { + throw new AggregateError([settlementError, closeError], + 'Agent worker could not reconcile relay settlement or close its node runtime'); + } + throw closeError; + } finally { + this.attached = false; + } + if (settlementError !== undefined) throw settlementError; } private readonly onDispatch = (dispatch: StepDispatchEvent): void => { if (this.closing) return; if (dispatch.step_type !== 'agent') return; + const relayTransport = (dispatch.spec as Partial).transport === 'relay'; const running: Promise = this.execute(dispatch).catch((error: unknown) => { this.emit('error', error, dispatch); }); this.inFlight.add(running); - void running.finally(() => { this.inFlight.delete(running); }); + void running.finally(() => { + this.inFlight.delete(running); + if (relayTransport) { + void this.reconcileRelaySettlements().catch(error => this.emit('error', error)); + } + }); }; private async execute(dispatch: StepDispatchEvent): Promise { @@ -173,7 +221,7 @@ export class AgentWorker extends EventEmitter { }, cwd.directory ?? (spec.transport === 'relay' ? undefined : this.options.runRoot), spec.transport === 'relay' ? 'relay' : 'direct', { runId: dispatch.run_id, stepId: dispatch.step_id, idempotencyKey: dispatch.idempotency_key, - dataDir: this.options.dataDir, resultSchema: spec.verification?.json_schema, + attempt: dispatch.attempt, dataDir: this.options.dataDir, resultSchema: spec.verification?.json_schema, nodeSpawnRuntime: this.options.nodeSpawnRuntime, nodeSpawnRuntimeProvider: signal => this.nodeSpawnRuntimeOwner.get(signal) }, this.options.environment, spec.cli_identity, spec.timeout_ms, spec.resume, spec.secrets, @@ -230,6 +278,7 @@ export class AgentWorker extends EventEmitter { ...(transport === undefined ? {} : { transport }), }; + await relayDelivery?.prepareSettlement?.({ attempt: dispatch.attempt, completionReason }); await this.client.stepComplete( dispatch.run_id, dispatch.step_id, @@ -253,6 +302,46 @@ export class AgentWorker extends EventEmitter { await relayDelivery?.ack(); } + private async reconcileRelaySettlements(): Promise { + if (this.settlementReconcile !== undefined) return this.settlementReconcile; + const running = this.reconcileRelaySettlementsOnce(); + this.settlementReconcile = running; + void running.finally(() => { + if (this.settlementReconcile === running) this.settlementReconcile = undefined; + }).catch(() => undefined); + return running; + } + + private async reconcileRelaySettlementsOnce(): Promise { + if (this.options.dataDir === undefined || (this.closing && !this.attached)) return; + const claims = await readNodeSpawnClaims(this.options.dataDir); + const hasPending = claims.some(entry => entry.claim.result?.settlement !== undefined + && entry.claim.result.settlement.settledAt === undefined); + if (!hasPending) return; + const runtime = await this.nodeSpawnRuntimeOwner.get(); + let pending: number; + try { + pending = await reconcileNodeResultSettlements(this.options.dataDir, runtime, this.client); + } catch (error) { + this.scheduleSettlementReconcile(); + throw error; + } + if (this.settlementTimer !== undefined) { + clearTimeout(this.settlementTimer); + this.settlementTimer = undefined; + } + if (pending > 0) this.scheduleSettlementReconcile(); + } + + private scheduleSettlementReconcile(): void { + if (this.closing || this.settlementTimer !== undefined) return; + this.settlementTimer = setTimeout(() => { + this.settlementTimer = undefined; + void this.reconcileRelaySettlements().catch(error => this.emit('error', error)); + }, 5_000); + this.settlementTimer.unref(); + } + private async retiredNodeSession(dispatch: StepDispatchEvent, error: string): Promise { await withWorkerLease(this.client, dispatch, async signal => { signal.throwIfAborted(); }); await this.client.stepComplete( diff --git a/packages/sdk/tests/agent-relay-transport.test.ts b/packages/sdk/tests/agent-relay-transport.test.ts index e9daeb850..044ffe9ef 100644 --- a/packages/sdk/tests/agent-relay-transport.test.ts +++ b/packages/sdk/tests/agent-relay-transport.test.ts @@ -11,10 +11,11 @@ import { EventEmitter } from 'node:events'; import { AgentWorker } from '../src/worker.js'; import type { JournalClient } from '../src/journal-client.js'; import type { StepDispatchEvent } from '../src/protocol.js'; +import { readNodeSpawnClaims, recordNodeSpawn } from '../src/node-spawn-state.js'; const request = (dataDir: string): AgentRelayTaskRequest => ({ cli: 'codex', task: 'Implement the change', runId: 'run_1', stepId: 'edit', - idempotencyKey: 'dispatch_1234567890', dataDir, timeoutMs: 10_000, + idempotencyKey: 'dispatch_1234567890', attempt: 1, dataDir, timeoutMs: 10_000, }); const roots: string[] = []; @@ -26,7 +27,8 @@ async function dataDirectory(label: string): Promise { } function fixture(result: 'completed' | 'failed' = 'completed'): { runtime: NodeSpawnRuntime; spawn: ReturnType; - release: ReturnType; status: ReturnType; ack: ReturnType } { + release: ReturnType; status: ReturnType; + ack: ReturnType; fail: ReturnType } { const receipt: NodeSpawnReceipt = { id: 'inv_spawn_1', action: 'spawn', status: 'completed', nodeId: 'node_1', nodeName: 'desktop-1', name: 'placeholder', agentId: 'agent_1', spawned: true, ready: true, @@ -36,6 +38,7 @@ function fixture(result: 'completed' | 'failed' = 'completed'): { runtime: NodeS status: 'completed' as const, spawned: false, ready: false, released: true })); const status = vi.fn(async () => ({ ...receipt, name: receipt.name })); const ack = vi.fn(async () => ({})); + const fail = vi.fn(async () => ({})); const port: NodeSpawnPort = { targets: async () => [], spawn, status, release, }; @@ -62,10 +65,10 @@ function fixture(result: 'completed' | 'failed' = 'completed'): { runtime: NodeS }; }, ack, - fail: vi.fn(async () => ({})), + fail, } }, }; - return { runtime, spawn, release, status, ack }; + return { runtime, spawn, release, status, ack, fail }; } describe('native node-spawn task transport', () => { @@ -84,18 +87,19 @@ describe('native node-spawn task transport', () => { task_execution: { agent_id: 'agent_1', delivery_id: 'delivery_1', release_invocation_id: 'inv_release_1' }, }); expect(f.ack).not.toHaveBeenCalled(); + await result.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); await result.delivery.ack(); expect(f.ack).toHaveBeenCalledWith(expect.objectContaining({ inboxItemId: 'delivery_1', state: 'read' })); }); - it('recovers a recorded spawn without issuing a second spawn or release', async () => { + it('replays a durable result without a second spawn, release, or remote status lookup', async () => { const dataDir = await dataDirectory('flows-node-recover-'); const f = fixture(); await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); const recovered = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); expect(f.spawn).toHaveBeenCalledTimes(1); expect(f.release).toHaveBeenCalledTimes(1); - expect(f.status).toHaveBeenCalledWith('spawn', 'inv_spawn_1'); + expect(f.status).not.toHaveBeenCalled(); expect(recovered.receipt.status).toBe('completed'); }); @@ -162,6 +166,49 @@ describe('native node-spawn task transport', () => { expect(f.status).not.toHaveBeenCalled(); }); + it('refuses to extend the original durable execution deadline on redispatch', async () => { + const dataDir = await dataDirectory('flows-node-deadline-identity-'); + const f = fixture(); + const controller = new AbortController(); + f.spawn.mockImplementationOnce(async (input: { name: string }) => { + queueMicrotask(() => controller.abort(new Error('worker stopped'))); + return { id: 'inv_spawn_1', action: 'spawn' as const, status: 'completed' as const, + nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, agentId: 'agent_1', + spawned: true as const, ready: true as const }; + }); + await expect(runAgentRelayTask({ ...request(dataDir), timeoutMs: 1_000 }, { + runtime: f.runtime, signal: controller.signal, + })).rejects.toThrow('worker stopped'); + await expect(runAgentRelayTask({ ...request(dataDir), timeoutMs: 60_000 }, { runtime: f.runtime })) + .rejects.toThrow('conflicts with its durable identity or input'); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).toHaveBeenCalledOnce(); + }); + + it('bounds a recovered spawn status lookup by the original deadline and releases exact identity', async () => { + vi.useFakeTimers(); + try { + const dataDir = await dataDirectory('flows-node-status-stall-'); + const f = fixture(); + const bounded = { ...request(dataDir), timeoutMs: 1_000 }; + await runAgentRelayTask(bounded, { runtime: f.runtime }); + const [entry] = await readNodeSpawnClaims(dataDir); + const { result: _result, release: _release, ...spawnOnly } = entry!.claim; + await recordNodeSpawn(entry!.path, spawnOnly); + f.status.mockImplementationOnce(() => new Promise(() => undefined)); + const pending = expect(runAgentRelayTask(bounded, { runtime: f.runtime })) + .rejects.toThrow(); + await vi.waitFor(() => expect(f.status).toHaveBeenCalledOnce()); + await vi.advanceTimersByTimeAsync(1_001); + await pending; + expect(f.release).toHaveBeenCalledTimes(2); + const [recovered] = await readNodeSpawnClaims(dataDir); + expect(recovered!.claim.release).toBeDefined(); + } finally { + vi.useRealTimers(); + } + }); + it('returns promptly from a cancelled confirmation and cleans up its late exact receipt', async () => { const dataDir = await dataDirectory('flows-node-confirm-cancel-'); const f = fixture(); @@ -235,6 +282,31 @@ describe('native node-spawn task transport', () => { expect(recovered.receipt.status).toBe('completed'); }); + it('bounds a stalled exact release and retries it from durable cleanup state', async () => { + vi.useFakeTimers(); + try { + const dataDir = await dataDirectory('flows-node-cleanup-stall-'); + const f = fixture(); + f.release.mockImplementationOnce(() => new Promise(() => undefined)); + const task = runAgentRelayTask({ ...request(dataDir), timeoutMs: 60_000 }, { runtime: f.runtime }); + const rejection = expect(task).rejects.toMatchObject({ cleanupPending: true }); + await vi.waitFor(() => expect(f.release).toHaveBeenCalledOnce()); + await vi.advanceTimersByTimeAsync(15_001); + await rejection; + expect(f.ack).not.toHaveBeenCalled(); + const claims = await readNodeSpawnClaims(dataDir); + expect(claims[0]!.claim.cleanupPending).toMatchObject({ + attempts: 1, + lastError: expect.stringContaining('independent deadline'), + }); + const recovered = await runAgentRelayTask({ ...request(dataDir), timeoutMs: 60_000 }, { runtime: f.runtime }); + expect(f.release).toHaveBeenCalledTimes(2); + expect(recovered.receipt.status).toBe('completed'); + } finally { + vi.useRealTimers(); + } + }); + it('leaves failed cleanup nonterminal so a worker redispatch can finish exact release', async () => { const dataDir = await dataDirectory('flows-node-worker-cleanup-'); const f = fixture(); @@ -298,7 +370,24 @@ describe('native node-spawn task transport', () => { expect(f.status).not.toHaveBeenCalled(); }); - it('keeps the original deadline across restart and bounds completed-result recovery', async () => { + it('releases a durably failed spawn before any status or inbox recovery', async () => { + const dataDir = await dataDirectory('flows-node-failure-before-cleanup-'); + const f = fixture(); + await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + const [entry] = await readNodeSpawnClaims(dataDir); + const { result: _result, release: _release, cleanupPending: _pending, ...spawned } = entry!.claim; + await recordNodeSpawn(entry!.path, { ...spawned, + failure: { code: 'spawn_cancelled', message: 'worker stopped after spawn' } }); + f.release.mockClear(); + f.status.mockClear(); + f.runtime.inbox.inbox.subscribe = async function* () { throw new Error('inbox must not be read'); }; + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('worker stopped after spawn'); + expect(f.release).toHaveBeenCalledOnce(); + expect(f.status).not.toHaveBeenCalled(); + }); + + it('keeps the original execution deadline but never expires a durably observed result', async () => { vi.useFakeTimers({ toFake: ['Date'] }); try { const started = new Date('2026-11-07T00:00:00.000Z'); @@ -309,10 +398,11 @@ describe('native node-spawn task transport', () => { await runAgentRelayTask(bounded, { runtime: f.runtime }); vi.setSystemTime(started.getTime() + 1_001); await expect(runAgentRelayTask(bounded, { runtime: f.runtime })).resolves.toBeDefined(); - expect(f.status).toHaveBeenCalledTimes(1); + expect(f.status).not.toHaveBeenCalled(); vi.setSystemTime(started.getTime() + 31_001); - await expect(runAgentRelayTask(bounded, { runtime: f.runtime })).rejects.toThrow('deadline elapsed'); - expect(f.status).toHaveBeenCalledTimes(1); + f.runtime.inbox.inbox.subscribe = async function* () { throw new Error('must use durable envelope'); }; + await expect(runAgentRelayTask(bounded, { runtime: f.runtime })).resolves.toBeDefined(); + expect(f.status).not.toHaveBeenCalled(); expect(f.spawn).toHaveBeenCalledOnce(); } finally { vi.useRealTimers(); @@ -349,4 +439,40 @@ describe('native node-spawn task transport', () => { trajectory_tail: { relay_task: expect.objectContaining({ status: 'failed', error: 'agent reported failure' }) }, })); }); + + it('journals a malformed exact-sender envelope before dead-lettering its delivery', async () => { + const dataDir = await dataDirectory('flows-node-malformed-result-'); + const f = fixture(); + f.runtime.inbox.inbox.subscribe = async function* () { + const name = (await f.spawn.mock.results[0]!.value).name; + yield { id: 'delivery_1', message: { id: 'message_1', + from: { id: 'agent_1', name }, + text: NODE_RESULT_PREFIX + '{bad', + } }; + }; + const order: string[] = []; + f.fail.mockImplementation(async () => { order.push('fail'); return {}; }); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), + stepComplete: vi.fn(async () => { expect(f.fail).not.toHaveBeenCalled(); order.push('journal'); return {}; }), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_1', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + const errors: unknown[] = []; + worker.on('error', error => errors.push(error)); + await worker.attach(); + client.emit('step.dispatch', { + run_id: 'run_1', step_id: 'edit', attempt: 1, step_type: 'agent', + spec: { cli: 'codex', instruction: 'Implement the change', transport: 'relay' }, + idempotency_key: 'dispatch_1234567890', lease_id: 'lease_1', + lease_deadline_ms: Date.now() + 30_000, pins: {}, + } satisfies StepDispatchEvent); + await worker.close(); + expect(errors).toEqual([]); + expect(order).toEqual(['journal', 'fail']); + expect(f.ack).not.toHaveBeenCalled(); + expect(f.fail).toHaveBeenCalledWith(expect.objectContaining({ inboxItemId: 'delivery_1', retry: false })); + }); }); diff --git a/packages/sdk/tests/node-spawn-ports.test.ts b/packages/sdk/tests/node-spawn-ports.test.ts index 3e59a85a4..cd65db5a5 100644 --- a/packages/sdk/tests/node-spawn-ports.test.ts +++ b/packages/sdk/tests/node-spawn-ports.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; +import { RelayPlacementError } from '@agent-relay/sdk/messaging'; import { connectCore, LocalNodeSpawnPort, type CoreClient } from '../src/node-spawn-local.js'; import { RemoteNodeSpawnPort } from '../src/node-spawn-remote.js'; @@ -201,6 +202,27 @@ describe('remote node-spawn port', () => { }); }); + it('retains cleanup identity from the real SDK placement error receipt', async () => { + const receipt = { invocationId: 'inv_1', actionName: 'spawn', status: 'completed', + dispatchedNodeId: 'node_1', output: { spawned: true, ready: true, + name: 'worker_1', agent_id: 'agent_1', warning: 'spawn_task_unconfirmed' } }; + const port = new RemoteNodeSpawnPort({ + agents: { get: async name => ({ id: 'agent_1', name }) }, + nodes: { list: async () => [] }, + placement: { spawn: async () => { throw new RelayPlacementError('spawn_failed', 'task unconfirmed', { + capability: 'spawn:codex', node: 'desktop-1', attempts: 1, + invocationId: 'inv_1', state: 'failed', dispatchState: 'dispatched', receipt, + }); } }, + commands: { getInvocation: vi.fn() }, + }); + await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', + idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ + code: 'spawn_task_unconfirmed', + cleanup: { id: 'inv_1', nodeId: 'node_1', nodeName: 'desktop-1', + name: 'worker_1', agentId: 'agent_1' }, + }); + }); + it('never substitutes a conflicting roster id for the handler cleanup identity', async () => { const port = new RemoteNodeSpawnPort({ agents: { get: async name => ({ id: 'agent_roster_replacement', name }) }, diff --git a/packages/sdk/tests/node-spawn-runtime.test.ts b/packages/sdk/tests/node-spawn-runtime.test.ts index 3a8308b51..b80839b02 100644 --- a/packages/sdk/tests/node-spawn-runtime.test.ts +++ b/packages/sdk/tests/node-spawn-runtime.test.ts @@ -112,6 +112,7 @@ describe('remote node-spawn runtime', () => { const pending = createRemoteNodeSpawnRuntime({ RELAY_AGENT_TOKEN: 'agent-token' }, { signal: controller.signal, }); + await vi.waitFor(() => expect(relay.me).toHaveBeenCalledOnce()); controller.abort(new Error('lease lost')); await expect(pending).rejects.toThrow('lease lost'); expect(relay.disconnect).toHaveBeenCalledOnce(); @@ -129,10 +130,23 @@ describe('remote node-spawn runtime', () => { const controller = new AbortController(); const pending = runAgentRelayTask({ cli: 'codex', task: 'work', runId: 'run_1', stepId: 'step_1', - idempotencyKey: 'dispatch_1234567890', dataDir: '/tmp/unused', + idempotencyKey: 'dispatch_1234567890', attempt: 1, dataDir: '/tmp/unused', }, { agentToken: 'agent-token', signal: controller.signal }); + await vi.waitFor(() => expect(relay.me).toHaveBeenCalledOnce()); controller.abort(new Error('lease lost')); await expect(pending).rejects.toThrow('lease lost'); expect(relay.disconnect).toHaveBeenCalledOnce(); }); + + it('bounds a runtime provider that ignores cancellation by the task deadline', async () => { + const runtimeProvider = vi.fn((_signal?: AbortSignal) => new Promise(() => undefined)); + await expect(runAgentRelayTask({ + cli: 'codex', task: 'work', runId: 'run_1', stepId: 'step_1', + idempotencyKey: 'dispatch_1234567890', attempt: 1, dataDir: '/tmp/unused', timeoutMs: 5, + }, { runtimeProvider })).rejects.toMatchObject({ + cause: expect.objectContaining({ code: 'task_timeout' }), + }); + expect(runtimeProvider).toHaveBeenCalledOnce(); + expect(runtimeProvider.mock.calls[0]![0]?.aborted).toBe(true); + }); }); diff --git a/packages/sdk/tests/node-spawn-settlement.test.ts b/packages/sdk/tests/node-spawn-settlement.test.ts new file mode 100644 index 000000000..c2d39721b --- /dev/null +++ b/packages/sdk/tests/node-spawn-settlement.test.ts @@ -0,0 +1,397 @@ +import { EventEmitter } from 'node:events'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { runAgentRelayTask, type AgentRelayTaskRequest } from '../src/agent-relay-transport.js'; +import type { JournalClient } from '../src/journal-client.js'; +import { NODE_RESULT_PREFIX } from '../src/node-spawn-result.js'; +import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; +import { readNodeSpawnClaims, recordNodeSpawn } from '../src/node-spawn-state.js'; +import { NodeSpawnError, type NodeSpawnPort, type NodeSpawnReceipt } from '../src/node-spawn.js'; +import type { StepDispatchEvent } from '../src/protocol.js'; +import { AgentWorker } from '../src/worker.js'; + +const request = (dataDir: string): AgentRelayTaskRequest => ({ + cli: 'codex', task: 'Implement the change', runId: 'run_1', stepId: 'edit', + idempotencyKey: 'dispatch_1234567890', attempt: 1, dataDir, timeoutMs: 10_000, +}); + +const roots: string[] = []; +afterEach(async () => { + await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); +}); + +async function dataDirectory(label: string): Promise { + const directory = await mkdtemp(join(tmpdir(), label)); + roots.push(directory); + return directory; +} + +function fixture(): { runtime: NodeSpawnRuntime; spawn: ReturnType; + release: ReturnType; ack: ReturnType; fail: ReturnType } { + const receipt: NodeSpawnReceipt = { + id: 'inv_spawn_1', action: 'spawn', status: 'completed', nodeId: 'node_1', nodeName: 'desktop-1', + name: 'placeholder', agentId: 'agent_1', spawned: true, ready: true, + }; + const spawn = vi.fn(async (input: { name: string }) => ({ ...receipt, name: input.name })); + const release = vi.fn(async () => ({ id: 'inv_release_1', action: 'release' as const, + status: 'completed' as const, spawned: false, ready: false, released: true })); + const ack = vi.fn(async () => ({})); + const fail = vi.fn(async () => ({})); + const port: NodeSpawnPort = { + targets: async () => [], spawn, + status: vi.fn(async () => ({ ...receipt })), + release, + }; + const runtime: NodeSpawnRuntime = { + port, + publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + inbox: { inbox: { + subscribe: async function* () { + const name = (await spawn.mock.results[0]!.value).name; + yield { id: 'delivery_1', message: { id: 'message_1', + from: { id: 'agent_1', name }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_1', + step_id: 'edit', dispatch_id: 'dispatch_1234567890', status: 'completed', + output: { summary: 'done' } }) } }; + }, + ack, + fail, + } }, + }; + return { runtime, spawn, release, ack, fail }; +} + +function journaledRelayResult(): Record { + return { + completionReason: 'success', + trajectory_tail: { relay_task: { task_execution: { + run_id: 'run_1', step_id: 'edit', dispatch_id: 'dispatch_1234567890', + delivery_id: 'delivery_1', message_id: 'message_1', + } } }, + }; +} + +function terminalEntry(payload: Record = journaledRelayResult()) { + return { + seq: 2, segment_id: 0, entry_type: 'step.completed', run_id: 'run_1', + step_id: 'edit', attempt: 1, at_ms: Date.now(), payload, + }; +} + +describe('durable node-result settlement', () => { + it('rejects an invalid resolved spawn receipt before reading the inbox and releases exact identity', async () => { + const dataDir = await dataDirectory('flows-node-invalid-resolved-spawn-'); + const f = fixture(); + f.spawn.mockImplementationOnce(async (input: { name: string }) => ({ + id: 'inv_spawn_1', action: 'spawn', status: 'completed', nodeId: 'node_1', nodeName: 'desktop-1', + name: input.name, agentId: 'agent_1', spawned: true, ready: false, + }) as never); + f.runtime.inbox.inbox.subscribe = async function* () { throw new Error('inbox must not be read'); }; + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('valid ready identity'); + expect(f.release).toHaveBeenCalledOnce(); + }); + + it('refuses malformed cleanup identity from a custom port without issuing a release', async () => { + const dataDir = await dataDirectory('flows-node-malformed-cleanup-'); + const f = fixture(); + f.spawn.mockRejectedValueOnce(new NodeSpawnError('custom port failed', + 'spawn_unconfirmed', undefined, { id: 'inv_spawn_1' } as never)); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('custom port failed'); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('ambiguous pre-ack'); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).not.toHaveBeenCalled(); + }); + + it('rejects a resolved but unconfirmed release and keeps the result unacked', async () => { + const dataDir = await dataDirectory('flows-node-cleanup-negative-receipt-'); + const f = fixture(); + f.release.mockResolvedValueOnce({ id: 'inv_release_1', action: 'release', status: 'failed', + spawned: true, ready: true, released: false, errorCode: 'release_refused' }); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toMatchObject({ cleanupPending: true }); + expect(f.ack).not.toHaveBeenCalled(); + const [claim] = await readNodeSpawnClaims(dataDir); + expect(claim!.claim.cleanupPending).toMatchObject({ attempts: 1, lastError: expect.any(String) }); + expect(claim!.claim.release).toBeUndefined(); + }); + + it('fails closed on an incomplete durable result instead of replaying or spawning', async () => { + const dataDir = await dataDirectory('flows-node-invalid-claim-'); + const f = fixture(); + await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + const [entry] = await readNodeSpawnClaims(dataDir); + const result = entry!.claim.result!; + const { envelope: _envelope, ...incomplete } = result; + await recordNodeSpawn(entry!.path, { ...entry!.claim, + result: incomplete as unknown as typeof result }); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('Invalid durable node-spawn claim'); + expect(f.spawn).toHaveBeenCalledOnce(); + }); + + it('fails closed on a corrupt durable spawn identity before attempting release', async () => { + const dataDir = await dataDirectory('flows-node-invalid-spawn-'); + const f = fixture(); + await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + const [entry] = await readNodeSpawnClaims(dataDir); + const spawn = entry!.claim.spawn!; + const { agentId: _agentId, ...incomplete } = spawn; + await recordNodeSpawn(entry!.path, { ...entry!.claim, + spawn: incomplete as unknown as typeof spawn, release: undefined }); + f.release.mockClear(); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('Invalid durable node-spawn claim'); + expect(f.release).not.toHaveBeenCalled(); + }); + + it('fails closed on an unconfirmed durable release before replaying a result', async () => { + const dataDir = await dataDirectory('flows-node-invalid-release-'); + const f = fixture(); + await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + const [entry] = await readNodeSpawnClaims(dataDir); + await recordNodeSpawn(entry!.path, { ...entry!.claim, + release: { ...entry!.claim.release!, status: 'failed' } }); + f.release.mockClear(); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('Invalid durable node-spawn claim'); + expect(f.release).not.toHaveBeenCalled(); + }); + + it('reconciles a journaled result left unacked by a crashed worker', async () => { + const dataDir = await dataDirectory('flows-node-settlement-crash-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + journalRead: vi.fn(async () => ({ entries: [terminalEntry()] })), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_recovery', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + await worker.attach(); + expect(f.ack).toHaveBeenCalledOnce(); + const claims = await readNodeSpawnClaims(dataDir); + expect(claims[0]!.claim.result?.settlement).toMatchObject({ settledAs: 'ack' }); + await worker.close(); + }); + + it('finds a terminal relay result beyond the first journal page before acknowledging', async () => { + const dataDir = await dataDirectory('flows-node-settlement-paged-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + const filler = Array.from({ length: 256 }, (_, index) => ({ + seq: index + 1, segment_id: 0, entry_type: 'channel.appended', run_id: 'run_1', + step_id: null, attempt: null, at_ms: Date.now(), payload: {}, + })); + const journalRead = vi.fn(async (_runId: string, fromSeq: number, limit?: number) => ({ + entries: fromSeq === 1 ? filler.slice(0, limit) : [{ ...terminalEntry(), seq: 257 }], + })); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + journalRead, + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_recovery', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + await worker.attach(); + expect(journalRead).toHaveBeenNthCalledWith(1, 'run_1', 1, 256); + expect(journalRead).toHaveBeenNthCalledWith(2, 'run_1', 257, 256); + expect(f.ack).toHaveBeenCalledOnce(); + const claims = await readNodeSpawnClaims(dataDir); + expect(claims[0]!.claim.result?.settlement).toMatchObject({ settledAs: 'ack' }); + await worker.close(); + }); + + it('fails a delivery when the journal terminal belongs to different relay evidence', async () => { + const dataDir = await dataDirectory('flows-node-settlement-mismatch-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + const mismatch = { + completionReason: 'success', + trajectory_tail: { relay_task: { task_execution: { + delivery_id: 'different_delivery', message_id: 'different_message', + } } }, + }; + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + journalRead: vi.fn(async () => ({ entries: [terminalEntry(mismatch)] })), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_recovery', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + await worker.attach(); + expect(f.ack).not.toHaveBeenCalled(); + expect(f.fail).toHaveBeenCalledWith(expect.objectContaining({ inboxItemId: 'delivery_1', retry: false })); + const claims = await readNodeSpawnClaims(dataDir); + expect(claims[0]!.claim.result?.settlement).toMatchObject({ settledAs: 'fail' }); + await worker.close(); + }); + + it('dead-letters matching delivery ids attached to a different dispatch', async () => { + const dataDir = await dataDirectory('flows-node-settlement-dispatch-mismatch-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + const mismatch = journaledRelayResult(); + const trajectory = mismatch.trajectory_tail as { relay_task: { task_execution: Record } }; + trajectory.relay_task.task_execution.dispatch_id = 'dispatch_different1'; + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + journalRead: vi.fn(async () => ({ entries: [terminalEntry(mismatch)] })), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_recovery', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + await worker.attach(); + expect(f.ack).not.toHaveBeenCalled(); + expect(f.fail).toHaveBeenCalledOnce(); + const claims = await readNodeSpawnClaims(dataDir); + expect(claims[0]!.claim.result?.settlement).toMatchObject({ settledAs: 'fail' }); + await worker.close(); + }); + + it('retries an idempotent inbox settlement when ack preceded its durable record', async () => { + const dataDir = await dataDirectory('flows-node-settlement-record-crash-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + await received.delivery.ack(); + const [entry] = await readNodeSpawnClaims(dataDir); + const result = entry!.claim.result!; + await recordNodeSpawn(entry!.path, { ...entry!.claim, result: { ...result, + settlement: { ...result.settlement!, settledAt: undefined, settledAs: undefined } } }); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + journalRead: vi.fn(async () => ({ entries: [terminalEntry()] })), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_recovery', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + await worker.attach(); + expect(f.ack).toHaveBeenCalledTimes(2); + await worker.close(); + }); + + it('replays an unjournaled result on a later kernel attempt without another spawn', async () => { + const dataDir = await dataDirectory('flows-node-settlement-redispatch-'); + const f = fixture(); + const first = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await first.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + const replayed = await runAgentRelayTask({ ...request(dataDir), attempt: 2 }, { runtime: f.runtime }); + await replayed.delivery.prepareSettlement!({ attempt: 2, completionReason: 'success' }); + const claims = await readNodeSpawnClaims(dataDir); + expect(claims[0]!.claim.result?.settlement).toMatchObject({ attempt: 2, completionReason: 'success' }); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).toHaveBeenCalledOnce(); + }); + + it('bounds a stalled inbox settlement and leaves its durable intent pending', async () => { + const dataDir = await dataDirectory('flows-node-settlement-stall-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + f.ack.mockImplementationOnce(() => new Promise(() => undefined)); + vi.useFakeTimers(); + try { + const pending = expect(received.delivery.ack()) + .rejects.toMatchObject({ code: 'result_settlement_unconfirmed' }); + await vi.advanceTimersByTimeAsync(10_001); + await pending; + const claims = await readNodeSpawnClaims(dataDir); + expect(claims[0]!.claim.result?.settlement).not.toHaveProperty('settledAt'); + } finally { + vi.useRealTimers(); + } + }); + + it('coalesces concurrent settlement callers onto one inbox operation', async () => { + const dataDir = await dataDirectory('flows-node-settlement-coalesce-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + let confirm!: () => void; + f.ack.mockImplementationOnce(() => new Promise(resolve => { confirm = resolve; })); + const first = received.delivery.ack(); + const second = received.delivery.ack(); + expect(first).toBe(second); + expect(f.ack).toHaveBeenCalledOnce(); + confirm(); + await expect(Promise.all([first, second])).resolves.toEqual([undefined, undefined]); + }); + + it('retries a rejected post-journal ack through the durable settlement outbox', async () => { + const dataDir = await dataDirectory('flows-node-settlement-reject-'); + const f = fixture(); + f.ack.mockRejectedValueOnce(new Error('temporary inbox outage')); + let completed = false; + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), + stepComplete: vi.fn(async () => { completed = true; return {}; }), + journalRead: vi.fn(async () => ({ entries: completed ? [terminalEntry()] : [] })), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_1', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + const errors: unknown[] = []; + worker.on('error', error => errors.push(error)); + await worker.attach(); + client.emit('step.dispatch', dispatch()); + await worker.close(); + expect(errors).toEqual([expect.objectContaining({ message: 'temporary inbox outage' })]); + expect(f.ack).toHaveBeenCalledTimes(2); + const claims = await readNodeSpawnClaims(dataDir); + expect(claims[0]!.claim.result?.settlement).toMatchObject({ settledAs: 'ack' }); + }); + + it('keeps retrying a journaled settlement after repeated inbox outages', async () => { + vi.useFakeTimers(); + try { + const dataDir = await dataDirectory('flows-node-settlement-repeat-'); + const f = fixture(); + f.ack.mockRejectedValueOnce(new Error('first inbox outage')) + .mockRejectedValueOnce(new Error('second inbox outage')); + let completed = false; + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), + stepComplete: vi.fn(async () => { completed = true; return {}; }), + journalRead: vi.fn(async () => ({ entries: completed ? [terminalEntry()] : [] })), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_1', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + const errors: unknown[] = []; + worker.on('error', error => errors.push(error)); + await worker.attach(); + client.emit('step.dispatch', dispatch()); + await vi.waitFor(() => expect(f.ack).toHaveBeenCalledTimes(2)); + expect(errors).toHaveLength(2); + await vi.advanceTimersByTimeAsync(5_001); + await vi.waitFor(() => expect(f.ack).toHaveBeenCalledTimes(3)); + const claims = await readNodeSpawnClaims(dataDir); + expect(claims[0]!.claim.result?.settlement).toMatchObject({ settledAs: 'ack' }); + await worker.close(); + } finally { + vi.useRealTimers(); + } + }); +}); + +function dispatch(): StepDispatchEvent { + return { + run_id: 'run_1', step_id: 'edit', attempt: 1, step_type: 'agent', + spec: { cli: 'codex', instruction: 'Implement the change', transport: 'relay' }, + idempotency_key: 'dispatch_1234567890', lease_id: 'lease_1', + lease_deadline_ms: Date.now() + 30_000, pins: {}, + }; +} diff --git a/packages/sdk/tests/worker-node-runtime-sharing.test.ts b/packages/sdk/tests/worker-node-runtime-sharing.test.ts index 7ace31043..6a3bb82e0 100644 --- a/packages/sdk/tests/worker-node-runtime-sharing.test.ts +++ b/packages/sdk/tests/worker-node-runtime-sharing.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { NODE_RESULT_PREFIX } from '../src/node-spawn-result.js'; +import { runAgentRelayTask } from '../src/agent-relay-transport.js'; import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; import type { NodeSpawnPort } from '../src/node-spawn.js'; import type { JournalClient } from '../src/journal-client.js'; @@ -26,6 +27,66 @@ afterEach(async () => { }); describe('AgentWorker node-spawn runtime ownership', () => { + it('closes the shared runtime even when final durable inbox settlement still fails', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-shared-node-runtime-settlement-failure-')); + roots.push(dataDir); + const ack = vi.fn(async () => { throw new Error('inbox unavailable'); }); + let spawnedName = ''; + const runtime: NodeSpawnRuntime = { + publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + port: { + targets: async () => [], + spawn: async input => { + spawnedName = input.name; + return { id: 'spawn_1', action: 'spawn', status: 'completed', + nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, agentId: 'agent_1', + spawned: true, ready: true }; + }, + status: vi.fn(), + release: async () => ({ id: 'release_1', action: 'release', status: 'completed', + spawned: false, ready: false, released: true }), + }, + inbox: { inbox: { + subscribe: async function* () { + yield { id: 'delivery_1', message: { id: 'message_1', + from: { id: 'agent_1', name: spawnedName }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_1', + step_id: 'edit', dispatch_id: 'dispatch_1234567890', status: 'completed', + output: { summary: 'done' } }) } }; + }, + ack, + fail: vi.fn(async () => ({})), + } }, + }; + const received = await runAgentRelayTask({ + cli: 'codex', task: 'Implement the change', runId: 'run_1', stepId: 'edit', + idempotencyKey: 'dispatch_1234567890', attempt: 1, dataDir, + }, { runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + owner.get.mockResolvedValue(runtime); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + journalRead: vi.fn(async () => ({ entries: [{ + seq: 2, segment_id: 0, entry_type: 'step.completed', run_id: 'run_1', + step_id: 'edit', attempt: 1, at_ms: Date.now(), payload: { + completionReason: 'success', trajectory_tail: { relay_task: { task_execution: { + run_id: 'run_1', step_id: 'edit', dispatch_id: 'dispatch_1234567890', + delivery_id: 'delivery_1', message_id: 'message_1', + } } }, + }, + }] })), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_recovery', pins: {}, dataDir, + }); + await expect(worker.attach()).rejects.toThrow('inbox unavailable'); + expect(owner.close).toHaveBeenCalledOnce(); + await expect(worker.close()).resolves.toBeUndefined(); + expect(ack).toHaveBeenCalledOnce(); + expect(owner.close).toHaveBeenCalledOnce(); + }); + it('shares one runtime and one concurrent close promise through dispatch drain', async () => { const dataDir = await mkdtemp(join(tmpdir(), 'flows-shared-node-runtime-')); roots.push(dataDir); From 74537c989f3951a3811359644558452b292a9859 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 8 Nov 2026 21:02:44 -0800 Subject: [PATCH 24/75] fix(sdk): reconcile relay settlements after dispatch drain --- packages/sdk/src/worker.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index a788e92ac..b25134bd5 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -169,14 +169,17 @@ export class AgentWorker extends EventEmitter { private readonly onDispatch = (dispatch: StepDispatchEvent): void => { if (this.closing) return; if (dispatch.step_type !== 'agent') return; - const relayTransport = (dispatch.spec as Partial).transport === 'relay'; const running: Promise = this.execute(dispatch).catch((error: unknown) => { this.emit('error', error, dispatch); }); this.inFlight.add(running); void running.finally(() => { this.inFlight.delete(running); - if (relayTransport) { + // A task's own ack settles its durable outbox entry after stepComplete. + // Reconcile only after the dispatch drain: scanning while another relay + // task is between its journal append and ack can race that task's own + // settlement and issue the same inbox operation twice. + if (this.inFlight.size === 0) { void this.reconcileRelaySettlements().catch(error => this.emit('error', error)); } }); From de6dbe889e1d7b4c7631bb3b16c4a2f99c771225 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 8 Nov 2026 21:09:50 -0800 Subject: [PATCH 25/75] fix(sdk): gate dispatches behind settlement recovery --- packages/sdk/src/worker.ts | 16 ++++- .../tests/worker-node-runtime-sharing.test.ts | 72 +++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index b25134bd5..bbb5f1e54 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -169,7 +169,7 @@ export class AgentWorker extends EventEmitter { private readonly onDispatch = (dispatch: StepDispatchEvent): void => { if (this.closing) return; if (dispatch.step_type !== 'agent') return; - const running: Promise = this.execute(dispatch).catch((error: unknown) => { + const running: Promise = this.executeAfterSettlementReconcile(dispatch).catch((error: unknown) => { this.emit('error', error, dispatch); }); this.inFlight.add(running); @@ -185,6 +185,16 @@ export class AgentWorker extends EventEmitter { }); }; + private async executeAfterSettlementReconcile(dispatch: StepDispatchEvent): Promise { + // A dispatch that arrives after a drain can race the drain's asynchronous + // claim scan. Keep it in the drain count, but do not let it prepare or + // settle a new delivery until the existing scan has finished. + while (this.settlementReconcile !== undefined) { + await this.settlementReconcile; + } + await this.execute(dispatch); + } + private async execute(dispatch: StepDispatchEvent): Promise { const spec = dispatch.spec as Partial; const helper = helperCall(spec); @@ -307,6 +317,10 @@ export class AgentWorker extends EventEmitter { private async reconcileRelaySettlements(): Promise { if (this.settlementReconcile !== undefined) return this.settlementReconcile; + if (this.inFlight.size > 0) { + this.scheduleSettlementReconcile(); + return; + } const running = this.reconcileRelaySettlementsOnce(); this.settlementReconcile = running; void running.finally(() => { diff --git a/packages/sdk/tests/worker-node-runtime-sharing.test.ts b/packages/sdk/tests/worker-node-runtime-sharing.test.ts index 6a3bb82e0..a2cf53270 100644 --- a/packages/sdk/tests/worker-node-runtime-sharing.test.ts +++ b/packages/sdk/tests/worker-node-runtime-sharing.test.ts @@ -221,6 +221,78 @@ describe('AgentWorker node-spawn runtime ownership', () => { expect(order.at(-1)).toBe('close'); }); + it('holds a new dispatch behind settlement reconciliation from the prior drain', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-shared-node-runtime-reconcile-barrier-')); + roots.push(dataDir); + const pending = dispatch('edit-after-drain', 'dispatch_afterdrain'); + let spawnedName = ''; + const spawn = vi.fn(async input => { + spawnedName = input.name; + return { id: 'spawn_after_drain', action: 'spawn', status: 'completed', + nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, agentId: 'agent_after_drain', + spawned: true, ready: true }; + }); + const runtime: NodeSpawnRuntime = { + publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + port: { + targets: async () => [], + spawn, + status: vi.fn(), + release: vi.fn(async () => ({ + id: 'release_after_drain', action: 'release', status: 'completed', + spawned: false, ready: false, released: true, + })), + }, + inbox: { inbox: { + subscribe: async function* () { + yield { id: 'delivery_after_drain', message: { id: 'message_after_drain', + from: { id: 'agent_after_drain', name: spawnedName }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: pending.run_id, + step_id: pending.step_id, dispatch_id: pending.idempotency_key, + status: 'completed', output: { step: pending.step_id } }) } }; + }, + ack: vi.fn(async () => ({})), + fail: vi.fn(async () => ({})), + } }, + }; + owner.get.mockResolvedValue(runtime); + const stepHeartbeat = vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat, + stepComplete: vi.fn(async () => ({})), + supportsFeature: vi.fn(() => true), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_reconcile_barrier', pins: {}, dataDir, + }); + const errors: unknown[] = []; + worker.on('error', error => errors.push(error)); + await worker.attach(); + + const state = worker as unknown as { settlementReconcile?: Promise }; + let finishReconcile!: () => void; + state.settlementReconcile = new Promise(resolve => { + finishReconcile = () => { + state.settlementReconcile = undefined; + resolve(); + }; + }); + client.emit('step.dispatch', pending); + await Promise.resolve(); + await Promise.resolve(); + expect(stepHeartbeat).not.toHaveBeenCalled(); + expect(spawn).not.toHaveBeenCalled(); + + const closing = worker.close(); + finishReconcile(); + await closing; + expect(errors).toEqual([]); + expect(stepHeartbeat).toHaveBeenCalledOnce(); + expect(spawn).toHaveBeenCalledOnce(); + }); + it('drains a six-task mixed-order fan with one lease cancellation before closing the shared runtime', async () => { const dataDir = await mkdtemp(join(tmpdir(), 'flows-shared-node-runtime-fan-')); roots.push(dataDir); From e07465c906be6fe3a600f52ffc72fbe52fd63af3 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 8 Nov 2026 21:22:07 -0800 Subject: [PATCH 26/75] fix(sdk): gate settlement retries by dispatch generation --- packages/sdk/src/worker.ts | 35 ++++++++++++------- .../tests/worker-node-runtime-sharing.test.ts | 23 ++++++------ 2 files changed, 35 insertions(+), 23 deletions(-) diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index bbb5f1e54..7480d0ecc 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -71,6 +71,7 @@ export class AgentWorker extends EventEmitter { private closing = false; private closePromise?: Promise; private settlementTimer?: NodeJS.Timeout; + private settlementDrain?: Promise; private settlementReconcile?: Promise; private readonly inFlight: Set> = new Set(); private readonly nodeSpawnRuntimeOwner: NodeSpawnRuntimeOwner; @@ -88,7 +89,7 @@ export class AgentWorker extends EventEmitter { if (this.closing) throw new Error('agent worker: cannot attach a closed worker (construct a new one)'); this.client.on('step.dispatch', this.onDispatch); try { - await this.reconcileRelaySettlements(); + await this.reconcileRelaySettlementsAfterDrain(); await this.client.workerAttach( this.options.workerId, ['agent'], @@ -148,7 +149,7 @@ export class AgentWorker extends EventEmitter { } let settlementError: unknown; try { - await this.reconcileRelaySettlements(); + await this.reconcileRelaySettlementsAfterDrain(); } catch (error) { settlementError = error; } @@ -169,7 +170,7 @@ export class AgentWorker extends EventEmitter { private readonly onDispatch = (dispatch: StepDispatchEvent): void => { if (this.closing) return; if (dispatch.step_type !== 'agent') return; - const running: Promise = this.executeAfterSettlementReconcile(dispatch).catch((error: unknown) => { + const running: Promise = this.executeAfterSettlementDrain(dispatch).catch((error: unknown) => { this.emit('error', error, dispatch); }); this.inFlight.add(running); @@ -180,17 +181,17 @@ export class AgentWorker extends EventEmitter { // task is between its journal append and ack can race that task's own // settlement and issue the same inbox operation twice. if (this.inFlight.size === 0) { - void this.reconcileRelaySettlements().catch(error => this.emit('error', error)); + void this.reconcileRelaySettlementsAfterDrain().catch(error => this.emit('error', error)); } }); }; - private async executeAfterSettlementReconcile(dispatch: StepDispatchEvent): Promise { + private async executeAfterSettlementDrain(dispatch: StepDispatchEvent): Promise { // A dispatch that arrives after a drain can race the drain's asynchronous // claim scan. Keep it in the drain count, but do not let it prepare or // settle a new delivery until the existing scan has finished. - while (this.settlementReconcile !== undefined) { - await this.settlementReconcile; + while (this.settlementDrain !== undefined) { + await this.settlementDrain; } await this.execute(dispatch); } @@ -315,12 +316,22 @@ export class AgentWorker extends EventEmitter { await relayDelivery?.ack(); } + private reconcileRelaySettlementsAfterDrain(): Promise { + if (this.settlementDrain !== undefined) return this.settlementDrain; + const generation = Array.from(this.inFlight); + const running = (async () => { + if (generation.length > 0) await Promise.allSettled(generation); + await this.reconcileRelaySettlements(); + })(); + this.settlementDrain = running; + void running.finally(() => { + if (this.settlementDrain === running) this.settlementDrain = undefined; + }).catch(() => undefined); + return running; + } + private async reconcileRelaySettlements(): Promise { if (this.settlementReconcile !== undefined) return this.settlementReconcile; - if (this.inFlight.size > 0) { - this.scheduleSettlementReconcile(); - return; - } const running = this.reconcileRelaySettlementsOnce(); this.settlementReconcile = running; void running.finally(() => { @@ -354,7 +365,7 @@ export class AgentWorker extends EventEmitter { if (this.closing || this.settlementTimer !== undefined) return; this.settlementTimer = setTimeout(() => { this.settlementTimer = undefined; - void this.reconcileRelaySettlements().catch(error => this.emit('error', error)); + void this.reconcileRelaySettlementsAfterDrain().catch(error => this.emit('error', error)); }, 5_000); this.settlementTimer.unref(); } diff --git a/packages/sdk/tests/worker-node-runtime-sharing.test.ts b/packages/sdk/tests/worker-node-runtime-sharing.test.ts index a2cf53270..ae2cc6282 100644 --- a/packages/sdk/tests/worker-node-runtime-sharing.test.ts +++ b/packages/sdk/tests/worker-node-runtime-sharing.test.ts @@ -221,7 +221,7 @@ describe('AgentWorker node-spawn runtime ownership', () => { expect(order.at(-1)).toBe('close'); }); - it('holds a new dispatch behind settlement reconciliation from the prior drain', async () => { + it('closes admission while a due settlement retry drains its current generation', async () => { const dataDir = await mkdtemp(join(tmpdir(), 'flows-shared-node-runtime-reconcile-barrier-')); roots.push(dataDir); const pending = dispatch('edit-after-drain', 'dispatch_afterdrain'); @@ -271,14 +271,14 @@ describe('AgentWorker node-spawn runtime ownership', () => { worker.on('error', error => errors.push(error)); await worker.attach(); - const state = worker as unknown as { settlementReconcile?: Promise }; - let finishReconcile!: () => void; - state.settlementReconcile = new Promise(resolve => { - finishReconcile = () => { - state.settlementReconcile = undefined; - resolve(); - }; - }); + const state = worker as unknown as { + inFlight: Set>; + reconcileRelaySettlementsAfterDrain(): Promise; + }; + let finishCurrentGeneration!: () => void; + const currentGeneration = new Promise(resolve => { finishCurrentGeneration = resolve; }); + state.inFlight.add(currentGeneration); + const reconciliation = state.reconcileRelaySettlementsAfterDrain(); client.emit('step.dispatch', pending); await Promise.resolve(); await Promise.resolve(); @@ -286,8 +286,9 @@ describe('AgentWorker node-spawn runtime ownership', () => { expect(spawn).not.toHaveBeenCalled(); const closing = worker.close(); - finishReconcile(); - await closing; + state.inFlight.delete(currentGeneration); + finishCurrentGeneration(); + await Promise.all([reconciliation, closing]); expect(errors).toEqual([]); expect(stepHeartbeat).toHaveBeenCalledOnce(); expect(spawn).toHaveBeenCalledOnce(); From fe7d2e0ad51c0e9f892582a3cb0afb4e38b7cca0 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 8 Nov 2026 22:17:08 -0800 Subject: [PATCH 27/75] fix(sdk): scope node results by spawn invocation --- packages/sdk/src/agent-relay-transport.ts | 1 + packages/sdk/src/node-spawn-result.ts | 6 ++-- .../sdk/tests/agent-relay-hardening.test.ts | 33 ++++++++++--------- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/packages/sdk/src/agent-relay-transport.ts b/packages/sdk/src/agent-relay-transport.ts index dc9e7f714..f5449e570 100644 --- a/packages/sdk/src/agent-relay-transport.ts +++ b/packages/sdk/src/agent-relay-transport.ts @@ -248,6 +248,7 @@ async function executeAgentRelayTask( ? await waitForNodeResult(runtime.inbox, { agentId: spawn.agentId, agentName: spawn.name, + invocationId: spawn.id, ...correlation, }, resultSignal) : undefined; diff --git a/packages/sdk/src/node-spawn-result.ts b/packages/sdk/src/node-spawn-result.ts index 0312b1dfa..9b2dd6538 100644 --- a/packages/sdk/src/node-spawn-result.ts +++ b/packages/sdk/src/node-spawn-result.ts @@ -27,7 +27,7 @@ interface InboxItem { export interface NodeResultInboxClient { inbox: { - subscribe(input?: { signal?: AbortSignal }): AsyncIterable; + subscribe(input?: { signal?: AbortSignal; invocationId?: string }): AsyncIterable; ack(input: { inboxItemId: string; state?: 'delivered' | 'read'; metadata?: Record }): Promise; fail(input: { inboxItemId: string; error: string; retry?: boolean; metadata?: Record }): Promise; }; @@ -61,10 +61,10 @@ export function nodeResultInstruction(correlation: NodeResultCorrelation): strin export async function waitForNodeResult( client: NodeResultInboxClient, - expected: { agentId: string; agentName: string } & NodeResultCorrelation, + expected: { agentId: string; agentName: string; invocationId: string } & NodeResultCorrelation, signal?: AbortSignal, ): Promise { - const iterator = client.inbox.subscribe({ signal })[Symbol.asyncIterator](); + const iterator = client.inbox.subscribe({ signal, invocationId: expected.invocationId })[Symbol.asyncIterator](); let closing: Promise | undefined; const close = (): Promise => { closing ??= Promise.resolve().then(() => iterator.return?.()).then(() => undefined); diff --git a/packages/sdk/tests/agent-relay-hardening.test.ts b/packages/sdk/tests/agent-relay-hardening.test.ts index 7e212459a..96f81ba91 100644 --- a/packages/sdk/tests/agent-relay-hardening.test.ts +++ b/packages/sdk/tests/agent-relay-hardening.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from 'vitest'; import { readNodeResult, waitForNodeResult, NODE_RESULT_PREFIX } from '../src/node-spawn-result.js'; const correlation = { runId: 'run_1', stepId: 'step_1', dispatchId: 'dispatch_1234567890' }; +const expected = { agentId: 'agent_1', agentName: 'worker', invocationId: 'inv_1', ...correlation }; describe('node-spawn result hardening', () => { it('requires immutable attempt correlation and terminal evidence', () => { @@ -18,19 +19,21 @@ describe('node-spawn result hardening', () => { it('ignores a forged name match without the exact immutable sender id', async () => { const ack = vi.fn(); + const subscribe = vi.fn(async function* () { + for (const [id, sender] of [['forged', 'attacker'], ['real', 'agent_1']] as const) yield { + id, message: { id: `message_${id}`, from: { id: sender, name: 'worker' }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_1', step_id: 'step_1', + dispatch_id: correlation.dispatchId, status: 'completed', output: { id } }) }, + }; + }); const delivery = await waitForNodeResult({ inbox: { - subscribe: async function* () { - for (const [id, sender] of [['forged', 'attacker'], ['real', 'agent_1']] as const) yield { - id, message: { id: `message_${id}`, from: { id: sender, name: 'worker' }, - text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_1', step_id: 'step_1', - dispatch_id: correlation.dispatchId, status: 'completed', output: { id } }) }, - }; - }, + subscribe, ack: async input => { ack(input); }, fail: async () => {}, }, - }, { agentId: 'agent_1', agentName: 'worker', ...correlation }); + }, expected); expect(delivery.envelope.output).toEqual({ id: 'real' }); + expect(subscribe).toHaveBeenCalledWith(expect.objectContaining({ invocationId: 'inv_1' })); expect(ack).not.toHaveBeenCalled(); }); @@ -40,7 +43,7 @@ describe('node-spawn result hardening', () => { subscribe: async function* () { yield { id: 'delivery_bad', message: { id: 'message_bad', from: { id: 'agent_1', name: 'worker' }, text: NODE_RESULT_PREFIX + '{bad' } }; }, ack: async () => ({}), fail, - } }, { agentId: 'agent_1', agentName: 'worker', ...correlation }); + } }, expected); expect(delivery.envelope.status).toBe('failed'); expect(fail).not.toHaveBeenCalled(); await delivery.ack(); @@ -65,7 +68,7 @@ describe('node-spawn result hardening', () => { return: returned, }) }), ack, fail: async () => ({}), - } }, { agentId: 'agent_1', agentName: 'worker', ...correlation }); + } }, expected); expect(returned).toHaveBeenCalledOnce(); expect(ack).not.toHaveBeenCalled(); await delivery.ack(); @@ -84,7 +87,7 @@ describe('node-spawn result hardening', () => { return: returned, }) }), ack: async () => ({}), fail: async () => ({}), - } }, { agentId: 'agent_1', agentName: 'worker', ...correlation }); + } }, expected); expect(delivery.envelope.output).toEqual({ ok: true }); expect(returned).toHaveBeenCalledOnce(); }); @@ -101,7 +104,7 @@ describe('node-spawn result hardening', () => { return: returned, }) }), ack: async () => ({}), fail: async () => ({}), - } }, { agentId: 'agent_1', agentName: 'worker', ...correlation }); + } }, expected); expect(delivery.envelope.output).toEqual({ ok: true }); expect(returned).toHaveBeenCalledOnce(); }); @@ -115,7 +118,7 @@ describe('node-spawn result hardening', () => { return: async () => { returned(); return { done: true, value: undefined }; }, }) }), ack: async () => ({}), fail: async () => ({}), - } }, { agentId: 'agent_1', agentName: 'worker', ...correlation }, controller.signal); + } }, expected, controller.signal); controller.abort(new Error('lease lost')); await expect(waiting).rejects.toThrow('lease lost'); expect(returned).toHaveBeenCalledOnce(); @@ -130,7 +133,7 @@ describe('node-spawn result hardening', () => { return: returned, }) }), ack: async () => ({}), fail: async () => ({}), - } }, { agentId: 'agent_1', agentName: 'worker', ...correlation }, controller.signal); + } }, expected, controller.signal); controller.abort(new Error('lease lost')); await expect(waiting).rejects.toThrow('lease lost'); expect(returned).toHaveBeenCalledOnce(); @@ -145,7 +148,7 @@ describe('node-spawn result hardening', () => { return: returned, }) }), ack: async () => ({}), fail: async () => ({}), - } }, { agentId: 'agent_1', agentName: 'worker', ...correlation }, controller.signal); + } }, expected, controller.signal); controller.abort(new Error('lease lost')); await expect(waiting).rejects.toThrow('lease lost'); expect(returned).toHaveBeenCalledOnce(); From aacb6a17a768d21d72a15852e71c6851a1b4020f Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 8 Nov 2026 23:01:57 -0800 Subject: [PATCH 28/75] feat(sdk): bind local node spawn to Desktop callers --- docs/MIGRATION-NODE-SPAWN.md | 30 +- packages/sdk/package-lock.json | 4 + packages/sdk/package.json | 4 + packages/sdk/src/agent-relay-transport.ts | 9 +- packages/sdk/src/cli-commands.ts | 4 + packages/sdk/src/cli.ts | 25 +- packages/sdk/src/cli/direct-run.ts | 2 +- packages/sdk/src/cli/run.ts | 8 +- packages/sdk/src/index.ts | 10 +- packages/sdk/src/local-agent.ts | 3 +- packages/sdk/src/node-spawn-local.ts | 347 +++++++++--------- packages/sdk/src/node-spawn-result.ts | 48 ++- packages/sdk/src/node-spawn-runtime.ts | 210 ++++++++++- packages/sdk/src/worker.ts | 15 +- packages/sdk/tests/node-spawn-ports.test.ts | 103 +++--- packages/sdk/tests/node-spawn-runtime.test.ts | 260 ++++++++++++- packages/sdk/tests/relay-cli-surface.test.ts | 6 +- packages/sdk/tests/worker-slots.test.ts | 21 ++ 18 files changed, 853 insertions(+), 256 deletions(-) diff --git a/docs/MIGRATION-NODE-SPAWN.md b/docs/MIGRATION-NODE-SPAWN.md index f210b325a..650b697cc 100644 --- a/docs/MIGRATION-NODE-SPAWN.md +++ b/docs/MIGRATION-NODE-SPAWN.md @@ -26,19 +26,31 @@ harness. Use `RELAYFLOW_NODE_NAME` for an exact node or - `RELAYFLOW_RELAY_BROKER_BIN`, broker connection files, helper sockets, and broker health checks are no longer read. -There is deliberately no compatibility fallback to broker 17.2.1. Retained -local sessions and linked communication remain unavailable until Desktop ships -the approved core-bridge client package and authorized headless Flows caller. -The local adapter requires both the Desktop-issued caller name and secret on -invoke, status, and release; an operator-supplied or borrowed coding-session -name is not a substitute. +There is deliberately no compatibility fallback to broker 17.2.1. A local +Desktop run uses a separately approved schema-v2 service caller. Pass its +private file explicitly with `--service-caller-file /absolute/caller.json`, +the SDK `serviceCallerFile` option, or `RELAYFLOW_SERVICE_CALLER_FILE`. Flows +never scans Desktop's app-data directory. It securely parses and +challenge-authenticates the handoff through optional +`@agent-relay/core-bridge` 17.3 or later, then sends no caller name or secret +on spawn or inbox calls. The handoff's harness and directory scope is enforced +before dispatch. + +`RELAYFLOW_SERVICE_CALLER_FILE` and `RELAY_AGENT_TOKEN` are mutually exclusive: +the former is local Desktop authority and the latter is remote fleet authority. +Setting both fails closed. A caller file whose publisher or workspace differs +from the authenticated core identity is also refused; rotate or reprovision it +in Desktop rather than editing the file. ## Hosted and self-hosted execution Hosted workers use the agent-token fleet path. Self-hosted workers use the -same path for remote nodes. Selecting the local Desktop bridge is withheld -until the approved client and service-caller contract land in a Desktop -release; the bridge adapter remains fail-closed in the meantime. +same path for remote nodes. The local Desktop mode requires a Desktop release +whose `hello.methods` contains the complete service-caller challenge, identity, +spawn, invocation-scoped inbox, settlement, close and disconnect contract. +Each result subscription uses its own authenticated connection. Flows closes +that iterator and connection before settling the item through another fresh +authenticated connection. Older Desktop builds fail closed before dispatch. Self-hosted Flows licensing is owned by relaycontrol's signed offline licence design. This SDK does not invent an entitlement store; deployments must fail diff --git a/packages/sdk/package-lock.json b/packages/sdk/package-lock.json index 9dcac116e..53b457943 100644 --- a/packages/sdk/package-lock.json +++ b/packages/sdk/package-lock.json @@ -36,9 +36,13 @@ "vitest": "^2.1.0" }, "peerDependencies": { + "@agent-relay/core-bridge": ">=17.3.0 <18", "@agent-relay/sdk": ">=17.2.1 <18" }, "peerDependenciesMeta": { + "@agent-relay/core-bridge": { + "optional": true + }, "@agent-relay/sdk": { "optional": true } diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 2626f0d16..698f3dfb2 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -76,9 +76,13 @@ "directory": "packages/sdk" }, "peerDependencies": { + "@agent-relay/core-bridge": ">=17.3.0 <18", "@agent-relay/sdk": ">=17.2.1 <18" }, "peerDependenciesMeta": { + "@agent-relay/core-bridge": { + "optional": true + }, "@agent-relay/sdk": { "optional": true } diff --git a/packages/sdk/src/agent-relay-transport.ts b/packages/sdk/src/agent-relay-transport.ts index f5449e570..1f5fcd7ed 100644 --- a/packages/sdk/src/agent-relay-transport.ts +++ b/packages/sdk/src/agent-relay-transport.ts @@ -3,7 +3,7 @@ import { claimNodeSpawn, recordNodeSpawn, type NodeSpawnClaim } from './node-spa import { nodeResultInstruction, waitForNodeResult } from './node-spawn-result.js'; import { durableNodeResultDelivery } from './node-spawn-settlement.js'; import { waitForNodeOperation, waitForNodeSpawn } from './node-spawn-deadline.js'; -import { createRemoteNodeSpawnRuntime, type NodeSpawnRuntime } from './node-spawn-runtime.js'; +import { createNodeSpawnRuntime, type NodeSpawnRuntime } from './node-spawn-runtime.js'; import { NodeSpawnError, nonempty, validIdempotencyKey, validNodeSpawnIdentity, validNodeSpawnReceipt, type NodeSpawnIdentity, type NodeSpawnReceipt } from './node-spawn.js'; @@ -27,6 +27,8 @@ export interface AgentRelayTaskRequest { export interface AgentRelayEnv { baseUrl?: string; agentToken?: string; + /** Absolute Desktop schema-v2 caller.json path; mutually exclusive with agentToken. */ + serviceCallerFile?: string; node?: string; repo?: string; sandboxOnly?: boolean; @@ -88,10 +90,13 @@ export async function runAgentRelayTask( } else if (options.runtimeProvider !== undefined) { runtime = await waitForNodeOperation(options.runtimeProvider(acquisitionSignal), acquisitionSignal); } else { - runtime = await createRemoteNodeSpawnRuntime({ + runtime = await createNodeSpawnRuntime({ ...process.env, ...(options.baseUrl === undefined ? {} : { RELAY_BASE_URL: options.baseUrl }), ...(options.agentToken === undefined ? {} : { RELAY_AGENT_TOKEN: options.agentToken }), + ...(options.serviceCallerFile === undefined + ? {} + : { RELAYFLOW_SERVICE_CALLER_FILE: options.serviceCallerFile }), ...(options.node === undefined ? {} : { RELAYFLOW_NODE_NAME: options.node }), ...(options.repo === undefined ? {} : { RELAYFLOW_NODE_REPO: options.repo }), ...(options.sandboxOnly === undefined diff --git a/packages/sdk/src/cli-commands.ts b/packages/sdk/src/cli-commands.ts index f7ea48e52..7d48525a1 100644 --- a/packages/sdk/src/cli-commands.ts +++ b/packages/sdk/src/cli-commands.ts @@ -95,6 +95,10 @@ const LOCAL_EXECUTION_OPTIONS = [ description: `With --local-agent, how many agent steps (and, separately, LLM steps) run at once (1-${MAX_LOCAL_AGENT_CAPACITY})`, defaultValue: String(DEFAULT_LOCAL_AGENT_CAPACITY), }, + { + flags: '--service-caller-file ', + description: 'With --local-agent, use this explicit Desktop schema-v2 service caller for relay agent steps', + }, { flags: '--no-spawn', description: 'Require a running relayflowd rather than starting one' }, { flags: '--no-observer-link', description: 'Do not mint an observer link for this run' }, { flags: '--cloud-mirror', description: 'Also put this run on the Cloud dashboard (also FLOWS_CLOUD_MIRROR=1)' }, diff --git a/packages/sdk/src/cli.ts b/packages/sdk/src/cli.ts index 3ba441e0b..71a1ba913 100644 --- a/packages/sdk/src/cli.ts +++ b/packages/sdk/src/cli.ts @@ -115,8 +115,8 @@ export type ParsedArgs = | { command: 'schedules'; json: boolean } | { command: 'unschedule'; scheduleId: string; json: boolean } | { command: 'check'; json: boolean; watch: boolean; explainWarnings: boolean; daemonValidation: DaemonValidationMode; value: string } - | { command: 'run'; detach?: boolean; bucket: string | undefined; reuseFromRunId: string | undefined; localAgent: boolean; agentCapacity: number | undefined; dataDir: string; input: string | undefined; json: boolean; spawn: boolean; noObserverLink: boolean; cloudMirror: boolean; allowHumanInfluenced: boolean; value: string } - | { command: 'resume'; detach?: boolean; localAgent: boolean; agentCapacity: number | undefined; dataDir: string; json: boolean; spawn: boolean; noObserverLink: boolean; cloudMirror: boolean; allowHumanInfluenced: boolean; value: string } + | { command: 'run'; detach?: boolean; bucket: string | undefined; reuseFromRunId: string | undefined; localAgent: boolean; agentCapacity: number | undefined; serviceCallerFile: string | undefined; dataDir: string; input: string | undefined; json: boolean; spawn: boolean; noObserverLink: boolean; cloudMirror: boolean; allowHumanInfluenced: boolean; value: string } + | { command: 'resume'; detach?: boolean; localAgent: boolean; agentCapacity: number | undefined; serviceCallerFile: string | undefined; dataDir: string; json: boolean; spawn: boolean; noObserverLink: boolean; cloudMirror: boolean; allowHumanInfluenced: boolean; value: string } | { command: 'cloud-answer'; runId: string; answer: boolean; json: boolean; note?: string; source?: string } | { command: 'answer'; dataDir: string; json: boolean; spawn: boolean; note: string | undefined; by: string | undefined; runId: string; waitId: string; answer: boolean } | RunsArgs @@ -151,14 +151,14 @@ const USAGE = [ 'flows run @sha256: [--bucket ] [--data-dir ] [--json] [--detach]', 'flows check [--against-daemon|--no-daemon-check] [--watch] [--json] [--explain-warnings] ', 'flows serve-webhook --data-dir --port

[--allow [,]]', - 'flows run [--json] [--no-spawn] [--no-observer-link] [--cloud-mirror] [--detach] [--data-dir

] [--local-agent [--agent-capacity ]] [--reuse-from ] ', + 'flows run [--json] [--no-spawn] [--no-observer-link] [--cloud-mirror] [--detach] [--data-dir ] [--local-agent [--agent-capacity ] [--service-caller-file ]] [--reuse-from ] ', 'flows run --cloud [--json] [--wait] [--sync-code] [--no-connect] [--budget-minutes ] ', 'flows run --cloud [--json] [--wait] [--sync-code] [--no-connect] [--budget-minutes ] --input ', 'flows sync [--json] [--dry-run] [--dir ] ', - 'flows run [--json] [--no-spawn] [--no-observer-link] [--cloud-mirror] [--detach] [--data-dir ] [--local-agent [--agent-capacity ]] --input ', + 'flows run [--json] [--no-spawn] [--no-observer-link] [--cloud-mirror] [--detach] [--data-dir ] [--local-agent [--agent-capacity ] [--service-caller-file ]] --input ', 'flows run [--input ]', 'flows tick start --schedule-id --interval-ms [--epoch-ms ] [--max-catch-up ] [--poll-interval-ms ] [--data-dir ] ', - 'flows resume [--allow-human-influenced] [--json] [--no-spawn] [--no-observer-link] [--cloud-mirror] [--detach] [--data-dir ] [--local-agent [--agent-capacity ]] ', + 'flows resume [--allow-human-influenced] [--json] [--no-spawn] [--no-observer-link] [--cloud-mirror] [--detach] [--data-dir ] [--local-agent [--agent-capacity ] [--service-caller-file ]] ', 'flows answer --cloud [--json] [--note ] [--source ] ', 'flows answer [--json] [--no-spawn] [--data-dir ] [--note ] [--by ] ', 'flows replay [--allow-human-influenced] [--json] [--data-dir ] [--at ]', @@ -445,6 +445,7 @@ export async function runCli( ...(parsed.command === 'run' && parsed.reuseFromRunId !== undefined ? { reuseFromRunId: parsed.reuseFromRunId } : {}), localAgent: parsed.localAgent, ...(parsed.agentCapacity === undefined ? {} : { agentCapacity: parsed.agentCapacity }), + ...(parsed.serviceCallerFile === undefined ? {} : { serviceCallerFile: parsed.serviceCallerFile }), onProgress: showProgress, ...(observer === undefined && mirror === undefined && options.detachedReceipt === undefined ? {} : { onJournalEntry: (entry: JournalEvent) => { @@ -761,6 +762,7 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { let localAgent = false; let detach = false; let agentCapacity: number | undefined; + let serviceCallerFile: string | undefined; let allowHumanInfluenced = false; let dataDir = DEFAULT_DATA_DIR; let sawDataDir = false; @@ -813,6 +815,12 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { if (!isAgentCapacity(agentCapacity)) return undefined; continue; } + if (argument === '--service-caller-file') { + const value = args[++index]; + if (command === 'check' || serviceCallerFile !== undefined || value === undefined || value.startsWith('-')) return undefined; + serviceCallerFile = value; + continue; + } if (argument === '--against-daemon' || argument === '--no-daemon-check') { if (command !== 'check' || daemonValidation !== undefined) return undefined; daemonValidation = argument === '--against-daemon' ? 'required' : 'off'; @@ -890,6 +898,7 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { if (positionals.length !== 1) return undefined; // It sizes the in-process worker, so without `--local-agent` it describes nothing. if (agentCapacity !== undefined && !localAgent) return undefined; + if (serviceCallerFile !== undefined && !localAgent) return undefined; if (bucket !== undefined && (cloud || !parseDigestReference(positionals[0]!))) return undefined; if (cloud) { @@ -898,7 +907,7 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { // observer-link opt-out -- describes nothing there and is refused rather // than ignored. `--input` is the authored body's argument and travels with // the source, so it is accepted exactly where a local run accepts it. - if (detach || allowHumanInfluenced || sawDataDir || !spawn || localAgent || noObserverLink || cloudMirror + if (detach || allowHumanInfluenced || sawDataDir || !spawn || localAgent || serviceCallerFile !== undefined || noObserverLink || cloudMirror || reuseFromRunId !== undefined) return undefined; if (sawInput && !isAuthoredFlowPath(positionals[0]!)) return undefined; return { command: 'cloud-run', value: positionals[0]!, json, wait, input, syncCode, noConnect, budgetMinutes }; @@ -912,8 +921,8 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { daemonValidation: daemonValidation ?? (process.env['FLOWS_NO_DAEMON_CHECK'] === '1' ? 'off' : process.env['FLOWS_CHECK_AGAINST_DAEMON'] === '1' ? 'required' : 'auto') } : command === 'run' - ? { command, ...(detach ? { detach } : {}), bucket, reuseFromRunId, localAgent, agentCapacity, dataDir, input, json, spawn, noObserverLink, cloudMirror, allowHumanInfluenced, value: positionals[0]! } - : { command, ...(detach ? { detach } : {}), localAgent, agentCapacity, dataDir, json, spawn, noObserverLink, cloudMirror, allowHumanInfluenced, value: positionals[0]! }; + ? { command, ...(detach ? { detach } : {}), bucket, reuseFromRunId, localAgent, agentCapacity, serviceCallerFile, dataDir, input, json, spawn, noObserverLink, cloudMirror, allowHumanInfluenced, value: positionals[0]! } + : { command, ...(detach ? { detach } : {}), localAgent, agentCapacity, serviceCallerFile, dataDir, json, spawn, noObserverLink, cloudMirror, allowHumanInfluenced, value: positionals[0]! }; } /** diff --git a/packages/sdk/src/cli/direct-run.ts b/packages/sdk/src/cli/direct-run.ts index 4e28ecd4f..9b9cff50f 100644 --- a/packages/sdk/src/cli/direct-run.ts +++ b/packages/sdk/src/cli/direct-run.ts @@ -87,7 +87,7 @@ export async function runDirectFlow( if (options.localAgent) { localAgent = await attachLocalAgent( client, dataDir, options.onPtyReady, authoredLocalAgentStream(admissionIdentity), workerCapacity, - undefined, [], agentEnvironment, + undefined, [], agentEnvironment, options.serviceCallerFile, ); // A session owns one worker registration. Keep the workspace-free LLM // worker on its own connection so it cannot replace the agent worker. diff --git a/packages/sdk/src/cli/run.ts b/packages/sdk/src/cli/run.ts index 33777dbf0..297c680e1 100644 --- a/packages/sdk/src/cli/run.ts +++ b/packages/sdk/src/cli/run.ts @@ -161,6 +161,8 @@ export interface RunLifecycleOptions { localAgent?: boolean; /** `--agent-capacity`: the local workers' concurrency; the default is `DEFAULT_LOCAL_AGENT_CAPACITY`. */ agentCapacity?: number; + /** Absolute Desktop schema-v2 caller.json path for local relay agent steps. */ + serviceCallerFile?: string; signal?: AbortSignal; onWait?: (progress: RunProgress) => void; /** @@ -235,7 +237,7 @@ async function executeCheckedFlow( if (options.localAgent) { localAgent = await attachLocalAgent( client, dataDir, options.onPtyReady, undefined, options.agentCapacity, - undefined, declaredLocalAgentStreams(spec), agentEnvironment, + undefined, declaredLocalAgentStreams(spec), agentEnvironment, options.serviceCallerFile, ); } if (options.localAgent && spec.steps.some(step => step.type === 'agent' && communicationInstruction(step.instruction))) { @@ -356,7 +358,7 @@ export async function resumeFlow( if (options.localAgent) { authoredAgent = await attachLocalAgent( client, dataDir, options.onPtyReady, authoredRoot.localAgentStream, workerCapacity, - undefined, [], agentEnvironment, + undefined, [], agentEnvironment, options.serviceCallerFile, ); authoredLlmClient = new JournalClient(socketPath); await authoredLlmClient.connect(); @@ -395,7 +397,7 @@ export async function resumeFlow( if (!spec) throw new Error('Cannot attach a local worker: the journaled run spec is missing'); authoredAgent = await attachLocalAgent( client, dataDir, options.onPtyReady, undefined, options.agentCapacity, - undefined, declaredLocalAgentStreams(spec), agentEnvironment, + undefined, declaredLocalAgentStreams(spec), agentEnvironment, options.serviceCallerFile, ); if (spec?.steps.some(step => step.type === 'agent' && communicationInstruction(step.instruction))) { const { attachCommunicationWorkers } = await import('../communication/local.js'); diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 139c61c16..1918883d1 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -253,13 +253,15 @@ export { } from './node-spawn.js'; export { LocalNodeSpawnPort, - connectCore, - coreSocketPath, - type CoreClient, - type LocalNodeSpawnCaller, + loadCoreBridge, + type CallerDescriptor, + type CoreBridgeApi, + type CoreBridgeClient, } from './node-spawn-local.js'; export { RemoteNodeSpawnPort, type RemoteNodeSpawnClient } from './node-spawn-remote.js'; export { + createLocalNodeSpawnRuntime, + createNodeSpawnRuntime, createRemoteNodeSpawnRuntime, type NodeSpawnPublisher, type NodeSpawnRuntime, diff --git a/packages/sdk/src/local-agent.ts b/packages/sdk/src/local-agent.ts index 97378c693..7ba563260 100644 --- a/packages/sdk/src/local-agent.ts +++ b/packages/sdk/src/local-agent.ts @@ -17,6 +17,7 @@ export async function attachLocalAgent( runRoot?: string, declaredStreams: readonly string[] = [], environment?: NodeJS.ProcessEnv, + serviceCallerFile?: string, ): Promise<{ stream: string; readonly failure: unknown; @@ -34,7 +35,7 @@ export async function attachLocalAgent( // second parking behind the first. Authored bodies size their admission to // this same number (worker-slots.ts), so they never ask for more. capacity, - dataDir, onPtyReady, runRoot, environment, + dataDir, onPtyReady, runRoot, environment, serviceCallerFile, pins: { workspace: [{ surface: LOCAL_AGENT_WORKSPACE, revision_id: LOCAL_AGENT_REVISION }], streams: [...new Set([stream, ...declaredStreams])].map(stream => ({ stream, read_offset: 0 })), diff --git a/packages/sdk/src/node-spawn-local.ts b/packages/sdk/src/node-spawn-local.ts index 7a46d0fd5..643858d92 100644 --- a/packages/sdk/src/node-spawn-local.ts +++ b/packages/sdk/src/node-spawn-local.ts @@ -1,11 +1,7 @@ -import { existsSync, lstatSync, statSync } from 'node:fs'; -import { homedir } from 'node:os'; -import { dirname, isAbsolute, join } from 'node:path'; -import { createConnection, type Socket } from 'node:net'; +import { isAbsolute, relative, sep } from 'node:path'; + import { NodeSpawnError, - isRecord, - nonempty, normalizeNodeSpawnStatus, terminalStatus, type NodeReleaseRequest, @@ -18,170 +14,172 @@ import { validIdempotencyKey, } from './node-spawn.js'; -const BRIDGE_VERSION = 2; -const DEFAULT_TIMEOUT_MS = 15_000; - -export interface CoreClient { - call(method: string, params?: Record): Promise; +/** The structural core-bridge surface Flows consumes. Kept here so the package stays optional. */ +export interface CoreBridgeClient { + readonly hello: { methods: string[] }; close(): void; } -export function coreSocketPath(env: NodeJS.ProcessEnv = process.env, home = homedir()): string { - const fallback = join(home, '.agentworkforce', 'desktop', 'core.sock'); - const configured = env.AGENT_RELAY_CORE_SOCKET?.trim(); - if (!configured) return fallback; - return isAbsolute(configured) ? configured : join(dirname(fallback), configured); -} - -function verifySocket(socket: string): void { - if (typeof process.getuid !== 'function') return; - const directory = dirname(socket); - const entry = lstatSync(directory); - const target = statSync(directory); - if (entry.uid !== process.getuid() || target.uid !== process.getuid() - || !target.isDirectory() || (target.mode & 0o022) !== 0) { - throw new NodeSpawnError('The Agent Relay core socket directory is not private to this user', 'untrusted_core_socket'); - } - const file = lstatSync(socket); - if (!file.isSocket() || file.uid !== process.getuid() || (file.mode & 0o077) !== 0) { - throw new NodeSpawnError('The Agent Relay core socket is not private to this user', 'untrusted_core_socket'); - } -} - -/** Connects to Desktop's credential-holding core bridge; never starts a daemon. */ -export async function connectCore(socket = coreSocketPath(), timeoutMs = DEFAULT_TIMEOUT_MS): Promise { - if (process.platform === 'win32') { - throw new NodeSpawnError( - 'Local node spawn is unavailable on Windows until the Desktop pipe server can be authenticated', - 'untrusted_core_socket', - ); - } - if (!existsSync(socket)) return undefined; - verifySocket(socket); - const stream = await openSocket(socket, timeoutMs); - if (!stream) return undefined; - let nextId = 1; - let buffer = ''; - let closed = false; - const pending = new Map(); - const fail = (error: Error) => { - closed = true; - for (const waiter of pending.values()) { clearTimeout(waiter.timer); waiter.reject(error); } - pending.clear(); +export interface CallerDescriptor { + readonly schema: 'agent-relay-service-caller/2'; + readonly schemaVersion: 2; + readonly auth: unknown; + readonly callerId: string; + readonly callerName: string; + readonly publisher: { readonly id: string; readonly name: string }; + readonly workspace: { + readonly site_url: string; + readonly base_url: string; + readonly workspace_id: string; + readonly relaycast_workspace_id: string; }; - stream.setEncoding('utf8'); - stream.on('data', (chunk: string) => { - buffer += chunk; - for (let newline = buffer.indexOf('\n'); newline >= 0; newline = buffer.indexOf('\n')) { - const line = buffer.slice(0, newline); - buffer = buffer.slice(newline + 1); - let message: { id?: unknown; result?: unknown; error?: { message?: string } }; - try { message = JSON.parse(line); } catch { continue; } - if (typeof message.id !== 'number') continue; - const waiter = pending.get(message.id); - if (!waiter) continue; - pending.delete(message.id); - clearTimeout(waiter.timer); - if (message.error) waiter.reject(new NodeSpawnError(message.error.message ?? 'The Agent Relay core refused the request')); - else waiter.resolve(message.result ?? null); - } - }); - stream.on('error', error => fail(error)); - stream.on('close', () => fail(new NodeSpawnError('The Agent Relay core closed the connection'))); - const client: CoreClient = { - call(method: string, params: Record = {}): Promise { - if (closed) return Promise.reject(new NodeSpawnError('The Agent Relay core connection is closed')); - const id = nextId++; - return new Promise((resolve, reject) => { - const timer = setTimeout(() => { - pending.delete(id); - reject(new NodeSpawnError(`The Agent Relay core did not answer ${method} in time`, 'core_timeout')); - }, timeoutMs); - pending.set(id, { resolve: value => resolve(value as T), reject, timer }); - stream.write(`${JSON.stringify({ jsonrpc: '2.0', id, method, params })}\n`); - }); - }, - close() { closed = true; stream.destroy(); }, + readonly account: { readonly user_id: string }; + readonly credential: unknown; + readonly createdAt: string; + readonly rotatedAt?: string; + readonly expiresAt: string; + readonly harnesses: readonly string[]; + readonly directoryScope: readonly string[]; + readonly ownsOnly: boolean; + readonly socket: string; + readonly inbox: { + readonly transport: string; + readonly open: string; + readonly next: string; + readonly ack: string; + readonly fail: string; + readonly close: string; + readonly disconnect: string; }; - try { - const hello = await client.call<{ bridge_version?: unknown }>('hello', { bridge_version: BRIDGE_VERSION }); - if (hello.bridge_version !== BRIDGE_VERSION) { - throw new NodeSpawnError('The process at the core socket does not speak Agent Relay bridge v2', 'core_version_mismatch'); - } - } catch (error) { client.close(); throw error; } - return client; + readonly desktopVersion: string; } -async function openSocket(address: string, timeoutMs: number): Promise { - return new Promise(resolve => { - const socket = createConnection(address); - const timer = setTimeout(() => { socket.destroy(); resolve(undefined); }, timeoutMs); - socket.once('connect', () => { clearTimeout(timer); resolve(socket); }); - socket.once('error', () => { clearTimeout(timer); resolve(undefined); }); - }); +interface CoreTarget { + id: string; + name: string; + live: boolean; + handlers_live: boolean; + stale: boolean; + capabilities: string[]; + active_agents: number; + reserved_agents: number; + max_agents: number; } interface CoreInvocation { id: string; - agent_id?: string; + agent_id: string | null; status: string; - node_id?: string; - name?: string; + node_id: string | null; + name: string | null; spawned: boolean; ready: boolean; - released?: boolean; + released: boolean; task_unconfirmed: boolean; - error_code?: string; + error_code: string | null; } -/** Desktop-issued identity for the authorized headless Flows service caller. */ -export interface LocalNodeSpawnCaller { - name: string; - secret: string; +export interface CoreInboxDelivery { + item_id: string; + message_id: string; + sender: { id: string; name: string }; + text?: string; + failure?: { code: 'result_too_large' | 'result_write_failed'; bytes: number; limit: number }; + invocation_id: string; + sent_at: string; } -const callerSecrets = new WeakMap(); - -/** - * Local native-spawn adapter. The caller identity is deliberately explicit: - * this class never borrows a Desktop registration or provisions a service - * caller. A core without the spawn-callers plugin refuses the secret. - */ -export class LocalNodeSpawnPort implements NodeSpawnPort { - private readonly callerName: string; +/** Typed API required from @agent-relay/core-bridge's schema-v2 release. */ +export interface CoreBridgeApi { + connectCore(options?: { socket?: string; signal?: AbortSignal }): Promise; + readCallerDescriptor(path: string): Promise; + authenticate(client: CoreBridgeClient, credential: unknown, options?: { signal?: AbortSignal }): Promise; + requireServiceCallerMethods(client: CoreBridgeClient): void; + assertServiceCallerIdentity( + descriptor: CallerDescriptor, + identity: Awaited>, + ): void; + serviceCallerIdentity(client: CoreBridgeClient, options?: { signal?: AbortSignal }): Promise<{ + publisher: { id: string; name: string }; + node: { id: string; name: string }; + base_url: string; + workspace_id: string; + relaycast_workspace_id: string; + }>; + spawnTargets(client: CoreBridgeClient, options?: { signal?: AbortSignal }): Promise<{ targets: CoreTarget[] }>; + spawnInvoke(client: CoreBridgeClient, params: Record, options?: { signal?: AbortSignal; actionTimeoutMs?: number }): Promise; + spawnStatus(client: CoreBridgeClient, params: Record, options?: { signal?: AbortSignal }): Promise; + spawnRelease(client: CoreBridgeClient, params: Record, options?: { signal?: AbortSignal; actionTimeoutMs?: number }): Promise; + serviceCallerInboxOpen(client: CoreBridgeClient, invocationId: string, options?: { signal?: AbortSignal }): Promise<{ iterator_id: string }>; + serviceCallerInboxNext(client: CoreBridgeClient, iteratorId: string, timeoutMs: number, options?: { signal?: AbortSignal }): Promise<{ delivery: CoreInboxDelivery | null }>; + serviceCallerInboxAck(client: CoreBridgeClient, itemId: string, options?: { signal?: AbortSignal }): Promise; + serviceCallerInboxFail(client: CoreBridgeClient, itemId: string, options?: { signal?: AbortSignal }): Promise; + serviceCallerInboxClose(client: CoreBridgeClient, iteratorId: string, options?: { signal?: AbortSignal }): Promise; + serviceCallerDisconnect(client: CoreBridgeClient, options?: { signal?: AbortSignal }): Promise; +} - constructor(private readonly core: CoreClient, caller: LocalNodeSpawnCaller) { - if (!caller.name.trim() || !caller.secret.trim()) { - throw new NodeSpawnError( - 'Local node spawn requires an explicitly authorized service caller', - 'caller_required', - ); - } - this.callerName = caller.name; - callerSecrets.set(this, caller.secret); +/** Loads the optional bridge without making remote-only installs depend on Desktop support. */ +export async function loadCoreBridge(): Promise { + const specifier = '@agent-relay/core-bridge'; + try { + return await import(specifier) as unknown as CoreBridgeApi; + } catch (error) { + throw new NodeSpawnError( + 'Local Desktop node spawn requires optional @agent-relay/core-bridge with schema-v2 service-caller support', + 'core_bridge_unavailable', + error, + ); } +} + +/** Native local spawn adapter over one already authenticated schema-v2 bridge connection. */ +export class LocalNodeSpawnPort implements NodeSpawnPort { + constructor( + private readonly core: CoreBridgeClient, + private readonly bridge: Pick, + private readonly scope: Pick, + ) {} async targets(): Promise { - const result = await this.core.call<{ targets: Array> }>('spawn.targets'); - if (!isRecord(result) || !Array.isArray(result.targets)) { + const result = await this.bridge.spawnTargets(this.core); + if (!result || !Array.isArray(result.targets)) { throw new NodeSpawnError('The core returned invalid spawn targets', 'invalid_core_response'); } return result.targets.map(value => { - if (!isRecord(value) || !nonempty(value.id) || !nonempty(value.name)) { + if (!value || typeof value.id !== 'string' || value.id === '' + || typeof value.name !== 'string' || value.name === '') { throw new NodeSpawnError('The core returned a spawn target without an exact identity', 'invalid_core_response'); } return { - id: value.id, name: value.name, live: value.live === true, - handlersLive: value.handlers_live === true, stale: value.stale === true, - capabilities: Array.isArray(value.capabilities) ? value.capabilities.filter(nonempty) : [], - activeAgents: Number(value.active_agents), reservedAgents: Number(value.reserved_agents), maxAgents: Number(value.max_agents), + id: value.id, + name: value.name, + live: value.live === true, + handlersLive: value.handlers_live === true, + stale: value.stale === true, + capabilities: Array.isArray(value.capabilities) + ? value.capabilities.filter((entry): entry is string => typeof entry === 'string' && entry.length > 0) + : [], + activeAgents: Number(value.active_agents), + reservedAgents: Number(value.reserved_agents), + maxAgents: Number(value.max_agents), }; }); } async spawn(request: NodeSpawnRequest): Promise { - if (!request.directory || !isAbsolute(request.directory)) throw new NodeSpawnError('Local node spawn requires an absolute directory', 'spawn_invalid_input'); - if (!validIdempotencyKey(request.idempotencyKey)) throw new NodeSpawnError('Invalid node-spawn idempotency key', 'spawn_invalid_input'); + if (!request.directory || !isAbsolute(request.directory)) { + throw new NodeSpawnError('Local node spawn requires an absolute directory', 'spawn_invalid_input'); + } + if (!this.scope.harnesses.includes(request.harness) + || !this.scope.directoryScope.some(root => withinDirectory(root, request.directory!))) { + throw new NodeSpawnError( + 'The Desktop service caller does not authorize this harness and directory', + 'outside_scope', + ); + } + if (!validIdempotencyKey(request.idempotencyKey)) { + throw new NodeSpawnError('Invalid node-spawn idempotency key', 'spawn_invalid_input'); + } if (request.model !== undefined || request.repo !== undefined || request.sandboxOnly !== undefined) { throw new NodeSpawnError( 'The Desktop core bridge does not accept model, repository, or sandbox placement overrides', @@ -189,18 +187,21 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { ); } const allTargets = await this.targets(); - const targets = allTargets.filter(candidate => candidate.live && candidate.handlersLive + const eligible = allTargets.filter(candidate => candidate.live && candidate.handlersLive && !candidate.stale && candidate.capabilities.includes(`spawn:${request.harness}`)); const target = request.node === undefined - ? targets[0] - : targets.find(candidate => candidate.name === request.node); + ? eligible[0] + : eligible.find(candidate => candidate.name === request.node); if (!target?.id) throw new NodeSpawnError('No eligible local Agent Relay spawn target', 'no_eligible_node'); - const raw = await this.core.call('spawn.invoke', { - caller_name: this.callerName, caller_secret: callerSecret(this), - node_id: target.id, node_name: target.name, - name: request.name, harness: request.harness, task: request.task, - directory: request.directory, idempotency_key: request.idempotencyKey, - }); + const raw = await this.bridge.spawnInvoke(this.core, { + node_id: target.id, + node_name: target.name, + name: request.name, + harness: request.harness, + task: request.task, + directory: request.directory, + idempotency_key: request.idempotencyKey, + }, { actionTimeoutMs: request.timeoutMs }); // spawn.invoke is the action receipt; spawn.status is also the boundary // that enriches it from the core's exact same-workspace registration. const settled = await this.wait('spawn', raw.id, request.timeoutMs); @@ -223,25 +224,33 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { } async status(action: NodeSpawnAction, invocationId: string): Promise { - const raw = await this.core.call('spawn.status', { - caller_name: this.callerName, caller_secret: callerSecret(this), - action, invocation_id: invocationId, - }); - return mapCoreInvocation(action, raw); + return mapCoreInvocation(action, await this.bridge.spawnStatus(this.core, { + action, + invocation_id: invocationId, + })); } async release(request: NodeReleaseRequest): Promise { - if (!validIdempotencyKey(request.idempotencyKey)) throw new NodeSpawnError('Invalid node-release idempotency key', 'spawn_invalid_input'); + if (!validIdempotencyKey(request.idempotencyKey)) { + throw new NodeSpawnError('Invalid node-release idempotency key', 'spawn_invalid_input'); + } const receipt = request.receipt; - if (!receipt.nodeId) throw new NodeSpawnError('Local release requires the exact spawn node id', 'release_identity_mismatch'); - const raw = await this.core.call('spawn.release', { - caller_name: this.callerName, caller_secret: callerSecret(this), - node_id: receipt.nodeId, node_name: receipt.nodeName, - name: receipt.name, expected_agent_id: receipt.agentId, idempotency_key: request.idempotencyKey, + if (!receipt.nodeId) { + throw new NodeSpawnError('Local release requires the exact spawn node id', 'release_identity_mismatch'); + } + const raw = await this.bridge.spawnRelease(this.core, { + node_id: receipt.nodeId, + node_name: receipt.nodeName, + name: receipt.name, + expected_agent_id: receipt.agentId, + idempotency_key: request.idempotencyKey, }); - const settled = raw.status === 'completed' ? mapCoreInvocation('release', raw) : await this.wait('release', raw.id); + const settled = raw.status === 'completed' + ? mapCoreInvocation('release', raw) + : await this.wait('release', raw.id); if (settled.status !== 'completed' || settled.released !== true) { - throw new NodeSpawnError('The local node did not confirm exact agent release', settled.errorCode ?? 'release_unconfirmed'); + throw new NodeSpawnError('The local node did not confirm exact agent release', + settled.errorCode ?? 'release_unconfirmed'); } return settled; } @@ -257,18 +266,24 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { } } -function callerSecret(port: LocalNodeSpawnPort): string { - const secret = callerSecrets.get(port); - if (secret === undefined) throw new NodeSpawnError('Local node spawn caller authority was lost', 'caller_required'); - return secret; +function withinDirectory(root: string, candidate: string): boolean { + const path = relative(root, candidate); + return path === '' || (path !== '..' && !path.startsWith(`..${sep}`) + && !isAbsolute(path)); } function mapCoreInvocation(action: NodeSpawnAction, value: CoreInvocation): NodeSpawnInvocation { return { - id: value.id, action, status: normalizeNodeSpawnStatus(value.status), - nodeId: value.node_id, name: value.name, agentId: value.agent_id, - spawned: value.spawned, ready: value.ready, taskUnconfirmed: value.task_unconfirmed, + id: value.id, + action, + status: normalizeNodeSpawnStatus(value.status), + nodeId: value.node_id ?? undefined, + name: value.name ?? undefined, + agentId: value.agent_id ?? undefined, + spawned: value.spawned, + ready: value.ready, + taskUnconfirmed: value.task_unconfirmed, released: value.released, - errorCode: value.error_code, + errorCode: value.error_code ?? undefined, }; } diff --git a/packages/sdk/src/node-spawn-result.ts b/packages/sdk/src/node-spawn-result.ts index 9b2dd6538..b4f456800 100644 --- a/packages/sdk/src/node-spawn-result.ts +++ b/packages/sdk/src/node-spawn-result.ts @@ -20,9 +20,18 @@ export interface NodeResultCorrelation { dispatchId: string; } +export interface NodeResultTerminalFailure { + code: 'result_too_large' | 'result_write_failed'; + bytes: number; + limit: number; +} + interface InboxItem { id: string; message: { id: string; text: string; from: { id?: string; name?: string } }; + terminalFailure?: NodeResultTerminalFailure; + /** Local Desktop delivery teardown; settlement uses a separate authenticated connection after this resolves. */ + settlementReady?: Promise<{ failure?: unknown }>; } export interface NodeResultInboxClient { @@ -77,6 +86,11 @@ export async function waitForNodeResult( const item = next.value; signal?.throwIfAborted(); if (item.message.from.id !== expected.agentId || item.message.from.name !== expected.agentName) continue; + if (item.terminalFailure !== undefined) { + const failure = item.terminalFailure; + const reason = `${failure.code} (bytes=${failure.bytes}, limit=${failure.limit})`; + return rejectedDelivery(client, item, expected, reason); + } if (!item.message.text.startsWith(NODE_RESULT_PREFIX)) continue; let raw: unknown; try { raw = JSON.parse(item.message.text.slice(NODE_RESULT_PREFIX.length)); } @@ -85,22 +99,14 @@ export async function waitForNodeResult( try { envelope = readNodeResult(raw, expected); } catch (error) { const reason = error instanceof Error ? error.message : 'Malformed Relayflows result'; - return { - inboxItemId: item.id, - messageId: item.message.id, - envelope: { version: 1, run_id: expected.runId, step_id: expected.stepId, - dispatch_id: expected.dispatchId, status: 'failed', error: reason }, - settlementAction: 'fail', - ack: async () => { await client.inbox.fail({ inboxItemId: item.id, error: reason, retry: false, - metadata: { relayflows_result: 'rejected' } }); }, - }; + return rejectedDelivery(client, item, expected, reason); } return { inboxItemId: item.id, messageId: item.message.id, envelope, settlementAction: 'ack', - ack: async () => { await client.inbox.ack({ inboxItemId: item.id, state: 'read', + ack: async () => { await awaitSettlementReady(item); await client.inbox.ack({ inboxItemId: item.id, state: 'read', metadata: { relayflows_result: 'journaled' } }); }, }; } @@ -121,6 +127,28 @@ export async function waitForNodeResult( } } +function rejectedDelivery( + client: NodeResultInboxClient, + item: InboxItem, + expected: NodeResultCorrelation, + reason: string, +): NodeResultDelivery { + return { + inboxItemId: item.id, + messageId: item.message.id, + envelope: { version: 1, run_id: expected.runId, step_id: expected.stepId, + dispatch_id: expected.dispatchId, status: 'failed', error: reason }, + settlementAction: 'fail', + ack: async () => { await awaitSettlementReady(item); await client.inbox.fail({ inboxItemId: item.id, error: reason, retry: false, + metadata: { relayflows_result: 'rejected' } }); }, + }; +} + +async function awaitSettlementReady(item: InboxItem): Promise { + const teardown = await item.settlementReady; + if (teardown?.failure !== undefined) throw teardown.failure; +} + async function abortableNext( iterator: AsyncIterator, signal: AbortSignal | undefined, diff --git a/packages/sdk/src/node-spawn-runtime.ts b/packages/sdk/src/node-spawn-runtime.ts index 4166c7401..cb2db58e5 100644 --- a/packages/sdk/src/node-spawn-runtime.ts +++ b/packages/sdk/src/node-spawn-runtime.ts @@ -1,5 +1,14 @@ import { isIP } from 'node:net'; -import type { NodeResultInboxClient } from './node-spawn-result.js'; +import { isAbsolute } from 'node:path'; +import type { NodeResultInboxClient, NodeResultTerminalFailure } from './node-spawn-result.js'; +import { + LocalNodeSpawnPort, + loadCoreBridge, + type CallerDescriptor, + type CoreBridgeApi, + type CoreBridgeClient, + type CoreInboxDelivery, +} from './node-spawn-local.js'; import { NodeSpawnError } from './node-spawn.js'; import { RemoteNodeSpawnPort, type RemoteNodeSpawnClient } from './node-spawn-remote.js'; import type { NodeSpawnPort } from './node-spawn.js'; @@ -76,11 +85,70 @@ export async function createRemoteNodeSpawnRuntime( }; } +/** + * Opens one explicitly configured Desktop service caller. The caller file is + * the authority: there is no directory scan and no fallback to a remote token. + */ +export async function createLocalNodeSpawnRuntime( + serviceCallerFile: string, + options: { signal?: AbortSignal; bridge?: CoreBridgeApi } = {}, +): Promise { + if (!serviceCallerFile || !isAbsolute(serviceCallerFile) || serviceCallerFile.includes('\0')) { + throw new NodeSpawnError( + 'RELAYFLOW_SERVICE_CALLER_FILE must be an absolute path to caller.json', + 'caller_file_invalid', + ); + } + const bridge = options.bridge ?? await loadCoreBridge(); + const descriptor = await bridge.readCallerDescriptor(serviceCallerFile); + const core = await bindCore(bridge, descriptor, options.signal); + try { + const identity = await bridge.serviceCallerIdentity(core, { signal: options.signal }); + bridge.assertServiceCallerIdentity(descriptor, identity); + if (!identity.node.id.trim() || !identity.node.name.trim()) { + throw new NodeSpawnError( + 'Desktop service-caller identity omitted its exact node identity', + 'caller_identity_mismatch', + ); + } + return { + port: new LocalNodeSpawnPort(core, bridge, descriptor), + inbox: localResultInbox(bridge, descriptor), + publisher: identity.publisher, + baseUrl: identity.base_url, + node: identity.node.name, + disconnect: async () => { + await disconnectBoundCore(bridge, core); + }, + }; + } catch (error) { + await disconnectBoundCore(bridge, core, error); + throw error; // unreachable; keeps the return type explicit + } +} + +/** Selects exactly one authority family and refuses ambiguous credentials. */ +export async function createNodeSpawnRuntime( + env: NodeJS.ProcessEnv = process.env, + options: { signal?: AbortSignal; bridge?: CoreBridgeApi } = {}, +): Promise { + const serviceCallerFile = env.RELAYFLOW_SERVICE_CALLER_FILE?.trim(); + const agentToken = env.RELAY_AGENT_TOKEN?.trim(); + if (serviceCallerFile && agentToken) { + throw new NodeSpawnError( + 'RELAYFLOW_SERVICE_CALLER_FILE and RELAY_AGENT_TOKEN name different node-spawn authorities; choose one', + 'ambiguous_publisher_authority', + ); + } + if (serviceCallerFile) return createLocalNodeSpawnRuntime(serviceCallerFile, options); + return createRemoteNodeSpawnRuntime(env, options); +} + /** One lazy publisher connection shared by every relay task on a worker. */ export function createNodeSpawnRuntimeOwner( provided?: NodeSpawnRuntime, factory: (signal: AbortSignal) => Promise = signal => - createRemoteNodeSpawnRuntime(process.env, { signal }), + createNodeSpawnRuntime(process.env, { signal }), ): NodeSpawnRuntimeOwner { const controller = new AbortController(); let pending: Promise | undefined; @@ -107,6 +175,144 @@ export function createNodeSpawnRuntimeOwner( return { get, close: () => closePromise ??= closeOnce() }; } +async function bindCore( + bridge: CoreBridgeApi, + descriptor: CallerDescriptor, + signal?: AbortSignal, +): Promise { + const core = await bridge.connectCore({ socket: descriptor.socket, signal }); + try { + bridge.requireServiceCallerMethods(core); + await bridge.authenticate(core, descriptor.credential, { signal }); + return core; + } catch (error) { + core.close(); + throw error; + } +} + +function localResultInbox(bridge: CoreBridgeApi, descriptor: CallerDescriptor): NodeResultInboxClient { + const settle = async (action: 'ack' | 'fail', itemId: string): Promise => { + const core = await bindCore(bridge, descriptor); + let failure: unknown; + try { + if (action === 'ack') await bridge.serviceCallerInboxAck(core, itemId); + else await bridge.serviceCallerInboxFail(core, itemId); + } catch (error) { failure = error; } + await disconnectBoundCore(bridge, core, failure); + }; + return { inbox: { + subscribe(input = {}) { + const invocationId = input.invocationId?.trim(); + if (!invocationId) { + throw new NodeSpawnError( + 'Local result inbox requires the exact spawn invocation id', + 'result_identity_missing', + ); + } + return localInboxSubscription( + bridge, + descriptor, + invocationId, + input.signal, + ); + }, + async ack(input) { await settle('ack', input.inboxItemId); }, + async fail(input) { await settle('fail', input.inboxItemId); }, + } }; +} + +function localInboxSubscription( + bridge: CoreBridgeApi, + descriptor: CallerDescriptor, + invocationId: string, + signal?: AbortSignal, +): AsyncIterable<{ id: string; message: { id: string; text: string; from: { id?: string; name?: string } } }> { + return { [Symbol.asyncIterator]() { + let core: CoreBridgeClient | undefined; + let iteratorId: string | undefined; + let closed = false; + let resolveClosed!: (result: { failure?: unknown }) => void; + const closedPromise = new Promise<{ failure?: unknown }>(resolve => { + resolveClosed = resolve; + }); + const close = async (): Promise => { + if (closed) return; + closed = true; + if (core === undefined) { + resolveClosed({}); + return; + } + let failure: unknown; + try { + if (iteratorId !== undefined) await bridge.serviceCallerInboxClose(core, iteratorId); + } catch (error) { failure = error; } + try { + await disconnectBoundCore(bridge, core, failure); + resolveClosed({}); + } catch (error) { + resolveClosed({ failure: error }); + throw error; + } + }; + return { + async next() { + for (;;) { + signal?.throwIfAborted(); + if (closed) return { done: true, value: undefined }; + if (core === undefined) { + core = await bindCore(bridge, descriptor, signal); + const opened = await bridge.serviceCallerInboxOpen(core, invocationId, { signal }); + iteratorId = opened.iterator_id; + } + const result = await bridge.serviceCallerInboxNext(core, iteratorId!, 30_000, { signal }); + if (result.delivery !== null) { + if (result.delivery.invocation_id !== invocationId) { + throw new NodeSpawnError( + 'Desktop returned a result outside the subscribed spawn invocation', + 'result_identity_mismatch', + ); + } + return { done: false as const, value: mapInboxDelivery(result.delivery, closedPromise) }; + } + } + }, + async return() { + await close(); + return { done: true as const, value: undefined }; + }, + }; + } }; +} + +async function disconnectBoundCore( + bridge: Pick, + core: CoreBridgeClient, + priorFailure?: unknown, +): Promise { + let disconnectFailure: unknown; + try { await bridge.serviceCallerDisconnect(core); } + catch (error) { disconnectFailure = error; } + finally { core.close(); } + if (priorFailure !== undefined) throw priorFailure; + if (disconnectFailure !== undefined) throw disconnectFailure; +} + +function mapInboxDelivery(delivery: CoreInboxDelivery, settlementReady: Promise<{ failure?: unknown }>) { + return { + id: delivery.item_id, + settlementReady, + message: { + id: delivery.message_id, + text: delivery.text ?? '', + from: { id: delivery.sender.id, name: delivery.sender.name }, + }, + ...(delivery.failure === undefined ? {} : { + terminalFailure: delivery.failure satisfies NodeResultTerminalFailure, + }), + }; +} + async function waitForRuntime(pending: Promise, signal?: AbortSignal): Promise { if (signal === undefined) return pending; let onAbort: (() => void) | undefined; diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index 7480d0ecc..1babc9f99 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -22,7 +22,7 @@ import { redactText, type TranscriptDigest, } from './agent-transcript.js'; -import { createNodeSpawnRuntimeOwner, type NodeSpawnRuntime, +import { createNodeSpawnRuntime, createNodeSpawnRuntimeOwner, type NodeSpawnRuntime, type NodeSpawnRuntimeOwner } from './node-spawn-runtime.js'; import { reconcileNodeResultSettlements } from './node-spawn-settlement.js'; import { readNodeSpawnClaims } from './node-spawn-state.js'; @@ -48,6 +48,8 @@ export interface AgentWorkerOptions { environment?: NodeJS.ProcessEnv; /** Optional approved node-spawn runtime; otherwise relay steps use the agent-token fleet client. */ nodeSpawnRuntime?: NodeSpawnRuntime; + /** Explicit Desktop schema-v2 handoff; never discovered from an app-data directory. */ + serviceCallerFile?: string; } /** @@ -81,7 +83,16 @@ export class AgentWorker extends EventEmitter { private readonly options: AgentWorkerOptions, ) { super(); - this.nodeSpawnRuntimeOwner = createNodeSpawnRuntimeOwner(options.nodeSpawnRuntime); + this.nodeSpawnRuntimeOwner = createNodeSpawnRuntimeOwner( + options.nodeSpawnRuntime, + signal => createNodeSpawnRuntime({ + ...process.env, + ...options.environment, + ...(options.serviceCallerFile === undefined + ? {} + : { RELAYFLOW_SERVICE_CALLER_FILE: options.serviceCallerFile }), + }, { signal }), + ); } async attach(): Promise { diff --git a/packages/sdk/tests/node-spawn-ports.test.ts b/packages/sdk/tests/node-spawn-ports.test.ts index cd65db5a5..22da54747 100644 --- a/packages/sdk/tests/node-spawn-ports.test.ts +++ b/packages/sdk/tests/node-spawn-ports.test.ts @@ -1,22 +1,34 @@ import { describe, expect, it, vi } from 'vitest'; import { RelayPlacementError } from '@agent-relay/sdk/messaging'; -import { connectCore, LocalNodeSpawnPort, type CoreClient } from '../src/node-spawn-local.js'; +import { + LocalNodeSpawnPort, + type CoreBridgeApi, + type CoreBridgeClient, +} from '../src/node-spawn-local.js'; import { RemoteNodeSpawnPort } from '../src/node-spawn-remote.js'; -describe('local node-spawn port', () => { - it('refuses Windows before opening an unauthenticated named pipe', async () => { - const platform = vi.spyOn(process, 'platform', 'get').mockReturnValue('win32'); - try { - await expect(connectCore()).rejects.toMatchObject({ code: 'untrusted_core_socket' }); - } finally { - platform.mockRestore(); - } +interface TestCore { + call(method: string, params?: Record): Promise; +} + +function localPort(core: TestCore): LocalNodeSpawnPort { + const client = { hello: { methods: [] }, close() {} } satisfies CoreBridgeClient; + const bridge = { + spawnTargets: () => core.call('spawn.targets'), + spawnInvoke: (_client, params) => core.call('spawn.invoke', params), + spawnStatus: (_client, params) => core.call('spawn.status', params), + spawnRelease: (_client, params) => core.call('spawn.release', params), + } satisfies Pick; + return new LocalNodeSpawnPort(client, bridge, { + harnesses: ['codex'], + directoryScope: ['/workspace'], }); +} - it('uses the bridge v2 spawn methods and exact release identity', async () => { +describe('local node-spawn port', () => { + it('uses bound bridge v2 spawn methods with no raw caller credential and exact release identity', async () => { const calls: Array<[string, Record | undefined]> = []; - const core: CoreClient = { - close() {}, + const core: TestCore = { async call(method: string, params?: Record): Promise { calls.push([method, params]); if (method === 'spawn.targets') return { targets: [{ id: 'node_1', name: 'desktop-1', live: true, @@ -28,35 +40,26 @@ describe('local node-spawn port', () => { ready: false, released: true, task_unconfirmed: false } as T; }, }; - const port = new LocalNodeSpawnPort(core, { - name: 'authorized-caller', - secret: 'desktop-service-secret', - }); - expect(JSON.stringify(port)).not.toContain('desktop-service-secret'); - expect(Object.values(port)).not.toContain('desktop-service-secret'); + const port = localPort(core); const receipt = await port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', directory: '/workspace', idempotencyKey: 'dispatch_1234567890' }); await port.release({ receipt, idempotencyKey: 'release_1234567890' }); - expect(calls[1]).toEqual(['spawn.invoke', expect.objectContaining({ caller_name: 'authorized-caller', - caller_secret: 'desktop-service-secret', node_id: 'node_1', name: 'worker_1' })]); - expect(calls[2]).toEqual(['spawn.status', expect.objectContaining({ caller_name: 'authorized-caller', - caller_secret: 'desktop-service-secret', invocation_id: 'inv_1' })]); + expect(calls[1]).toEqual(['spawn.invoke', expect.objectContaining({ node_id: 'node_1', name: 'worker_1' })]); + expect(calls[2]).toEqual(['spawn.status', expect.objectContaining({ invocation_id: 'inv_1' })]); expect(calls[3]).toEqual(['spawn.release', expect.objectContaining({ - expected_agent_id: 'agent_1', caller_name: 'authorized-caller', - caller_secret: 'desktop-service-secret', node_id: 'node_1', node_name: 'desktop-1', + expected_agent_id: 'agent_1', node_id: 'node_1', node_name: 'desktop-1', name: 'worker_1', idempotency_key: 'release_1234567890' })]); - }); - - it('refuses a local service caller without its Desktop-issued secret', () => { - const core: CoreClient = { close() {}, async call(): Promise { throw new Error('unused'); } }; - expect(() => new LocalNodeSpawnPort(core, { name: 'authorized-caller', secret: ' ' })) - .toThrow(expect.objectContaining({ code: 'caller_required' })); + for (const [, params] of calls) { + if (params === undefined) continue; + expect(params).not.toHaveProperty('caller_name'); + expect(params).not.toHaveProperty('caller_secret'); + } }); it('refuses local-only fields the Desktop core bridge cannot preserve', async () => { const calls = vi.fn(); - const core: CoreClient = { close() {}, async call(): Promise { calls(); throw new Error('unused'); } }; - const port = new LocalNodeSpawnPort(core, { name: 'authorized-caller', secret: 'desktop-service-secret' }); + const core: TestCore = { async call(): Promise { calls(); throw new Error('unused'); } }; + const port = localPort(core); await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', directory: '/workspace', model: 'codex-medium', idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ code: 'spawn_invalid_input', @@ -64,12 +67,25 @@ describe('local node-spawn port', () => { expect(calls).not.toHaveBeenCalled(); }); + it.each([ + { harness: 'claude', directory: '/workspace', label: 'harness' }, + { harness: 'codex', directory: '/other', label: 'directory' }, + { harness: 'codex', directory: '/workspace-sibling', label: 'directory prefix' }, + ])('refuses a $label outside the caller-file scope before bridge dispatch', async request => { + const calls = vi.fn(); + const port = localPort({ async call(): Promise { calls(); throw new Error('unused'); } }); + await expect(port.spawn({ name: 'worker_1', harness: request.harness, task: 'work', + directory: request.directory, idempotencyKey: 'dispatch_1234567890' })) + .rejects.toMatchObject({ code: 'outside_scope' }); + expect(calls).not.toHaveBeenCalled(); + }); + it('refuses local targets without an exact string id instead of fabricating one', async () => { - const core: CoreClient = { close() {}, async call(): Promise { + const core: TestCore = { async call(): Promise { return { targets: [{ id: undefined, name: 'desktop-1', live: true, handlers_live: true, capabilities: ['spawn:codex'] }] } as T; } }; - const port = new LocalNodeSpawnPort(core, { name: 'authorized-caller', secret: 'desktop-service-secret' }); + const port = localPort(core); await expect(port.targets()).rejects.toMatchObject({ code: 'invalid_core_response' }); }); @@ -78,16 +94,13 @@ describe('local node-spawn port', () => { { live: true, handlers_live: false }, ])('does not dispatch to an explicitly named node unless both liveness checks pass (%j)', async readiness => { const calls = vi.fn(); - const core: CoreClient = { close() {}, async call(method: string): Promise { + const core: TestCore = { async call(method: string): Promise { calls(method); if (method === 'spawn.targets') return { targets: [{ id: 'node_1', name: 'desktop-1', ...readiness, stale: false, capabilities: ['spawn:codex'] }] } as T; throw new Error('spawn must not be invoked'); } }; - const port = new LocalNodeSpawnPort(core, { - name: 'authorized-caller', - secret: 'desktop-service-secret', - }); + const port = localPort(core); await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', node: 'desktop-1', directory: '/workspace', idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ code: 'no_eligible_node', @@ -97,7 +110,7 @@ describe('local node-spawn port', () => { it('stops polling on a denied terminal status', async () => { const status = vi.fn(); - const core: CoreClient = { close() {}, async call(method: string): Promise { + const core: TestCore = { async call(method: string): Promise { if (method === 'spawn.targets') return { targets: [{ id: 'node_1', name: 'desktop-1', live: true, handlers_live: true, stale: false, capabilities: ['spawn:codex'] }] } as T; if (method === 'spawn.invoke') return { id: 'inv_1', status: 'pending', spawned: false, @@ -109,14 +122,14 @@ describe('local node-spawn port', () => { } throw new Error(`unexpected core method: ${method}`); } }; - const port = new LocalNodeSpawnPort(core, { name: 'authorized-caller', secret: 'desktop-service-secret' }); + const port = localPort(core); await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', directory: '/workspace', idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ code: 'outside_scope' }); expect(status).toHaveBeenCalledOnce(); }); it('refuses a completed local spawn whose returned node identity changed', async () => { - const core: CoreClient = { close() {}, async call(method: string): Promise { + const core: TestCore = { async call(method: string): Promise { if (method === 'spawn.targets') return { targets: [ { id: 'node_1', name: 'desktop-1', live: true, handlers_live: true, stale: false, capabilities: ['spawn:codex'] }, @@ -128,7 +141,7 @@ describe('local node-spawn port', () => { return { id: 'inv_1', status: 'completed', node_id: 'node_other', agent_id: 'agent_1', name: 'worker_1', spawned: true, ready: true, task_unconfirmed: false } as T; } }; - const port = new LocalNodeSpawnPort(core, { name: 'authorized-caller', secret: 'desktop-service-secret' }); + const port = localPort(core); await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', directory: '/workspace', idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ code: 'spawn_identity_mismatch', @@ -137,7 +150,7 @@ describe('local node-spawn port', () => { }); it('refuses a completed local spawn without explicit task delivery confirmation', async () => { - const core: CoreClient = { close() {}, async call(method: string): Promise { + const core: TestCore = { async call(method: string): Promise { if (method === 'spawn.targets') return { targets: [{ id: 'node_1', name: 'desktop-1', live: true, handlers_live: true, stale: false, capabilities: ['spawn:codex'] }] } as T; if (method === 'spawn.invoke') return { id: 'inv_1', status: 'pending', spawned: false, @@ -146,7 +159,7 @@ describe('local node-spawn port', () => { agent_id: 'agent_1', name: 'worker_1', spawned: true, ready: true } as T; throw new Error(`unexpected core method: ${method}`); } }; - const port = new LocalNodeSpawnPort(core, { name: 'authorized-caller', secret: 'desktop-service-secret' }); + const port = localPort(core); await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', directory: '/workspace', idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ code: 'spawn_task_unconfirmed', diff --git a/packages/sdk/tests/node-spawn-runtime.test.ts b/packages/sdk/tests/node-spawn-runtime.test.ts index b80839b02..3d9c5e4ee 100644 --- a/packages/sdk/tests/node-spawn-runtime.test.ts +++ b/packages/sdk/tests/node-spawn-runtime.test.ts @@ -21,8 +21,10 @@ vi.mock('@agent-relay/sdk', () => ({ }, })); -import { createNodeSpawnRuntimeOwner, createRemoteNodeSpawnRuntime, +import { createLocalNodeSpawnRuntime, createNodeSpawnRuntime, createNodeSpawnRuntimeOwner, createRemoteNodeSpawnRuntime, type NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; +import type { CallerDescriptor, CoreBridgeApi, CoreBridgeClient } from '../src/node-spawn-local.js'; +import { waitForNodeResult } from '../src/node-spawn-result.js'; import { runAgentRelayTask } from '../src/agent-relay-transport.js'; describe('remote node-spawn runtime', () => { @@ -150,3 +152,259 @@ describe('remote node-spawn runtime', () => { expect(runtimeProvider.mock.calls[0]![0]?.aborted).toBe(true); }); }); + +describe('Desktop service-caller runtime', () => { + const descriptor: CallerDescriptor = { + schema: 'agent-relay-service-caller/2', + schemaVersion: 2, + auth: { type: 'challenge' }, + credential: { callerId: 'sc_0123456789abcdef' }, + callerId: 'sc_0123456789abcdef', + callerName: 'flows-project', + publisher: { id: 'agent_publisher', name: 'flows-project' }, + workspace: { + site_url: 'https://agentrelay.com', + base_url: 'https://cast.agentrelay.com', + workspace_id: 'workspace_1', + relaycast_workspace_id: 'relay_workspace_1', + }, + account: { user_id: 'user_1' }, + createdAt: '2026-11-08T00:00:00.000Z', + expiresAt: '2026-12-08T00:00:00.000Z', + harnesses: ['codex'], + directoryScope: ['/workspace'], + ownsOnly: true, + socket: '/tmp/core.sock', + inbox: { + transport: 'core-bridge', + open: 'caller.inbox.open', + next: 'caller.inbox.next', + ack: 'caller.inbox.ack', + fail: 'caller.inbox.fail', + close: 'caller.inbox.close', + disconnect: 'caller.disconnect', + }, + desktopVersion: '17.3.0', + }; + + function fixture() { + const clients: CoreBridgeClient[] = []; + const close = vi.fn(); + const bridge: CoreBridgeApi = { + connectCore: vi.fn(async () => { + const client = { hello: { methods: [] }, close } satisfies CoreBridgeClient; + clients.push(client); + return client; + }), + readCallerDescriptor: vi.fn(async () => descriptor), + authenticate: vi.fn(async () => ({ caller_id: descriptor.callerId })), + requireServiceCallerMethods: vi.fn(), + assertServiceCallerIdentity: vi.fn((caller, identity) => { + if (caller.publisher.id !== identity.publisher.id + || caller.publisher.name !== identity.publisher.name + || caller.workspace.base_url !== identity.base_url + || caller.workspace.workspace_id !== identity.workspace_id + || caller.workspace.relaycast_workspace_id !== identity.relaycast_workspace_id) { + throw Object.assign(new Error('identity mismatch'), { code: 'caller_identity_mismatch' }); + } + }), + serviceCallerIdentity: vi.fn(async () => ({ + publisher: descriptor.publisher, + node: { id: 'node_1', name: 'desktop-1' }, + base_url: descriptor.workspace.base_url, + workspace_id: descriptor.workspace.workspace_id, + relaycast_workspace_id: descriptor.workspace.relaycast_workspace_id, + })), + spawnTargets: vi.fn(async () => ({ targets: [] })), + spawnInvoke: vi.fn(), + spawnStatus: vi.fn(), + spawnRelease: vi.fn(), + serviceCallerInboxOpen: vi.fn(async () => ({ iterator_id: 'it_1' })), + serviceCallerInboxNext: vi.fn(async () => ({ delivery: { + item_id: 'item_1', message_id: 'message_1', invocation_id: 'inv_1', sent_at: 'now', + sender: { id: 'agent_1', name: 'worker_1' }, text: 'RELAYFLOWS_RESULT_V1 {}', + } })), + serviceCallerInboxAck: vi.fn(async () => ({})), + serviceCallerInboxFail: vi.fn(async () => ({})), + serviceCallerInboxClose: vi.fn(async () => ({})), + serviceCallerDisconnect: vi.fn(async () => ({})), + }; + return { bridge, clients, close }; + } + + it('refuses ambiguous local and remote authority before loading either runtime', async () => { + const { bridge } = fixture(); + await expect(createNodeSpawnRuntime({ + RELAYFLOW_SERVICE_CALLER_FILE: '/tmp/caller.json', + RELAY_AGENT_TOKEN: 'agent-token', + }, { bridge })).rejects.toMatchObject({ code: 'ambiguous_publisher_authority' }); + expect(bridge.readCallerDescriptor).not.toHaveBeenCalled(); + expect(relay.constructorOptions).toHaveLength(0); + }); + + it('does not scan for a Desktop caller when neither authority was explicit', async () => { + const { bridge } = fixture(); + await expect(createNodeSpawnRuntime({}, { bridge })) + .rejects.toMatchObject({ code: 'publisher_token_required' }); + expect(bridge.readCallerDescriptor).not.toHaveBeenCalled(); + expect(bridge.connectCore).not.toHaveBeenCalled(); + }); + + it('refuses a relative caller file before loading or connecting the optional bridge', async () => { + const { bridge } = fixture(); + await expect(createLocalNodeSpawnRuntime('caller.json', { bridge })) + .rejects.toMatchObject({ code: 'caller_file_invalid' }); + expect(bridge.readCallerDescriptor).not.toHaveBeenCalled(); + expect(bridge.connectCore).not.toHaveBeenCalled(); + }); + + it('binds schema-v2 authority and checks the live identity against caller.json', async () => { + const { bridge, close } = fixture(); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge }); + expect(bridge.readCallerDescriptor).toHaveBeenCalledWith('/tmp/caller.json'); + expect(bridge.readCallerDescriptor).toHaveBeenCalledOnce(); + expect(bridge.connectCore).toHaveBeenCalledWith({ socket: '/tmp/core.sock', signal: undefined }); + expect(bridge.requireServiceCallerMethods).toHaveBeenCalledOnce(); + expect(bridge.authenticate).toHaveBeenCalledWith(expect.anything(), descriptor.credential, { signal: undefined }); + expect(runtime).toMatchObject({ + publisher: descriptor.publisher, + baseUrl: descriptor.workspace.base_url, + node: 'desktop-1', + }); + await runtime.disconnect?.(); + expect(bridge.serviceCallerDisconnect).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + }); + + it('fails closed when the live publisher or workspace differs from caller.json', async () => { + const { bridge, close } = fixture(); + vi.mocked(bridge.serviceCallerIdentity).mockResolvedValue({ + publisher: { ...descriptor.publisher, id: 'agent_other' }, + node: { id: 'node_1', name: 'desktop-1' }, + base_url: descriptor.workspace.base_url, + workspace_id: descriptor.workspace.workspace_id, + relaycast_workspace_id: descriptor.workspace.relaycast_workspace_id, + }); + await expect(createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge })) + .rejects.toMatchObject({ code: 'caller_identity_mismatch' }); + expect(close).toHaveBeenCalledOnce(); + }); + + it('fails closed when Desktop omits the exact live node identity', async () => { + const { bridge, close } = fixture(); + vi.mocked(bridge.serviceCallerIdentity).mockResolvedValue({ + publisher: descriptor.publisher, + node: { id: '', name: '' }, + base_url: descriptor.workspace.base_url, + workspace_id: descriptor.workspace.workspace_id, + relaycast_workspace_id: descriptor.workspace.relaycast_workspace_id, + }); + await expect(createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge })) + .rejects.toMatchObject({ code: 'caller_identity_mismatch' }); + expect(bridge.serviceCallerDisconnect).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + }); + + it('opens the exact invocation and settles after iterator teardown on a fresh binding', async () => { + const { bridge, clients, close } = fixture(); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge }); + const iterator = runtime.inbox.inbox.subscribe({ invocationId: 'inv_1' })[Symbol.asyncIterator](); + await expect(iterator.next()).resolves.toMatchObject({ + done: false, + value: { id: 'item_1', message: { id: 'message_1', from: { id: 'agent_1', name: 'worker_1' } } }, + }); + await iterator.return?.(); + await runtime.inbox.inbox.ack({ inboxItemId: 'item_1', metadata: { relayflows_result: 'journaled' } }); + expect(bridge.serviceCallerInboxOpen).toHaveBeenCalledWith(clients[1], 'inv_1', { signal: undefined }); + expect(bridge.serviceCallerInboxClose).toHaveBeenCalledWith(clients[1], 'it_1'); + expect(bridge.serviceCallerInboxAck).toHaveBeenCalledWith(clients[2], 'item_1'); + expect(bridge.connectCore).toHaveBeenCalledTimes(3); + expect(bridge.authenticate).toHaveBeenCalledTimes(3); + expect(bridge.serviceCallerDisconnect).toHaveBeenCalledTimes(2); + expect(close).toHaveBeenCalledTimes(2); + await runtime.disconnect?.(); + expect(bridge.serviceCallerDisconnect).toHaveBeenCalledTimes(3); + expect(close).toHaveBeenCalledTimes(3); + }); + + it('does not open the fresh settlement binding until the invocation iterator is closed', async () => { + const { bridge } = fixture(); + let finishClose!: () => void; + vi.mocked(bridge.serviceCallerInboxClose).mockImplementation( + () => new Promise(resolve => { finishClose = () => resolve({}); }), + ); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge }); + const delivery = await waitForNodeResult(runtime.inbox, { + agentId: 'agent_1', agentName: 'worker_1', invocationId: 'inv_1', + runId: 'run_1', stepId: 'step_1', dispatchId: 'dispatch_1', + }); + await vi.waitFor(() => expect(bridge.serviceCallerInboxClose).toHaveBeenCalledOnce()); + const settlement = delivery.ack(); + expect(bridge.connectCore).toHaveBeenCalledTimes(2); + finishClose(); + await settlement; + expect(bridge.connectCore).toHaveBeenCalledTimes(3); + }); + + it('keeps polling the same invocation after empty long polls without opening another binding', async () => { + const { bridge } = fixture(); + vi.mocked(bridge.serviceCallerInboxNext) + .mockResolvedValueOnce({ delivery: null }) + .mockResolvedValueOnce({ delivery: null }); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge }); + const iterator = runtime.inbox.inbox.subscribe({ invocationId: 'inv_1' })[Symbol.asyncIterator](); + await expect(iterator.next()).resolves.toMatchObject({ done: false, value: { id: 'item_1' } }); + expect(bridge.serviceCallerInboxOpen).toHaveBeenCalledOnce(); + expect(bridge.serviceCallerInboxNext).toHaveBeenCalledTimes(3); + await iterator.return?.(); + }); + + it.each(['result_too_large', 'result_write_failed'] as const)( + 'preserves typed %s details with exact correlation and settles it as failed on a fresh binding', + async code => { + const { bridge, clients } = fixture(); + vi.mocked(bridge.serviceCallerInboxNext).mockResolvedValue({ delivery: { + item_id: 'item_failure', message_id: 'message_failure', invocation_id: 'inv_1', sent_at: 'now', + sender: { id: 'agent_1', name: 'worker_1' }, failure: { code, bytes: 1025, limit: 1024 }, + } }); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge }); + const delivery = await waitForNodeResult(runtime.inbox, { + agentId: 'agent_1', agentName: 'worker_1', invocationId: 'inv_1', + runId: 'run_1', stepId: 'step_1', dispatchId: 'dispatch_1', + }); + expect(delivery).toMatchObject({ + settlementAction: 'fail', + envelope: { + version: 1, run_id: 'run_1', step_id: 'step_1', dispatch_id: 'dispatch_1', + status: 'failed', error: `${code} (bytes=1025, limit=1024)`, + }, + }); + await delivery.ack(); + await vi.waitFor(() => expect(bridge.serviceCallerInboxClose).toHaveBeenCalledOnce()); + expect(bridge.serviceCallerInboxFail).toHaveBeenCalledWith(clients[2], 'item_failure'); + expect(bridge.connectCore).toHaveBeenCalledTimes(3); + }, + ); + + it('requires an invocation id before opening a local inbox connection', async () => { + const { bridge } = fixture(); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge }); + expect(() => runtime.inbox.inbox.subscribe({})).toThrow( + expect.objectContaining({ code: 'result_identity_missing' }), + ); + expect(bridge.connectCore).toHaveBeenCalledOnce(); + }); + + it('refuses a delivery outside the exact opened invocation', async () => { + const { bridge } = fixture(); + vi.mocked(bridge.serviceCallerInboxNext).mockResolvedValue({ delivery: { + item_id: 'item_1', message_id: 'message_1', invocation_id: 'inv_other', sent_at: 'now', + sender: { id: 'agent_1', name: 'worker_1' }, text: 'RELAYFLOWS_RESULT_V1 {}', + } }); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge }); + const iterator = runtime.inbox.inbox.subscribe({ invocationId: 'inv_1' })[Symbol.asyncIterator](); + await expect(iterator.next()).rejects.toMatchObject({ code: 'result_identity_mismatch' }); + await iterator.return?.(); + expect(bridge.serviceCallerInboxClose).toHaveBeenCalledOnce(); + }); +}); diff --git a/packages/sdk/tests/relay-cli-surface.test.ts b/packages/sdk/tests/relay-cli-surface.test.ts index 6f76038ff..9a5e9317d 100644 --- a/packages/sdk/tests/relay-cli-surface.test.ts +++ b/packages/sdk/tests/relay-cli-surface.test.ts @@ -142,14 +142,16 @@ const INVOCATIONS: readonly { verb: string; argv: readonly string[]; variant: Pa { verb: 'resume', argv: ['resume', RUN_ID], variant: 'resume' }, { verb: 'resume', - argv: ['resume', '--json', '--data-dir', '.relayflowd', '--local-agent', '--agent-capacity', '8', '--no-spawn', + argv: ['resume', '--json', '--data-dir', '.relayflowd', '--local-agent', '--agent-capacity', '8', + '--service-caller-file', '/tmp/caller.json', '--no-spawn', '--no-observer-link', '--cloud-mirror', '--detach', '--allow-human-influenced', RUN_ID], variant: 'resume', }, { verb: 'run', argv: ['run', 'flow.yaml'], variant: 'run' }, { verb: 'run', - argv: ['run', '--json', '--data-dir', '.relayflowd', '--local-agent', '--agent-capacity', '8', '--no-spawn', + argv: ['run', '--json', '--data-dir', '.relayflowd', '--local-agent', '--agent-capacity', '8', + '--service-caller-file', '/tmp/caller.json', '--no-spawn', '--no-observer-link', '--cloud-mirror', '--detach', '--allow-human-influenced', '--input', '{"a":1}', 'review.flow.ts'], variant: 'run', }, diff --git a/packages/sdk/tests/worker-slots.test.ts b/packages/sdk/tests/worker-slots.test.ts index 52fb55da2..2180abc3c 100644 --- a/packages/sdk/tests/worker-slots.test.ts +++ b/packages/sdk/tests/worker-slots.test.ts @@ -128,3 +128,24 @@ describe('--agent-capacity', () => { } }); }); + +describe('--service-caller-file', () => { + it('carries an explicit Desktop handoff only for local run and resume workers', () => { + expect(parseCliArgs(['run', '--local-agent', '--service-caller-file', '/tmp/caller.json', 'flow.yaml'])) + .toMatchObject({ command: 'run', localAgent: true, serviceCallerFile: '/tmp/caller.json' }); + expect(parseCliArgs(['resume', '--local-agent', '--service-caller-file', '/tmp/caller.json', 'run-1'])) + .toMatchObject({ command: 'resume', localAgent: true, serviceCallerFile: '/tmp/caller.json' }); + }); + + it('refuses a handoff that would be ignored or sent to Cloud', () => { + for (const argv of [ + ['run', '--service-caller-file', '/tmp/caller.json', 'flow.yaml'], + ['run', '--cloud', '--local-agent', '--service-caller-file', '/tmp/caller.json', 'flow.yaml'], + ['check', '--service-caller-file', '/tmp/caller.json', 'flow.yaml'], + ['run', '--local-agent', '--service-caller-file', 'flow.yaml'], + ['run', '--local-agent', '--service-caller-file', '/tmp/a', '--service-caller-file', '/tmp/b', 'flow.yaml'], + ]) { + expect(parseCliArgs(argv), argv.join(' ')).toBeUndefined(); + } + }); +}); From 6a610b1b0d0f124713bd5912c140a27446f2b485 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 8 Nov 2026 23:20:43 -0800 Subject: [PATCH 29/75] fix(sdk): close Desktop node-spawn lifecycle gaps --- docs/MIGRATION-NODE-SPAWN.md | 5 +- packages/sdk/package-lock.json | 2 +- packages/sdk/package.json | 2 +- packages/sdk/src/agent-relay-transport.ts | 2 +- packages/sdk/src/node-spawn-local.ts | 20 ++++--- packages/sdk/src/node-spawn-result.ts | 8 +++ packages/sdk/src/node-spawn-runtime.ts | 53 ++++++++++++----- packages/sdk/src/node-spawn-settlement.ts | 12 ++++ packages/sdk/src/worker.ts | 16 ++++- packages/sdk/tests/node-spawn-ports.test.ts | 16 ++++- packages/sdk/tests/node-spawn-runtime.test.ts | 58 ++++++++++++++++++- .../sdk/tests/node-spawn-settlement.test.ts | 45 ++++++++++++++ .../tests/worker-node-runtime-sharing.test.ts | 56 ++++++++++++++++++ 13 files changed, 266 insertions(+), 29 deletions(-) diff --git a/docs/MIGRATION-NODE-SPAWN.md b/docs/MIGRATION-NODE-SPAWN.md index 650b697cc..b4223896c 100644 --- a/docs/MIGRATION-NODE-SPAWN.md +++ b/docs/MIGRATION-NODE-SPAWN.md @@ -32,9 +32,10 @@ private file explicitly with `--service-caller-file /absolute/caller.json`, the SDK `serviceCallerFile` option, or `RELAYFLOW_SERVICE_CALLER_FILE`. Flows never scans Desktop's app-data directory. It securely parses and challenge-authenticates the handoff through optional -`@agent-relay/core-bridge` 17.3 or later, then sends no caller name or secret +`@agent-relay/core-bridge` 18.x, then sends no caller name or secret on spawn or inbox calls. The handoff's harness and directory scope is enforced -before dispatch. +before dispatch. A local relay step with no authored `cwd` spawns in that run's +root; remote relay placement still receives no local directory. `RELAYFLOW_SERVICE_CALLER_FILE` and `RELAY_AGENT_TOKEN` are mutually exclusive: the former is local Desktop authority and the latter is remote fleet authority. diff --git a/packages/sdk/package-lock.json b/packages/sdk/package-lock.json index 53b457943..7f6e5985c 100644 --- a/packages/sdk/package-lock.json +++ b/packages/sdk/package-lock.json @@ -36,7 +36,7 @@ "vitest": "^2.1.0" }, "peerDependencies": { - "@agent-relay/core-bridge": ">=17.3.0 <18", + "@agent-relay/core-bridge": ">=18.0.0 <19", "@agent-relay/sdk": ">=17.2.1 <18" }, "peerDependenciesMeta": { diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 698f3dfb2..edf69877a 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -76,7 +76,7 @@ "directory": "packages/sdk" }, "peerDependencies": { - "@agent-relay/core-bridge": ">=17.3.0 <18", + "@agent-relay/core-bridge": ">=18.0.0 <19", "@agent-relay/sdk": ">=17.2.1 <18" }, "peerDependenciesMeta": { diff --git a/packages/sdk/src/agent-relay-transport.ts b/packages/sdk/src/agent-relay-transport.ts index 1f5fcd7ed..a936d106d 100644 --- a/packages/sdk/src/agent-relay-transport.ts +++ b/packages/sdk/src/agent-relay-transport.ts @@ -281,7 +281,7 @@ async function executeAgentRelayTask( // Construct the settlement wrapper from the post-release claim. Otherwise // preparing its journal intent could rewrite the file from a stale copy // and silently discard the exact release receipt. - const delivery = durableNodeResultDelivery(runtime, claimed.path, claim); + const delivery = durableNodeResultDelivery(runtime, claimed.path, claim, observed); resultSignal?.throwIfAborted(); const envelope = delivery.envelope; const error = envelope.status === 'failed' ? envelope.error! : null; diff --git a/packages/sdk/src/node-spawn-local.ts b/packages/sdk/src/node-spawn-local.ts index 643858d92..fe4f1e8bd 100644 --- a/packages/sdk/src/node-spawn-local.ts +++ b/packages/sdk/src/node-spawn-local.ts @@ -138,6 +138,7 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { private readonly core: CoreBridgeClient, private readonly bridge: Pick, private readonly scope: Pick, + private readonly exactNode: { readonly id: string; readonly name: string }, ) {} async targets(): Promise { @@ -187,11 +188,14 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { ); } const allTargets = await this.targets(); - const eligible = allTargets.filter(candidate => candidate.live && candidate.handlersLive - && !candidate.stale && candidate.capabilities.includes(`spawn:${request.harness}`)); - const target = request.node === undefined - ? eligible[0] - : eligible.find(candidate => candidate.name === request.node); + if (request.node !== undefined && request.node !== this.exactNode.name) { + throw new NodeSpawnError('The requested node differs from the authenticated Desktop node', + 'spawn_identity_mismatch'); + } + const target = allTargets.find(candidate => candidate.id === this.exactNode.id + && candidate.name === this.exactNode.name + && candidate.live && candidate.handlersLive && !candidate.stale + && candidate.capabilities.includes(`spawn:${request.harness}`)); if (!target?.id) throw new NodeSpawnError('No eligible local Agent Relay spawn target', 'no_eligible_node'); const raw = await this.bridge.spawnInvoke(this.core, { node_id: target.id, @@ -235,8 +239,10 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { throw new NodeSpawnError('Invalid node-release idempotency key', 'spawn_invalid_input'); } const receipt = request.receipt; - if (!receipt.nodeId) { - throw new NodeSpawnError('Local release requires the exact spawn node id', 'release_identity_mismatch'); + if (!receipt.nodeId || receipt.nodeId !== this.exactNode.id + || receipt.nodeName !== this.exactNode.name) { + throw new NodeSpawnError('Local release requires the authenticated Desktop node identity', + 'release_identity_mismatch'); } const raw = await this.bridge.spawnRelease(this.core, { node_id: receipt.nodeId, diff --git a/packages/sdk/src/node-spawn-result.ts b/packages/sdk/src/node-spawn-result.ts index b4f456800..b46d0d59d 100644 --- a/packages/sdk/src/node-spawn-result.ts +++ b/packages/sdk/src/node-spawn-result.ts @@ -47,6 +47,8 @@ export interface NodeResultDelivery { messageId: string; envelope: NodeResultEnvelope; settlementAction: 'ack' | 'fail'; + /** Non-rejecting current-process teardown outcome. Durable replay intentionally omits it. */ + settlementReady?: Promise<{ failure?: unknown }>; /** Durably records the journal settlement that must precede inbox settlement. */ prepareSettlement?(input: { attempt: number; completionReason: string }): Promise; /** Called only after the result has been durably appended to the run journal. */ @@ -106,6 +108,7 @@ export async function waitForNodeResult( messageId: item.message.id, envelope, settlementAction: 'ack', + settlementReady: settlementOutcome(item), ack: async () => { await awaitSettlementReady(item); await client.inbox.ack({ inboxItemId: item.id, state: 'read', metadata: { relayflows_result: 'journaled' } }); }, }; @@ -139,11 +142,16 @@ function rejectedDelivery( envelope: { version: 1, run_id: expected.runId, step_id: expected.stepId, dispatch_id: expected.dispatchId, status: 'failed', error: reason }, settlementAction: 'fail', + settlementReady: settlementOutcome(item), ack: async () => { await awaitSettlementReady(item); await client.inbox.fail({ inboxItemId: item.id, error: reason, retry: false, metadata: { relayflows_result: 'rejected' } }); }, }; } +function settlementOutcome(item: InboxItem): Promise<{ failure?: unknown }> | undefined { + return item.settlementReady?.then(outcome => outcome, failure => ({ failure })); +} + async function awaitSettlementReady(item: InboxItem): Promise { const teardown = await item.settlementReady; if (teardown?.failure !== undefined) throw teardown.failure; diff --git a/packages/sdk/src/node-spawn-runtime.ts b/packages/sdk/src/node-spawn-runtime.ts index cb2db58e5..adb79f275 100644 --- a/packages/sdk/src/node-spawn-runtime.ts +++ b/packages/sdk/src/node-spawn-runtime.ts @@ -22,6 +22,8 @@ export interface NodeSpawnRuntime { node?: string; repo?: string; sandboxOnly?: boolean; + /** Explicit authority family; Desktop alone may receive the worker's local run root. */ + authority?: 'desktop' | 'remote'; /** Closes the publisher connection after every task using it has drained. */ disconnect?(): Promise; } @@ -78,6 +80,7 @@ export async function createRemoteNodeSpawnRuntime( inbox: { inbox: client.inbox }, publisher, baseUrl, + authority: 'remote', disconnect: () => client.events.disconnect(), ...(env.RELAYFLOW_NODE_NAME?.trim() ? { node: env.RELAYFLOW_NODE_NAME.trim() } : {}), ...(env.RELAYFLOW_NODE_REPO?.trim() ? { repo: env.RELAYFLOW_NODE_REPO.trim() } : {}), @@ -112,10 +115,11 @@ export async function createLocalNodeSpawnRuntime( ); } return { - port: new LocalNodeSpawnPort(core, bridge, descriptor), + port: new LocalNodeSpawnPort(core, bridge, descriptor, identity.node), inbox: localResultInbox(bridge, descriptor), publisher: identity.publisher, baseUrl: identity.base_url, + authority: 'desktop', node: identity.node.name, disconnect: async () => { await disconnectBoundCore(bridge, core); @@ -230,42 +234,65 @@ function localInboxSubscription( ): AsyncIterable<{ id: string; message: { id: string; text: string; from: { id?: string; name?: string } } }> { return { [Symbol.asyncIterator]() { let core: CoreBridgeClient | undefined; + let binding: Promise | undefined; + let opening: Promise | undefined; let iteratorId: string | undefined; let closed = false; + let closePromise: Promise | undefined; let resolveClosed!: (result: { failure?: unknown }) => void; const closedPromise = new Promise<{ failure?: unknown }>(resolve => { resolveClosed = resolve; }); - const close = async (): Promise => { - if (closed) return; + const closeOnce = async (): Promise => { closed = true; - if (core === undefined) { - resolveClosed({}); + let failure: unknown; + if (opening !== undefined) { + try { await opening; } + catch (error) { failure = error; } + } + const bound = core; + if (bound === undefined) { + resolveClosed(failure === undefined ? {} : { failure }); + if (failure !== undefined) throw failure; return; } - let failure: unknown; try { - if (iteratorId !== undefined) await bridge.serviceCallerInboxClose(core, iteratorId); - } catch (error) { failure = error; } + if (iteratorId !== undefined) await bridge.serviceCallerInboxClose(bound, iteratorId); + } catch (error) { failure ??= error; } try { - await disconnectBoundCore(bridge, core, failure); + await disconnectBoundCore(bridge, bound, failure); resolveClosed({}); } catch (error) { resolveClosed({ failure: error }); throw error; } }; + const close = (): Promise => closePromise ??= closeOnce(); + const ensureOpen = (): Promise => opening ??= (async () => { + binding ??= bindCore(bridge, descriptor, signal); + core = await binding; + if (closed) return; + const opened = await bridge.serviceCallerInboxOpen(core, invocationId, { signal }); + iteratorId = opened.iterator_id; + })(); return { async next() { for (;;) { signal?.throwIfAborted(); if (closed) return { done: true, value: undefined }; if (core === undefined) { - core = await bindCore(bridge, descriptor, signal); - const opened = await bridge.serviceCallerInboxOpen(core, invocationId, { signal }); - iteratorId = opened.iterator_id; + await ensureOpen(); + if (closed) { + await close(); + return { done: true, value: undefined }; + } + } + const bound = core; + if (bound === undefined || iteratorId === undefined) { + throw new NodeSpawnError('Desktop inbox binding did not return an iterator', + 'invalid_core_response'); } - const result = await bridge.serviceCallerInboxNext(core, iteratorId!, 30_000, { signal }); + const result = await bridge.serviceCallerInboxNext(bound, iteratorId, 30_000, { signal }); if (result.delivery !== null) { if (result.delivery.invocation_id !== invocationId) { throw new NodeSpawnError( diff --git a/packages/sdk/src/node-spawn-settlement.ts b/packages/sdk/src/node-spawn-settlement.ts index c8a5447bc..7f2365a4b 100644 --- a/packages/sdk/src/node-spawn-settlement.ts +++ b/packages/sdk/src/node-spawn-settlement.ts @@ -11,9 +11,16 @@ export function durableNodeResultDelivery( runtime: NodeSpawnRuntime, path: string, initial: NodeSpawnClaim, + observed?: NodeResultDelivery, ): NodeResultDelivery { const result = initial.result; if (result === undefined) throw new NodeSpawnError('Node result settlement requires a durable result'); + if (observed !== undefined && (observed.inboxItemId !== result.inboxItemId + || observed.messageId !== result.messageId + || observed.settlementAction !== result.settlementAction)) { + throw new NodeSpawnError('Observed node result differs from its durable settlement identity', + 'result_identity_mismatch'); + } let claim = initial; let settling: Promise | undefined; const settleOnce = async (): Promise => { @@ -23,6 +30,10 @@ export function durableNodeResultDelivery( throw new NodeSpawnError('Node result cannot settle before its journal intent is durable', 'result_settlement_unprepared'); } + if (observed?.settlementReady !== undefined) { + const teardown = await observed.settlementReady; + if (teardown.failure !== undefined) throw teardown.failure; + } await settleInbox(runtime, current.settlementAction, current.inboxItemId, current.envelope.error ?? 'Relayflows rejected an invalid result'); claim = { ...claim, result: { ...current, settlement: { @@ -43,6 +54,7 @@ export function durableNodeResultDelivery( messageId: result.messageId, envelope: result.envelope, settlementAction: result.settlementAction, + ...(observed?.settlementReady === undefined ? {} : { settlementReady: observed.settlementReady }), prepareSettlement: async input => { const current = claim.result; if (current === undefined) throw new NodeSpawnError('Durable node result disappeared before settlement'); diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index 1babc9f99..8ae87978a 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -235,7 +235,9 @@ export class AgentWorker extends EventEmitter { // the agent's filesystem. A refusal completes the step the way a missing // CLI does — journaled as `worker_error` with the reason — rather than // spawning into a directory nobody established is inside the run root. - const cwd = agentStepCwd(spec, dispatch.step_id, this.options.runRoot); + const localDesktopRelay = spec.transport === 'relay' && this.usesLocalDesktopAuthority(); + const cwd = agentStepCwd(localDesktopRelay ? { ...spec, transport: 'direct' } : spec, + dispatch.step_id, this.options.runRoot); const completed: WorkerCliResult = await withWorkerLease(this.client, dispatch, signal => 'refusal' in cwd ? Promise.resolve({ exit_code: null, stdout_tail: '', stderr_tail: cwd.refusal }) @@ -243,7 +245,9 @@ export class AgentWorker extends EventEmitter { ? runAgentCli(spec.cli, workerInstruction(spec.instruction, dispatch), dispatch.wake_context, effectiveModel, undefined, signal, 'agent', this.options.dataDir === undefined ? undefined : { dataDir: this.options.dataDir, runId: dispatch.run_id, stepId: dispatch.step_id, attempt: dispatch.attempt, onReady: this.options.onPtyReady, onDrive: () => { humanIntervention = true; }, - }, cwd.directory ?? (spec.transport === 'relay' ? undefined : this.options.runRoot), + }, cwd.directory ?? (localDesktopRelay + ? (this.options.runRoot ?? process.cwd()) + : (spec.transport === 'relay' ? undefined : this.options.runRoot)), spec.transport === 'relay' ? 'relay' : 'direct', { runId: dispatch.run_id, stepId: dispatch.step_id, idempotencyKey: dispatch.idempotency_key, attempt: dispatch.attempt, dataDir: this.options.dataDir, resultSchema: spec.verification?.json_schema, @@ -327,6 +331,14 @@ export class AgentWorker extends EventEmitter { await relayDelivery?.ack(); } + /** Explicit Desktop authority is the only relay mode allowed to receive a local run root. */ + private usesLocalDesktopAuthority(): boolean { + return this.options.nodeSpawnRuntime?.authority === 'desktop' + || Boolean(this.options.serviceCallerFile?.trim() + || this.options.environment?.RELAYFLOW_SERVICE_CALLER_FILE?.trim() + || process.env.RELAYFLOW_SERVICE_CALLER_FILE?.trim()); + } + private reconcileRelaySettlementsAfterDrain(): Promise { if (this.settlementDrain !== undefined) return this.settlementDrain; const generation = Array.from(this.inFlight); diff --git a/packages/sdk/tests/node-spawn-ports.test.ts b/packages/sdk/tests/node-spawn-ports.test.ts index 22da54747..2353de6fe 100644 --- a/packages/sdk/tests/node-spawn-ports.test.ts +++ b/packages/sdk/tests/node-spawn-ports.test.ts @@ -22,7 +22,7 @@ function localPort(core: TestCore): LocalNodeSpawnPort { return new LocalNodeSpawnPort(client, bridge, { harnesses: ['codex'], directoryScope: ['/workspace'], - }); + }, { id: 'node_1', name: 'desktop-1' }); } describe('local node-spawn port', () => { @@ -108,6 +108,20 @@ describe('local node-spawn port', () => { expect(calls).toHaveBeenCalledTimes(1); }); + it('does not dispatch to a duplicate node name with a different authenticated id', async () => { + const calls = vi.fn(); + const core: TestCore = { async call(method: string): Promise { + calls(method); + if (method === 'spawn.targets') return { targets: [{ id: 'node_other', name: 'desktop-1', + live: true, handlers_live: true, stale: false, capabilities: ['spawn:codex'] }] } as T; + throw new Error('spawn must not be invoked'); + } }; + await expect(localPort(core).spawn({ name: 'worker_1', harness: 'codex', task: 'work', + node: 'desktop-1', directory: '/workspace', idempotencyKey: 'dispatch_1234567890' })) + .rejects.toMatchObject({ code: 'no_eligible_node' }); + expect(calls).toHaveBeenCalledTimes(1); + }); + it('stops polling on a denied terminal status', async () => { const status = vi.fn(); const core: TestCore = { async call(method: string): Promise { diff --git a/packages/sdk/tests/node-spawn-runtime.test.ts b/packages/sdk/tests/node-spawn-runtime.test.ts index 3d9c5e4ee..913c4c615 100644 --- a/packages/sdk/tests/node-spawn-runtime.test.ts +++ b/packages/sdk/tests/node-spawn-runtime.test.ts @@ -184,7 +184,7 @@ describe('Desktop service-caller runtime', () => { close: 'caller.inbox.close', disconnect: 'caller.disconnect', }, - desktopVersion: '17.3.0', + desktopVersion: '18.0.0', }; function fixture() { @@ -359,6 +359,62 @@ describe('Desktop service-caller runtime', () => { await iterator.return?.(); }); + it('disconnects a late inbox binding without opening an iterator after close', async () => { + const { bridge, close } = fixture(); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge }); + let resolveBinding!: (client: CoreBridgeClient) => void; + vi.mocked(bridge.connectCore).mockImplementationOnce(() => new Promise(resolve => { + resolveBinding = resolve; + })); + const iterator = runtime.inbox.inbox.subscribe({ invocationId: 'inv_1' })[Symbol.asyncIterator](); + const next = iterator.next(); + await vi.waitFor(() => expect(bridge.connectCore).toHaveBeenCalledTimes(2)); + const closing = iterator.return!(); + resolveBinding({ hello: { methods: [] }, close }); + await expect(next).resolves.toEqual({ done: true, value: undefined }); + await expect(closing).resolves.toEqual({ done: true, value: undefined }); + expect(bridge.serviceCallerInboxOpen).not.toHaveBeenCalled(); + expect(bridge.serviceCallerInboxClose).not.toHaveBeenCalled(); + expect(bridge.serviceCallerDisconnect).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + }); + + it('closes an iterator whose uncooperative open resolves after close', async () => { + const { bridge, close } = fixture(); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge }); + let finishOpen!: () => void; + vi.mocked(bridge.serviceCallerInboxOpen).mockImplementationOnce( + () => new Promise(resolve => { finishOpen = () => resolve({ iterator_id: 'it_late' }); }), + ); + const iterator = runtime.inbox.inbox.subscribe({ invocationId: 'inv_1' })[Symbol.asyncIterator](); + const next = iterator.next(); + await vi.waitFor(() => expect(bridge.serviceCallerInboxOpen).toHaveBeenCalledOnce()); + const closing = iterator.return!(); + expect(bridge.serviceCallerInboxNext).not.toHaveBeenCalled(); + expect(bridge.serviceCallerDisconnect).not.toHaveBeenCalled(); + finishOpen(); + await expect(next).resolves.toEqual({ done: true, value: undefined }); + await expect(closing).resolves.toEqual({ done: true, value: undefined }); + expect(bridge.serviceCallerInboxNext).not.toHaveBeenCalled(); + expect(bridge.serviceCallerInboxClose).toHaveBeenCalledWith(expect.anything(), 'it_late'); + expect(bridge.serviceCallerDisconnect).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + }); + + it('disconnects an authenticated binding when inbox open rejects', async () => { + const { bridge, close } = fixture(); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge }); + const openFailure = new Error('open rejected'); + vi.mocked(bridge.serviceCallerInboxOpen).mockRejectedValueOnce(openFailure); + const iterator = runtime.inbox.inbox.subscribe({ invocationId: 'inv_1' })[Symbol.asyncIterator](); + await expect(iterator.next()).rejects.toBe(openFailure); + await expect(iterator.return!()).rejects.toBe(openFailure); + expect(bridge.serviceCallerInboxNext).not.toHaveBeenCalled(); + expect(bridge.serviceCallerInboxClose).not.toHaveBeenCalled(); + expect(bridge.serviceCallerDisconnect).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + }); + it.each(['result_too_large', 'result_write_failed'] as const)( 'preserves typed %s details with exact correlation and settles it as failed on a fresh binding', async code => { diff --git a/packages/sdk/tests/node-spawn-settlement.test.ts b/packages/sdk/tests/node-spawn-settlement.test.ts index c2d39721b..2b994d979 100644 --- a/packages/sdk/tests/node-spawn-settlement.test.ts +++ b/packages/sdk/tests/node-spawn-settlement.test.ts @@ -82,6 +82,51 @@ function terminalEntry(payload: Record = journaledRelayResult() } describe('durable node-result settlement', () => { + it('carries the observed teardown barrier through durable settlement', async () => { + const dataDir = await dataDirectory('flows-node-settlement-teardown-barrier-'); + const f = fixture(); + const order: string[] = []; + let finishClose!: () => void; + const closeGate = new Promise(resolve => { finishClose = resolve; }); + let reportClosed!: (value: { failure?: unknown }) => void; + const settlementReady = new Promise<{ failure?: unknown }>(resolve => { reportClosed = resolve; }); + let delivered = false; + f.runtime.inbox.inbox.subscribe = () => ({ [Symbol.asyncIterator]() { + return { + async next() { + if (delivered) return new Promise>(() => undefined); + delivered = true; + const name = (await f.spawn.mock.results[0]!.value).name; + return { done: false as const, value: { + id: 'delivery_1', settlementReady, + message: { id: 'message_1', from: { id: 'agent_1', name }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_1', + step_id: 'edit', dispatch_id: 'dispatch_1234567890', status: 'completed', + output: { summary: 'done' } }) }, + } }; + }, + async return() { + order.push('inbox.close'); + await closeGate; + order.push('caller.disconnect'); + reportClosed({}); + return { done: true as const, value: undefined }; + }, + }; + } }); + f.ack.mockImplementation(async () => { order.push('settle'); return {}; }); + + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await vi.waitFor(() => expect(order).toEqual(['inbox.close'])); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + const settling = received.delivery.ack(); + await Promise.resolve(); + expect(f.ack).not.toHaveBeenCalled(); + finishClose(); + await settling; + expect(order).toEqual(['inbox.close', 'caller.disconnect', 'settle']); + }); + it('rejects an invalid resolved spawn receipt before reading the inbox and releases exact identity', async () => { const dataDir = await dataDirectory('flows-node-invalid-resolved-spawn-'); const f = fixture(); diff --git a/packages/sdk/tests/worker-node-runtime-sharing.test.ts b/packages/sdk/tests/worker-node-runtime-sharing.test.ts index ae2cc6282..cfc0c63ef 100644 --- a/packages/sdk/tests/worker-node-runtime-sharing.test.ts +++ b/packages/sdk/tests/worker-node-runtime-sharing.test.ts @@ -27,6 +27,62 @@ afterEach(async () => { }); describe('AgentWorker node-spawn runtime ownership', () => { + it.each([ + { label: 'Desktop caller', serviceCallerFile: '/tmp/caller.json', authority: undefined, + provideRuntime: false, expectedDirectory: 'run-root' }, + { label: 'provided Desktop runtime', serviceCallerFile: undefined, authority: 'desktop' as const, + provideRuntime: true, expectedDirectory: 'run-root' }, + { label: 'remote publisher', serviceCallerFile: undefined, authority: 'remote' as const, + provideRuntime: true, expectedDirectory: undefined }, + ])('passes the local run root only to the $label relay spawn', async testCase => { + const runRoot = await mkdtemp(join(tmpdir(), 'flows-node-worker-run-root-')); + roots.push(runRoot); + let spawnedName = ''; + const spawn = vi.fn(async input => { + spawnedName = input.name; + return { id: 'spawn_worker_path', action: 'spawn', status: 'completed', + nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, agentId: 'agent_worker_path', + spawned: true, ready: true }; + }); + const runtime: NodeSpawnRuntime = { + publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + ...(testCase.authority === undefined ? {} : { authority: testCase.authority }), + port: { targets: async () => [], spawn, status: vi.fn(), + release: vi.fn(async () => ({ id: 'release_worker_path', action: 'release' as const, + status: 'completed' as const, spawned: false, ready: false, released: true })) }, + inbox: { inbox: { + subscribe: async function* () { + yield { id: 'delivery_worker_path', message: { id: 'message_worker_path', + from: { id: 'agent_worker_path', name: spawnedName }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_parallel', + step_id: 'edit-local-path', dispatch_id: 'dispatch_localpath', status: 'completed', + output: { summary: 'done' } }) } }; + }, + ack: vi.fn(async () => ({})), fail: vi.fn(async () => ({})), + } }, + }; + owner.get.mockResolvedValue(runtime); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), + stepComplete: vi.fn(async () => ({})), + supportsFeature: vi.fn(() => true), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: `worker_${testCase.label}`, pins: {}, dataDir: join(runRoot, 'data'), runRoot, + ...(testCase.serviceCallerFile === undefined ? {} : { serviceCallerFile: testCase.serviceCallerFile }), + ...(testCase.provideRuntime ? { nodeSpawnRuntime: runtime } : {}), + }); + await worker.attach(); + client.emit('step.dispatch', dispatch('edit-local-path', 'dispatch_localpath')); + await worker.close(); + expect(spawn).toHaveBeenCalledOnce(); + expect(spawn.mock.calls[0]![0].directory).toBe( + testCase.expectedDirectory === 'run-root' ? runRoot : undefined, + ); + }); + it('closes the shared runtime even when final durable inbox settlement still fails', async () => { const dataDir = await mkdtemp(join(tmpdir(), 'flows-shared-node-runtime-settlement-failure-')); roots.push(dataDir); From e2e0b42a6906e7c8785fdb2d24651b7cf4e00a4c Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 8 Nov 2026 23:25:58 -0800 Subject: [PATCH 30/75] fix(sdk): bound Desktop result teardown --- packages/sdk/src/node-spawn-settlement.ts | 25 ++++++++---- .../sdk/tests/node-spawn-settlement.test.ts | 40 +++++++++++++++++++ .../tests/worker-node-runtime-sharing.test.ts | 32 ++++++++++----- 3 files changed, 80 insertions(+), 17 deletions(-) diff --git a/packages/sdk/src/node-spawn-settlement.ts b/packages/sdk/src/node-spawn-settlement.ts index 7f2365a4b..fbd9ff09c 100644 --- a/packages/sdk/src/node-spawn-settlement.ts +++ b/packages/sdk/src/node-spawn-settlement.ts @@ -30,12 +30,14 @@ export function durableNodeResultDelivery( throw new NodeSpawnError('Node result cannot settle before its journal intent is durable', 'result_settlement_unprepared'); } - if (observed?.settlementReady !== undefined) { - const teardown = await observed.settlementReady; - if (teardown.failure !== undefined) throw teardown.failure; - } - await settleInbox(runtime, current.settlementAction, current.inboxItemId, - current.envelope.error ?? 'Relayflows rejected an invalid result'); + await bounded((async () => { + if (observed?.settlementReady !== undefined) { + const teardown = await observed.settlementReady; + if (teardown.failure !== undefined) throw teardown.failure; + } + await issueInboxSettlement(runtime, current.settlementAction, current.inboxItemId, + current.envelope.error ?? 'Relayflows rejected an invalid result'); + })()); claim = { ...claim, result: { ...current, settlement: { ...current.settlement, settledAt: Date.now(), settledAs: current.settlementAction, @@ -159,11 +161,20 @@ async function settleInbox( inboxItemId: string, error: string, ): Promise { + await bounded(issueInboxSettlement(runtime, action, inboxItemId, error)); +} + +function issueInboxSettlement( + runtime: NodeSpawnRuntime, + action: 'ack' | 'fail', + inboxItemId: string, + error: string, +): Promise { const pending = action === 'ack' ? runtime.inbox.inbox.ack({ inboxItemId, state: 'read', metadata: { relayflows_result: 'journaled' } }) : runtime.inbox.inbox.fail({ inboxItemId, error, retry: false, metadata: { relayflows_result: 'rejected' } }); - await bounded(pending); + return pending; } async function bounded(pending: Promise): Promise { diff --git a/packages/sdk/tests/node-spawn-settlement.test.ts b/packages/sdk/tests/node-spawn-settlement.test.ts index 2b994d979..b2be14558 100644 --- a/packages/sdk/tests/node-spawn-settlement.test.ts +++ b/packages/sdk/tests/node-spawn-settlement.test.ts @@ -127,6 +127,46 @@ describe('durable node-result settlement', () => { expect(order).toEqual(['inbox.close', 'caller.disconnect', 'settle']); }); + it('bounds an observed teardown that never settles before opening settlement', async () => { + const dataDir = await dataDirectory('flows-node-settlement-teardown-timeout-'); + const f = fixture(); + let delivered = false; + const settlementReady = new Promise<{ failure?: unknown }>(() => undefined); + f.runtime.inbox.inbox.subscribe = () => ({ [Symbol.asyncIterator]() { + return { + async next() { + if (delivered) return new Promise>(() => undefined); + delivered = true; + const name = (await f.spawn.mock.results[0]!.value).name; + return { done: false as const, value: { + id: 'delivery_1', settlementReady, + message: { id: 'message_1', from: { id: 'agent_1', name }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_1', + step_id: 'edit', dispatch_id: 'dispatch_1234567890', status: 'completed', + output: { summary: 'done' } }) }, + } }; + }, + return: () => new Promise>(() => undefined), + }; + } }); + + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + vi.useFakeTimers(); + try { + const settlement = expect(received.delivery.ack()) + .rejects.toMatchObject({ code: 'result_settlement_unconfirmed' }); + await vi.advanceTimersByTimeAsync(10_001); + await settlement; + expect(f.ack).not.toHaveBeenCalled(); + expect(f.fail).not.toHaveBeenCalled(); + const claims = await readNodeSpawnClaims(dataDir); + expect(claims[0]!.claim.result?.settlement).not.toHaveProperty('settledAt'); + } finally { + vi.useRealTimers(); + } + }); + it('rejects an invalid resolved spawn receipt before reading the inbox and releases exact identity', async () => { const dataDir = await dataDirectory('flows-node-invalid-resolved-spawn-'); const f = fixture(); diff --git a/packages/sdk/tests/worker-node-runtime-sharing.test.ts b/packages/sdk/tests/worker-node-runtime-sharing.test.ts index cfc0c63ef..de337540d 100644 --- a/packages/sdk/tests/worker-node-runtime-sharing.test.ts +++ b/packages/sdk/tests/worker-node-runtime-sharing.test.ts @@ -1,5 +1,5 @@ import { EventEmitter } from 'node:events'; -import { mkdtemp, rm } from 'node:fs/promises'; +import { mkdir, mkdtemp, realpath, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; @@ -29,14 +29,21 @@ afterEach(async () => { describe('AgentWorker node-spawn runtime ownership', () => { it.each([ { label: 'Desktop caller', serviceCallerFile: '/tmp/caller.json', authority: undefined, - provideRuntime: false, expectedDirectory: 'run-root' }, + provideRuntime: false, expectedDirectory: 'run-root', expectedSpawn: true }, { label: 'provided Desktop runtime', serviceCallerFile: undefined, authority: 'desktop' as const, - provideRuntime: true, expectedDirectory: 'run-root' }, + provideRuntime: true, expectedDirectory: 'run-root', expectedSpawn: true }, { label: 'remote publisher', serviceCallerFile: undefined, authority: 'remote' as const, - provideRuntime: true, expectedDirectory: undefined }, - ])('passes the local run root only to the $label relay spawn', async testCase => { + provideRuntime: true, expectedDirectory: undefined, expectedSpawn: true }, + { label: 'Desktop caller with declared cwd', serviceCallerFile: '/tmp/caller.json', authority: undefined, + provideRuntime: false, cwd: 'checkout', expectedDirectory: 'declared', expectedSpawn: true }, + { label: 'remote publisher with declared cwd', serviceCallerFile: undefined, authority: 'remote' as const, + provideRuntime: true, cwd: 'checkout', expectedDirectory: undefined, expectedSpawn: false }, + ])('routes the run directory only for the $label', async testCase => { const runRoot = await mkdtemp(join(tmpdir(), 'flows-node-worker-run-root-')); roots.push(runRoot); + const declaredDirectory = join(runRoot, 'checkout'); + await mkdir(declaredDirectory); + const exactDeclaredDirectory = await realpath(declaredDirectory); let spawnedName = ''; const spawn = vi.fn(async input => { spawnedName = input.name; @@ -75,12 +82,17 @@ describe('AgentWorker node-spawn runtime ownership', () => { ...(testCase.provideRuntime ? { nodeSpawnRuntime: runtime } : {}), }); await worker.attach(); - client.emit('step.dispatch', dispatch('edit-local-path', 'dispatch_localpath')); + const value = dispatch('edit-local-path', 'dispatch_localpath'); + if (testCase.cwd !== undefined) { + value.spec = { ...(value.spec as Record), cwd: testCase.cwd }; + } + client.emit('step.dispatch', value); await worker.close(); - expect(spawn).toHaveBeenCalledOnce(); - expect(spawn.mock.calls[0]![0].directory).toBe( - testCase.expectedDirectory === 'run-root' ? runRoot : undefined, - ); + expect(spawn).toHaveBeenCalledTimes(testCase.expectedSpawn ? 1 : 0); + if (testCase.expectedSpawn) { + expect(spawn.mock.calls[0]![0].directory).toBe(testCase.expectedDirectory === 'run-root' + ? runRoot : testCase.expectedDirectory === 'declared' ? exactDeclaredDirectory : undefined); + } }); it('closes the shared runtime even when final durable inbox settlement still fails', async () => { From 83033c923796712d648be5f6a601fbef5069681c Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 8 Nov 2026 23:39:19 -0800 Subject: [PATCH 31/75] fix(sdk): preserve live Desktop teardown state --- packages/sdk/src/node-spawn-runtime.ts | 23 +++- packages/sdk/src/node-spawn-settlement.ts | 101 ++++++++++++++-- packages/sdk/src/worker.ts | 25 ++-- .../sdk/tests/node-spawn-settlement.test.ts | 78 ++++++++++++- .../tests/worker-node-runtime-sharing.test.ts | 108 +++++++++++++++++- 5 files changed, 306 insertions(+), 29 deletions(-) diff --git a/packages/sdk/src/node-spawn-runtime.ts b/packages/sdk/src/node-spawn-runtime.ts index adb79f275..2d5fc858a 100644 --- a/packages/sdk/src/node-spawn-runtime.ts +++ b/packages/sdk/src/node-spawn-runtime.ts @@ -12,6 +12,7 @@ import { import { NodeSpawnError } from './node-spawn.js'; import { RemoteNodeSpawnPort, type RemoteNodeSpawnClient } from './node-spawn-remote.js'; import type { NodeSpawnPort } from './node-spawn.js'; +import { clearLiveNodeResultTeardowns } from './node-spawn-settlement.js'; export interface NodeSpawnPublisher { id: string; name: string } export interface NodeSpawnRuntime { @@ -33,6 +34,21 @@ export interface NodeSpawnRuntimeOwner { close(): Promise; } +/** Applies the same authority precedence for runtime creation and worker directory policy. */ +export function mergeNodeSpawnEnvironment( + environment: NodeJS.ProcessEnv | undefined, + serviceCallerFile: string | undefined, + inherited: NodeJS.ProcessEnv = process.env, +): NodeJS.ProcessEnv { + return { + ...inherited, + ...environment, + ...(serviceCallerFile === undefined + ? {} + : { RELAYFLOW_SERVICE_CALLER_FILE: serviceCallerFile }), + }; +} + interface RelayRuntimeClient extends RemoteNodeSpawnClient, NodeResultInboxClient { agents: RemoteNodeSpawnClient['agents'] & { me(): Promise }; events: { disconnect(): Promise }; @@ -174,7 +190,12 @@ export function createNodeSpawnRuntimeOwner( controller.abort(new Error('Node-spawn runtime owner closed')); if (provided !== undefined) return; const runtime = await pending?.catch(() => undefined); - await runtime?.disconnect?.().catch(() => undefined); + if (runtime?.disconnect !== undefined) { + let disconnected = false; + try { await runtime.disconnect(); disconnected = true; } + catch { /* Owner shutdown remains best-effort, but live teardown stays fail-closed. */ } + if (disconnected) clearLiveNodeResultTeardowns(runtime); + } }; return { get, close: () => closePromise ??= closeOnce() }; } diff --git a/packages/sdk/src/node-spawn-settlement.ts b/packages/sdk/src/node-spawn-settlement.ts index fbd9ff09c..fd5d3100e 100644 --- a/packages/sdk/src/node-spawn-settlement.ts +++ b/packages/sdk/src/node-spawn-settlement.ts @@ -7,6 +7,15 @@ import { NodeSpawnError, isRecord } from './node-spawn.js'; const SETTLEMENT_ATTEMPT_TIMEOUT_MS = 10_000; +interface LiveTeardown { + settled: boolean; + failure?: unknown; + outcome: Promise<{ failure?: unknown }>; +} + +/** Current-process iterator teardown is not durable and intentionally vanishes on restart. */ +const liveTeardowns = new WeakMap>>(); + export function durableNodeResultDelivery( runtime: NodeSpawnRuntime, path: string, @@ -21,6 +30,9 @@ export function durableNodeResultDelivery( throw new NodeSpawnError('Observed node result differs from its durable settlement identity', 'result_identity_mismatch'); } + if (observed?.settlementReady !== undefined) { + rememberLiveTeardown(runtime, path, observed.settlementReady); + } let claim = initial; let settling: Promise | undefined; const settleOnce = async (): Promise => { @@ -30,14 +42,18 @@ export function durableNodeResultDelivery( throw new NodeSpawnError('Node result cannot settle before its journal intent is durable', 'result_settlement_unprepared'); } - await bounded((async () => { - if (observed?.settlementReady !== undefined) { - const teardown = await observed.settlementReady; - if (teardown.failure !== undefined) throw teardown.failure; - } - await issueInboxSettlement(runtime, current.settlementAction, current.inboxItemId, + const live = waitForLiveTeardowns(runtime, path); + if (live === undefined) { + await settleInbox(runtime, current.settlementAction, current.inboxItemId, current.envelope.error ?? 'Relayflows rejected an invalid result'); - })()); + } else { + await boundedOperation(async signal => { + await live; + signal.throwIfAborted(); + await issueInboxSettlement(runtime, current.settlementAction, current.inboxItemId, + current.envelope.error ?? 'Relayflows rejected an invalid result'); + }); + } claim = { ...claim, result: { ...current, settlement: { ...current.settlement, settledAt: Date.now(), settledAs: current.settlementAction, @@ -102,6 +118,12 @@ export async function reconcileNodeResultSettlements( throw new NodeSpawnError('Pending node-result settlement belongs to a different publisher identity', 'publisher_identity_mismatch'); } + const live = liveTeardowns.get(runtime)?.get(entry.path); + if (live !== undefined) { + const failed = [...live].find(item => item.settled && item.failure !== undefined); + if (failed !== undefined) throw failed.failure; + if ([...live].some(item => !item.settled)) { pending += 1; continue; } + } const terminal = await terminalEntry(journal, entry.claim.runId, entry.claim.stepId, settlement.attempt); if (terminal === undefined) { pending += 1; continue; } const exact = terminalMatchesResult(terminal, entry.claim, settlement, result); @@ -117,6 +139,49 @@ export async function reconcileNodeResultSettlements( return pending; } +/** A proven owner disconnect makes every invocation connection for that runtime stale. */ +export function clearLiveNodeResultTeardowns(runtime: NodeSpawnRuntime): void { + liveTeardowns.delete(runtime); +} + +function rememberLiveTeardown( + runtime: NodeSpawnRuntime, + path: string, + outcome: Promise<{ failure?: unknown }>, +): void { + const normalized = outcome.then(result => result, failure => ({ failure })); + const live: LiveTeardown = { settled: false, outcome: normalized }; + let byPath = liveTeardowns.get(runtime); + if (byPath === undefined) { + byPath = new Map(); + liveTeardowns.set(runtime, byPath); + } + let barriers = byPath.get(path); + if (barriers === undefined) { + barriers = new Set(); + byPath.set(path, barriers); + } + barriers.add(live); + void normalized.then(result => { + live.settled = true; + live.failure = result.failure; + if (result.failure === undefined) { + const current = liveTeardowns.get(runtime)?.get(path); + current?.delete(live); + if (current?.size === 0) liveTeardowns.get(runtime)?.delete(path); + } + }); +} + +function waitForLiveTeardowns(runtime: NodeSpawnRuntime, path: string): Promise | undefined { + const barriers = liveTeardowns.get(runtime)?.get(path); + if (barriers === undefined) return undefined; + return Promise.all([...barriers].map(item => item.outcome)).then(outcomes => { + const failed = outcomes.find(item => item.failure !== undefined); + if (failed?.failure !== undefined) throw failed.failure; + }); +} + function terminalMatchesResult( terminal: JournalEvent, claim: NodeSpawnClaim, @@ -193,3 +258,25 @@ async function bounded(pending: Promise): Promise { if (timer !== undefined) clearTimeout(timer); } } + +async function boundedOperation(operation: (signal: AbortSignal) => Promise): Promise { + const controller = new AbortController(); + let timer: NodeJS.Timeout | undefined; + const deadline = new Promise((_resolve, reject) => { + timer = setTimeout(() => { + const error = new NodeSpawnError( + 'Node-result inbox settlement did not confirm within its independent deadline', + 'result_settlement_unconfirmed'); + controller.abort(error); + reject(error); + }, SETTLEMENT_ATTEMPT_TIMEOUT_MS); + }); + const pending = operation(controller.signal) + .then(() => ({ ok: true as const }), error => ({ ok: false as const, error })); + try { + const settled = await Promise.race([pending, deadline]); + if (!settled.ok) throw settled.error; + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index 8ae87978a..2d1ae6965 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -22,7 +22,7 @@ import { redactText, type TranscriptDigest, } from './agent-transcript.js'; -import { createNodeSpawnRuntime, createNodeSpawnRuntimeOwner, type NodeSpawnRuntime, +import { createNodeSpawnRuntime, createNodeSpawnRuntimeOwner, mergeNodeSpawnEnvironment, type NodeSpawnRuntime, type NodeSpawnRuntimeOwner } from './node-spawn-runtime.js'; import { reconcileNodeResultSettlements } from './node-spawn-settlement.js'; import { readNodeSpawnClaims } from './node-spawn-state.js'; @@ -75,6 +75,7 @@ export class AgentWorker extends EventEmitter { private settlementTimer?: NodeJS.Timeout; private settlementDrain?: Promise; private settlementReconcile?: Promise; + private readonly localDesktopAuthority: boolean; private readonly inFlight: Set> = new Set(); private readonly nodeSpawnRuntimeOwner: NodeSpawnRuntimeOwner; @@ -83,15 +84,13 @@ export class AgentWorker extends EventEmitter { private readonly options: AgentWorkerOptions, ) { super(); + const runtimeEnvironment = mergeNodeSpawnEnvironment(options.environment, options.serviceCallerFile); + this.localDesktopAuthority = options.nodeSpawnRuntime === undefined + ? Boolean(runtimeEnvironment.RELAYFLOW_SERVICE_CALLER_FILE?.trim()) + : options.nodeSpawnRuntime.authority === 'desktop'; this.nodeSpawnRuntimeOwner = createNodeSpawnRuntimeOwner( options.nodeSpawnRuntime, - signal => createNodeSpawnRuntime({ - ...process.env, - ...options.environment, - ...(options.serviceCallerFile === undefined - ? {} - : { RELAYFLOW_SERVICE_CALLER_FILE: options.serviceCallerFile }), - }, { signal }), + signal => createNodeSpawnRuntime(runtimeEnvironment, { signal }), ); } @@ -235,7 +234,7 @@ export class AgentWorker extends EventEmitter { // the agent's filesystem. A refusal completes the step the way a missing // CLI does — journaled as `worker_error` with the reason — rather than // spawning into a directory nobody established is inside the run root. - const localDesktopRelay = spec.transport === 'relay' && this.usesLocalDesktopAuthority(); + const localDesktopRelay = spec.transport === 'relay' && this.localDesktopAuthority; const cwd = agentStepCwd(localDesktopRelay ? { ...spec, transport: 'direct' } : spec, dispatch.step_id, this.options.runRoot); const completed: WorkerCliResult = await withWorkerLease(this.client, dispatch, signal => @@ -331,14 +330,6 @@ export class AgentWorker extends EventEmitter { await relayDelivery?.ack(); } - /** Explicit Desktop authority is the only relay mode allowed to receive a local run root. */ - private usesLocalDesktopAuthority(): boolean { - return this.options.nodeSpawnRuntime?.authority === 'desktop' - || Boolean(this.options.serviceCallerFile?.trim() - || this.options.environment?.RELAYFLOW_SERVICE_CALLER_FILE?.trim() - || process.env.RELAYFLOW_SERVICE_CALLER_FILE?.trim()); - } - private reconcileRelaySettlementsAfterDrain(): Promise { if (this.settlementDrain !== undefined) return this.settlementDrain; const generation = Array.from(this.inFlight); diff --git a/packages/sdk/tests/node-spawn-settlement.test.ts b/packages/sdk/tests/node-spawn-settlement.test.ts index b2be14558..3e8274c48 100644 --- a/packages/sdk/tests/node-spawn-settlement.test.ts +++ b/packages/sdk/tests/node-spawn-settlement.test.ts @@ -5,8 +5,9 @@ import { join } from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { runAgentRelayTask, type AgentRelayTaskRequest } from '../src/agent-relay-transport.js'; import type { JournalClient } from '../src/journal-client.js'; -import { NODE_RESULT_PREFIX } from '../src/node-spawn-result.js'; -import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; +import { NODE_RESULT_PREFIX, type NodeResultDelivery } from '../src/node-spawn-result.js'; +import { createNodeSpawnRuntimeOwner, type NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; +import { durableNodeResultDelivery, reconcileNodeResultSettlements } from '../src/node-spawn-settlement.js'; import { readNodeSpawnClaims, recordNodeSpawn } from '../src/node-spawn-state.js'; import { NodeSpawnError, type NodeSpawnPort, type NodeSpawnReceipt } from '../src/node-spawn.js'; import type { StepDispatchEvent } from '../src/protocol.js'; @@ -131,7 +132,10 @@ describe('durable node-result settlement', () => { const dataDir = await dataDirectory('flows-node-settlement-teardown-timeout-'); const f = fixture(); let delivered = false; - const settlementReady = new Promise<{ failure?: unknown }>(() => undefined); + let finishTeardown!: () => void; + const settlementReady = new Promise<{ failure?: unknown }>(resolve => { + finishTeardown = () => resolve({}); + }); f.runtime.inbox.inbox.subscribe = () => ({ [Symbol.asyncIterator]() { return { async next() { @@ -162,11 +166,79 @@ describe('durable node-result settlement', () => { expect(f.fail).not.toHaveBeenCalled(); const claims = await readNodeSpawnClaims(dataDir); expect(claims[0]!.claim.result?.settlement).not.toHaveProperty('settledAt'); + finishTeardown(); + await Promise.resolve(); } finally { vi.useRealTimers(); } }); + it('retains every live teardown registered for the same durable claim', async () => { + const dataDir = await dataDirectory('flows-node-settlement-multiple-teardowns-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + const [entry] = await readNodeSpawnClaims(dataDir); + let finishFirst!: () => void; + let finishSecond!: () => void; + const firstReady = new Promise<{ failure?: unknown }>(resolve => { finishFirst = () => resolve({}); }); + const secondReady = new Promise<{ failure?: unknown }>(resolve => { finishSecond = () => resolve({}); }); + const observed = (settlementReady: Promise<{ failure?: unknown }>): NodeResultDelivery => ({ + ...received.delivery, settlementReady, + }); + durableNodeResultDelivery(f.runtime, entry!.path, entry!.claim, observed(firstReady)); + const delivery = durableNodeResultDelivery(f.runtime, entry!.path, entry!.claim, observed(secondReady)); + await delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + finishSecond(); + await Promise.resolve(); + vi.useFakeTimers(); + try { + const blocked = expect(delivery.ack()) + .rejects.toMatchObject({ code: 'result_settlement_unconfirmed' }); + await vi.advanceTimersByTimeAsync(10_001); + await blocked; + expect(f.ack).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + finishFirst(); + await Promise.resolve(); + await expect(delivery.ack()).resolves.toBeUndefined(); + expect(f.ack).toHaveBeenCalledOnce(); + }); + + it.each([ + { label: 'successful', disconnectFails: false, expectedPending: 0, expectedAcks: 1 }, + { label: 'failed', disconnectFails: true, expectedPending: 1, expectedAcks: 0 }, + ])('$label owner disconnect controls whether a live teardown survives', async testCase => { + const dataDir = await dataDirectory(`flows-node-settlement-owner-${testCase.label}-`); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + const [entry] = await readNodeSpawnClaims(dataDir); + let finishTeardown!: () => void; + const settlementReady = new Promise<{ failure?: unknown }>(resolve => { + finishTeardown = () => resolve({}); + }); + const delivery = durableNodeResultDelivery(f.runtime, entry!.path, entry!.claim, { + ...received.delivery, settlementReady, + }); + await delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + const disconnect = vi.fn(async () => { + if (testCase.disconnectFails) throw new Error('disconnect failed'); + }); + f.runtime.disconnect = disconnect; + const owner = createNodeSpawnRuntimeOwner(undefined, async () => f.runtime); + await owner.get(); + await owner.close(); + const pending = await reconcileNodeResultSettlements(dataDir, f.runtime, { + journalRead: vi.fn(async () => ({ entries: [terminalEntry()] })), + }); + expect(pending).toBe(testCase.expectedPending); + expect(f.ack).toHaveBeenCalledTimes(testCase.expectedAcks); + expect(disconnect).toHaveBeenCalledOnce(); + finishTeardown(); + await Promise.resolve(); + }); + it('rejects an invalid resolved spawn receipt before reading the inbox and releases exact identity', async () => { const dataDir = await dataDirectory('flows-node-invalid-resolved-spawn-'); const f = fixture(); diff --git a/packages/sdk/tests/worker-node-runtime-sharing.test.ts b/packages/sdk/tests/worker-node-runtime-sharing.test.ts index de337540d..89ce5f0b2 100644 --- a/packages/sdk/tests/worker-node-runtime-sharing.test.ts +++ b/packages/sdk/tests/worker-node-runtime-sharing.test.ts @@ -21,6 +21,7 @@ import { AgentWorker } from '../src/worker.js'; const roots: string[] = []; afterEach(async () => { + vi.unstubAllEnvs(); owner.get.mockReset(); owner.close.mockClear(); await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); @@ -33,7 +34,10 @@ describe('AgentWorker node-spawn runtime ownership', () => { { label: 'provided Desktop runtime', serviceCallerFile: undefined, authority: 'desktop' as const, provideRuntime: true, expectedDirectory: 'run-root', expectedSpawn: true }, { label: 'remote publisher', serviceCallerFile: undefined, authority: 'remote' as const, - provideRuntime: true, expectedDirectory: undefined, expectedSpawn: true }, + provideRuntime: true, inheritedCaller: true, expectedDirectory: undefined, expectedSpawn: true }, + { label: 'inherited caller cleared by worker environment', serviceCallerFile: undefined, + authority: undefined, provideRuntime: false, inheritedCaller: true, environmentCaller: '', + expectedDirectory: undefined, expectedSpawn: true }, { label: 'Desktop caller with declared cwd', serviceCallerFile: '/tmp/caller.json', authority: undefined, provideRuntime: false, cwd: 'checkout', expectedDirectory: 'declared', expectedSpawn: true }, { label: 'remote publisher with declared cwd', serviceCallerFile: undefined, authority: 'remote' as const, @@ -41,6 +45,9 @@ describe('AgentWorker node-spawn runtime ownership', () => { ])('routes the run directory only for the $label', async testCase => { const runRoot = await mkdtemp(join(tmpdir(), 'flows-node-worker-run-root-')); roots.push(runRoot); + if (testCase.inheritedCaller) { + vi.stubEnv('RELAYFLOW_SERVICE_CALLER_FILE', '/tmp/inherited-caller.json'); + } const declaredDirectory = join(runRoot, 'checkout'); await mkdir(declaredDirectory); const exactDeclaredDirectory = await realpath(declaredDirectory); @@ -80,6 +87,9 @@ describe('AgentWorker node-spawn runtime ownership', () => { workerId: `worker_${testCase.label}`, pins: {}, dataDir: join(runRoot, 'data'), runRoot, ...(testCase.serviceCallerFile === undefined ? {} : { serviceCallerFile: testCase.serviceCallerFile }), ...(testCase.provideRuntime ? { nodeSpawnRuntime: runtime } : {}), + ...(testCase.environmentCaller === undefined + ? {} + : { environment: { RELAYFLOW_SERVICE_CALLER_FILE: testCase.environmentCaller } }), }); await worker.attach(); const value = dispatch('edit-local-path', 'dispatch_localpath'); @@ -95,6 +105,102 @@ describe('AgentWorker node-spawn runtime ownership', () => { } }); + it('does not reconcile a journaled delivery while its live iterator teardown is unresolved', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-live-teardown-reconcile-')); + roots.push(dataDir); + let finishTeardown!: () => void; + const settlementReady = new Promise<{ failure?: unknown }>(resolve => { + finishTeardown = () => resolve({}); + }); + let spawnedName = ''; + let delivered = false; + const ack = vi.fn(async () => ({})); + const fail = vi.fn(async () => ({})); + const runtime: NodeSpawnRuntime = { + authority: 'desktop', publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + port: { + targets: async () => [], + spawn: async input => { + spawnedName = input.name; + return { id: 'spawn_live_teardown', action: 'spawn', status: 'completed', + nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, + agentId: 'agent_live_teardown', spawned: true, ready: true }; + }, + status: vi.fn(), + release: async () => ({ id: 'release_live_teardown', action: 'release', + status: 'completed', spawned: false, ready: false, released: true }), + }, + inbox: { inbox: { + subscribe: () => ({ [Symbol.asyncIterator]() { return { + async next() { + if (delivered) return new Promise>(() => undefined); + delivered = true; + return { done: false as const, value: { + id: 'delivery_live_teardown', settlementReady, + message: { id: 'message_live_teardown', + from: { id: 'agent_live_teardown', name: spawnedName }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_parallel', + step_id: 'edit-live-teardown', dispatch_id: 'dispatch_liveteardown', + status: 'completed', output: { summary: 'done' } }) }, + } }; + }, + async return() { await settlementReady; return { done: true as const, value: undefined }; }, + }; } }), + ack, + fail, + } }, + }; + owner.get.mockResolvedValue(runtime); + let completed = false; + let terminalPayload: Record = {}; + const stepComplete = vi.fn(async ( + _runId, _stepId, _attempt, _idempotencyKey, completionReason, payload, + ) => { + completed = true; + terminalPayload = { ...payload, completionReason }; + return {} as Awaited>; + }); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), + stepComplete, + journalRead: vi.fn(async () => ({ entries: completed ? [{ + seq: 2, segment_id: 0, entry_type: 'step.completed', run_id: 'run_parallel', + step_id: 'edit-live-teardown', attempt: 1, at_ms: Date.now(), payload: terminalPayload, + }] : [] })), + supportsFeature: vi.fn(() => true), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_live_teardown', pins: {}, dataDir, nodeSpawnRuntime: runtime, + }); + const errors: unknown[] = []; + worker.on('error', error => errors.push(error)); + await worker.attach(); + vi.useFakeTimers(); + try { + client.emit('step.dispatch', dispatch('edit-live-teardown', 'dispatch_liveteardown')); + await vi.waitFor(() => expect(client.stepComplete).toHaveBeenCalledOnce()); + expect(ack).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(10_001); + await vi.waitFor(() => expect(errors).toHaveLength(1)); + const state = worker as unknown as { reconcileRelaySettlementsAfterDrain(): Promise }; + await state.reconcileRelaySettlementsAfterDrain(); + expect(ack).not.toHaveBeenCalled(); + expect(fail).not.toHaveBeenCalled(); + + finishTeardown(); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(5_001); + await vi.waitFor(() => expect(ack).toHaveBeenCalledOnce()); + expect(ack).toHaveBeenCalledOnce(); + expect(fail).not.toHaveBeenCalled(); + await worker.close(); + } finally { + vi.useRealTimers(); + } + }); + it('closes the shared runtime even when final durable inbox settlement still fails', async () => { const dataDir = await mkdtemp(join(tmpdir(), 'flows-shared-node-runtime-settlement-failure-')); roots.push(dataDir); From 849db394d1b40d848e6e591c8b06b8a0c044ef78 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 8 Nov 2026 23:41:29 -0800 Subject: [PATCH 32/75] fix(sdk): retain live inbox teardown barriers --- packages/sdk/src/node-spawn-runtime.ts | 8 +------- packages/sdk/src/node-spawn-settlement.ts | 5 ----- packages/sdk/tests/node-spawn-settlement.test.ts | 16 +++++++++++----- 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/packages/sdk/src/node-spawn-runtime.ts b/packages/sdk/src/node-spawn-runtime.ts index 2d5fc858a..b90ba4568 100644 --- a/packages/sdk/src/node-spawn-runtime.ts +++ b/packages/sdk/src/node-spawn-runtime.ts @@ -12,7 +12,6 @@ import { import { NodeSpawnError } from './node-spawn.js'; import { RemoteNodeSpawnPort, type RemoteNodeSpawnClient } from './node-spawn-remote.js'; import type { NodeSpawnPort } from './node-spawn.js'; -import { clearLiveNodeResultTeardowns } from './node-spawn-settlement.js'; export interface NodeSpawnPublisher { id: string; name: string } export interface NodeSpawnRuntime { @@ -190,12 +189,7 @@ export function createNodeSpawnRuntimeOwner( controller.abort(new Error('Node-spawn runtime owner closed')); if (provided !== undefined) return; const runtime = await pending?.catch(() => undefined); - if (runtime?.disconnect !== undefined) { - let disconnected = false; - try { await runtime.disconnect(); disconnected = true; } - catch { /* Owner shutdown remains best-effort, but live teardown stays fail-closed. */ } - if (disconnected) clearLiveNodeResultTeardowns(runtime); - } + await runtime?.disconnect?.().catch(() => undefined); }; return { get, close: () => closePromise ??= closeOnce() }; } diff --git a/packages/sdk/src/node-spawn-settlement.ts b/packages/sdk/src/node-spawn-settlement.ts index fd5d3100e..6e45fd53e 100644 --- a/packages/sdk/src/node-spawn-settlement.ts +++ b/packages/sdk/src/node-spawn-settlement.ts @@ -139,11 +139,6 @@ export async function reconcileNodeResultSettlements( return pending; } -/** A proven owner disconnect makes every invocation connection for that runtime stale. */ -export function clearLiveNodeResultTeardowns(runtime: NodeSpawnRuntime): void { - liveTeardowns.delete(runtime); -} - function rememberLiveTeardown( runtime: NodeSpawnRuntime, path: string, diff --git a/packages/sdk/tests/node-spawn-settlement.test.ts b/packages/sdk/tests/node-spawn-settlement.test.ts index 3e8274c48..96a868dd8 100644 --- a/packages/sdk/tests/node-spawn-settlement.test.ts +++ b/packages/sdk/tests/node-spawn-settlement.test.ts @@ -207,9 +207,9 @@ describe('durable node-result settlement', () => { }); it.each([ - { label: 'successful', disconnectFails: false, expectedPending: 0, expectedAcks: 1 }, - { label: 'failed', disconnectFails: true, expectedPending: 1, expectedAcks: 0 }, - ])('$label owner disconnect controls whether a live teardown survives', async testCase => { + { label: 'successful', disconnectFails: false }, + { label: 'failed', disconnectFails: true }, + ])('$label publisher disconnect leaves a separate live inbox teardown blocked', async testCase => { const dataDir = await dataDirectory(`flows-node-settlement-owner-${testCase.label}-`); const f = fixture(); const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); @@ -232,9 +232,15 @@ describe('durable node-result settlement', () => { const pending = await reconcileNodeResultSettlements(dataDir, f.runtime, { journalRead: vi.fn(async () => ({ entries: [terminalEntry()] })), }); - expect(pending).toBe(testCase.expectedPending); - expect(f.ack).toHaveBeenCalledTimes(testCase.expectedAcks); + expect(pending).toBe(1); + expect(f.ack).not.toHaveBeenCalled(); expect(disconnect).toHaveBeenCalledOnce(); + + const freshRuntime = { ...f.runtime } satisfies NodeSpawnRuntime; + await expect(reconcileNodeResultSettlements(dataDir, freshRuntime, { + journalRead: vi.fn(async () => ({ entries: [terminalEntry()] })), + })).resolves.toBe(0); + expect(f.ack).toHaveBeenCalledOnce(); finishTeardown(); await Promise.resolve(); }); From 64a89ec2800186ef5b7e31a49dd7be84fa18d390 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 8 Nov 2026 23:55:39 -0800 Subject: [PATCH 33/75] fix(sdk): serialize live result settlement --- packages/sdk/src/node-spawn-settlement.ts | 174 +++++++++++------- .../sdk/tests/node-spawn-settlement.test.ts | 71 +++++-- 2 files changed, 159 insertions(+), 86 deletions(-) diff --git a/packages/sdk/src/node-spawn-settlement.ts b/packages/sdk/src/node-spawn-settlement.ts index 6e45fd53e..610495fd4 100644 --- a/packages/sdk/src/node-spawn-settlement.ts +++ b/packages/sdk/src/node-spawn-settlement.ts @@ -13,8 +13,14 @@ interface LiveTeardown { outcome: Promise<{ failure?: unknown }>; } -/** Current-process iterator teardown is not durable and intentionally vanishes on restart. */ -const liveTeardowns = new WeakMap>>(); +interface LiveTeardownGroup { + generation: number; + barriers: Set; + settlementStarted: boolean; +} + +/** Process-generation state keyed by durable claim path; a real restart intentionally resets it. */ +const liveTeardowns = new Map(); export function durableNodeResultDelivery( runtime: NodeSpawnRuntime, @@ -31,7 +37,7 @@ export function durableNodeResultDelivery( 'result_identity_mismatch'); } if (observed?.settlementReady !== undefined) { - rememberLiveTeardown(runtime, path, observed.settlementReady); + rememberLiveTeardown(path, observed.settlementReady); } let claim = initial; let settling: Promise | undefined; @@ -42,23 +48,19 @@ export function durableNodeResultDelivery( throw new NodeSpawnError('Node result cannot settle before its journal intent is durable', 'result_settlement_unprepared'); } - const live = waitForLiveTeardowns(runtime, path); - if (live === undefined) { - await settleInbox(runtime, current.settlementAction, current.inboxItemId, - current.envelope.error ?? 'Relayflows rejected an invalid result'); - } else { - await boundedOperation(async signal => { - await live; - signal.throwIfAborted(); - await issueInboxSettlement(runtime, current.settlementAction, current.inboxItemId, - current.envelope.error ?? 'Relayflows rejected an invalid result'); - }); - } + const liveSettlement = await settleAfterLiveTeardowns(runtime, path, current.settlementAction, + current.inboxItemId, current.envelope.error ?? 'Relayflows rejected an invalid result'); claim = { ...claim, result: { ...current, settlement: { ...current.settlement, settledAt: Date.now(), settledAs: current.settlementAction, } } }; - await recordNodeSpawn(path, claim); + try { + await recordNodeSpawn(path, claim); + } catch (error) { + releaseLiveSettlement(path, liveSettlement); + throw error; + } + completeLiveSettlement(path, liveSettlement); }; const settle = (): Promise => { if (settling !== undefined) return settling; @@ -118,65 +120,122 @@ export async function reconcileNodeResultSettlements( throw new NodeSpawnError('Pending node-result settlement belongs to a different publisher identity', 'publisher_identity_mismatch'); } - const live = liveTeardowns.get(runtime)?.get(entry.path); + const live = liveTeardowns.get(entry.path); if (live !== undefined) { - const failed = [...live].find(item => item.settled && item.failure !== undefined); + const failed = [...live.barriers].find(item => item.settled && item.failure !== undefined); if (failed !== undefined) throw failed.failure; - if ([...live].some(item => !item.settled)) { pending += 1; continue; } + if (live.settlementStarted || [...live.barriers].some(item => !item.settled)) { + pending += 1; + continue; + } } const terminal = await terminalEntry(journal, entry.claim.runId, entry.claim.stepId, settlement.attempt); if (terminal === undefined) { pending += 1; continue; } const exact = terminalMatchesResult(terminal, entry.claim, settlement, result); const action = exact ? result.settlementAction : 'fail'; - await settleInbox(runtime, action, result.inboxItemId, - exact ? (result.envelope.error ?? 'Relayflows rejected an invalid result') - : 'The journal completed this attempt with a different terminal outcome'); + const error = exact ? (result.envelope.error ?? 'Relayflows rejected an invalid result') + : 'The journal completed this attempt with a different terminal outcome'; + const liveSettlement = await settleAfterLiveTeardowns( + runtime, entry.path, action, result.inboxItemId, error); const updated: NodeSpawnClaim = { ...entry.claim, result: { ...result, settlement: { ...settlement, settledAt: Date.now(), settledAs: action, } } }; - await recordNodeSpawn(entry.path, updated); + try { + await recordNodeSpawn(entry.path, updated); + } catch (error) { + releaseLiveSettlement(entry.path, liveSettlement); + throw error; + } + completeLiveSettlement(entry.path, liveSettlement); } return pending; } function rememberLiveTeardown( - runtime: NodeSpawnRuntime, path: string, outcome: Promise<{ failure?: unknown }>, ): void { const normalized = outcome.then(result => result, failure => ({ failure })); const live: LiveTeardown = { settled: false, outcome: normalized }; - let byPath = liveTeardowns.get(runtime); - if (byPath === undefined) { - byPath = new Map(); - liveTeardowns.set(runtime, byPath); + let group = liveTeardowns.get(path); + if (group === undefined) { + group = { generation: 0, barriers: new Set(), settlementStarted: false }; + liveTeardowns.set(path, group); } - let barriers = byPath.get(path); - if (barriers === undefined) { - barriers = new Set(); - byPath.set(path, barriers); + if (group.settlementStarted) { + throw new NodeSpawnError('Node result gained a live teardown after settlement started', + 'result_settlement_unconfirmed'); } - barriers.add(live); + group.generation += 1; + group.barriers.add(live); void normalized.then(result => { live.settled = true; live.failure = result.failure; if (result.failure === undefined) { - const current = liveTeardowns.get(runtime)?.get(path); - current?.delete(live); - if (current?.size === 0) liveTeardowns.get(runtime)?.delete(path); + liveTeardowns.get(path)?.barriers.delete(live); } }); } -function waitForLiveTeardowns(runtime: NodeSpawnRuntime, path: string): Promise | undefined { - const barriers = liveTeardowns.get(runtime)?.get(path); - if (barriers === undefined) return undefined; - return Promise.all([...barriers].map(item => item.outcome)).then(outcomes => { - const failed = outcomes.find(item => item.failure !== undefined); - if (failed?.failure !== undefined) throw failed.failure; +interface LiveSettlement { + group: LiveTeardownGroup; + generation: number; +} + +async function settleAfterLiveTeardowns( + runtime: NodeSpawnRuntime, + path: string, + action: 'ack' | 'fail', + inboxItemId: string, + error: string, +): Promise { + return boundedOperation(async signal => { + for (;;) { + let group = liveTeardowns.get(path); + if (group === undefined) { + group = { generation: 0, barriers: new Set(), settlementStarted: false }; + liveTeardowns.set(path, group); + } + if (group.settlementStarted) { + throw new NodeSpawnError('Node result settlement is already in progress', + 'result_settlement_unconfirmed'); + } + const generation = group.generation; + const outcomes = group.barriers.size === 0 + ? [] + : await Promise.all([...group.barriers].map(item => item.outcome)); + const failed = outcomes.find(item => item.failure !== undefined); + if (failed?.failure !== undefined) throw failed.failure; + signal.throwIfAborted(); + const current = liveTeardowns.get(path); + if (current !== group || current.generation !== generation) continue; + current.settlementStarted = true; + try { + await issueInboxSettlement(runtime, action, inboxItemId, error); + return { group: current, generation }; + } catch (settlementError) { + current.settlementStarted = false; + throw settlementError; + } + } }); } +function releaseLiveSettlement(path: string, settlement: LiveSettlement): void { + const current = liveTeardowns.get(path); + if (current === settlement.group && current.generation === settlement.generation) { + current.settlementStarted = false; + } +} + +function completeLiveSettlement(path: string, settlement: LiveSettlement): void { + const current = liveTeardowns.get(path); + if (current === settlement.group && current.generation === settlement.generation + && current.settlementStarted && current.barriers.size === 0) { + liveTeardowns.delete(path); + } +} + function terminalMatchesResult( terminal: JournalEvent, claim: NodeSpawnClaim, @@ -215,15 +274,6 @@ async function terminalEntry( } } -async function settleInbox( - runtime: NodeSpawnRuntime, - action: 'ack' | 'fail', - inboxItemId: string, - error: string, -): Promise { - await bounded(issueInboxSettlement(runtime, action, inboxItemId, error)); -} - function issueInboxSettlement( runtime: NodeSpawnRuntime, action: 'ack' | 'fail', @@ -237,24 +287,7 @@ function issueInboxSettlement( return pending; } -async function bounded(pending: Promise): Promise { - let timer: NodeJS.Timeout | undefined; - const deadline = new Promise((_resolve, reject) => { - timer = setTimeout(() => reject(new NodeSpawnError( - 'Node-result inbox settlement did not confirm within its independent deadline', - 'result_settlement_unconfirmed')), - SETTLEMENT_ATTEMPT_TIMEOUT_MS); - }); - const outcome = pending.then(() => ({ ok: true as const }), error => ({ ok: false as const, error })); - try { - const settled = await Promise.race([outcome, deadline]); - if (!settled.ok) throw settled.error; - } finally { - if (timer !== undefined) clearTimeout(timer); - } -} - -async function boundedOperation(operation: (signal: AbortSignal) => Promise): Promise { +async function boundedOperation(operation: (signal: AbortSignal) => Promise): Promise { const controller = new AbortController(); let timer: NodeJS.Timeout | undefined; const deadline = new Promise((_resolve, reject) => { @@ -267,10 +300,11 @@ async function boundedOperation(operation: (signal: AbortSignal) => Promise ({ ok: true as const }), error => ({ ok: false as const, error })); + .then(value => ({ ok: true as const, value }), error => ({ ok: false as const, error })); try { const settled = await Promise.race([pending, deadline]); if (!settled.ok) throw settled.error; + return settled.value; } finally { if (timer !== undefined) clearTimeout(timer); } diff --git a/packages/sdk/tests/node-spawn-settlement.test.ts b/packages/sdk/tests/node-spawn-settlement.test.ts index 96a868dd8..7dbcacff1 100644 --- a/packages/sdk/tests/node-spawn-settlement.test.ts +++ b/packages/sdk/tests/node-spawn-settlement.test.ts @@ -1,5 +1,5 @@ import { EventEmitter } from 'node:events'; -import { mkdtemp, rm } from 'node:fs/promises'; +import { mkdtemp, rename, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; @@ -185,25 +185,60 @@ describe('durable node-result settlement', () => { const observed = (settlementReady: Promise<{ failure?: unknown }>): NodeResultDelivery => ({ ...received.delivery, settlementReady, }); - durableNodeResultDelivery(f.runtime, entry!.path, entry!.claim, observed(firstReady)); - const delivery = durableNodeResultDelivery(f.runtime, entry!.path, entry!.claim, observed(secondReady)); + const delivery = durableNodeResultDelivery(f.runtime, entry!.path, entry!.claim, observed(firstReady)); await delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + const settling = delivery.ack(); + durableNodeResultDelivery(f.runtime, entry!.path, entry!.claim, observed(secondReady)); + finishFirst(); + await Promise.resolve(); + expect(f.ack).not.toHaveBeenCalled(); finishSecond(); await Promise.resolve(); - vi.useFakeTimers(); + await expect(settling).resolves.toBeUndefined(); + expect(f.ack).toHaveBeenCalledOnce(); + }); + + it('refuses a new live observation after inbox settlement has started', async () => { + const dataDir = await dataDirectory('flows-node-settlement-late-observation-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + const [entry] = await readNodeSpawnClaims(dataDir); + let finishAck!: () => void; + f.ack.mockImplementationOnce(() => new Promise(resolve => { finishAck = () => resolve({}); })); + const delivery = durableNodeResultDelivery(f.runtime, entry!.path, entry!.claim); + await delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + const settling = delivery.ack(); + await vi.waitFor(() => expect(f.ack).toHaveBeenCalledOnce()); + + expect(() => durableNodeResultDelivery(f.runtime, entry!.path, entry!.claim, { + ...received.delivery, settlementReady: Promise.resolve({}), + })).toThrowError(expect.objectContaining({ code: 'result_settlement_unconfirmed' })); + finishAck(); + await expect(settling).resolves.toBeUndefined(); + }); + + it('releases the process settlement lock when persisting settledAt fails', async () => { + const dataDir = await dataDirectory('flows-node-settlement-record-retry-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + const claimsDirectory = join(dataDir, 'node-spawns'); + const unavailableDirectory = `${claimsDirectory}.unavailable`; + await rename(claimsDirectory, unavailableDirectory); + let restored = false; try { - const blocked = expect(delivery.ack()) - .rejects.toMatchObject({ code: 'result_settlement_unconfirmed' }); - await vi.advanceTimersByTimeAsync(10_001); - await blocked; - expect(f.ack).not.toHaveBeenCalled(); + await expect(received.delivery.ack()).rejects.toMatchObject({ code: 'ENOENT' }); + expect(f.ack).toHaveBeenCalledOnce(); + await rename(unavailableDirectory, claimsDirectory); + restored = true; } finally { - vi.useRealTimers(); + if (!restored) await rename(unavailableDirectory, claimsDirectory).catch(() => undefined); } - finishFirst(); - await Promise.resolve(); - await expect(delivery.ack()).resolves.toBeUndefined(); - expect(f.ack).toHaveBeenCalledOnce(); + + await expect(received.delivery.ack()).resolves.toBeUndefined(); + expect(f.ack).toHaveBeenCalledTimes(2); + const [settled] = await readNodeSpawnClaims(dataDir); + expect(settled!.claim.result?.settlement).toMatchObject({ settledAs: 'ack' }); }); it.each([ @@ -239,10 +274,14 @@ describe('durable node-result settlement', () => { const freshRuntime = { ...f.runtime } satisfies NodeSpawnRuntime; await expect(reconcileNodeResultSettlements(dataDir, freshRuntime, { journalRead: vi.fn(async () => ({ entries: [terminalEntry()] })), - })).resolves.toBe(0); - expect(f.ack).toHaveBeenCalledOnce(); + })).resolves.toBe(1); + expect(f.ack).not.toHaveBeenCalled(); finishTeardown(); await Promise.resolve(); + await expect(reconcileNodeResultSettlements(dataDir, freshRuntime, { + journalRead: vi.fn(async () => ({ entries: [terminalEntry()] })), + })).resolves.toBe(0); + expect(f.ack).toHaveBeenCalledOnce(); }); it('rejects an invalid resolved spawn receipt before reading the inbox and releases exact identity', async () => { From 611f8e1ccae34ce0d47a9bd8ab9eaeb1eb1be946 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 8 Nov 2026 23:59:09 -0800 Subject: [PATCH 34/75] fix(sdk): serialize node result observation --- packages/sdk/src/agent-relay-transport.ts | 34 +++++++++++++- packages/sdk/src/node-spawn-state.ts | 19 ++++++-- .../sdk/tests/node-spawn-settlement.test.ts | 47 +++++++++++++------ 3 files changed, 80 insertions(+), 20 deletions(-) diff --git a/packages/sdk/src/agent-relay-transport.ts b/packages/sdk/src/agent-relay-transport.ts index a936d106d..8be5cba34 100644 --- a/packages/sdk/src/agent-relay-transport.ts +++ b/packages/sdk/src/agent-relay-transport.ts @@ -1,5 +1,6 @@ import { createHash } from 'node:crypto'; -import { claimNodeSpawn, recordNodeSpawn, type NodeSpawnClaim } from './node-spawn-state.js'; +import { claimNodeSpawn, nodeSpawnClaimPath, recordNodeSpawn, + type NodeSpawnClaim } from './node-spawn-state.js'; import { nodeResultInstruction, waitForNodeResult } from './node-spawn-result.js'; import { durableNodeResultDelivery } from './node-spawn-settlement.js'; import { waitForNodeOperation, waitForNodeSpawn } from './node-spawn-deadline.js'; @@ -46,6 +47,7 @@ export class AgentRelayTransportError extends Error { } const CLEANUP_ATTEMPT_TIMEOUT_MS = 15_000; +const claimExecutions = new Map>(); export function readAgentRelayEnv(env: NodeJS.ProcessEnv = process.env): Required> { const agentToken = env.RELAY_AGENT_TOKEN?.trim(); @@ -73,6 +75,21 @@ export async function runAgentRelayTask( || !validIdempotencyKey(request.idempotencyKey)) { throw new AgentRelayTransportError('Node-spawn task requires durable dispatch identity and a valid deadline'); } + const claimPath = nodeSpawnClaimPath( + request.dataDir, request.runId, request.stepId, request.idempotencyKey); + return serializeClaimExecution(claimPath, async () => { + options.signal?.throwIfAborted(); + return runAgentRelayTaskExclusive(request, options, taskStartedAt, timeoutMs); + }); +} + +async function runAgentRelayTaskExclusive( + request: AgentRelayTaskRequest, + options: AgentRelayEnv & { runtime?: NodeSpawnRuntime; + runtimeProvider?: (signal?: AbortSignal) => Promise; signal?: AbortSignal }, + taskStartedAt: number, + timeoutMs: number, +): Promise { const taskDeadline = taskStartedAt + timeoutMs; let acquisitionRemaining: number; try { acquisitionRemaining = remainingDeadline(taskDeadline); } @@ -117,6 +134,21 @@ export async function runAgentRelayTask( } } +async function serializeClaimExecution(path: string, execute: () => Promise): Promise { + const prior = claimExecutions.get(path) ?? Promise.resolve(); + let release!: () => void; + const occupied = new Promise(resolve => { release = resolve; }); + const queued = prior.then(() => occupied, () => occupied); + claimExecutions.set(path, queued); + await prior.catch(() => undefined); + try { + return await execute(); + } finally { + release(); + if (claimExecutions.get(path) === queued) claimExecutions.delete(path); + } +} + async function executeAgentRelayTask( request: AgentRelayTaskRequest, runtime: NodeSpawnRuntime, diff --git a/packages/sdk/src/node-spawn-state.ts b/packages/sdk/src/node-spawn-state.ts index da99fa9bb..96804a48a 100644 --- a/packages/sdk/src/node-spawn-state.ts +++ b/packages/sdk/src/node-spawn-state.ts @@ -35,10 +35,9 @@ export async function claimNodeSpawn(dataDir: string, proposed: NodeSpawnClaim): const directory = join(dataDir, 'node-spawns'); await mkdir(directory, { recursive: true }); await syncDirectory(dataDir); - const digest = createHash('sha256') - .update(canonicalize([proposed.runId, proposed.stepId, proposed.idempotencyKey])) - .digest('hex'); - const path = join(directory, `${digest}.json`); + const path = nodeSpawnClaimPath( + dataDir, proposed.runId, proposed.stepId, proposed.idempotencyKey); + const digest = path.slice(directory.length + 1, -'.json'.length); const temporary = join(directory, `.${digest}.${randomUUID()}.tmp`); const file = await open(temporary, 'wx', 0o600); try { await file.writeFile(canonicalize(proposed)); await file.sync(); } @@ -59,6 +58,18 @@ export async function claimNodeSpawn(dataDir: string, proposed: NodeSpawnClaim): } } +export function nodeSpawnClaimPath( + dataDir: string, + runId: string, + stepId: string, + idempotencyKey: string, +): string { + const digest = createHash('sha256') + .update(canonicalize([runId, stepId, idempotencyKey])) + .digest('hex'); + return join(dataDir, 'node-spawns', `${digest}.json`); +} + export async function recordNodeSpawn(path: string, claim: NodeSpawnClaim): Promise { const temporary = `${path}.${randomUUID()}.tmp`; const file = await open(temporary, 'wx', 0o600); diff --git a/packages/sdk/tests/node-spawn-settlement.test.ts b/packages/sdk/tests/node-spawn-settlement.test.ts index 7dbcacff1..95f95ff58 100644 --- a/packages/sdk/tests/node-spawn-settlement.test.ts +++ b/packages/sdk/tests/node-spawn-settlement.test.ts @@ -198,23 +198,40 @@ describe('durable node-result settlement', () => { expect(f.ack).toHaveBeenCalledOnce(); }); - it('refuses a new live observation after inbox settlement has started', async () => { - const dataDir = await dataDirectory('flows-node-settlement-late-observation-'); + it('serializes concurrent executions before opening a second result iterator', async () => { + const dataDir = await dataDirectory('flows-node-settlement-concurrent-observation-'); const f = fixture(); - const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); - const [entry] = await readNodeSpawnClaims(dataDir); - let finishAck!: () => void; - f.ack.mockImplementationOnce(() => new Promise(resolve => { finishAck = () => resolve({}); })); - const delivery = durableNodeResultDelivery(f.runtime, entry!.path, entry!.claim); - await delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); - const settling = delivery.ack(); - await vi.waitFor(() => expect(f.ack).toHaveBeenCalledOnce()); + let reportResult!: () => void; + const resultGate = new Promise(resolve => { reportResult = resolve; }); + const subscribe = vi.fn(async function* () { + await resultGate; + const name = (await f.spawn.mock.results[0]!.value).name; + yield { id: 'delivery_1', message: { id: 'message_1', + from: { id: 'agent_1', name }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_1', + step_id: 'edit', dispatch_id: 'dispatch_1234567890', status: 'completed', + output: { summary: 'done' } }) } }; + }); + f.runtime.inbox.inbox.subscribe = subscribe; - expect(() => durableNodeResultDelivery(f.runtime, entry!.path, entry!.claim, { - ...received.delivery, settlementReady: Promise.resolve({}), - })).toThrowError(expect.objectContaining({ code: 'result_settlement_unconfirmed' })); - finishAck(); - await expect(settling).resolves.toBeUndefined(); + const first = runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await vi.waitFor(() => expect(subscribe).toHaveBeenCalledOnce()); + const second = runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await Promise.resolve(); + expect(subscribe).toHaveBeenCalledOnce(); + expect(f.spawn).toHaveBeenCalledOnce(); + + reportResult(); + const [firstResult, secondResult] = await Promise.all([first, second]); + expect(subscribe).toHaveBeenCalledOnce(); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).toHaveBeenCalledOnce(); + expect(secondResult.delivery).toMatchObject({ + inboxItemId: firstResult.delivery.inboxItemId, + messageId: firstResult.delivery.messageId, + }); + await firstResult.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + await firstResult.delivery.ack(); }); it('releases the process settlement lock when persisting settledAt fails', async () => { From f329e4550e1391668c01b4eb4db1d01f9138f18b Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Mon, 9 Nov 2026 00:02:03 -0800 Subject: [PATCH 35/75] fix(sdk): cancel queued claim execution --- packages/sdk/src/agent-relay-transport.ts | 36 ++++++++++++++++--- .../sdk/tests/node-spawn-settlement.test.ts | 27 +++++++++++--- 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/packages/sdk/src/agent-relay-transport.ts b/packages/sdk/src/agent-relay-transport.ts index 8be5cba34..8e3cc32f2 100644 --- a/packages/sdk/src/agent-relay-transport.ts +++ b/packages/sdk/src/agent-relay-transport.ts @@ -77,7 +77,8 @@ export async function runAgentRelayTask( } const claimPath = nodeSpawnClaimPath( request.dataDir, request.runId, request.stepId, request.idempotencyKey); - return serializeClaimExecution(claimPath, async () => { + const taskDeadline = taskStartedAt + timeoutMs; + return serializeClaimExecution(claimPath, taskDeadline, options.signal, async () => { options.signal?.throwIfAborted(); return runAgentRelayTaskExclusive(request, options, taskStartedAt, timeoutMs); }); @@ -134,18 +135,45 @@ async function runAgentRelayTaskExclusive( } } -async function serializeClaimExecution(path: string, execute: () => Promise): Promise { +async function serializeClaimExecution( + path: string, + deadline: number, + signal: AbortSignal | undefined, + execute: () => Promise, +): Promise { const prior = claimExecutions.get(path) ?? Promise.resolve(); let release!: () => void; const occupied = new Promise(resolve => { release = resolve; }); const queued = prior.then(() => occupied, () => occupied); claimExecutions.set(path, queued); - await prior.catch(() => undefined); + let acquired = false; try { + let remaining: number; + try { remaining = remainingDeadline(deadline); } + catch (error) { + throw new AgentRelayTransportError('The node-spawn task deadline elapsed', error); + } + const queueTimeout = AbortSignal.timeout(remaining); + const queueSignal = signal === undefined ? queueTimeout : AbortSignal.any([signal, queueTimeout]); + try { await waitForNodeOperation(prior, queueSignal); } + catch (error) { + if (signal?.aborted !== true && queueTimeout.aborted) { + const timeout = new NodeSpawnError('The node-spawn task deadline elapsed', 'task_timeout', error); + throw new AgentRelayTransportError(timeout.message, timeout); + } + throw error; + } + acquired = true; return await execute(); } finally { release(); - if (claimExecutions.get(path) === queued) claimExecutions.delete(path); + if (acquired && claimExecutions.get(path) === queued) { + claimExecutions.delete(path); + } else if (!acquired) { + void queued.then(() => { + if (claimExecutions.get(path) === queued) claimExecutions.delete(path); + }); + } } } diff --git a/packages/sdk/tests/node-spawn-settlement.test.ts b/packages/sdk/tests/node-spawn-settlement.test.ts index 95f95ff58..f5b0df870 100644 --- a/packages/sdk/tests/node-spawn-settlement.test.ts +++ b/packages/sdk/tests/node-spawn-settlement.test.ts @@ -198,7 +198,10 @@ describe('durable node-result settlement', () => { expect(f.ack).toHaveBeenCalledOnce(); }); - it('serializes concurrent executions before opening a second result iterator', async () => { + it.each([ + { label: 'cancelled', timeoutMs: 10_000 }, + { label: 'deadline-expired', timeoutMs: 50 }, + ])('skips a $label queued execution without opening another result iterator', async testCase => { const dataDir = await dataDirectory('flows-node-settlement-concurrent-observation-'); const f = fixture(); let reportResult!: () => void; @@ -216,17 +219,33 @@ describe('durable node-result settlement', () => { const first = runAgentRelayTask(request(dataDir), { runtime: f.runtime }); await vi.waitFor(() => expect(subscribe).toHaveBeenCalledOnce()); - const second = runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + const cancelled = new AbortController(); + const cancellation = new Error('worker lease lost'); + const queuedRequest = { ...request(dataDir), timeoutMs: testCase.timeoutMs }; + const second = runAgentRelayTask(queuedRequest, { runtime: f.runtime, + ...(testCase.label === 'cancelled' ? { signal: cancelled.signal } : {}) }); + const third = runAgentRelayTask(request(dataDir), { runtime: f.runtime }); await Promise.resolve(); expect(subscribe).toHaveBeenCalledOnce(); expect(f.spawn).toHaveBeenCalledOnce(); + if (testCase.label === 'cancelled') { + cancelled.abort(cancellation); + await expect(second).rejects.toBe(cancellation); + } else { + await expect(second).rejects.toMatchObject({ + message: 'The node-spawn task deadline elapsed', + cause: { code: 'task_timeout' }, + }); + } + expect(subscribe).toHaveBeenCalledOnce(); + expect(f.spawn).toHaveBeenCalledOnce(); reportResult(); - const [firstResult, secondResult] = await Promise.all([first, second]); + const [firstResult, thirdResult] = await Promise.all([first, third]); expect(subscribe).toHaveBeenCalledOnce(); expect(f.spawn).toHaveBeenCalledOnce(); expect(f.release).toHaveBeenCalledOnce(); - expect(secondResult.delivery).toMatchObject({ + expect(thirdResult.delivery).toMatchObject({ inboxItemId: firstResult.delivery.inboxItemId, messageId: firstResult.delivery.messageId, }); From 2df85dc53d3c8a1ad4d95d655e3d955c9a4844d6 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Mon, 9 Nov 2026 00:13:13 -0800 Subject: [PATCH 36/75] fix(sdk): serialize durable result claims --- packages/sdk/src/agent-relay-transport.ts | 13 +- packages/sdk/src/node-spawn-settlement.ts | 165 +++++++++++++----- packages/sdk/src/node-spawn-state.ts | 44 +++-- .../sdk/tests/node-spawn-settlement.test.ts | 62 +++++-- 4 files changed, 208 insertions(+), 76 deletions(-) diff --git a/packages/sdk/src/agent-relay-transport.ts b/packages/sdk/src/agent-relay-transport.ts index 8e3cc32f2..7787eda12 100644 --- a/packages/sdk/src/agent-relay-transport.ts +++ b/packages/sdk/src/agent-relay-transport.ts @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { claimNodeSpawn, nodeSpawnClaimPath, recordNodeSpawn, +import { claimNodeSpawnAtPath, nodeSpawnClaimPath, recordNodeSpawn, type NodeSpawnClaim } from './node-spawn-state.js'; import { nodeResultInstruction, waitForNodeResult } from './node-spawn-result.js'; import { durableNodeResultDelivery } from './node-spawn-settlement.js'; @@ -75,17 +75,18 @@ export async function runAgentRelayTask( || !validIdempotencyKey(request.idempotencyKey)) { throw new AgentRelayTransportError('Node-spawn task requires durable dispatch identity and a valid deadline'); } - const claimPath = nodeSpawnClaimPath( + const claimPath = await nodeSpawnClaimPath( request.dataDir, request.runId, request.stepId, request.idempotencyKey); const taskDeadline = taskStartedAt + timeoutMs; return serializeClaimExecution(claimPath, taskDeadline, options.signal, async () => { options.signal?.throwIfAborted(); - return runAgentRelayTaskExclusive(request, options, taskStartedAt, timeoutMs); + return runAgentRelayTaskExclusive(request, claimPath, options, taskStartedAt, timeoutMs); }); } async function runAgentRelayTaskExclusive( request: AgentRelayTaskRequest, + claimPath: string, options: AgentRelayEnv & { runtime?: NodeSpawnRuntime; runtimeProvider?: (signal?: AbortSignal) => Promise; signal?: AbortSignal }, taskStartedAt: number, @@ -129,7 +130,8 @@ async function runAgentRelayTaskExclusive( throw error; } try { - return await executeAgentRelayTask(request, runtime, options.signal, taskStartedAt, timeoutMs); + return await executeAgentRelayTask( + request, claimPath, runtime, options.signal, taskStartedAt, timeoutMs); } finally { if (ownsRuntime) await runtime.disconnect?.().catch(() => undefined); } @@ -179,6 +181,7 @@ async function serializeClaimExecution( async function executeAgentRelayTask( request: AgentRelayTaskRequest, + claimPath: string, runtime: NodeSpawnRuntime, signal: AbortSignal | undefined, taskStartedAt: number, @@ -211,7 +214,7 @@ async function executeAgentRelayTask( }, startedAt: taskStartedAt, }; - const claimed = await claimNodeSpawn(request.dataDir, proposed); + const claimed = await claimNodeSpawnAtPath(claimPath, proposed); let claim = claimed.claim; let releaseAttempted = false; const deadline = claim.startedAt + claim.timeoutMs; diff --git a/packages/sdk/src/node-spawn-settlement.ts b/packages/sdk/src/node-spawn-settlement.ts index 610495fd4..d470df272 100644 --- a/packages/sdk/src/node-spawn-settlement.ts +++ b/packages/sdk/src/node-spawn-settlement.ts @@ -1,7 +1,8 @@ import type { JournalClient } from './journal-client.js'; import type { JournalEvent } from './journal-reader.js'; import type { NodeResultDelivery } from './node-spawn-result.js'; -import { readNodeSpawnClaims, recordNodeSpawn, type NodeSpawnClaim } from './node-spawn-state.js'; +import { readNodeSpawnClaim, readNodeSpawnClaims, recordNodeSpawn, + type NodeSpawnClaim } from './node-spawn-state.js'; import type { NodeSpawnRuntime } from './node-spawn-runtime.js'; import { NodeSpawnError, isRecord } from './node-spawn.js'; @@ -21,6 +22,7 @@ interface LiveTeardownGroup { /** Process-generation state keyed by durable claim path; a real restart intentionally resets it. */ const liveTeardowns = new Map(); +const settlementMutations = new Map>(); export function durableNodeResultDelivery( runtime: NodeSpawnRuntime, @@ -41,27 +43,43 @@ export function durableNodeResultDelivery( } let claim = initial; let settling: Promise | undefined; - const settleOnce = async (): Promise => { - const current = claim.result; + const settleOnce = (): Promise => serializeSettlementMutation(path, async () => { + const durable = await loadDurableClaim(path, result); + const current = durable.result; if (current === undefined) throw new NodeSpawnError('Durable node result disappeared before settlement'); - if (current.settlement === undefined) { + const expected = claim.result?.settlement; + if (expected === undefined || current.settlement === undefined) { throw new NodeSpawnError('Node result cannot settle before its journal intent is durable', 'result_settlement_unprepared'); } + if (current.settlement.attempt !== expected.attempt + || current.settlement.completionReason !== expected.completionReason) { + throw new NodeSpawnError('Node result settlement conflicts with its journal identity', + 'result_identity_mismatch'); + } + if (current.settlement.settledAt !== undefined) { + if (current.settlement.settledAs !== current.settlementAction) { + throw new NodeSpawnError('Node result settled with a different inbox action', + 'result_identity_mismatch'); + } + claim = durable; + return; + } const liveSettlement = await settleAfterLiveTeardowns(runtime, path, current.settlementAction, current.inboxItemId, current.envelope.error ?? 'Relayflows rejected an invalid result'); - claim = { ...claim, result: { ...current, settlement: { + const updated: NodeSpawnClaim = { ...durable, result: { ...current, settlement: { ...current.settlement, settledAt: Date.now(), settledAs: current.settlementAction, } } }; try { - await recordNodeSpawn(path, claim); + await recordNodeSpawn(path, updated); } catch (error) { releaseLiveSettlement(path, liveSettlement); throw error; } + claim = updated; completeLiveSettlement(path, liveSettlement); - }; + }); const settle = (): Promise => { if (settling !== undefined) return settling; const attempt = settleOnce(); @@ -75,8 +93,9 @@ export function durableNodeResultDelivery( envelope: result.envelope, settlementAction: result.settlementAction, ...(observed?.settlementReady === undefined ? {} : { settlementReady: observed.settlementReady }), - prepareSettlement: async input => { - const current = claim.result; + prepareSettlement: input => serializeSettlementMutation(path, async () => { + const durable = await loadDurableClaim(path, result); + const current = durable.result; if (current === undefined) throw new NodeSpawnError('Durable node result disappeared before settlement'); const prior = current.settlement; if (prior !== undefined && prior.completionReason !== input.completionReason) { @@ -87,22 +106,63 @@ export function durableNodeResultDelivery( throw new NodeSpawnError('A settled node result cannot move to a different journal attempt', 'result_identity_mismatch'); } + if (prior?.settledAt !== undefined && prior.settledAs !== current.settlementAction) { + throw new NodeSpawnError('Node result settled with a different inbox action', + 'result_identity_mismatch'); + } if (prior !== undefined && input.attempt < prior.attempt) { throw new NodeSpawnError('Node result settlement cannot move back to an older journal attempt', 'result_identity_mismatch'); } - claim = { ...claim, result: { ...current, settlement: + if (prior !== undefined && prior.attempt === input.attempt) { + claim = durable; + return; + } + const updated: NodeSpawnClaim = { ...durable, result: { ...current, settlement: prior !== undefined && prior.attempt === input.attempt ? prior : { attempt: input.attempt, completionReason: input.completionReason, preparedAt: Date.now(), } } }; - await recordNodeSpawn(path, claim); - }, + await recordNodeSpawn(path, updated); + claim = updated; + }), ack: settle, }; } +async function loadDurableClaim( + path: string, + expected: NonNullable, +): Promise { + const durable = await readNodeSpawnClaim(path); + const current = durable.result; + if (current === undefined + || current.inboxItemId !== expected.inboxItemId + || current.messageId !== expected.messageId + || current.status !== expected.status + || current.settlementAction !== expected.settlementAction) { + throw new NodeSpawnError('Durable node result differs from its delivery identity', + 'result_identity_mismatch'); + } + return durable; +} + +async function serializeSettlementMutation(path: string, mutate: () => Promise): Promise { + const prior = settlementMutations.get(path) ?? Promise.resolve(); + let release!: () => void; + const occupied = new Promise(resolve => { release = resolve; }); + const queued = prior.then(() => occupied, () => occupied); + settlementMutations.set(path, queued); + await prior.catch(() => undefined); + try { + return await mutate(); + } finally { + release(); + if (settlementMutations.get(path) === queued) settlementMutations.delete(path); + } +} + /** Reconciles journaled result settlements left by a crash or failed inbox call. */ export async function reconcileNodeResultSettlements( dataDir: string, @@ -111,46 +171,54 @@ export async function reconcileNodeResultSettlements( ): Promise { let pending = 0; for (const entry of await readNodeSpawnClaims(dataDir)) { - const result = entry.claim.result; - const settlement = result?.settlement; - if (result === undefined || settlement === undefined || settlement.settledAt !== undefined) continue; - if (entry.claim.baseUrl !== runtime.baseUrl - || entry.claim.publisherId !== runtime.publisher.id - || entry.claim.publisherName !== runtime.publisher.name) { - throw new NodeSpawnError('Pending node-result settlement belongs to a different publisher identity', - 'publisher_identity_mismatch'); - } - const live = liveTeardowns.get(entry.path); - if (live !== undefined) { - const failed = [...live.barriers].find(item => item.settled && item.failure !== undefined); - if (failed !== undefined) throw failed.failure; - if (live.settlementStarted || [...live.barriers].some(item => !item.settled)) { - pending += 1; - continue; - } - } - const terminal = await terminalEntry(journal, entry.claim.runId, entry.claim.stepId, settlement.attempt); - if (terminal === undefined) { pending += 1; continue; } - const exact = terminalMatchesResult(terminal, entry.claim, settlement, result); - const action = exact ? result.settlementAction : 'fail'; - const error = exact ? (result.envelope.error ?? 'Relayflows rejected an invalid result') - : 'The journal completed this attempt with a different terminal outcome'; - const liveSettlement = await settleAfterLiveTeardowns( - runtime, entry.path, action, result.inboxItemId, error); - const updated: NodeSpawnClaim = { ...entry.claim, result: { ...result, settlement: { - ...settlement, settledAt: Date.now(), settledAs: action, - } } }; - try { - await recordNodeSpawn(entry.path, updated); - } catch (error) { - releaseLiveSettlement(entry.path, liveSettlement); - throw error; - } - completeLiveSettlement(entry.path, liveSettlement); + if (await serializeSettlementMutation(entry.path, + () => reconcileNodeResultSettlement(entry.path, runtime, journal))) pending += 1; } return pending; } +async function reconcileNodeResultSettlement( + path: string, + runtime: NodeSpawnRuntime, + journal: Pick, +): Promise { + const claim = await readNodeSpawnClaim(path); + const result = claim.result; + const settlement = result?.settlement; + if (result === undefined || settlement === undefined || settlement.settledAt !== undefined) return false; + if (claim.baseUrl !== runtime.baseUrl + || claim.publisherId !== runtime.publisher.id + || claim.publisherName !== runtime.publisher.name) { + throw new NodeSpawnError('Pending node-result settlement belongs to a different publisher identity', + 'publisher_identity_mismatch'); + } + const live = liveTeardowns.get(path); + if (live !== undefined) { + const failed = [...live.barriers].find(item => item.settled && item.failure !== undefined); + if (failed !== undefined) throw failed.failure; + if (live.settlementStarted || [...live.barriers].some(item => !item.settled)) return true; + } + const terminal = await terminalEntry(journal, claim.runId, claim.stepId, settlement.attempt); + if (terminal === undefined) return true; + const exact = terminalMatchesResult(terminal, claim, settlement, result); + const action = exact ? result.settlementAction : 'fail'; + const error = exact ? (result.envelope.error ?? 'Relayflows rejected an invalid result') + : 'The journal completed this attempt with a different terminal outcome'; + const liveSettlement = await settleAfterLiveTeardowns( + runtime, path, action, result.inboxItemId, error); + const updated: NodeSpawnClaim = { ...claim, result: { ...result, settlement: { + ...settlement, settledAt: Date.now(), settledAs: action, + } } }; + try { + await recordNodeSpawn(path, updated); + } catch (writeError) { + releaseLiveSettlement(path, liveSettlement); + throw writeError; + } + completeLiveSettlement(path, liveSettlement); + return false; +} + function rememberLiveTeardown( path: string, outcome: Promise<{ failure?: unknown }>, @@ -212,6 +280,7 @@ async function settleAfterLiveTeardowns( current.settlementStarted = true; try { await issueInboxSettlement(runtime, action, inboxItemId, error); + signal.throwIfAborted(); return { group: current, generation }; } catch (settlementError) { current.settlementStarted = false; diff --git a/packages/sdk/src/node-spawn-state.ts b/packages/sdk/src/node-spawn-state.ts index 96804a48a..b1bea9ec1 100644 --- a/packages/sdk/src/node-spawn-state.ts +++ b/packages/sdk/src/node-spawn-state.ts @@ -1,6 +1,6 @@ import { createHash, randomUUID } from 'node:crypto'; -import { link, mkdir, open, readFile, readdir, rename, unlink } from 'node:fs/promises'; -import { dirname, join } from 'node:path'; +import { link, mkdir, open, readFile, readdir, realpath, rename, unlink } from 'node:fs/promises'; +import { basename, dirname, join } from 'node:path'; import { canonicalize } from './canonical.js'; import { readNodeResult, type NodeResultEnvelope } from './node-spawn-result.js'; import { isRecord, nonempty, validIdempotencyKey, validNodeSpawnIdentity, validNodeSpawnReceipt, @@ -32,12 +32,18 @@ export interface NodeSpawnClaim { export async function claimNodeSpawn(dataDir: string, proposed: NodeSpawnClaim): Promise<{ claim: NodeSpawnClaim; created: boolean; path: string; }> { - const directory = join(dataDir, 'node-spawns'); - await mkdir(directory, { recursive: true }); - await syncDirectory(dataDir); - const path = nodeSpawnClaimPath( + const path = await nodeSpawnClaimPath( dataDir, proposed.runId, proposed.stepId, proposed.idempotencyKey); - const digest = path.slice(directory.length + 1, -'.json'.length); + return claimNodeSpawnAtPath(path, proposed); +} + +export async function claimNodeSpawnAtPath(path: string, proposed: NodeSpawnClaim): Promise<{ + claim: NodeSpawnClaim; created: boolean; path: string; +}> { + const directory = dirname(path); + await mkdir(directory, { recursive: true }); + await syncDirectory(dirname(directory)); + const digest = basename(path, '.json'); const temporary = join(directory, `.${digest}.${randomUUID()}.tmp`); const file = await open(temporary, 'wx', 0o600); try { await file.writeFile(canonicalize(proposed)); await file.sync(); } @@ -58,16 +64,18 @@ export async function claimNodeSpawn(dataDir: string, proposed: NodeSpawnClaim): } } -export function nodeSpawnClaimPath( +export async function nodeSpawnClaimPath( dataDir: string, runId: string, stepId: string, idempotencyKey: string, -): string { +): Promise { + await mkdir(dataDir, { recursive: true }); + const canonicalDataDir = await realpath(dataDir); const digest = createHash('sha256') .update(canonicalize([runId, stepId, idempotencyKey])) .digest('hex'); - return join(dataDir, 'node-spawns', `${digest}.json`); + return join(canonicalDataDir, 'node-spawns', `${digest}.json`); } export async function recordNodeSpawn(path: string, claim: NodeSpawnClaim): Promise { @@ -80,7 +88,13 @@ export async function recordNodeSpawn(path: string, claim: NodeSpawnClaim): Prom } export async function readNodeSpawnClaims(dataDir: string): Promise> { - const directory = join(dataDir, 'node-spawns'); + let canonicalDataDir: string; + try { canonicalDataDir = await realpath(dataDir); } + catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; + throw error; + } + const directory = join(canonicalDataDir, 'node-spawns'); let names: string[]; try { names = await readdir(directory); } catch (error) { @@ -91,12 +105,16 @@ export async function readNodeSpawnClaims(dataDir: string): Promise { + const value = JSON.parse(await readFile(path, 'utf8')) as unknown; + return decodeNodeSpawnClaim(value, path); +} + function sameClaim(prior: NodeSpawnClaim, proposed: NodeSpawnClaim): boolean { const { spawn: _priorSpawn, cleanup: _priorCleanup, failure: _priorFailure, result: _priorResult, cleanupPending: _priorCleanupPending, diff --git a/packages/sdk/tests/node-spawn-settlement.test.ts b/packages/sdk/tests/node-spawn-settlement.test.ts index f5b0df870..ebb994821 100644 --- a/packages/sdk/tests/node-spawn-settlement.test.ts +++ b/packages/sdk/tests/node-spawn-settlement.test.ts @@ -1,7 +1,7 @@ import { EventEmitter } from 'node:events'; import { mkdtemp, rename, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { join, relative } from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { runAgentRelayTask, type AgentRelayTaskRequest } from '../src/agent-relay-transport.js'; import type { JournalClient } from '../src/journal-client.js'; @@ -29,6 +29,14 @@ async function dataDirectory(label: string): Promise { return directory; } +async function waitForSettlementTimer(): Promise { + for (let attempt = 0; attempt < 50; attempt += 1) { + if (vi.getTimerCount() > 0) return; + await new Promise(resolve => setImmediate(resolve)); + } + throw new Error('Settlement deadline was not armed'); +} + function fixture(): { runtime: NodeSpawnRuntime; spawn: ReturnType; release: ReturnType; ack: ReturnType; fail: ReturnType } { const receipt: NodeSpawnReceipt = { @@ -156,10 +164,11 @@ describe('durable node-result settlement', () => { const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); - vi.useFakeTimers(); + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); try { const settlement = expect(received.delivery.ack()) .rejects.toMatchObject({ code: 'result_settlement_unconfirmed' }); + await waitForSettlementTimer(); await vi.advanceTimersByTimeAsync(10_001); await settlement; expect(f.ack).not.toHaveBeenCalled(); @@ -219,12 +228,13 @@ describe('durable node-result settlement', () => { const first = runAgentRelayTask(request(dataDir), { runtime: f.runtime }); await vi.waitFor(() => expect(subscribe).toHaveBeenCalledOnce()); + const aliasedRequest = request(relative(process.cwd(), dataDir)); const cancelled = new AbortController(); const cancellation = new Error('worker lease lost'); - const queuedRequest = { ...request(dataDir), timeoutMs: testCase.timeoutMs }; + const queuedRequest = { ...aliasedRequest, timeoutMs: testCase.timeoutMs }; const second = runAgentRelayTask(queuedRequest, { runtime: f.runtime, ...(testCase.label === 'cancelled' ? { signal: cancelled.signal } : {}) }); - const third = runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + const third = runAgentRelayTask(aliasedRequest, { runtime: f.runtime }); await Promise.resolve(); expect(subscribe).toHaveBeenCalledOnce(); expect(f.spawn).toHaveBeenCalledOnce(); @@ -249,8 +259,21 @@ describe('durable node-result settlement', () => { inboxItemId: firstResult.delivery.inboxItemId, messageId: firstResult.delivery.messageId, }); - await firstResult.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); - await firstResult.delivery.ack(); + await Promise.all([ + firstResult.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }), + thirdResult.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }), + ]); + await Promise.all([firstResult.delivery.ack(), thirdResult.delivery.ack()]); + expect(f.ack).toHaveBeenCalledOnce(); + const [settled] = await readNodeSpawnClaims(dataDir); + expect(settled!.claim.result?.settlement).toMatchObject({ + attempt: 1, completionReason: 'success', settledAs: 'ack', settledAt: expect.any(Number), + }); + await expect(thirdResult.delivery.prepareSettlement!({ + attempt: 2, completionReason: 'success', + })).rejects.toMatchObject({ code: 'result_identity_mismatch' }); + const [unchanged] = await readNodeSpawnClaims(dataDir); + expect(unchanged!.claim.result?.settlement).toEqual(settled!.claim.result?.settlement); }); it('releases the process settlement lock when persisting settledAt fails', async () => { @@ -258,12 +281,17 @@ describe('durable node-result settlement', () => { const f = fixture(); const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + let confirmAck!: () => void; + f.ack.mockImplementationOnce(() => new Promise(resolve => { confirmAck = resolve; })); + const attempt = received.delivery.ack(); + await vi.waitFor(() => expect(f.ack).toHaveBeenCalledOnce()); const claimsDirectory = join(dataDir, 'node-spawns'); const unavailableDirectory = `${claimsDirectory}.unavailable`; await rename(claimsDirectory, unavailableDirectory); let restored = false; try { - await expect(received.delivery.ack()).rejects.toMatchObject({ code: 'ENOENT' }); + confirmAck(); + await expect(attempt).rejects.toMatchObject({ code: 'ENOENT' }); expect(f.ack).toHaveBeenCalledOnce(); await rename(unavailableDirectory, claimsDirectory); restored = true; @@ -537,15 +565,29 @@ describe('durable node-result settlement', () => { const f = fixture(); const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); - f.ack.mockImplementationOnce(() => new Promise(() => undefined)); - vi.useFakeTimers(); + let confirmLate!: () => void; + f.ack.mockImplementationOnce(() => new Promise(resolve => { confirmLate = resolve; })); + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); try { const pending = expect(received.delivery.ack()) .rejects.toMatchObject({ code: 'result_settlement_unconfirmed' }); + await waitForSettlementTimer(); + expect(f.ack).toHaveBeenCalledOnce(); await vi.advanceTimersByTimeAsync(10_001); await pending; const claims = await readNodeSpawnClaims(dataDir); expect(claims[0]!.claim.result?.settlement).not.toHaveProperty('settledAt'); + confirmLate(); + await Promise.resolve(); + await Promise.resolve(); + const afterLateSuccess = await readNodeSpawnClaims(dataDir); + expect(afterLateSuccess[0]!.claim.result?.settlement).not.toHaveProperty('settledAt'); + await received.delivery.ack(); + expect(f.ack).toHaveBeenCalledTimes(2); + const settled = await readNodeSpawnClaims(dataDir); + expect(settled[0]!.claim.result?.settlement).toMatchObject({ + settledAt: expect.any(Number), settledAs: 'ack', + }); } finally { vi.useRealTimers(); } @@ -561,7 +603,7 @@ describe('durable node-result settlement', () => { const first = received.delivery.ack(); const second = received.delivery.ack(); expect(first).toBe(second); - expect(f.ack).toHaveBeenCalledOnce(); + await vi.waitFor(() => expect(f.ack).toHaveBeenCalledOnce()); confirm(); await expect(Promise.all([first, second])).resolves.toEqual([undefined, undefined]); }); From cdef5e6e44dcb33822485e3d569245ab04a07b4a Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Mon, 9 Nov 2026 00:18:10 -0800 Subject: [PATCH 37/75] test(sdk): prove canonical claim aliases --- packages/sdk/tests/node-spawn-runtime.test.ts | 2 +- packages/sdk/tests/node-spawn-settlement.test.ts | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/packages/sdk/tests/node-spawn-runtime.test.ts b/packages/sdk/tests/node-spawn-runtime.test.ts index 913c4c615..1d592b9c8 100644 --- a/packages/sdk/tests/node-spawn-runtime.test.ts +++ b/packages/sdk/tests/node-spawn-runtime.test.ts @@ -144,7 +144,7 @@ describe('remote node-spawn runtime', () => { const runtimeProvider = vi.fn((_signal?: AbortSignal) => new Promise(() => undefined)); await expect(runAgentRelayTask({ cli: 'codex', task: 'work', runId: 'run_1', stepId: 'step_1', - idempotencyKey: 'dispatch_1234567890', attempt: 1, dataDir: '/tmp/unused', timeoutMs: 5, + idempotencyKey: 'dispatch_1234567890', attempt: 1, dataDir: '/tmp/unused', timeoutMs: 100, }, { runtimeProvider })).rejects.toMatchObject({ cause: expect.objectContaining({ code: 'task_timeout' }), }); diff --git a/packages/sdk/tests/node-spawn-settlement.test.ts b/packages/sdk/tests/node-spawn-settlement.test.ts index ebb994821..b610a49f8 100644 --- a/packages/sdk/tests/node-spawn-settlement.test.ts +++ b/packages/sdk/tests/node-spawn-settlement.test.ts @@ -1,7 +1,7 @@ import { EventEmitter } from 'node:events'; -import { mkdtemp, rename, rm } from 'node:fs/promises'; +import { mkdtemp, rename, rm, symlink } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { join, relative } from 'node:path'; +import { join } from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { runAgentRelayTask, type AgentRelayTaskRequest } from '../src/agent-relay-transport.js'; import type { JournalClient } from '../src/journal-client.js'; @@ -30,9 +30,11 @@ async function dataDirectory(label: string): Promise { } async function waitForSettlementTimer(): Promise { - for (let attempt = 0; attempt < 50; attempt += 1) { + for (let attempt = 0; attempt < 500; attempt += 1) { if (vi.getTimerCount() > 0) return; - await new Promise(resolve => setImmediate(resolve)); + await new Promise(resolve => { + const handle = setInterval(() => { clearInterval(handle); resolve(); }, 1); + }); } throw new Error('Settlement deadline was not armed'); } @@ -212,6 +214,9 @@ describe('durable node-result settlement', () => { { label: 'deadline-expired', timeoutMs: 50 }, ])('skips a $label queued execution without opening another result iterator', async testCase => { const dataDir = await dataDirectory('flows-node-settlement-concurrent-observation-'); + const aliasedDataDir = `${dataDir}-alias`; + await symlink(dataDir, aliasedDataDir, process.platform === 'win32' ? 'junction' : 'dir'); + roots.push(aliasedDataDir); const f = fixture(); let reportResult!: () => void; const resultGate = new Promise(resolve => { reportResult = resolve; }); @@ -228,7 +233,7 @@ describe('durable node-result settlement', () => { const first = runAgentRelayTask(request(dataDir), { runtime: f.runtime }); await vi.waitFor(() => expect(subscribe).toHaveBeenCalledOnce()); - const aliasedRequest = request(relative(process.cwd(), dataDir)); + const aliasedRequest = request(aliasedDataDir); const cancelled = new AbortController(); const cancellation = new Error('worker lease lost'); const queuedRequest = { ...aliasedRequest, timeoutMs: testCase.timeoutMs }; From 18e811672bce886ed9cd45ee3055dd0469b838ed Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Mon, 9 Nov 2026 01:39:54 -0800 Subject: [PATCH 38/75] fix(sdk): validate Desktop caller before admission --- docs/AGENT-RELAY-TRANSPORT.md | 32 ++++++++++++------- packages/sdk/src/cli.ts | 4 ++- packages/sdk/src/worker.ts | 7 ++++ .../tests/worker-node-runtime-sharing.test.ts | 17 ++++++++++ packages/sdk/tests/worker-slots.test.ts | 2 ++ 5 files changed, 49 insertions(+), 13 deletions(-) diff --git a/docs/AGENT-RELAY-TRANSPORT.md b/docs/AGENT-RELAY-TRANSPORT.md index cfd621704..4268a2073 100644 --- a/docs/AGENT-RELAY-TRANSPORT.md +++ b/docs/AGENT-RELAY-TRANSPORT.md @@ -75,20 +75,28 @@ cannot turn Relay acknowledgement into fabricated execution truth. ## Local Desktop bridge -The SDK contains a local `NodeSpawnPort` adapter for the Desktop core's bridge -v2 methods: `spawn.targets`, `spawn.invoke`, `spawn.status`, and -`spawn.release`. On Unix it connects to `core.sock`, validates socket -ownership/mode, performs the `hello` handshake, and requires an explicit -Desktop-issued caller name and secret on invoke, status, and release. The -secret is kept in module-private storage rather than on the exported port -instance. Local node spawn is refused on Windows until Desktop exposes an -authenticated named-pipe server contract; a bridge-version hello alone is not -server authentication. +The SDK contains a local `NodeSpawnPort` adapter for Desktop's schema-v2 +service-caller bridge. The caller is an explicit private `caller.json` path +supplied with `--service-caller-file`, the SDK `serviceCallerFile` option, or +`RELAYFLOW_SERVICE_CALLER_FILE`; Flows never scans Desktop's app-data +directory. Optional `@agent-relay/core-bridge` validates the handoff and local +socket or named pipe, challenge-authenticates a fresh connection, verifies the +bound publisher/workspace/node identity, and then calls `spawn.targets`, +`spawn.invoke`, `spawn.status`, and `spawn.release` without sending a caller +name or secret. The same authenticated bridge provides the invocation-scoped +result inbox and its ack/fail/close/disconnect lifecycle. + +An explicit caller handoff is bound before the local worker registers with the +journal, so a missing, expired, malformed, revoked or identity-mismatched file +cannot admit a run and fail only at its first Relay step. The bridge package +owns platform-specific socket and Windows named-pipe trust checks; a bridge +version hello by itself is not authentication. The current bridge does not accept model, repository, or sandbox placement overrides, so the local adapter refuses those fields instead of dropping them. -It is not selected by the CLI yet. Flows does not borrow another session's -name and does not invent or provision its caller secret. Local admission stays -fail-closed until Desktop supplies the service credential path and file shape. +The CLI selects this adapter only when the caller file is explicit and +absolute. It is mutually exclusive with `RELAY_AGENT_TOKEN`; Flows does not +borrow another session's name and does not invent or provision authority. +Desktop creates, rotates and revokes the private handoff. See [the migration note](MIGRATION-NODE-SPAWN.md) for removed broker surfaces. diff --git a/packages/sdk/src/cli.ts b/packages/sdk/src/cli.ts index 71a1ba913..bd6d24bf7 100644 --- a/packages/sdk/src/cli.ts +++ b/packages/sdk/src/cli.ts @@ -22,6 +22,7 @@ import { type CloudMirrorReceipt, } from './cli/cloud-mirror-session.js'; import { realpathSync } from 'node:fs'; +import { isAbsolute } from 'node:path'; import { pathToFileURL } from 'node:url'; import { checkFlow, @@ -817,7 +818,8 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { } if (argument === '--service-caller-file') { const value = args[++index]; - if (command === 'check' || serviceCallerFile !== undefined || value === undefined || value.startsWith('-')) return undefined; + if (command === 'check' || serviceCallerFile !== undefined || value === undefined + || value.startsWith('-') || value.includes('\0') || !isAbsolute(value)) return undefined; serviceCallerFile = value; continue; } diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index 2d1ae6965..429756cff 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -99,6 +99,13 @@ export class AgentWorker extends EventEmitter { if (this.closing) throw new Error('agent worker: cannot attach a closed worker (construct a new one)'); this.client.on('step.dispatch', this.onDispatch); try { + // An explicit Desktop handoff is fully checkable before this worker is + // admitted. Bind and validate it now so a missing, expired or mismatched + // caller cannot let earlier deterministic steps run and fail only when + // the first Relay dispatch eventually asks for the lazy runtime. + // Remote fleet authority stays lazy: a direct-only local worker does not + // need to contact Relay merely because RELAY_AGENT_TOKEN is present. + if (this.localDesktopAuthority) await this.nodeSpawnRuntimeOwner.get(); await this.reconcileRelaySettlementsAfterDrain(); await this.client.workerAttach( this.options.workerId, diff --git a/packages/sdk/tests/worker-node-runtime-sharing.test.ts b/packages/sdk/tests/worker-node-runtime-sharing.test.ts index 89ce5f0b2..609222af8 100644 --- a/packages/sdk/tests/worker-node-runtime-sharing.test.ts +++ b/packages/sdk/tests/worker-node-runtime-sharing.test.ts @@ -28,6 +28,23 @@ afterEach(async () => { }); describe('AgentWorker node-spawn runtime ownership', () => { + it('validates an explicit Desktop caller before registering the worker', async () => { + const callerFailure = new Error('Desktop caller is expired'); + owner.get.mockRejectedValue(callerFailure); + const workerAttach = vi.fn(async () => ({})); + const client = Object.assign(new EventEmitter(), { workerAttach }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_invalid_desktop_caller', + pins: {}, + serviceCallerFile: '/tmp/caller.json', + }); + + await expect(worker.attach()).rejects.toBe(callerFailure); + expect(owner.get).toHaveBeenCalledOnce(); + expect(workerAttach).not.toHaveBeenCalled(); + expect(owner.close).toHaveBeenCalledOnce(); + }); + it.each([ { label: 'Desktop caller', serviceCallerFile: '/tmp/caller.json', authority: undefined, provideRuntime: false, expectedDirectory: 'run-root', expectedSpawn: true }, diff --git a/packages/sdk/tests/worker-slots.test.ts b/packages/sdk/tests/worker-slots.test.ts index 2180abc3c..c8bfc667c 100644 --- a/packages/sdk/tests/worker-slots.test.ts +++ b/packages/sdk/tests/worker-slots.test.ts @@ -142,6 +142,8 @@ describe('--service-caller-file', () => { ['run', '--service-caller-file', '/tmp/caller.json', 'flow.yaml'], ['run', '--cloud', '--local-agent', '--service-caller-file', '/tmp/caller.json', 'flow.yaml'], ['check', '--service-caller-file', '/tmp/caller.json', 'flow.yaml'], + ['run', '--local-agent', '--service-caller-file', 'relative/caller.json', 'flow.yaml'], + ['run', '--local-agent', '--service-caller-file', '/tmp/caller\0.json', 'flow.yaml'], ['run', '--local-agent', '--service-caller-file', 'flow.yaml'], ['run', '--local-agent', '--service-caller-file', '/tmp/a', '--service-caller-file', '/tmp/b', 'flow.yaml'], ]) { From ec97042d9ff17387479c06d88ac355a15eb5a90e Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Mon, 9 Nov 2026 02:46:30 -0800 Subject: [PATCH 39/75] fix(sdk): preserve exact caller capability paths --- packages/sdk/src/node-spawn-runtime.ts | 12 +++++++--- packages/sdk/src/worker.ts | 6 ++--- packages/sdk/tests/node-spawn-runtime.test.ts | 23 +++++++++++++++++++ .../tests/worker-node-runtime-sharing.test.ts | 22 ++++++++++++++++++ 4 files changed, 57 insertions(+), 6 deletions(-) diff --git a/packages/sdk/src/node-spawn-runtime.ts b/packages/sdk/src/node-spawn-runtime.ts index b90ba4568..20dc9103a 100644 --- a/packages/sdk/src/node-spawn-runtime.ts +++ b/packages/sdk/src/node-spawn-runtime.ts @@ -48,6 +48,12 @@ export function mergeNodeSpawnEnvironment( }; } +/** Returns an exact nonempty Desktop capability path; paths are never normalized. */ +export function explicitServiceCallerFile(env: NodeJS.ProcessEnv): string | undefined { + const serviceCallerFile = env.RELAYFLOW_SERVICE_CALLER_FILE; + return serviceCallerFile === undefined || serviceCallerFile === '' ? undefined : serviceCallerFile; +} + interface RelayRuntimeClient extends RemoteNodeSpawnClient, NodeResultInboxClient { agents: RemoteNodeSpawnClient['agents'] & { me(): Promise }; events: { disconnect(): Promise }; @@ -151,15 +157,15 @@ export async function createNodeSpawnRuntime( env: NodeJS.ProcessEnv = process.env, options: { signal?: AbortSignal; bridge?: CoreBridgeApi } = {}, ): Promise { - const serviceCallerFile = env.RELAYFLOW_SERVICE_CALLER_FILE?.trim(); + const serviceCallerFile = explicitServiceCallerFile(env); const agentToken = env.RELAY_AGENT_TOKEN?.trim(); - if (serviceCallerFile && agentToken) { + if (serviceCallerFile !== undefined && agentToken) { throw new NodeSpawnError( 'RELAYFLOW_SERVICE_CALLER_FILE and RELAY_AGENT_TOKEN name different node-spawn authorities; choose one', 'ambiguous_publisher_authority', ); } - if (serviceCallerFile) return createLocalNodeSpawnRuntime(serviceCallerFile, options); + if (serviceCallerFile !== undefined) return createLocalNodeSpawnRuntime(serviceCallerFile, options); return createRemoteNodeSpawnRuntime(env, options); } diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index 429756cff..16f8a0a63 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -22,8 +22,8 @@ import { redactText, type TranscriptDigest, } from './agent-transcript.js'; -import { createNodeSpawnRuntime, createNodeSpawnRuntimeOwner, mergeNodeSpawnEnvironment, type NodeSpawnRuntime, - type NodeSpawnRuntimeOwner } from './node-spawn-runtime.js'; +import { createNodeSpawnRuntime, createNodeSpawnRuntimeOwner, explicitServiceCallerFile, + mergeNodeSpawnEnvironment, type NodeSpawnRuntime, type NodeSpawnRuntimeOwner } from './node-spawn-runtime.js'; import { reconcileNodeResultSettlements } from './node-spawn-settlement.js'; import { readNodeSpawnClaims } from './node-spawn-state.js'; @@ -86,7 +86,7 @@ export class AgentWorker extends EventEmitter { super(); const runtimeEnvironment = mergeNodeSpawnEnvironment(options.environment, options.serviceCallerFile); this.localDesktopAuthority = options.nodeSpawnRuntime === undefined - ? Boolean(runtimeEnvironment.RELAYFLOW_SERVICE_CALLER_FILE?.trim()) + ? explicitServiceCallerFile(runtimeEnvironment) !== undefined : options.nodeSpawnRuntime.authority === 'desktop'; this.nodeSpawnRuntimeOwner = createNodeSpawnRuntimeOwner( options.nodeSpawnRuntime, diff --git a/packages/sdk/tests/node-spawn-runtime.test.ts b/packages/sdk/tests/node-spawn-runtime.test.ts index 1d592b9c8..b7575065f 100644 --- a/packages/sdk/tests/node-spawn-runtime.test.ts +++ b/packages/sdk/tests/node-spawn-runtime.test.ts @@ -242,6 +242,29 @@ describe('Desktop service-caller runtime', () => { expect(relay.constructorOptions).toHaveLength(0); }); + it('passes the exact caller capability path without trimming it', async () => { + const { bridge } = fixture(); + const exactPath = '/tmp/caller.json '; + const runtime = await createNodeSpawnRuntime({ + RELAYFLOW_SERVICE_CALLER_FILE: exactPath, + }, { bridge }); + expect(bridge.readCallerDescriptor).toHaveBeenCalledWith(exactPath); + await runtime.disconnect?.(); + }); + + it('refuses whitespace-only caller authority instead of selecting the remote path', async () => { + const { bridge } = fixture(); + await expect(createNodeSpawnRuntime({ + RELAYFLOW_SERVICE_CALLER_FILE: ' ', + }, { bridge })).rejects.toMatchObject({ code: 'caller_file_invalid' }); + await expect(createNodeSpawnRuntime({ + RELAYFLOW_SERVICE_CALLER_FILE: ' ', + RELAY_AGENT_TOKEN: 'agent-token', + }, { bridge })).rejects.toMatchObject({ code: 'ambiguous_publisher_authority' }); + expect(bridge.readCallerDescriptor).not.toHaveBeenCalled(); + expect(relay.constructorOptions).toHaveLength(0); + }); + it('does not scan for a Desktop caller when neither authority was explicit', async () => { const { bridge } = fixture(); await expect(createNodeSpawnRuntime({}, { bridge })) diff --git a/packages/sdk/tests/worker-node-runtime-sharing.test.ts b/packages/sdk/tests/worker-node-runtime-sharing.test.ts index 609222af8..b7aabe901 100644 --- a/packages/sdk/tests/worker-node-runtime-sharing.test.ts +++ b/packages/sdk/tests/worker-node-runtime-sharing.test.ts @@ -45,6 +45,28 @@ describe('AgentWorker node-spawn runtime ownership', () => { expect(owner.close).toHaveBeenCalledOnce(); }); + it.each([ + { label: 'whitespace-only caller', environment: { RELAYFLOW_SERVICE_CALLER_FILE: ' ' } }, + { label: 'whitespace caller plus remote token', environment: { + RELAYFLOW_SERVICE_CALLER_FILE: ' ', RELAY_AGENT_TOKEN: 'agent-token', + } }, + ])('validates a $label before registering the worker', async ({ environment }) => { + const callerFailure = new Error('Desktop caller authority is invalid'); + owner.get.mockRejectedValue(callerFailure); + const workerAttach = vi.fn(async () => ({})); + const client = Object.assign(new EventEmitter(), { workerAttach }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_invalid_whitespace_caller', + pins: {}, + environment, + }); + + await expect(worker.attach()).rejects.toBe(callerFailure); + expect(owner.get).toHaveBeenCalledOnce(); + expect(workerAttach).not.toHaveBeenCalled(); + expect(owner.close).toHaveBeenCalledOnce(); + }); + it.each([ { label: 'Desktop caller', serviceCallerFile: '/tmp/caller.json', authority: undefined, provideRuntime: false, expectedDirectory: 'run-root', expectedSpawn: true }, From 8c49413198565616bc8ba7bee5f687b6e748460e Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Mon, 9 Nov 2026 03:18:38 -0800 Subject: [PATCH 40/75] fix(sdk): preserve relay deadlines and settlement retries --- docs/SURFACE.md | 5 +- packages/sdk/src/agent-relay-transport.ts | 7 -- packages/sdk/src/node-spawn-settlement.ts | 82 +++++++++------- packages/sdk/src/worker-cli.ts | 29 +++++- .../sdk/tests/node-spawn-settlement.test.ts | 38 ++++++- .../tests/worker-node-runtime-timeout.test.ts | 98 +++++++++++++++++++ packages/sdk/tsconfig.tests.json | 2 + 7 files changed, 212 insertions(+), 49 deletions(-) create mode 100644 packages/sdk/tests/worker-node-runtime-timeout.test.ts diff --git a/docs/SURFACE.md b/docs/SURFACE.md index 820e23eb4..d7768cdf6 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -426,8 +426,9 @@ The retired `transport: "broker"` retained-session surface (`keepAlive`, broker fallback. Native retained sessions remain fail-closed until Agent Relay Desktop exposes an authorized headless Flows caller/delegation contract; Flows will not borrow a live coding session's registration. The local adapter carries -the Desktop-issued caller name and secret on invoke, status, and release. See -[the migration note](MIGRATION-NODE-SPAWN.md). +the Desktop-issued private handoff into a challenge-authenticated bridge; it +sends no caller name or secret on invoke, status, or release. See [the migration +note](MIGRATION-NODE-SPAWN.md). ### Agent retry and recovery controls diff --git a/packages/sdk/src/agent-relay-transport.ts b/packages/sdk/src/agent-relay-transport.ts index 7787eda12..b0e625273 100644 --- a/packages/sdk/src/agent-relay-transport.ts +++ b/packages/sdk/src/agent-relay-transport.ts @@ -49,13 +49,6 @@ export class AgentRelayTransportError extends Error { const CLEANUP_ATTEMPT_TIMEOUT_MS = 15_000; const claimExecutions = new Map>(); -export function readAgentRelayEnv(env: NodeJS.ProcessEnv = process.env): Required> { - const agentToken = env.RELAY_AGENT_TOKEN?.trim(); - if (!agentToken) throw new AgentRelayTransportError( - 'Node-spawn transport requires a dedicated publisher RELAY_AGENT_TOKEN.'); - return { baseUrl: env.RELAY_BASE_URL?.trim() || 'https://cast.agentrelay.com', agentToken }; -} - /** * Dispatches exactly once through an Agent Relay node and waits on the * publisher's durable inbox. AgentWorker acks the returned delivery only diff --git a/packages/sdk/src/node-spawn-settlement.ts b/packages/sdk/src/node-spawn-settlement.ts index d470df272..d56b05fd0 100644 --- a/packages/sdk/src/node-spawn-settlement.ts +++ b/packages/sdk/src/node-spawn-settlement.ts @@ -17,7 +17,7 @@ interface LiveTeardown { interface LiveTeardownGroup { generation: number; barriers: Set; - settlementStarted: boolean; + settlementToken?: object; } /** Process-generation state keyed by durable claim path; a real restart intentionally resets it. */ @@ -196,7 +196,7 @@ async function reconcileNodeResultSettlement( if (live !== undefined) { const failed = [...live.barriers].find(item => item.settled && item.failure !== undefined); if (failed !== undefined) throw failed.failure; - if (live.settlementStarted || [...live.barriers].some(item => !item.settled)) return true; + if (live.settlementToken !== undefined || [...live.barriers].some(item => !item.settled)) return true; } const terminal = await terminalEntry(journal, claim.runId, claim.stepId, settlement.attempt); if (terminal === undefined) return true; @@ -227,10 +227,10 @@ function rememberLiveTeardown( const live: LiveTeardown = { settled: false, outcome: normalized }; let group = liveTeardowns.get(path); if (group === undefined) { - group = { generation: 0, barriers: new Set(), settlementStarted: false }; + group = { generation: 0, barriers: new Set() }; liveTeardowns.set(path, group); } - if (group.settlementStarted) { + if (group.settlementToken !== undefined) { throw new NodeSpawnError('Node result gained a live teardown after settlement started', 'result_settlement_unconfirmed'); } @@ -248,6 +248,7 @@ function rememberLiveTeardown( interface LiveSettlement { group: LiveTeardownGroup; generation: number; + token: object; } async function settleAfterLiveTeardowns( @@ -257,50 +258,59 @@ async function settleAfterLiveTeardowns( inboxItemId: string, error: string, ): Promise { - return boundedOperation(async signal => { - for (;;) { - let group = liveTeardowns.get(path); - if (group === undefined) { - group = { generation: 0, barriers: new Set(), settlementStarted: false }; - liveTeardowns.set(path, group); - } - if (group.settlementStarted) { - throw new NodeSpawnError('Node result settlement is already in progress', - 'result_settlement_unconfirmed'); - } - const generation = group.generation; - const outcomes = group.barriers.size === 0 - ? [] - : await Promise.all([...group.barriers].map(item => item.outcome)); - const failed = outcomes.find(item => item.failure !== undefined); - if (failed?.failure !== undefined) throw failed.failure; - signal.throwIfAborted(); - const current = liveTeardowns.get(path); - if (current !== group || current.generation !== generation) continue; - current.settlementStarted = true; - try { - await issueInboxSettlement(runtime, action, inboxItemId, error); + let active: LiveSettlement | undefined; + try { + return await boundedOperation(async signal => { + for (;;) { + let group = liveTeardowns.get(path); + if (group === undefined) { + group = { generation: 0, barriers: new Set() }; + liveTeardowns.set(path, group); + } + if (group.settlementToken !== undefined) { + throw new NodeSpawnError('Node result settlement is already in progress', + 'result_settlement_unconfirmed'); + } + const generation = group.generation; + const outcomes = group.barriers.size === 0 + ? [] + : await Promise.all([...group.barriers].map(item => item.outcome)); + const failed = outcomes.find(item => item.failure !== undefined); + if (failed?.failure !== undefined) throw failed.failure; signal.throwIfAborted(); - return { group: current, generation }; - } catch (settlementError) { - current.settlementStarted = false; - throw settlementError; + const current = liveTeardowns.get(path); + if (current !== group || current.generation !== generation) continue; + const token = {}; + current.settlementToken = token; + active = { group: current, generation, token }; + try { + await issueInboxSettlement(runtime, action, inboxItemId, error); + signal.throwIfAborted(); + return active; + } catch (settlementError) { + releaseLiveSettlement(path, active); + throw settlementError; + } } - } - }); + }); + } catch (error) { + if (active !== undefined) releaseLiveSettlement(path, active); + throw error; + } } function releaseLiveSettlement(path: string, settlement: LiveSettlement): void { const current = liveTeardowns.get(path); - if (current === settlement.group && current.generation === settlement.generation) { - current.settlementStarted = false; + if (current === settlement.group && current.generation === settlement.generation + && current.settlementToken === settlement.token) { + current.settlementToken = undefined; } } function completeLiveSettlement(path: string, settlement: LiveSettlement): void { const current = liveTeardowns.get(path); if (current === settlement.group && current.generation === settlement.generation - && current.settlementStarted && current.barriers.size === 0) { + && current.settlementToken === settlement.token && current.barriers.size === 0) { liveTeardowns.delete(path); } } diff --git a/packages/sdk/src/worker-cli.ts b/packages/sdk/src/worker-cli.ts index b21e8e946..6b3991ecf 100644 --- a/packages/sdk/src/worker-cli.ts +++ b/packages/sdk/src/worker-cli.ts @@ -48,6 +48,7 @@ import { pinCliAlias } from './cli/pinned-cli-alias.js'; import { prepareDevinRuntime } from './devin-runtime.js'; import { promptOperand } from './adapters/base.js'; import type { NodeSpawnRuntime } from './node-spawn-runtime.js'; +import { NodeSpawnError } from './node-spawn.js'; export type { CliTransportCause, CliTransportEvidence, CliTransportPhase } from './cli-transport-evidence.js'; export { agentCompletionReason } from './cli-transport-evidence.js'; @@ -186,7 +187,10 @@ export async function runAgentCli( stderr_tail: `resume is refused for CLI ${JSON.stringify(cli)} (${mode}, ${transport}): only a directly spawned Claude or Codex agent can continue a recorded session` }; } if (mode === 'agent' && transport === 'relay') { - return runViaAgentRelay(kind, instruction, wakeContext, effectiveModel, relayContext, cwd, signal); + return runViaAgentRelay( + kind, instruction, wakeContext, effectiveModel, relayContext, cwd, signal, stepTimeoutMs, + childEnvironment, + ); } // Artifact detection brackets the spawn: the directory the CLI runs in is @@ -395,6 +399,8 @@ async function runViaAgentRelay( kind: CliAdapterKind, instruction: string, wakeContext: unknown, model: string | undefined, context: AgentRelayContext | undefined, worker_cwd: string | undefined, signal: AbortSignal | undefined, + timeoutMs: number | undefined, + environment: NodeJS.ProcessEnv, ): Promise { try { if (kind === 'relayflows-wrapper-v1') throw new Error('Relay task transport does not support same-process wrappers.'); @@ -404,6 +410,7 @@ async function runViaAgentRelay( cli: kind, task, model, worker_cwd, result_schema: context.resultSchema, runId: context.runId, stepId: context.stepId, idempotencyKey: context.idempotencyKey, attempt: context.attempt, dataDir: context.dataDir, + ...(timeoutMs === undefined ? {} : { timeoutMs }), }, { signal, runtime: context.nodeSpawnRuntime, runtimeProvider: context.nodeSpawnRuntimeProvider }); const receipt = { ...received.receipt, error: received.receipt.error === null ? null : redactRelayError(received.receipt.error) }; @@ -425,8 +432,26 @@ async function runViaAgentRelay( if (error instanceof AgentRelayTransportError && error.cleanupPending) throw error; const detail = error instanceof AgentRelayTransportError ? error.message : error instanceof Error ? error.message : 'Relay task transport failed'; - return { exit_code: null, stdout_tail: '', stderr_tail: redactRelayError(detail) }; + const timedOut = error instanceof AgentRelayTransportError && relayTaskTimedOut(error); + return { exit_code: null, stdout_tail: '', stderr_tail: redactRelayError(detail), + ...(timedOut ? { transport: transportEvidence({ + phase: 'timeout', cause: 'timeout', exitCode: null, signal: null, + stderr: detail, retryable: false, + }, environment) } : {}) }; + } +} + +function relayTaskTimedOut(error: AgentRelayTransportError): boolean { + let current: unknown = error; + const seen = new Set(); + while (current !== undefined && current !== null && !seen.has(current)) { + seen.add(current); + if (current instanceof NodeSpawnError && current.code === 'task_timeout') return true; + if (current instanceof DOMException && current.name === 'TimeoutError') return true; + current = current instanceof AgentRelayTransportError || current instanceof NodeSpawnError + ? current.cause : undefined; } + return false; } /** diff --git a/packages/sdk/tests/node-spawn-settlement.test.ts b/packages/sdk/tests/node-spawn-settlement.test.ts index b610a49f8..e1369bf46 100644 --- a/packages/sdk/tests/node-spawn-settlement.test.ts +++ b/packages/sdk/tests/node-spawn-settlement.test.ts @@ -571,7 +571,10 @@ describe('durable node-result settlement', () => { const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); let confirmLate!: () => void; - f.ack.mockImplementationOnce(() => new Promise(resolve => { confirmLate = resolve; })); + let confirmRetry!: () => void; + f.ack + .mockImplementationOnce(() => new Promise(resolve => { confirmLate = resolve; })) + .mockImplementationOnce(() => new Promise(resolve => { confirmRetry = resolve; })); vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); try { const pending = expect(received.delivery.ack()) @@ -582,12 +585,43 @@ describe('durable node-result settlement', () => { await pending; const claims = await readNodeSpawnClaims(dataDir); expect(claims[0]!.claim.result?.settlement).not.toHaveProperty('settledAt'); + await Promise.resolve(); + const retry = received.delivery.ack(); + await vi.waitFor(() => expect(f.ack).toHaveBeenCalledTimes(2)); confirmLate(); await Promise.resolve(); await Promise.resolve(); const afterLateSuccess = await readNodeSpawnClaims(dataDir); expect(afterLateSuccess[0]!.claim.result?.settlement).not.toHaveProperty('settledAt'); - await received.delivery.ack(); + expect(() => durableNodeResultDelivery(f.runtime, claims[0]!.path, afterLateSuccess[0]!.claim, { + ...received.delivery, settlementReady: Promise.resolve({}), + })).toThrow('gained a live teardown after settlement started'); + confirmRetry(); + await retry; + const settled = await readNodeSpawnClaims(dataDir); + expect(settled[0]!.claim.result?.settlement).toMatchObject({ + settledAt: expect.any(Number), settledAs: 'ack', + }); + } finally { + vi.useRealTimers(); + } + }); + + it('retries after a timed-out inbox settlement that never resolves', async () => { + const dataDir = await dataDirectory('flows-node-settlement-permanent-stall-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + f.ack.mockImplementationOnce(() => new Promise(() => undefined)); + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + try { + const stalled = expect(received.delivery.ack()) + .rejects.toMatchObject({ code: 'result_settlement_unconfirmed' }); + await waitForSettlementTimer(); + await vi.advanceTimersByTimeAsync(10_001); + await stalled; + await Promise.resolve(); + await expect(received.delivery.ack()).resolves.toBeUndefined(); expect(f.ack).toHaveBeenCalledTimes(2); const settled = await readNodeSpawnClaims(dataDir); expect(settled[0]!.claim.result?.settlement).toMatchObject({ diff --git a/packages/sdk/tests/worker-node-runtime-timeout.test.ts b/packages/sdk/tests/worker-node-runtime-timeout.test.ts new file mode 100644 index 000000000..62453ba03 --- /dev/null +++ b/packages/sdk/tests/worker-node-runtime-timeout.test.ts @@ -0,0 +1,98 @@ +import { EventEmitter } from 'node:events'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { JournalClient } from '../src/journal-client.js'; +import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; +import type { NodeSpawnPort, NodeSpawnReceipt } from '../src/node-spawn.js'; +import type { StepDispatchEvent } from '../src/protocol.js'; +import { AgentWorker } from '../src/worker.js'; + +const roots: string[] = []; +afterEach(async () => { + await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); +}); + +function dispatch(stepId: string, idempotencyKey: string, timeoutMs: number): StepDispatchEvent { + return { + run_id: 'run_timeout', step_id: stepId, attempt: 1, step_type: 'agent', + spec: { cli: 'codex', instruction: 'Implement the change', transport: 'relay', + timeout_ms: timeoutMs }, + idempotency_key: idempotencyKey, lease_id: `lease_${stepId}`, + lease_deadline_ms: Date.now() + 30_000, pins: {}, + }; +} + +function client(stepComplete: JournalClient['stepComplete']): JournalClient & EventEmitter { + return Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), + stepComplete, + supportsFeature: vi.fn(() => true), + }) as unknown as JournalClient & EventEmitter; +} + +function runtime(spawn: NodeSpawnPort['spawn']): NodeSpawnRuntime { + return { + authority: 'remote', publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + port: { targets: async () => [], spawn, status: vi.fn(), release: vi.fn() }, + inbox: { inbox: { + subscribe: async function* () { return; }, + ack: vi.fn(async () => ({})), fail: vi.fn(async () => ({})), + } }, + }; +} + +describe('AgentWorker relay deadlines', () => { + it('carries an admitted journal timeout into the native node-spawn deadline', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-worker-timeout-')); + roots.push(dataDir); + const declaredTimeoutMs = 5_000; + const spawn = vi.fn(async () => { + throw new Error('stop after observing timeout'); + }); + const stepComplete = vi.fn(async () => ( + {} as Awaited> + )); + const journal = client(stepComplete); + const worker = new AgentWorker(journal, { + workerId: 'worker_timeout', pins: {}, dataDir, nodeSpawnRuntime: runtime(spawn), + }); + await worker.attach(); + journal.emit('step.dispatch', dispatch('edit-timeout', 'dispatch_timeout000', declaredTimeoutMs)); + await vi.waitFor(() => expect(stepComplete).toHaveBeenCalledOnce()); + await worker.close(); + + expect(spawn).toHaveBeenCalledOnce(); + const observedTimeoutMs = spawn.mock.calls[0]![0].timeoutMs; + expect(observedTimeoutMs).toBeGreaterThan(0); + expect(observedTimeoutMs).toBeLessThanOrEqual(declaredTimeoutMs); + }); + + it('journals a native node-spawn deadline as timeout rather than worker_error', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-worker-timeout-reason-')); + roots.push(dataDir); + const spawn = vi.fn( + () => new Promise(() => undefined)); + const stepComplete = vi.fn(async () => ( + {} as Awaited> + )); + const journal = client(stepComplete); + const worker = new AgentWorker(journal, { + workerId: 'worker_timeout_reason', pins: {}, dataDir, nodeSpawnRuntime: runtime(spawn), + }); + await worker.attach(); + journal.emit('step.dispatch', dispatch('edit-timeout-reason', 'dispatch_timeout001', 100)); + await vi.waitFor(() => expect(stepComplete).toHaveBeenCalledOnce()); + await worker.close(); + + expect(stepComplete).toHaveBeenCalledWith( + 'run_timeout', 'edit-timeout-reason', 1, 'dispatch_timeout001', 'timeout', + expect.objectContaining({ trajectory_tail: expect.objectContaining({ + transport: expect.objectContaining({ phase: 'timeout', cause: 'timeout', retryable: false }), + }) }), + ); + }); +}); diff --git a/packages/sdk/tsconfig.tests.json b/packages/sdk/tsconfig.tests.json index c231f7ca6..ea5fc042d 100644 --- a/packages/sdk/tsconfig.tests.json +++ b/packages/sdk/tsconfig.tests.json @@ -76,8 +76,10 @@ "tests/agent-relay-hardening.test.ts", "tests/node-spawn-ports.test.ts", "tests/node-spawn-runtime.test.ts", + "tests/node-spawn-settlement.test.ts", "tests/node-spawn-live.test.ts", "tests/worker-node-runtime-sharing.test.ts", + "tests/worker-node-runtime-timeout.test.ts", "tests/webhook-hardening.test.ts", "tests/adapters/claude.test.ts", "tests/adapters/codex.test.ts", From 051b3efb36cad093b26a473287e6ac76908aa3cd Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Mon, 9 Nov 2026 03:32:28 -0800 Subject: [PATCH 41/75] test(sdk): drain node spawn acknowledgement --- packages/sdk/tests/live-kernel.test.ts | 2 +- packages/sdk/tests/node-spawn-live.test.ts | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/sdk/tests/live-kernel.test.ts b/packages/sdk/tests/live-kernel.test.ts index 75f2b5768..a711a5fb6 100644 --- a/packages/sdk/tests/live-kernel.test.ts +++ b/packages/sdk/tests/live-kernel.test.ts @@ -1364,7 +1364,7 @@ steps: const unreachable = invokeCli([ 'run', '--no-spawn', '--data-dir', absentDir, join(TESTDATA, 'hello-deterministic.flow.yaml'), ]); - expect(unreachable.status).toBe(2); + expect(unreachable.status, unreachable.stderr).toBe(2); expect(unreachable.stderr).toContain('REFUSED [daemon_unreachable]'); expect(unreachable.stderr).toContain(absentSocket); expect(unreachable.stderr).toContain('relayflowd --data-dir'); diff --git a/packages/sdk/tests/node-spawn-live.test.ts b/packages/sdk/tests/node-spawn-live.test.ts index 9060a68f0..5eec6dc1b 100644 --- a/packages/sdk/tests/node-spawn-live.test.ts +++ b/packages/sdk/tests/node-spawn-live.test.ts @@ -66,6 +66,10 @@ describe('node-spawn transport through the real journal', () => { idempotencyKey: string; }> = []; const ackStates: string[] = []; + let allowAck!: () => void; + const ackReady = new Promise((resolveReady) => { + allowAck = resolveReady; + }); const runtime: NodeSpawnRuntime = { baseUrl: 'https://cast.agentrelay.com', publisher: { id: 'publisher_1', name: 'flows-publisher' }, @@ -130,6 +134,7 @@ describe('node-spawn transport through the real journal', () => { }; }, ack: async () => { + await ackReady; const snapshot = await client.runGet(runId); ackStates.push(snapshot.steps['edit']?.state ?? 'missing'); return {}; @@ -197,9 +202,13 @@ steps: idempotencyKey: expect.stringMatching(/^release_[a-f0-9]{32}$/), }, ]); + expect(ackStates).toEqual([]); + allowAck(); + await worker.close(); expect(ackStates).toEqual(['done']); } finally { allowSpawn(); + allowAck(); await worker.close(); } }, 20_000); From 03691e9230f24ae16d880d6ff0e7f95e5101ab85 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Mon, 9 Nov 2026 03:44:35 -0800 Subject: [PATCH 42/75] test(sdk): isolate live lifecycle fixtures --- packages/sdk/tests/live-kernel.test.ts | 39 ++++++++++--------- packages/sdk/tests/node-spawn-live.test.ts | 12 ++++-- .../tests/yaml-declared-streams-live.test.ts | 3 +- .../sdk/tests/yaml-local-agent-live.test.ts | 3 +- 4 files changed, 32 insertions(+), 25 deletions(-) diff --git a/packages/sdk/tests/live-kernel.test.ts b/packages/sdk/tests/live-kernel.test.ts index a711a5fb6..98a644f61 100644 --- a/packages/sdk/tests/live-kernel.test.ts +++ b/packages/sdk/tests/live-kernel.test.ts @@ -1159,7 +1159,6 @@ steps: // exported. That is exactly the ambient-state failure this field was // added to remove, so it has to be enforced by the spawn, not by luck. // Without this test nothing else would catch it. - process.env.RELAYFLOW_MODEL = 'leaked-parent-model'; const dataDir = temporaryDirectory('flows-live-model-unset-'); await startDaemon(dataDir); const cli = join(TESTDATA, 'preflight', 'echo-model-cli'); @@ -1171,7 +1170,10 @@ steps: }); await worker.attach(); - const started = await client.runStart(toKernelSpec(compileYaml(` + const inheritedModel = process.env.RELAYFLOW_MODEL; + process.env.RELAYFLOW_MODEL = 'leaked-parent-model'; + try { + const started = await client.runStart(toKernelSpec(compileYaml(` version: '0.1.0' steps: - id: probe @@ -1180,22 +1182,23 @@ steps: instruction: Report the model env var. `))); - expect(await waitForStep(client, started.run_id, 'probe', 'done')).toMatchObject({ - type: 'agent', - state: 'done', - }); - const entries = (await client.journalRead(started.run_id)).entries; - const completed = entries.find( - (entry) => (entry as { entry_type: string; step_id?: string }).entry_type === 'step.completed' - && (entry as { step_id?: string }).step_id === 'probe', - ) as { payload: { output: { story_title: string } } } | undefined; - expect(completed).toBeDefined(); - // UNSET, not EMPTY and not the leaked parent value. - expect(completed!.payload.output.story_title).toBe('model:UNSET'); - - delete process.env.RELAYFLOW_MODEL; - - await worker.close(); + expect(await waitForStep(client, started.run_id, 'probe', 'done')).toMatchObject({ + type: 'agent', + state: 'done', + }); + const entries = (await client.journalRead(started.run_id)).entries; + const completed = entries.find( + (entry) => (entry as { entry_type: string; step_id?: string }).entry_type === 'step.completed' + && (entry as { step_id?: string }).step_id === 'probe', + ) as { payload: { output: { story_title: string } } } | undefined; + expect(completed).toBeDefined(); + // UNSET, not EMPTY and not the leaked parent value. + expect(completed!.payload.output.story_title).toBe('model:UNSET'); + } finally { + if (inheritedModel === undefined) delete process.env.RELAYFLOW_MODEL; + else process.env.RELAYFLOW_MODEL = inheritedModel; + await worker.close(); + } }, 30_000); it('hn-monitor analyze-story reaches done through the real Claude analyzer CLI', async (context) => { diff --git a/packages/sdk/tests/node-spawn-live.test.ts b/packages/sdk/tests/node-spawn-live.test.ts index 5eec6dc1b..bb86cf784 100644 --- a/packages/sdk/tests/node-spawn-live.test.ts +++ b/packages/sdk/tests/node-spawn-live.test.ts @@ -66,6 +66,10 @@ describe('node-spawn transport through the real journal', () => { idempotencyKey: string; }> = []; const ackStates: string[] = []; + let acknowledgeEntered!: () => void; + const ackEntered = new Promise((resolveEntered) => { + acknowledgeEntered = resolveEntered; + }); let allowAck!: () => void; const ackReady = new Promise((resolveReady) => { allowAck = resolveReady; @@ -134,9 +138,10 @@ describe('node-spawn transport through the real journal', () => { }; }, ack: async () => { - await ackReady; const snapshot = await client.runGet(runId); ackStates.push(snapshot.steps['edit']?.state ?? 'missing'); + acknowledgeEntered(); + await ackReady; return {}; }, fail: async () => { @@ -202,12 +207,13 @@ steps: idempotencyKey: expect.stringMatching(/^release_[a-f0-9]{32}$/), }, ]); - expect(ackStates).toEqual([]); + await ackEntered; + expect(ackStates).toEqual(['done']); allowAck(); await worker.close(); - expect(ackStates).toEqual(['done']); } finally { allowSpawn(); + acknowledgeEntered(); allowAck(); await worker.close(); } diff --git a/packages/sdk/tests/yaml-declared-streams-live.test.ts b/packages/sdk/tests/yaml-declared-streams-live.test.ts index 91ecc7afd..bb3934bd0 100644 --- a/packages/sdk/tests/yaml-declared-streams-live.test.ts +++ b/packages/sdk/tests/yaml-declared-streams-live.test.ts @@ -1,5 +1,5 @@ import { execFileSync, spawnSync } from 'node:child_process'; -import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { stringify } from 'yaml'; @@ -30,7 +30,6 @@ function fixture(source: 'step' | 'named' | 'flow' | 'project' = 'step', instruc const root = mkdtempSync(join(tmpdir(), 'flows-yaml-local-')); roots.push(root); mkdirSync(join(root, 'flows')); - chmodSync(wrapper, 0o755); symlinkSync(wrapper, join(root, 'agent.mjs')); writeFileSync(join(root, 'flows.json'), JSON.stringify({ models: ['yaml-test-model'], ...(source === 'project' ? { cli: './agent.mjs' } : {}), diff --git a/packages/sdk/tests/yaml-local-agent-live.test.ts b/packages/sdk/tests/yaml-local-agent-live.test.ts index b2cfa0dd4..50c5f5dc1 100644 --- a/packages/sdk/tests/yaml-local-agent-live.test.ts +++ b/packages/sdk/tests/yaml-local-agent-live.test.ts @@ -1,5 +1,5 @@ import { execFileSync, spawnSync } from 'node:child_process'; -import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { stringify } from 'yaml'; @@ -30,7 +30,6 @@ function fixture(source: 'step' | 'named' | 'flow' | 'project' = 'step', instruc const root = mkdtempSync(join(tmpdir(), 'flows-yaml-local-')); roots.push(root); mkdirSync(join(root, 'flows')); - chmodSync(wrapper, 0o755); symlinkSync(wrapper, join(root, 'agent.mjs')); writeFileSync(join(root, 'flows.json'), JSON.stringify({ models: ['yaml-test-model'], ...(source === 'project' ? { cli: './agent.mjs' } : {}), From 708e994725405b5a9fcd00dce1e99e9f75bad0a3 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Mon, 9 Nov 2026 04:19:49 -0800 Subject: [PATCH 43/75] fix(sdk): pin the Desktop bridge type contract --- packages/sdk/src/node-spawn-local.ts | 40 +++++++++++++++++-- packages/sdk/tests/node-spawn-ports.test.ts | 11 +++-- packages/sdk/tests/node-spawn-runtime.test.ts | 10 ++++- 3 files changed, 51 insertions(+), 10 deletions(-) diff --git a/packages/sdk/src/node-spawn-local.ts b/packages/sdk/src/node-spawn-local.ts index fe4f1e8bd..901e0cceb 100644 --- a/packages/sdk/src/node-spawn-local.ts +++ b/packages/sdk/src/node-spawn-local.ts @@ -16,7 +16,13 @@ import { /** The structural core-bridge surface Flows consumes. Kept here so the package stays optional. */ export interface CoreBridgeClient { - readonly hello: { methods: string[] }; + readonly hello: { + bridge_version: number; + daemon_version: string; + pid: number; + methods: string[]; + topics: Array<'status' | 'activity' | 'shutdown' | 'invalidated' | 'notify' | 'sign_in'>; + }; close(): void; } @@ -79,6 +85,32 @@ interface CoreInvocation { error_code: string | null; } +interface CoreSpawnInvokeParams { + caller_name?: string; + node_id: string; + node_name: string; + name: string; + harness: string; + task: string; + directory: string; + idempotency_key: string; +} + +interface CoreSpawnStatusParams { + caller_name?: string; + action: NodeSpawnAction; + invocation_id: string; +} + +interface CoreSpawnReleaseParams { + caller_name?: string; + node_id: string; + node_name: string; + name: string; + expected_agent_id: string; + idempotency_key: string; +} + export interface CoreInboxDelivery { item_id: string; message_id: string; @@ -107,9 +139,9 @@ export interface CoreBridgeApi { relaycast_workspace_id: string; }>; spawnTargets(client: CoreBridgeClient, options?: { signal?: AbortSignal }): Promise<{ targets: CoreTarget[] }>; - spawnInvoke(client: CoreBridgeClient, params: Record, options?: { signal?: AbortSignal; actionTimeoutMs?: number }): Promise; - spawnStatus(client: CoreBridgeClient, params: Record, options?: { signal?: AbortSignal }): Promise; - spawnRelease(client: CoreBridgeClient, params: Record, options?: { signal?: AbortSignal; actionTimeoutMs?: number }): Promise; + spawnInvoke(client: CoreBridgeClient, params: CoreSpawnInvokeParams, options?: { signal?: AbortSignal; actionTimeoutMs?: number }): Promise; + spawnStatus(client: CoreBridgeClient, params: CoreSpawnStatusParams, options?: { signal?: AbortSignal }): Promise; + spawnRelease(client: CoreBridgeClient, params: CoreSpawnReleaseParams, options?: { signal?: AbortSignal; actionTimeoutMs?: number }): Promise; serviceCallerInboxOpen(client: CoreBridgeClient, invocationId: string, options?: { signal?: AbortSignal }): Promise<{ iterator_id: string }>; serviceCallerInboxNext(client: CoreBridgeClient, iteratorId: string, timeoutMs: number, options?: { signal?: AbortSignal }): Promise<{ delivery: CoreInboxDelivery | null }>; serviceCallerInboxAck(client: CoreBridgeClient, itemId: string, options?: { signal?: AbortSignal }): Promise; diff --git a/packages/sdk/tests/node-spawn-ports.test.ts b/packages/sdk/tests/node-spawn-ports.test.ts index 2353de6fe..f7426c2c9 100644 --- a/packages/sdk/tests/node-spawn-ports.test.ts +++ b/packages/sdk/tests/node-spawn-ports.test.ts @@ -12,12 +12,15 @@ interface TestCore { } function localPort(core: TestCore): LocalNodeSpawnPort { - const client = { hello: { methods: [] }, close() {} } satisfies CoreBridgeClient; + const client = { + hello: { bridge_version: 2, daemon_version: 'test', pid: 1, methods: [], topics: [] }, + close() {}, + } satisfies CoreBridgeClient; const bridge = { spawnTargets: () => core.call('spawn.targets'), - spawnInvoke: (_client, params) => core.call('spawn.invoke', params), - spawnStatus: (_client, params) => core.call('spawn.status', params), - spawnRelease: (_client, params) => core.call('spawn.release', params), + spawnInvoke: (_client, params) => core.call('spawn.invoke', { ...params }), + spawnStatus: (_client, params) => core.call('spawn.status', { ...params }), + spawnRelease: (_client, params) => core.call('spawn.release', { ...params }), } satisfies Pick; return new LocalNodeSpawnPort(client, bridge, { harnesses: ['codex'], diff --git a/packages/sdk/tests/node-spawn-runtime.test.ts b/packages/sdk/tests/node-spawn-runtime.test.ts index b7575065f..f6d900723 100644 --- a/packages/sdk/tests/node-spawn-runtime.test.ts +++ b/packages/sdk/tests/node-spawn-runtime.test.ts @@ -192,7 +192,10 @@ describe('Desktop service-caller runtime', () => { const close = vi.fn(); const bridge: CoreBridgeApi = { connectCore: vi.fn(async () => { - const client = { hello: { methods: [] }, close } satisfies CoreBridgeClient; + const client = { + hello: { bridge_version: 2, daemon_version: 'test', pid: 1, methods: [], topics: [] }, + close, + } satisfies CoreBridgeClient; clients.push(client); return client; }), @@ -393,7 +396,10 @@ describe('Desktop service-caller runtime', () => { const next = iterator.next(); await vi.waitFor(() => expect(bridge.connectCore).toHaveBeenCalledTimes(2)); const closing = iterator.return!(); - resolveBinding({ hello: { methods: [] }, close }); + resolveBinding({ + hello: { bridge_version: 2, daemon_version: 'test', pid: 1, methods: [], topics: [] }, + close, + }); await expect(next).resolves.toEqual({ done: true, value: undefined }); await expect(closing).resolves.toEqual({ done: true, value: undefined }); expect(bridge.serviceCallerInboxOpen).not.toHaveBeenCalled(); From 6bbd670d35f9ae65985aca408900bd84c0d0e677 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Mon, 9 Nov 2026 06:01:26 -0800 Subject: [PATCH 44/75] fix(sdk): preserve relay harness model defaults --- packages/sdk/src/cli/check.ts | 14 +++-- packages/sdk/src/worker.ts | 6 ++- packages/sdk/tests/cli.test.ts | 53 +++++++++++++++++++ .../tests/worker-node-runtime-sharing.test.ts | 22 +++++--- 4 files changed, 85 insertions(+), 10 deletions(-) diff --git a/packages/sdk/src/cli/check.ts b/packages/sdk/src/cli/check.ts index d4466bcda..20479539d 100644 --- a/packages/sdk/src/cli/check.ts +++ b/packages/sdk/src/cli/check.ts @@ -541,9 +541,17 @@ function bindResolvedCliPaths( if (executable !== undefined && isAbsolute(executable)) { identities.set(step.id, resolution.cli); } - return { ...step, cli: boundCli, - ...(resolution.modelSource === 'adapter' && resolution.model !== undefined - ? { model: resolution.model } : {}) }; + // Relay nodes own their harness default. Materializing the local CLI + // adapter's default here turns an omitted model into a placement + // override; Desktop correctly refuses that override. Explicit step + // models remain on `step`; named-agent models remain in the declaration + // map and are materialized by `toKernelSpec` at the journal boundary. + const adapterModel = resolution.modelSource === 'adapter' + && resolution.model !== undefined + && (step.type !== 'agent' || step.transport !== 'relay') + ? { model: resolution.model } + : {}; + return { ...step, cli: boundCli, ...adapterModel }; }), }; return rememberResolvedCliIdentities(bound, identities); diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index 16f8a0a63..9cd248d68 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -242,13 +242,17 @@ export class AgentWorker extends EventEmitter { // CLI does — journaled as `worker_error` with the reason — rather than // spawning into a directory nobody established is inside the run root. const localDesktopRelay = spec.transport === 'relay' && this.localDesktopAuthority; + // An adapter default is useful for readiness, pricing and direct CLI + // execution, but is not an authored placement override. Relay harnesses + // select their own default when the journaled step omitted `model`. + const executionModel = spec.transport === 'relay' ? spec.model : effectiveModel; const cwd = agentStepCwd(localDesktopRelay ? { ...spec, transport: 'direct' } : spec, dispatch.step_id, this.options.runRoot); const completed: WorkerCliResult = await withWorkerLease(this.client, dispatch, signal => 'refusal' in cwd ? Promise.resolve({ exit_code: null, stdout_tail: '', stderr_tail: cwd.refusal }) : typeof spec.cli === 'string' && typeof spec.instruction === 'string' - ? runAgentCli(spec.cli, workerInstruction(spec.instruction, dispatch), dispatch.wake_context, effectiveModel, undefined, signal, 'agent', this.options.dataDir === undefined ? undefined : { + ? runAgentCli(spec.cli, workerInstruction(spec.instruction, dispatch), dispatch.wake_context, executionModel, undefined, signal, 'agent', this.options.dataDir === undefined ? undefined : { dataDir: this.options.dataDir, runId: dispatch.run_id, stepId: dispatch.step_id, attempt: dispatch.attempt, onReady: this.options.onPtyReady, onDrive: () => { humanIntervention = true; }, }, cwd.directory ?? (localDesktopRelay diff --git a/packages/sdk/tests/cli.test.ts b/packages/sdk/tests/cli.test.ts index cd1ad962c..e3ddfa1c3 100644 --- a/packages/sdk/tests/cli.test.ts +++ b/packages/sdk/tests/cli.test.ts @@ -9,6 +9,7 @@ import { parse as parseYaml, stringify as stringifyYaml } from 'yaml'; import { afterEach, describe, expect, it } from 'vitest'; import { runCli, type CheckReport, type CliIo } from '../src/cli.js'; import { checkFlow } from '../src/cli/check.js'; +import { toKernelSpec } from '../src/compile.js'; import { socketPathFor } from '../src/daemon-connection.js'; import { runFlow } from '../src/cli/run.js'; import { runAgentCli } from '../src/worker-cli.js'; @@ -460,6 +461,58 @@ steps: ]); }); + it('does not turn an adapter default into a relay placement override', () => { + const directory = temporaryProject('flows-relay-default-model-'); + const cli = executableFixture(directory, 'claude', `if [ "$1 $2" = "auth status" ]; then exit 0; fi +if [ "$1" = "-p" ]; then exit 0; fi +exit 7`); + writeFileSync(join(directory, 'flows.json'), JSON.stringify({ + models: ['claude-opus-5', 'declared-relay-model', 'named-relay-model'], + })); + const path = join(directory, 'relay.flow.yaml'); + writeFileSync(path, `version: '0.1.0' +agents: + reviewer: + cli: ${JSON.stringify(cli)} + model: named-relay-model +steps: + - id: defaulted + type: agent + cli: ${JSON.stringify(cli)} + transport: relay + instruction: Review. + - id: declared + type: agent + cli: ${JSON.stringify(cli)} + model: declared-relay-model + transport: relay + instruction: Review. + - id: named + type: agent + agent: reviewer + transport: relay + instruction: Review. +`); + + const checked = checkFlow(path); + expect(checked.report.ok, checked.report.diagnostics).toBe(true); + expect(checked.report.resolutions).toEqual([ + { stepId: 'defaulted', cli, source: 'step', model: 'claude-opus-5', modelSource: 'adapter' }, + { stepId: 'declared', cli, source: 'step', model: 'declared-relay-model', modelSource: 'step' }, + { stepId: 'named', cli, source: 'named', model: 'named-relay-model', modelSource: 'named' }, + ]); + expect(checked.flow?.steps).toEqual([ + expect.objectContaining({ id: 'defaulted', transport: 'relay', cli }), + expect.objectContaining({ id: 'declared', transport: 'relay', cli, model: 'declared-relay-model' }), + expect.objectContaining({ id: 'named', transport: 'relay', cli, agent: 'reviewer' }), + ]); + expect(checked.flow?.steps[0]).not.toHaveProperty('model'); + expect(checked.flow?.steps[2]).not.toHaveProperty('model'); + expect(toKernelSpec(checked.flow!).steps[2]).toMatchObject({ + id: 'named', transport: 'relay', cli, model: 'named-relay-model', + }); + }); + it('keeps the project config path with the CLI clause when the step declares the model', async () => { const directory = temporaryProject('flows-project-model-'); const flowDirectory = join(directory, 'nested'); diff --git a/packages/sdk/tests/worker-node-runtime-sharing.test.ts b/packages/sdk/tests/worker-node-runtime-sharing.test.ts index b7aabe901..7a7805288 100644 --- a/packages/sdk/tests/worker-node-runtime-sharing.test.ts +++ b/packages/sdk/tests/worker-node-runtime-sharing.test.ts @@ -69,18 +69,24 @@ describe('AgentWorker node-spawn runtime ownership', () => { it.each([ { label: 'Desktop caller', serviceCallerFile: '/tmp/caller.json', authority: undefined, - provideRuntime: false, expectedDirectory: 'run-root', expectedSpawn: true }, + provideRuntime: false, expectedDirectory: 'run-root', expectedSpawn: true, expectedModel: 'codex-medium' }, + { label: 'Desktop caller with adapter default', serviceCallerFile: '/tmp/caller.json', authority: undefined, + provideRuntime: false, expectedDirectory: 'run-root', expectedSpawn: true, expectedModel: undefined, + omitModel: true }, { label: 'provided Desktop runtime', serviceCallerFile: undefined, authority: 'desktop' as const, - provideRuntime: true, expectedDirectory: 'run-root', expectedSpawn: true }, + provideRuntime: true, expectedDirectory: 'run-root', expectedSpawn: true, expectedModel: 'codex-medium' }, { label: 'remote publisher', serviceCallerFile: undefined, authority: 'remote' as const, - provideRuntime: true, inheritedCaller: true, expectedDirectory: undefined, expectedSpawn: true }, + provideRuntime: true, inheritedCaller: true, expectedDirectory: undefined, expectedSpawn: true, + expectedModel: 'codex-medium' }, { label: 'inherited caller cleared by worker environment', serviceCallerFile: undefined, authority: undefined, provideRuntime: false, inheritedCaller: true, environmentCaller: '', - expectedDirectory: undefined, expectedSpawn: true }, + expectedDirectory: undefined, expectedSpawn: true, expectedModel: 'codex-medium' }, { label: 'Desktop caller with declared cwd', serviceCallerFile: '/tmp/caller.json', authority: undefined, - provideRuntime: false, cwd: 'checkout', expectedDirectory: 'declared', expectedSpawn: true }, + provideRuntime: false, cwd: 'checkout', expectedDirectory: 'declared', expectedSpawn: true, + expectedModel: 'codex-medium' }, { label: 'remote publisher with declared cwd', serviceCallerFile: undefined, authority: 'remote' as const, - provideRuntime: true, cwd: 'checkout', expectedDirectory: undefined, expectedSpawn: false }, + provideRuntime: true, cwd: 'checkout', expectedDirectory: undefined, expectedSpawn: false, + expectedModel: 'codex-medium' }, ])('routes the run directory only for the $label', async testCase => { const runRoot = await mkdtemp(join(tmpdir(), 'flows-node-worker-run-root-')); roots.push(runRoot); @@ -132,6 +138,9 @@ describe('AgentWorker node-spawn runtime ownership', () => { }); await worker.attach(); const value = dispatch('edit-local-path', 'dispatch_localpath'); + if ('omitModel' in testCase && testCase.omitModel) { + delete (value.spec as { model?: string }).model; + } if (testCase.cwd !== undefined) { value.spec = { ...(value.spec as Record), cwd: testCase.cwd }; } @@ -141,6 +150,7 @@ describe('AgentWorker node-spawn runtime ownership', () => { if (testCase.expectedSpawn) { expect(spawn.mock.calls[0]![0].directory).toBe(testCase.expectedDirectory === 'run-root' ? runRoot : testCase.expectedDirectory === 'declared' ? exactDeclaredDirectory : undefined); + expect(spawn.mock.calls[0]![0].model).toBe(testCase.expectedModel); } }); From 6ed9eed5279d12096321ecac52bbf6da1655341a Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Mon, 9 Nov 2026 07:40:21 -0800 Subject: [PATCH 45/75] fix(sdk): keep relay harness model defaults remote --- packages/sdk/src/worker-cli.ts | 9 ++++++++- packages/sdk/tests/worker-node-runtime-sharing.test.ts | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/sdk/src/worker-cli.ts b/packages/sdk/src/worker-cli.ts index 6b3991ecf..dcc8ee53c 100644 --- a/packages/sdk/src/worker-cli.ts +++ b/packages/sdk/src/worker-cli.ts @@ -178,7 +178,14 @@ export async function runAgentCli( return { exit_code: null, stdout_tail: '', stderr_tail: error instanceof Error ? error.message : 'step secret grant is invalid' }; } - const effectiveModel = resolveCliModel(cliIdentity ?? cli, model); + // A relay node owns its harness default. `model === undefined` therefore + // has to survive this boundary unchanged: resolving the local adapter's + // default here would turn an omitted model into a Desktop placement + // override. Direct CLI execution still needs the adapter default for its + // invocation, usage and pricing paths. + const effectiveModel = transport === 'relay' + ? model + : resolveCliModel(cliIdentity ?? cli, model); const argv0 = cliInvocationArgv0(cli, cliIdentity); if (resume !== undefined && (mode !== 'agent' || transport === 'relay' || !agentResumable(kind))) { diff --git a/packages/sdk/tests/worker-node-runtime-sharing.test.ts b/packages/sdk/tests/worker-node-runtime-sharing.test.ts index 7a7805288..53f92f925 100644 --- a/packages/sdk/tests/worker-node-runtime-sharing.test.ts +++ b/packages/sdk/tests/worker-node-runtime-sharing.test.ts @@ -139,6 +139,7 @@ describe('AgentWorker node-spawn runtime ownership', () => { await worker.attach(); const value = dispatch('edit-local-path', 'dispatch_localpath'); if ('omitModel' in testCase && testCase.omitModel) { + value.spec = { ...(value.spec as Record), cli: 'claude', cli_identity: 'claude' }; delete (value.spec as { model?: string }).model; } if (testCase.cwd !== undefined) { From fb4000ea8e3b38996fe4f0f8c4af1ce898c0964c Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Mon, 9 Nov 2026 08:16:44 -0800 Subject: [PATCH 46/75] fix(sdk): harden node-spawn recovery ownership --- docs/MIGRATION-NODE-SPAWN.md | 9 ++++-- packages/sdk/src/agent-relay-transport.ts | 5 ++- packages/sdk/src/node-spawn-settlement.ts | 20 ++++++------ packages/sdk/src/node-spawn-state.ts | 8 ++++- packages/sdk/src/worker-cli.ts | 9 ++++-- packages/sdk/src/worker-input.ts | 15 ++++++++- packages/sdk/src/worker.ts | 5 +-- .../sdk/tests/agent-relay-transport.test.ts | 21 ++++++++++++ packages/sdk/tests/input-binding.test.ts | 15 ++++++++- .../sdk/tests/node-spawn-settlement.test.ts | 32 +++++++++++++++++++ 10 files changed, 119 insertions(+), 20 deletions(-) diff --git a/docs/MIGRATION-NODE-SPAWN.md b/docs/MIGRATION-NODE-SPAWN.md index b4223896c..ad8cb251b 100644 --- a/docs/MIGRATION-NODE-SPAWN.md +++ b/docs/MIGRATION-NODE-SPAWN.md @@ -28,11 +28,14 @@ harness. Use `RELAYFLOW_NODE_NAME` for an exact node or There is deliberately no compatibility fallback to broker 17.2.1. A local Desktop run uses a separately approved schema-v2 service caller. Pass its -private file explicitly with `--service-caller-file /absolute/caller.json`, +private file explicitly with +`--local-agent --service-caller-file /absolute/caller.json`, the SDK `serviceCallerFile` option, or `RELAYFLOW_SERVICE_CALLER_FILE`. Flows never scans Desktop's app-data directory. It securely parses and -challenge-authenticates the handoff through optional -`@agent-relay/core-bridge` 18.x, then sends no caller name or secret +challenge-authenticates the handoff through `@agent-relay/core-bridge` 18.x. +Install a compatible version (`npm install @agent-relay/core-bridge@^18`) when +using Desktop mode; the peer is optional only when Desktop mode is not used. +Flows then sends no caller name or secret on spawn or inbox calls. The handoff's harness and directory scope is enforced before dispatch. A local relay step with no authored `cwd` spawns in that run's root; remote relay placement still receives no local directory. diff --git a/packages/sdk/src/agent-relay-transport.ts b/packages/sdk/src/agent-relay-transport.ts index b0e625273..1ed639bb4 100644 --- a/packages/sdk/src/agent-relay-transport.ts +++ b/packages/sdk/src/agent-relay-transport.ts @@ -14,6 +14,8 @@ export type AgentTransport = 'direct' | 'relay'; export interface AgentRelayTaskRequest { cli: string; task: string; + /** Stable task input without per-attempt recovery evidence. */ + taskIdentity?: string; model?: string; worker_cwd?: string; result_schema?: unknown; @@ -199,7 +201,8 @@ async function executeAgentRelayTask( agentName, timeoutMs, request: { - cli: request.cli, task, ...(request.model === undefined ? {} : { model: request.model }), + cli: request.cli, task, task_identity: request.taskIdentity ?? request.task, + ...(request.model === undefined ? {} : { model: request.model }), ...(request.worker_cwd === undefined ? {} : { worker_cwd: request.worker_cwd }), ...(runtime.node === undefined ? {} : { node: runtime.node }), ...(runtime.repo === undefined ? {} : { repo: runtime.repo }), diff --git a/packages/sdk/src/node-spawn-settlement.ts b/packages/sdk/src/node-spawn-settlement.ts index d56b05fd0..7636aa865 100644 --- a/packages/sdk/src/node-spawn-settlement.ts +++ b/packages/sdk/src/node-spawn-settlement.ts @@ -20,6 +20,8 @@ interface LiveTeardownGroup { settlementToken?: object; } +type Reconciliation = 'complete' | 'pending' | 'foreign'; + /** Process-generation state keyed by durable claim path; a real restart intentionally resets it. */ const liveTeardowns = new Map(); const settlementMutations = new Map>(); @@ -171,8 +173,9 @@ export async function reconcileNodeResultSettlements( ): Promise { let pending = 0; for (const entry of await readNodeSpawnClaims(dataDir)) { - if (await serializeSettlementMutation(entry.path, - () => reconcileNodeResultSettlement(entry.path, runtime, journal))) pending += 1; + const status = await serializeSettlementMutation(entry.path, + () => reconcileNodeResultSettlement(entry.path, runtime, journal)); + if (status === 'pending') pending += 1; } return pending; } @@ -181,25 +184,24 @@ async function reconcileNodeResultSettlement( path: string, runtime: NodeSpawnRuntime, journal: Pick, -): Promise { +): Promise { const claim = await readNodeSpawnClaim(path); const result = claim.result; const settlement = result?.settlement; - if (result === undefined || settlement === undefined || settlement.settledAt !== undefined) return false; + if (result === undefined || settlement === undefined || settlement.settledAt !== undefined) return 'complete'; if (claim.baseUrl !== runtime.baseUrl || claim.publisherId !== runtime.publisher.id || claim.publisherName !== runtime.publisher.name) { - throw new NodeSpawnError('Pending node-result settlement belongs to a different publisher identity', - 'publisher_identity_mismatch'); + return 'foreign'; } const live = liveTeardowns.get(path); if (live !== undefined) { const failed = [...live.barriers].find(item => item.settled && item.failure !== undefined); if (failed !== undefined) throw failed.failure; - if (live.settlementToken !== undefined || [...live.barriers].some(item => !item.settled)) return true; + if (live.settlementToken !== undefined || [...live.barriers].some(item => !item.settled)) return 'pending'; } const terminal = await terminalEntry(journal, claim.runId, claim.stepId, settlement.attempt); - if (terminal === undefined) return true; + if (terminal === undefined) return 'pending'; const exact = terminalMatchesResult(terminal, claim, settlement, result); const action = exact ? result.settlementAction : 'fail'; const error = exact ? (result.envelope.error ?? 'Relayflows rejected an invalid result') @@ -216,7 +218,7 @@ async function reconcileNodeResultSettlement( throw writeError; } completeLiveSettlement(path, liveSettlement); - return false; + return 'complete'; } function rememberLiveTeardown( diff --git a/packages/sdk/src/node-spawn-state.ts b/packages/sdk/src/node-spawn-state.ts index b1bea9ec1..4f5c9c7c7 100644 --- a/packages/sdk/src/node-spawn-state.ts +++ b/packages/sdk/src/node-spawn-state.ts @@ -121,7 +121,12 @@ function sameClaim(prior: NodeSpawnClaim, proposed: NodeSpawnClaim): boolean { release: _priorRelease, ...priorIdentity } = prior; const { spawn: _spawn, cleanup: _cleanup, failure: _failure, result: _result, cleanupPending: _cleanupPending, release: _release, ...proposedIdentity } = proposed; - return canonicalize({ ...priorIdentity, startedAt: proposed.startedAt }) === canonicalize(proposedIdentity); + const { task: _priorRenderedTask, ...priorRequestIdentity } = priorIdentity.request; + const { task: _renderedTask, ...proposedRequestIdentity } = proposedIdentity.request; + return canonicalize({ ...priorIdentity, request: priorRequestIdentity, + startedAt: proposed.startedAt }) === canonicalize({ + ...proposedIdentity, request: proposedRequestIdentity, + }); } function decodeNodeSpawnClaim(value: unknown, label: string): NodeSpawnClaim { @@ -133,6 +138,7 @@ function decodeNodeSpawnClaim(value: unknown, label: string): NodeSpawnClaim { || !validIdempotencyKey(record.idempotencyKey) || !positiveInteger(record.timeoutMs) || record.timeoutMs > 86_400_000 || !isRecord(record.request) + || ![record.request.cli, record.request.task, record.request.task_identity].every(nonempty) || !safeTime(record.startedAt)) invalid(); const claim = record as unknown as NodeSpawnClaim; if (record.spawn !== undefined && !validNodeSpawnReceipt(record.spawn)) invalid(); diff --git a/packages/sdk/src/worker-cli.ts b/packages/sdk/src/worker-cli.ts index dcc8ee53c..23d67622f 100644 --- a/packages/sdk/src/worker-cli.ts +++ b/packages/sdk/src/worker-cli.ts @@ -127,6 +127,8 @@ export interface AgentRelayContext { stepId: string; idempotencyKey: string; attempt: number; + /** Authored journal input without per-attempt recovery evidence. */ + taskIdentity?: string; dataDir?: string; resultSchema?: unknown; nodeSpawnRuntime?: NodeSpawnRuntime; @@ -412,9 +414,12 @@ async function runViaAgentRelay( try { if (kind === 'relayflows-wrapper-v1') throw new Error('Relay task transport does not support same-process wrappers.'); if (!context?.dataDir) throw new Error('Relay task transport requires a durable data directory and journal dispatch identity.'); - const task = instruction + (wakeContext === undefined ? '' : `\n\nWake context (journaled):\n${JSON.stringify(wakeContext)}`); + const wake = wakeContext === undefined ? '' + : `\n\nWake context (journaled):\n${JSON.stringify(wakeContext)}`; + const task = instruction + wake; + const taskIdentity = (context.taskIdentity ?? instruction) + wake; const received = await runAgentRelayTask({ - cli: kind, task, model, worker_cwd, result_schema: context.resultSchema, + cli: kind, task, taskIdentity, model, worker_cwd, result_schema: context.resultSchema, runId: context.runId, stepId: context.stepId, idempotencyKey: context.idempotencyKey, attempt: context.attempt, dataDir: context.dataDir, ...(timeoutMs === undefined ? {} : { timeoutMs }), diff --git a/packages/sdk/src/worker-input.ts b/packages/sdk/src/worker-input.ts index 6cc68f5ff..1a5c7408e 100644 --- a/packages/sdk/src/worker-input.ts +++ b/packages/sdk/src/worker-input.ts @@ -2,9 +2,22 @@ import type { StepDispatchEvent } from './protocol.js'; /** Memory precedes the final, visibly delimited JSON input segment. */ export function workerInstruction(instruction: string, dispatch: StepDispatchEvent): string { + return renderWorkerInstruction(instruction, dispatch, true); +} + +/** Stable dispatch input, excluding recovery evidence that changes between attempts. */ +export function workerInstructionIdentity(instruction: string, dispatch: StepDispatchEvent): string { + return renderWorkerInstruction(instruction, dispatch, false); +} + +function renderWorkerInstruction( + instruction: string, + dispatch: StepDispatchEvent, + includeRecovery: boolean, +): string { const withMemory = dispatch.memory === undefined ? instruction : `${instruction}\n\nMemory context (journaled):\n${JSON.stringify(dispatch.memory.pack)}`; - const noted = `${withMemory}${recoveryNote(dispatch)}`; + const noted = includeRecovery ? `${withMemory}${recoveryNote(dispatch)}` : withMemory; if (dispatch.input === undefined) { if (typeof dispatch.spec === 'object' && dispatch.spec !== null && 'input' in dispatch.spec) { diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index 9cd248d68..006beef72 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -12,7 +12,7 @@ import { agentCompletionReason } from './cli-transport-evidence.js'; import { agentStepCwd } from './agent-cwd.js'; import { resolveCliModel } from './cli-adapter.js'; import { withWorkerLease } from './worker-lease.js'; -import { workerInstruction } from './worker-input.js'; +import { workerInstruction, workerInstructionIdentity } from './worker-input.js'; import { helperCall } from './yaml-helpers.js'; import { completeHelperDispatch } from './yaml-helper-effect.js'; import { @@ -260,7 +260,8 @@ export class AgentWorker extends EventEmitter { : (spec.transport === 'relay' ? undefined : this.options.runRoot)), spec.transport === 'relay' ? 'relay' : 'direct', { runId: dispatch.run_id, stepId: dispatch.step_id, idempotencyKey: dispatch.idempotency_key, - attempt: dispatch.attempt, dataDir: this.options.dataDir, resultSchema: spec.verification?.json_schema, + attempt: dispatch.attempt, taskIdentity: workerInstructionIdentity(spec.instruction, dispatch), + dataDir: this.options.dataDir, resultSchema: spec.verification?.json_schema, nodeSpawnRuntime: this.options.nodeSpawnRuntime, nodeSpawnRuntimeProvider: signal => this.nodeSpawnRuntimeOwner.get(signal) }, this.options.environment, spec.cli_identity, spec.timeout_ms, spec.resume, spec.secrets, diff --git a/packages/sdk/tests/agent-relay-transport.test.ts b/packages/sdk/tests/agent-relay-transport.test.ts index 044ffe9ef..2c18d727d 100644 --- a/packages/sdk/tests/agent-relay-transport.test.ts +++ b/packages/sdk/tests/agent-relay-transport.test.ts @@ -103,6 +103,27 @@ describe('native node-spawn task transport', () => { expect(recovered.receipt.status).toBe('completed'); }); + it('replays across changed recovery text while stable task identity remains exact', async () => { + const dataDir = await dataDirectory('flows-node-recover-task-identity-'); + const f = fixture(); + const initial = { ...request(dataDir), taskIdentity: 'Implement the change' }; + await runAgentRelayTask(initial, { runtime: f.runtime }); + const recovered = await runAgentRelayTask({ + ...initial, + task: `${initial.task}\n\nPrevious attempt (journaled):\n${JSON.stringify({ + completion_reason: 'crashed', trajectory_tail: { note: 'retry' }, + })}`, + attempt: 2, + }, { runtime: f.runtime }); + expect(recovered.receipt.status).toBe('completed'); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).toHaveBeenCalledOnce(); + + await expect(runAgentRelayTask({ + ...initial, taskIdentity: 'A different authored task', attempt: 2, + }, { runtime: f.runtime })).rejects.toThrow('conflicts with its durable identity or input'); + }); + it('fails closed across an ambiguous pre-ack crash instead of spawning twice', async () => { const dataDir = await dataDirectory('flows-node-ambiguous-'); const f = fixture(); diff --git a/packages/sdk/tests/input-binding.test.ts b/packages/sdk/tests/input-binding.test.ts index e0a8f20d5..819699c04 100644 --- a/packages/sdk/tests/input-binding.test.ts +++ b/packages/sdk/tests/input-binding.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest'; import { compileSpec, compileYaml, kernelToAuthoring, toKernelSpec } from '../src/compile.js'; import type { FlowSpec } from '../src/spec.js'; import { preflight } from '../src/preflight.js'; -import { workerInstruction } from '../src/worker-input.js'; +import { workerInstruction, workerInstructionIdentity } from '../src/worker-input.js'; import type { StepDispatchEvent } from '../src/protocol.js'; const schema = { @@ -85,4 +85,17 @@ steps: expect(prompt).toBe(`Task\n\nMemory context (journaled):\n"memory"\n\ninput:\n${JSON.stringify(input)}`); expect(() => workerInstruction('Task', { spec: { input: consumer.input } } as StepDispatchEvent)).toThrow('did not resolve'); }); + + it('keeps memory and input in task identity while excluding per-attempt recovery evidence', () => { + const dispatch = { + spec: { input: consumer.input }, + input: { message: 'stable input' }, + memory: { pack: 'stable memory' }, + recovery: { previous_completion_reason: 'crashed', trajectory_tail: { attempt: 1 } }, + } as unknown as StepDispatchEvent; + expect(workerInstruction('Task', dispatch)).toContain('Previous attempt (journaled)'); + expect(workerInstructionIdentity('Task', dispatch)).toBe( + `Task\n\nMemory context (journaled):\n"stable memory"\n\ninput:\n${JSON.stringify(dispatch.input)}`, + ); + }); }); diff --git a/packages/sdk/tests/node-spawn-settlement.test.ts b/packages/sdk/tests/node-spawn-settlement.test.ts index e1369bf46..b78dcfd0f 100644 --- a/packages/sdk/tests/node-spawn-settlement.test.ts +++ b/packages/sdk/tests/node-spawn-settlement.test.ts @@ -453,6 +453,38 @@ describe('durable node-result settlement', () => { await worker.close(); }); + it('skips a foreign pending settlement without blocking or rearming this runtime', async () => { + const dataDir = await dataDirectory('flows-node-settlement-foreign-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + const [own] = await readNodeSpawnClaims(dataDir); + const foreignPath = join(dataDir, 'node-spawns', `${'0'.repeat(64)}.json`); + await recordNodeSpawn(foreignPath, { + ...own!.claim, + publisherId: 'publisher_foreign', + publisherName: 'foreign-publisher', + }); + const journal = { journalRead: vi.fn(async () => ({ entries: [terminalEntry()] })) }; + + await expect(reconcileNodeResultSettlements(dataDir, f.runtime, journal)).resolves.toBe(0); + expect(f.ack).toHaveBeenCalledOnce(); + const afterOwn = await readNodeSpawnClaims(dataDir); + expect(afterOwn.find(entry => entry.claim.publisherId === 'publisher_1')! + .claim.result?.settlement).toMatchObject({ settledAs: 'ack' }); + expect(afterOwn.find(entry => entry.claim.publisherId === 'publisher_foreign')! + .claim.result?.settlement?.settledAt).toBeUndefined(); + + const foreignAck = vi.fn(async () => ({})); + const foreignRuntime: NodeSpawnRuntime = { + ...f.runtime, + publisher: { id: 'publisher_foreign', name: 'foreign-publisher' }, + inbox: { inbox: { ...f.runtime.inbox.inbox, ack: foreignAck } }, + }; + await expect(reconcileNodeResultSettlements(dataDir, foreignRuntime, journal)).resolves.toBe(0); + expect(foreignAck).toHaveBeenCalledOnce(); + }); + it('finds a terminal relay result beyond the first journal page before acknowledging', async () => { const dataDir = await dataDirectory('flows-node-settlement-paged-'); const f = fixture(); From c142c0cf488920290270cb50ac941893e187ad87 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Mon, 9 Nov 2026 08:38:07 -0800 Subject: [PATCH 47/75] fix(sdk): preserve result schema claim identity --- packages/sdk/src/agent-relay-transport.ts | 1 + packages/sdk/tests/agent-relay-transport.test.ts | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/sdk/src/agent-relay-transport.ts b/packages/sdk/src/agent-relay-transport.ts index 1ed639bb4..1a13e77b9 100644 --- a/packages/sdk/src/agent-relay-transport.ts +++ b/packages/sdk/src/agent-relay-transport.ts @@ -204,6 +204,7 @@ async function executeAgentRelayTask( cli: request.cli, task, task_identity: request.taskIdentity ?? request.task, ...(request.model === undefined ? {} : { model: request.model }), ...(request.worker_cwd === undefined ? {} : { worker_cwd: request.worker_cwd }), + ...(request.result_schema === undefined ? {} : { result_schema: request.result_schema }), ...(runtime.node === undefined ? {} : { node: runtime.node }), ...(runtime.repo === undefined ? {} : { repo: runtime.repo }), ...(runtime.sandboxOnly === undefined ? {} : { sandbox_only: runtime.sandboxOnly }), diff --git a/packages/sdk/tests/agent-relay-transport.test.ts b/packages/sdk/tests/agent-relay-transport.test.ts index 2c18d727d..7085851c2 100644 --- a/packages/sdk/tests/agent-relay-transport.test.ts +++ b/packages/sdk/tests/agent-relay-transport.test.ts @@ -106,7 +106,8 @@ describe('native node-spawn task transport', () => { it('replays across changed recovery text while stable task identity remains exact', async () => { const dataDir = await dataDirectory('flows-node-recover-task-identity-'); const f = fixture(); - const initial = { ...request(dataDir), taskIdentity: 'Implement the change' }; + const initial = { ...request(dataDir), taskIdentity: 'Implement the change', + result_schema: { type: 'object', required: ['summary'] } }; await runAgentRelayTask(initial, { runtime: f.runtime }); const recovered = await runAgentRelayTask({ ...initial, @@ -122,6 +123,9 @@ describe('native node-spawn task transport', () => { await expect(runAgentRelayTask({ ...initial, taskIdentity: 'A different authored task', attempt: 2, }, { runtime: f.runtime })).rejects.toThrow('conflicts with its durable identity or input'); + await expect(runAgentRelayTask({ + ...initial, result_schema: { type: 'object', required: ['different'] }, attempt: 2, + }, { runtime: f.runtime })).rejects.toThrow('conflicts with its durable identity or input'); }); it('fails closed across an ambiguous pre-ack crash instead of spawning twice', async () => { From 5dfda3618a7121f16ac9f7dd37c23bc1d0c15dad Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Mon, 9 Nov 2026 10:05:27 -0800 Subject: [PATCH 48/75] fix(sdk): retain Relay 18 node-spawn compatibility --- docs/AGENT-RELAY-TRANSPORT.md | 2 +- docs/MIGRATION-NODE-SPAWN.md | 2 +- packages/sdk/src/communication/relay.ts | 2 +- packages/sdk/src/node-spawn-runtime.ts | 2 +- packages/sdk/tests/broker-retirement.test.ts | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/AGENT-RELAY-TRANSPORT.md b/docs/AGENT-RELAY-TRANSPORT.md index 4268a2073..fea54753d 100644 --- a/docs/AGENT-RELAY-TRANSPORT.md +++ b/docs/AGENT-RELAY-TRANSPORT.md @@ -12,7 +12,7 @@ cannot replace it. `RELAY_BASE_URL` defaults to `RELAYFLOW_NODE_NAME`, `RELAYFLOW_NODE_REPO`, and `RELAYFLOW_NODE_SANDBOX_ONLY=1`. -The remote path uses `@agent-relay/sdk >=17.2.1 <18`: +The remote path uses `@agent-relay/sdk >=17.2.1 <19`: 1. `placement.spawn` selects a live node advertising `spawn:`. 2. Flows requests `confirm: true` and `verifyReady: true` and refuses an diff --git a/docs/MIGRATION-NODE-SPAWN.md b/docs/MIGRATION-NODE-SPAWN.md index ad8cb251b..2eeae49ef 100644 --- a/docs/MIGRATION-NODE-SPAWN.md +++ b/docs/MIGRATION-NODE-SPAWN.md @@ -6,7 +6,7 @@ all broker/PTy launch paths from `@relayflows/sdk`. ## One-shot remote agents Keep `transport: 'relay'`, install `@agent-relay/sdk` in the supported range -`>=17.2.1 <18`, and provide a dedicated publisher `RELAY_AGENT_TOKEN`. The +`>=17.2.1 <19`, and provide a dedicated publisher `RELAY_AGENT_TOKEN`. The transport now uses fleet node placement and the durable result inbox instead of Relaycast `task.run`. diff --git a/packages/sdk/src/communication/relay.ts b/packages/sdk/src/communication/relay.ts index d75440564..478e56ce8 100644 --- a/packages/sdk/src/communication/relay.ts +++ b/packages/sdk/src/communication/relay.ts @@ -26,7 +26,7 @@ export async function loadRelayModules(): Promise<{ RelaycastMessagingClient: un const sdk = await import('@agent-relay/sdk'); return { RelaycastMessagingClient: sdk.RelaycastMessagingClient }; } catch (error) { - throw new Error('Agent Relay node spawn requires optional @agent-relay/sdk >=17.2.1 <18.', { cause: error }); + throw new Error('Agent Relay node spawn requires optional @agent-relay/sdk >=17.2.1 <19.', { cause: error }); } } diff --git a/packages/sdk/src/node-spawn-runtime.ts b/packages/sdk/src/node-spawn-runtime.ts index 20dc9103a..98edc2242 100644 --- a/packages/sdk/src/node-spawn-runtime.ts +++ b/packages/sdk/src/node-spawn-runtime.ts @@ -76,7 +76,7 @@ export async function createRemoteNodeSpawnRuntime( }); } catch (error) { if (options.signal?.aborted === true) throw options.signal.reason ?? error; - throw new NodeSpawnError('Node-spawn transport requires optional @agent-relay/sdk >=17.2.1 <18.', + throw new NodeSpawnError('Node-spawn transport requires optional @agent-relay/sdk >=17.2.1 <19.', 'relay_sdk_unavailable', error); } const sandbox = env.RELAYFLOW_NODE_SANDBOX_ONLY?.trim(); diff --git a/packages/sdk/tests/broker-retirement.test.ts b/packages/sdk/tests/broker-retirement.test.ts index 7f190134c..66a9bf5b2 100644 --- a/packages/sdk/tests/broker-retirement.test.ts +++ b/packages/sdk/tests/broker-retirement.test.ts @@ -16,7 +16,7 @@ it('removes harness-driver from the published dependency contract', async () => const manifest = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8')); expect(JSON.stringify({ dependencies: manifest.dependencies, peerDependencies: manifest.peerDependencies, optionalDependencies: manifest.optionalDependencies })).not.toContain('@agent-relay/harness-driver'); - expect(manifest.peerDependencies['@agent-relay/sdk']).toBe('>=17.2.1 <18'); + expect(manifest.peerDependencies['@agent-relay/sdk']).toBe('>=17.2.1 <19'); }); it('fails legacy retained sessions before creating a journal child run', async () => { From 17fe0ddb61edab4922744583db0902cbe50e7a09 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Mon, 9 Nov 2026 11:14:29 -0800 Subject: [PATCH 49/75] fix(sdk): isolate lease snapshot cancellation --- packages/sdk/src/cli/running-step.ts | 13 ++++++- .../tests/authored-parallel-agents.test.ts | 38 +++++++++++++++++-- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/packages/sdk/src/cli/running-step.ts b/packages/sdk/src/cli/running-step.ts index 2b3ca12cf..315bfccd1 100644 --- a/packages/sdk/src/cli/running-step.ts +++ b/packages/sdk/src/cli/running-step.ts @@ -1,3 +1,4 @@ +import { AsyncResource } from 'node:async_hooks'; import { JournalFrameError, JournalProtocolError, type JournalClient } from '../journal-client.js'; import { AuthoredFlowExecutionError } from '../authored-flow-error.js'; import type { JournalEvent } from '../journal-reader.js'; @@ -8,6 +9,14 @@ const LEASE_POLL_MS = 2_000; const STALE_LEASE_MS = 30_000 + LEASE_SWEEP_GRACE_MS; export interface RunningStep extends ParkedStep { leaseDeadlineMs: number } +// Completion deliberately aborts a lease snapshot that is no longer needed. +// The snapshot transport is runtime work, not an authored callback: create its +// promises in this module-load context while the enclosing wait remains visible +// to the authored lifecycle and still propagates every real read failure. +const leaseSnapshotContext = new AsyncResource('RelayflowsLeaseSnapshot'); +const readRunSnapshot = (client: JournalClient, runId: string, signal?: AbortSignal) => + leaseSnapshotContext.runInAsyncScope(() => client.runGet(runId, signal === undefined ? {} : { signal })); + /** Watch completion; snapshots only refresh the non-journaled live lease deadline. */ export async function waitForRunningStep(client: JournalClient, runId: string, runningStep: RunningStep, options: RunLifecycleOptions): Promise { @@ -82,7 +91,7 @@ export async function waitForRunningStep(client: JournalClient, runId: string, `could not establish the completion watch: ${error instanceof Error ? error.message : String(error)}`); }), completed]); if (unconfirmed !== undefined) { - const snapshot = await client.runGet(runId, options.signal === undefined ? {} : { signal: options.signal }); + const snapshot = await readRunSnapshot(client, runId, options.signal); if (snapshot.steps[runningStep.id]?.state !== 'running') return; throw new AuthoredFlowExecutionError('daemon_unresponsive', `could not establish the completion watch: ${unconfirmed instanceof Error ? unconfirmed.message : String(unconfirmed)}`); @@ -105,7 +114,7 @@ export async function waitForRunningStep(client: JournalClient, runId: string, const read = snapshotRead = new AbortController(); const signal = options.signal === undefined ? read.signal : AbortSignal.any([options.signal, read.signal]); - const snapshot = await client.runGet(runId, { signal }) + const snapshot = await readRunSnapshot(client, runId, signal) .catch(error => { if (options.signal?.aborted) throw canceled(); if (protocolFailure !== undefined) throw protocolFailure; diff --git a/packages/sdk/tests/authored-parallel-agents.test.ts b/packages/sdk/tests/authored-parallel-agents.test.ts index c3503368e..11fe97cd0 100644 --- a/packages/sdk/tests/authored-parallel-agents.test.ts +++ b/packages/sdk/tests/authored-parallel-agents.test.ts @@ -1,6 +1,6 @@ import { existsSync, mkdirSync, readFileSync, symlinkSync, writeFileSync } from 'node:fs'; import { join, resolve } from 'node:path'; -import { afterEach, describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { flow } from '@relayflows/surface'; import { AuthoredFlowExecutionError, executeAuthoredFlow } from '../src/authored-flow-executor.js'; import { attachLocalAgent } from '../src/local-agent.js'; @@ -15,7 +15,7 @@ afterEach(async () => { }); /** The fixture's wrapper, but each session holds for a while and journals when it ran. */ -async function slowAgents(capacity: number, failFirstSession = false) { +async function slowAgents(capacity: number, failFirstSession = false, holdMs = 400) { const fixture = chainFixture(); closes.push(() => fixture.close()); const spans = join(fixture.root, 'spans.jsonl'); @@ -28,7 +28,7 @@ const request = await receiveWrapperRequest(); if (request) { writeFileSync(${JSON.stringify(started)}, ''); const start = Date.now(); - await new Promise(done => setTimeout(done, 400)); + await new Promise(done => setTimeout(done, ${holdMs})); appendFileSync(${JSON.stringify(spans)}, JSON.stringify({ start, end: Date.now() }) + '\\n'); ${failFirstSession ? `if (!existsSync(${JSON.stringify(join(fixture.root, 'failed-once'))})) { writeFileSync(${JSON.stringify(join(fixture.root, 'failed-once'))}, ''); @@ -143,6 +143,38 @@ describe('authored steps under local workers with capacity', () => { expect(peakOverlap(readSpans())).toBe(1); }); + it('does not attribute an intentionally aborted lease snapshot to the authored agent callback', async () => { + const { fixture, client, agent } = await slowAgents(1, false, 2_500); + const originalRunGet = client.runGet.bind(client); + let snapshotHeld = false; + let snapshotsAbortedByCompletion = 0; + vi.spyOn(client, 'runGet').mockImplementation(async (runId, options) => { + const snapshot = await originalRunGet(runId, options); + if (snapshotHeld || options?.signal === undefined || snapshot.steps['agent-1']?.state !== 'running') { + return snapshot; + } + snapshotHeld = true; + await new Promise((resolve, reject) => { + const timer = setTimeout(resolve, 5_000); + options?.signal?.addEventListener('abort', () => { + snapshotsAbortedByCompletion += 1; + clearTimeout(timer); + reject(options.signal!.reason); + }, { once: true }); + }); + return snapshot; + }); + const completesDuringSnapshot = flow('completes-during-snapshot', async f => { + await f.agent('agent', { task: 'finish while the lease snapshot is pending' }); + f.done('success'); + }); + const result = await executeAuthoredFlow(completesDuringSnapshot, client, undefined, { + flowPath: fixture.flowPath, localAgentStream: agent.stream, workerCapacity: 1, + }); + expect(result.completionReason).toBe('success'); + expect(snapshotsAbortedByCompletion).toBe(1); + }, 10_000); + it('never starts queued agents once the body has failed', async () => { const { fixture, client, agent, readSpans, firstStarted } = await slowAgents(1); const failing = flow('fails-while-queued', async f => { From 7897907659ce11da0f8fa67015bb7a9426234f4e Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Mon, 9 Nov 2026 12:00:26 -0800 Subject: [PATCH 50/75] fix(sdk): preserve node spawn recovery state --- packages/sdk/src/agent-relay-transport.ts | 8 +- packages/sdk/src/node-spawn-local.ts | 56 +++++++++++-- packages/sdk/src/worker.ts | 4 +- .../sdk/tests/agent-relay-transport.test.ts | 45 +++++++++++ packages/sdk/tests/node-spawn-ports.test.ts | 80 +++++++++++++++++++ .../tests/worker-node-runtime-sharing.test.ts | 11 ++- 6 files changed, 188 insertions(+), 16 deletions(-) diff --git a/packages/sdk/src/agent-relay-transport.ts b/packages/sdk/src/agent-relay-transport.ts index 1a13e77b9..bb7c68304 100644 --- a/packages/sdk/src/agent-relay-transport.ts +++ b/packages/sdk/src/agent-relay-transport.ts @@ -332,16 +332,16 @@ async function executeAgentRelayTask( receivedAt: Date.now() } }; await recordNodeSpawn(claimed.path, claim); } + // Register the live inbox teardown before exact release. A failed release + // is recoverable from the durable result, but a same-process retry cannot + // recreate this process-generation barrier after the iterator is gone. + const delivery = durableNodeResultDelivery(runtime, claimed.path, claim, observed); if (claim.release === undefined) { releaseAttempted = true; const cleanup = await tryRelease(runtime, claimed.path, claim, request.idempotencyKey); claim = cleanup.claim; if (cleanup.error !== undefined) throw cleanup.error; } - // Construct the settlement wrapper from the post-release claim. Otherwise - // preparing its journal intent could rewrite the file from a stale copy - // and silently discard the exact release receipt. - const delivery = durableNodeResultDelivery(runtime, claimed.path, claim, observed); resultSignal?.throwIfAborted(); const envelope = delivery.envelope; const error = envelope.status === 'failed' ? envelope.error! : null; diff --git a/packages/sdk/src/node-spawn-local.ts b/packages/sdk/src/node-spawn-local.ts index 901e0cceb..73f568c4d 100644 --- a/packages/sdk/src/node-spawn-local.ts +++ b/packages/sdk/src/node-spawn-local.ts @@ -7,6 +7,7 @@ import { type NodeReleaseRequest, type NodeSpawnAction, type NodeSpawnInvocation, + type NodeSpawnIdentity, type NodeSpawnPort, type NodeSpawnReceipt, type NodeSpawnRequest, @@ -240,12 +241,14 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { }, { actionTimeoutMs: request.timeoutMs }); // spawn.invoke is the action receipt; spawn.status is also the boundary // that enriches it from the core's exact same-workspace registration. - const settled = await this.wait('spawn', raw.id, request.timeoutMs); + const confirmation = await this.wait('spawn', raw.id, request.timeoutMs, + mapCoreInvocation('spawn', raw)); + const settled = confirmation.invocation; const settledTarget = allTargets.find(candidate => candidate.id === settled.nodeId); - const cleanup = settled.agentId && settled.name && settledTarget?.id + const cleanup = confirmation.cleanup ?? (settled.agentId && settled.name && settledTarget?.id ? { id: settled.id, nodeId: settledTarget.id, nodeName: settledTarget.name, name: settled.name, agentId: settled.agentId } - : undefined; + : undefined); if (settled.status !== 'completed' || !settled.spawned || !settled.ready || settled.taskUnconfirmed !== false || !settled.agentId || !settled.name) { throw new NodeSpawnError('The local node did not prove a ready spawned agent', @@ -285,7 +288,7 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { }); const settled = raw.status === 'completed' ? mapCoreInvocation('release', raw) - : await this.wait('release', raw.id); + : (await this.wait('release', raw.id)).invocation; if (settled.status !== 'completed' || settled.released !== true) { throw new NodeSpawnError('The local node did not confirm exact agent release', settled.errorCode ?? 'release_unconfirmed'); @@ -293,14 +296,51 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { return settled; } - private async wait(action: NodeSpawnAction, id: string, timeoutMs = 360_000): Promise { + private async wait( + action: NodeSpawnAction, + id: string, + timeoutMs = 360_000, + initial?: NodeSpawnInvocation, + ): Promise<{ invocation: NodeSpawnInvocation; cleanup?: NodeSpawnIdentity }> { const deadline = Date.now() + timeoutMs; + let cleanup = this.cleanupIdentity(action, initial); while (Date.now() < deadline) { - const value = await this.status(action, id); - if (terminalStatus(value.status)) return value; + let value: NodeSpawnInvocation; + try { + value = await this.status(action, id); + } catch (error) { + if (cleanup === undefined) throw error; + if (error instanceof NodeSpawnError) { + throw new NodeSpawnError(error.message, error.code, error.cause ?? error, cleanup); + } + throw new NodeSpawnError(`Could not confirm local ${action}`, `${action}_unconfirmed`, error, cleanup); + } + const observed = this.cleanupIdentity(action, value); + if (cleanup !== undefined && value.spawned === true + && ((value.id !== '' && cleanup.id !== value.id) + || (value.nodeId !== undefined && cleanup.nodeId !== value.nodeId) + || (value.name !== undefined && cleanup.name !== value.name) + || (value.agentId !== undefined && cleanup.agentId !== value.agentId))) { + throw new NodeSpawnError('The local spawn confirmation changed its exact identity', + 'spawn_identity_mismatch', undefined, cleanup); + } + cleanup ??= observed; + if (terminalStatus(value.status)) return { invocation: value, cleanup }; await new Promise(resolve => setTimeout(resolve, 500)); } - throw new NodeSpawnError(`Timed out waiting for local ${action} confirmation`, `${action}_unconfirmed`); + throw new NodeSpawnError(`Timed out waiting for local ${action} confirmation`, + `${action}_unconfirmed`, undefined, cleanup); + } + + private cleanupIdentity( + action: NodeSpawnAction, + value?: NodeSpawnInvocation, + ): NodeSpawnIdentity | undefined { + if (action !== 'spawn' || value?.spawned !== true || value.id.trim() === '' + || value.nodeId !== this.exactNode.id || !value.name || value.name.trim() === '' + || !value.agentId || value.agentId.trim() === '') return undefined; + return { id: value.id, nodeId: value.nodeId, nodeName: this.exactNode.name, + name: value.name, agentId: value.agentId }; } } diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index 006beef72..11f457f99 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -267,8 +267,8 @@ export class AgentWorker extends EventEmitter { this.options.environment, spec.cli_identity, spec.timeout_ms, spec.resume, spec.secrets, this.client.supportsFeature?.('step_secret_scoping') ?? true) : Promise.resolve({ exit_code: null, stdout_tail: '', stderr_tail: 'agent step has no declared CLI' })); - const { result, usage } = workerSpend(completed, effectiveModel); - const cost = reportedCost(completed, effectiveModel); + const { result, usage } = workerSpend(completed, executionModel); + const cost = reportedCost(completed, executionModel); const completionReason = agentCompletionReason(result); // Output shape: if the CLI's stdout parses as JSON, promote THAT diff --git a/packages/sdk/tests/agent-relay-transport.test.ts b/packages/sdk/tests/agent-relay-transport.test.ts index 7085851c2..ffabfd233 100644 --- a/packages/sdk/tests/agent-relay-transport.test.ts +++ b/packages/sdk/tests/agent-relay-transport.test.ts @@ -103,6 +103,51 @@ describe('native node-spawn task transport', () => { expect(recovered.receipt.status).toBe('completed'); }); + it('retains the original inbox teardown barrier across a failed release and cached retry', async () => { + const dataDir = await dataDirectory('flows-node-release-retry-teardown-'); + const f = fixture(); + let finishTeardown!: () => void; + const settlementReady = new Promise<{ failure?: unknown }>(resolve => { + finishTeardown = () => resolve({}); + }); + let delivered = false; + const subscribe = vi.fn(() => ({ [Symbol.asyncIterator]() { return { + async next() { + if (delivered) return new Promise>(() => undefined); + delivered = true; + const name = (await f.spawn.mock.results[0]!.value).name; + return { done: false as const, value: { id: 'delivery_1', settlementReady, + message: { id: 'message_1', from: { id: 'agent_1', name }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_1', + step_id: 'edit', dispatch_id: 'dispatch_1234567890', status: 'completed', + output: { summary: 'done' }, accounting: { tokens_input: 10, tokens_output: 5 } }) } } }; + }, + async return() { return { done: true as const, value: undefined }; }, + }; } })); + f.runtime.inbox.inbox.subscribe = subscribe; + f.release.mockRejectedValueOnce(new Error('release unavailable')); + + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('release unavailable'); + const recovered = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + expect(subscribe).toHaveBeenCalledOnce(); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).toHaveBeenCalledTimes(2); + await recovered.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + const settling = recovered.delivery.ack(); + const settledBeforeTeardown = await Promise.race([ + settling.then(() => true), + new Promise(resolve => setTimeout(() => resolve(false), 100)), + ]); + expect(settledBeforeTeardown).toBe(false); + expect(f.ack).not.toHaveBeenCalled(); + expect(f.fail).not.toHaveBeenCalled(); + finishTeardown(); + await settling; + expect(f.ack).toHaveBeenCalledOnce(); + expect(f.fail).not.toHaveBeenCalled(); + }); + it('replays across changed recovery text while stable task identity remains exact', async () => { const dataDir = await dataDirectory('flows-node-recover-task-identity-'); const f = fixture(); diff --git a/packages/sdk/tests/node-spawn-ports.test.ts b/packages/sdk/tests/node-spawn-ports.test.ts index f7426c2c9..2ed2ae101 100644 --- a/packages/sdk/tests/node-spawn-ports.test.ts +++ b/packages/sdk/tests/node-spawn-ports.test.ts @@ -183,6 +183,86 @@ describe('local node-spawn port', () => { cleanup: { nodeId: 'node_1', nodeName: 'desktop-1', name: 'worker_1', agentId: 'agent_1' }, }); }); + + it('retains exact cleanup identity when local spawn confirmation times out', async () => { + vi.useFakeTimers(); + try { + const status = vi.fn(); + const core: TestCore = { async call(method: string): Promise { + if (method === 'spawn.targets') return { targets: [{ id: 'node_1', name: 'desktop-1', live: true, + handlers_live: true, stale: false, capabilities: ['spawn:codex'] }] } as T; + if (method === 'spawn.invoke') return { id: 'inv_1', status: 'pending', spawned: false, + ready: false, task_unconfirmed: false } as T; + if (method === 'spawn.status') { + status(); + return { id: 'inv_1', status: 'running', node_id: 'node_1', agent_id: 'agent_1', + name: 'worker_1', spawned: true, ready: false, task_unconfirmed: false } as T; + } + throw new Error(`unexpected core method: ${method}`); + } }; + const failure = expect(localPort(core).spawn({ name: 'worker_1', harness: 'codex', task: 'work', + directory: '/workspace', idempotencyKey: 'dispatch_1234567890', timeoutMs: 750 })) + .rejects.toMatchObject({ code: 'spawn_unconfirmed', cleanup: { + id: 'inv_1', nodeId: 'node_1', nodeName: 'desktop-1', name: 'worker_1', agentId: 'agent_1', + } }); + await vi.waitFor(() => expect(status).toHaveBeenCalledOnce()); + await vi.advanceTimersByTimeAsync(1_000); + await failure; + } finally { + vi.useRealTimers(); + } + }); + + it('retains exact cleanup identity when later local spawn polling fails', async () => { + vi.useFakeTimers(); + try { + const pollFailure = new Error('core disconnected'); + let statusCalls = 0; + const core: TestCore = { async call(method: string): Promise { + if (method === 'spawn.targets') return { targets: [{ id: 'node_1', name: 'desktop-1', live: true, + handlers_live: true, stale: false, capabilities: ['spawn:codex'] }] } as T; + if (method === 'spawn.invoke') return { id: 'inv_1', status: 'pending', spawned: false, + ready: false, task_unconfirmed: false } as T; + if (method === 'spawn.status' && statusCalls++ === 0) return { id: 'inv_1', status: 'running', + node_id: 'node_1', agent_id: 'agent_1', name: 'worker_1', spawned: true, + ready: false, task_unconfirmed: false } as T; + if (method === 'spawn.status') throw pollFailure; + throw new Error(`unexpected core method: ${method}`); + } }; + const failure = expect(localPort(core).spawn({ name: 'worker_1', harness: 'codex', task: 'work', + directory: '/workspace', idempotencyKey: 'dispatch_1234567890' })) + .rejects.toMatchObject({ code: 'spawn_unconfirmed', cause: pollFailure, cleanup: { + id: 'inv_1', nodeId: 'node_1', nodeName: 'desktop-1', name: 'worker_1', agentId: 'agent_1', + } }); + await vi.waitFor(() => expect(statusCalls).toBe(1)); + await vi.advanceTimersByTimeAsync(500); + await failure; + } finally { + vi.useRealTimers(); + } + }); + + it('fails closed on a conflicting later spawn identity and retains the first exact cleanup', async () => { + let statusCalls = 0; + const core: TestCore = { async call(method: string): Promise { + if (method === 'spawn.targets') return { targets: [{ id: 'node_1', name: 'desktop-1', live: true, + handlers_live: true, stale: false, capabilities: ['spawn:codex'] }] } as T; + if (method === 'spawn.invoke') return { id: 'inv_1', status: 'pending', spawned: false, + ready: false, task_unconfirmed: false } as T; + if (method === 'spawn.status' && statusCalls++ === 0) return { id: 'inv_1', status: 'running', + node_id: 'node_1', agent_id: 'agent_1', name: 'worker_1', spawned: true, + ready: false, task_unconfirmed: false } as T; + if (method === 'spawn.status') return { id: 'inv_1', status: 'completed', node_id: 'node_1', + agent_id: 'agent_other', name: 'worker_1', spawned: true, + ready: true, task_unconfirmed: false } as T; + throw new Error(`unexpected core method: ${method}`); + } }; + const pending = localPort(core).spawn({ name: 'worker_1', harness: 'codex', task: 'work', + directory: '/workspace', idempotencyKey: 'dispatch_1234567890' }); + await expect(pending).rejects.toMatchObject({ code: 'spawn_identity_mismatch', cleanup: { + id: 'inv_1', nodeId: 'node_1', nodeName: 'desktop-1', name: 'worker_1', agentId: 'agent_1', + } }); + }); }); describe('remote node-spawn port', () => { diff --git a/packages/sdk/tests/worker-node-runtime-sharing.test.ts b/packages/sdk/tests/worker-node-runtime-sharing.test.ts index 53f92f925..fffbbb51b 100644 --- a/packages/sdk/tests/worker-node-runtime-sharing.test.ts +++ b/packages/sdk/tests/worker-node-runtime-sharing.test.ts @@ -116,7 +116,7 @@ describe('AgentWorker node-spawn runtime ownership', () => { from: { id: 'agent_worker_path', name: spawnedName }, text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_parallel', step_id: 'edit-local-path', dispatch_id: 'dispatch_localpath', status: 'completed', - output: { summary: 'done' } }) } }; + output: { summary: 'done' }, accounting: { tokens_input: 10, tokens_output: 5 } }) } }; }, ack: vi.fn(async () => ({})), fail: vi.fn(async () => ({})), } }, @@ -125,7 +125,8 @@ describe('AgentWorker node-spawn runtime ownership', () => { const client = Object.assign(new EventEmitter(), { workerAttach: vi.fn(async () => ({})), stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), - stepComplete: vi.fn(async () => ({})), + stepComplete: vi.fn(async () => ({} as Awaited< + ReturnType>)), supportsFeature: vi.fn(() => true), }); const worker = new AgentWorker(client as unknown as JournalClient, { @@ -153,6 +154,12 @@ describe('AgentWorker node-spawn runtime ownership', () => { ? runRoot : testCase.expectedDirectory === 'declared' ? exactDeclaredDirectory : undefined); expect(spawn.mock.calls[0]![0].model).toBe(testCase.expectedModel); } + if ('omitModel' in testCase && testCase.omitModel) { + expect(client.stepComplete.mock.calls[0]?.[5]).toMatchObject({ + usage: { tokens_in: 10, tokens_out: 5, dollars_unmetered: true }, + }); + expect(client.stepComplete.mock.calls[0]?.[5]).not.toHaveProperty('reported_cost'); + } }); it('does not reconcile a journaled delivery while its live iterator teardown is unresolved', async () => { From d8c1a58b641aa93b651fdc5e6284fd66652bee0d Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Mon, 9 Nov 2026 16:36:41 -0800 Subject: [PATCH 51/75] fix(sdk): require complete core bridge surface --- docs/AGENT-RELAY-TRANSPORT.md | 8 +- docs/MIGRATION-NODE-SPAWN.md | 7 +- packages/sdk/package-lock.json | 2 +- packages/sdk/package.json | 2 +- packages/sdk/src/node-spawn-local.ts | 43 ++++++++++- packages/sdk/tests/broker-retirement.test.ts | 2 + .../sdk/tests/node-spawn-core-bridge.test.ts | 74 +++++++++++++++++++ 7 files changed, 127 insertions(+), 11 deletions(-) create mode 100644 packages/sdk/tests/node-spawn-core-bridge.test.ts diff --git a/docs/AGENT-RELAY-TRANSPORT.md b/docs/AGENT-RELAY-TRANSPORT.md index fea54753d..ce85e3759 100644 --- a/docs/AGENT-RELAY-TRANSPORT.md +++ b/docs/AGENT-RELAY-TRANSPORT.md @@ -79,10 +79,10 @@ The SDK contains a local `NodeSpawnPort` adapter for Desktop's schema-v2 service-caller bridge. The caller is an explicit private `caller.json` path supplied with `--service-caller-file`, the SDK `serviceCallerFile` option, or `RELAYFLOW_SERVICE_CALLER_FILE`; Flows never scans Desktop's app-data -directory. Optional `@agent-relay/core-bridge` validates the handoff and local -socket or named pipe, challenge-authenticates a fresh connection, verifies the -bound publisher/workspace/node identity, and then calls `spawn.targets`, -`spawn.invoke`, `spawn.status`, and `spawn.release` without sending a caller +directory. Optional `@agent-relay/core-bridge` `>=18.1.0 <19` validates the +handoff and local socket or named pipe, challenge-authenticates a fresh +connection, verifies the bound publisher/workspace/node identity, and then +calls `spawn.targets`, `spawn.invoke`, `spawn.status`, and `spawn.release` without sending a caller name or secret. The same authenticated bridge provides the invocation-scoped result inbox and its ack/fail/close/disconnect lifecycle. diff --git a/docs/MIGRATION-NODE-SPAWN.md b/docs/MIGRATION-NODE-SPAWN.md index 2eeae49ef..3663ff38a 100644 --- a/docs/MIGRATION-NODE-SPAWN.md +++ b/docs/MIGRATION-NODE-SPAWN.md @@ -32,9 +32,10 @@ private file explicitly with `--local-agent --service-caller-file /absolute/caller.json`, the SDK `serviceCallerFile` option, or `RELAYFLOW_SERVICE_CALLER_FILE`. Flows never scans Desktop's app-data directory. It securely parses and -challenge-authenticates the handoff through `@agent-relay/core-bridge` 18.x. -Install a compatible version (`npm install @agent-relay/core-bridge@^18`) when -using Desktop mode; the peer is optional only when Desktop mode is not used. +challenge-authenticates the handoff through `@agent-relay/core-bridge` 18.1 or +newer. Install a compatible version +(`npm install @agent-relay/core-bridge@^18.1.0`) when using Desktop mode; the +peer is optional only when Desktop mode is not used. Flows then sends no caller name or secret on spawn or inbox calls. The handoff's harness and directory scope is enforced before dispatch. A local relay step with no authored `cwd` spawns in that run's diff --git a/packages/sdk/package-lock.json b/packages/sdk/package-lock.json index c58cd401e..a64303179 100644 --- a/packages/sdk/package-lock.json +++ b/packages/sdk/package-lock.json @@ -36,7 +36,7 @@ "vitest": "^2.1.0" }, "peerDependencies": { - "@agent-relay/core-bridge": ">=18.0.0 <19", + "@agent-relay/core-bridge": ">=18.1.0 <19", "@agent-relay/sdk": ">=17.2.1 <19" }, "peerDependenciesMeta": { diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 768cb6ae9..2947b3036 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -76,7 +76,7 @@ "directory": "packages/sdk" }, "peerDependencies": { - "@agent-relay/core-bridge": ">=18.0.0 <19", + "@agent-relay/core-bridge": ">=18.1.0 <19", "@agent-relay/sdk": ">=17.2.1 <19" }, "peerDependenciesMeta": { diff --git a/packages/sdk/src/node-spawn-local.ts b/packages/sdk/src/node-spawn-local.ts index 73f568c4d..64dd92eab 100644 --- a/packages/sdk/src/node-spawn-local.ts +++ b/packages/sdk/src/node-spawn-local.ts @@ -151,18 +151,57 @@ export interface CoreBridgeApi { serviceCallerDisconnect(client: CoreBridgeClient, options?: { signal?: AbortSignal }): Promise; } +const REQUIRED_CORE_BRIDGE_METHODS = { + connectCore: true, + readCallerDescriptor: true, + authenticate: true, + requireServiceCallerMethods: true, + assertServiceCallerIdentity: true, + serviceCallerIdentity: true, + spawnTargets: true, + spawnInvoke: true, + spawnStatus: true, + spawnRelease: true, + serviceCallerInboxOpen: true, + serviceCallerInboxNext: true, + serviceCallerInboxAck: true, + serviceCallerInboxFail: true, + serviceCallerInboxClose: true, + serviceCallerDisconnect: true, +} as const satisfies Record; + +/** Fails before local admission when an installed optional bridge lacks the required contract. */ +export function assertCoreBridgeApi(value: unknown): asserts value is CoreBridgeApi { + const module = (typeof value === 'object' && value !== null) + ? value as Record + : {}; + const missingMethods = Object.keys(REQUIRED_CORE_BRIDGE_METHODS) + .filter(method => typeof module[method] !== 'function'); + if (missingMethods.length === 0) return; + throw new NodeSpawnError( + `Installed @agent-relay/core-bridge is missing required schema-v2 methods: ${missingMethods.join(', ')}. ` + + 'Install >=18.1.0 <19.', + 'core_bridge_unavailable', + { missingMethods, requiredVersion: '>=18.1.0 <19' }, + ); +} + /** Loads the optional bridge without making remote-only installs depend on Desktop support. */ export async function loadCoreBridge(): Promise { const specifier = '@agent-relay/core-bridge'; + let bridge: unknown; try { - return await import(specifier) as unknown as CoreBridgeApi; + bridge = await import(specifier); } catch (error) { throw new NodeSpawnError( - 'Local Desktop node spawn requires optional @agent-relay/core-bridge with schema-v2 service-caller support', + 'Local Desktop node spawn requires optional @agent-relay/core-bridge >=18.1.0 <19 ' + + 'with schema-v2 service-caller support', 'core_bridge_unavailable', error, ); } + assertCoreBridgeApi(bridge); + return bridge; } /** Native local spawn adapter over one already authenticated schema-v2 bridge connection. */ diff --git a/packages/sdk/tests/broker-retirement.test.ts b/packages/sdk/tests/broker-retirement.test.ts index 66a9bf5b2..bd015f238 100644 --- a/packages/sdk/tests/broker-retirement.test.ts +++ b/packages/sdk/tests/broker-retirement.test.ts @@ -17,6 +17,8 @@ it('removes harness-driver from the published dependency contract', async () => expect(JSON.stringify({ dependencies: manifest.dependencies, peerDependencies: manifest.peerDependencies, optionalDependencies: manifest.optionalDependencies })).not.toContain('@agent-relay/harness-driver'); expect(manifest.peerDependencies['@agent-relay/sdk']).toBe('>=17.2.1 <19'); + expect(manifest.peerDependencies['@agent-relay/core-bridge']).toBe('>=18.1.0 <19'); + expect(manifest.peerDependenciesMeta['@agent-relay/core-bridge']).toEqual({ optional: true }); }); it('fails legacy retained sessions before creating a journal child run', async () => { diff --git a/packages/sdk/tests/node-spawn-core-bridge.test.ts b/packages/sdk/tests/node-spawn-core-bridge.test.ts new file mode 100644 index 000000000..8912ead2b --- /dev/null +++ b/packages/sdk/tests/node-spawn-core-bridge.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { assertCoreBridgeApi, loadCoreBridge } from '../src/node-spawn-local.js'; + +// 18.0.0 implements only these six CoreBridgeApi methods. Undefined exports let +// Vitest expose the same missing lifecycle surface through a virtual module. +const publicCoreBridge180Shape = vi.hoisted(() => ({ + connectCore: vi.fn(), + authenticate: vi.fn(), + spawnTargets: vi.fn(), + spawnInvoke: vi.fn(), + spawnStatus: vi.fn(), + spawnRelease: vi.fn(), + readCallerDescriptor: undefined, + requireServiceCallerMethods: undefined, + assertServiceCallerIdentity: undefined, + serviceCallerIdentity: undefined, + serviceCallerInboxOpen: undefined, + serviceCallerInboxNext: undefined, + serviceCallerInboxAck: undefined, + serviceCallerInboxFail: undefined, + serviceCallerInboxClose: undefined, + serviceCallerDisconnect: undefined, +})); + +vi.mock('@agent-relay/core-bridge', () => publicCoreBridge180Shape); + +const schemaV2CoreBridge181Shape = { + ...publicCoreBridge180Shape, + readCallerDescriptor: vi.fn(), + requireServiceCallerMethods: vi.fn(), + assertServiceCallerIdentity: vi.fn(), + serviceCallerIdentity: vi.fn(), + serviceCallerInboxOpen: vi.fn(), + serviceCallerInboxNext: vi.fn(), + serviceCallerInboxAck: vi.fn(), + serviceCallerInboxFail: vi.fn(), + serviceCallerInboxClose: vi.fn(), + serviceCallerDisconnect: vi.fn(), +}; + +describe('optional core-bridge contract', () => { + it('fails closed while loading the incomplete public 18.0 module shape', async () => { + let failure: unknown; + try { + await loadCoreBridge(); + } catch (error) { + failure = error; + } + expect(failure).toMatchObject({ + code: 'core_bridge_unavailable', + message: expect.stringContaining('Install >=18.1.0 <19'), + cause: { + missingMethods: [ + 'readCallerDescriptor', + 'requireServiceCallerMethods', + 'assertServiceCallerIdentity', + 'serviceCallerIdentity', + 'serviceCallerInboxOpen', + 'serviceCallerInboxNext', + 'serviceCallerInboxAck', + 'serviceCallerInboxFail', + 'serviceCallerInboxClose', + 'serviceCallerDisconnect', + ], + requiredVersion: '>=18.1.0 <19', + }, + }); + }); + + it('accepts the complete 18.1 schema-v2 lifecycle module shape', () => { + expect(() => assertCoreBridgeApi(schemaV2CoreBridge181Shape)).not.toThrow(); + }); +}); From ba0c633176a54b84695ed8b879715601389c58e6 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Mon, 9 Nov 2026 16:38:11 -0800 Subject: [PATCH 52/75] test(sdk): typecheck core bridge contract --- packages/sdk/tsconfig.tests.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/sdk/tsconfig.tests.json b/packages/sdk/tsconfig.tests.json index ea5fc042d..bc59c4dc8 100644 --- a/packages/sdk/tsconfig.tests.json +++ b/packages/sdk/tsconfig.tests.json @@ -74,6 +74,7 @@ "tests/agent-relay-transport.test.ts", "tests/broker-retirement.test.ts", "tests/agent-relay-hardening.test.ts", + "tests/node-spawn-core-bridge.test.ts", "tests/node-spawn-ports.test.ts", "tests/node-spawn-runtime.test.ts", "tests/node-spawn-settlement.test.ts", From d34f6a7c4d327ea7971ebb8fddfcfafd82700863 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Mon, 9 Nov 2026 19:33:28 -0800 Subject: [PATCH 53/75] fix(sdk): fence durable node spawn dispatch --- docs/AGENT-RELAY-TRANSPORT.md | 30 +- docs/MIGRATION-NODE-SPAWN.md | 13 + packages/sdk/src/agent-relay-transport.ts | 60 ++-- packages/sdk/src/index.ts | 2 + packages/sdk/src/node-spawn-dispatch.ts | 78 +++++ packages/sdk/src/node-spawn-local.ts | 71 +++-- packages/sdk/src/node-spawn-remote.ts | 10 +- packages/sdk/src/node-spawn-state.ts | 106 ++++++- packages/sdk/src/node-spawn.ts | 19 +- .../sdk/tests/agent-relay-transport.test.ts | 34 +- .../tests/node-spawn-dispatch-fence.test.ts | 291 ++++++++++++++++++ packages/sdk/tests/node-spawn-live.test.ts | 4 +- packages/sdk/tests/node-spawn-ports.test.ts | 76 +++-- .../sdk/tests/node-spawn-settlement.test.ts | 25 +- .../tests/worker-node-runtime-sharing.test.ts | 26 +- .../tests/worker-node-runtime-timeout.test.ts | 8 +- packages/sdk/tsconfig.tests.json | 1 + 17 files changed, 730 insertions(+), 124 deletions(-) create mode 100644 packages/sdk/src/node-spawn-dispatch.ts create mode 100644 packages/sdk/tests/node-spawn-dispatch-fence.test.ts diff --git a/docs/AGENT-RELAY-TRANSPORT.md b/docs/AGENT-RELAY-TRANSPORT.md index ce85e3759..5b81f1485 100644 --- a/docs/AGENT-RELAY-TRANSPORT.md +++ b/docs/AGENT-RELAY-TRANSPORT.md @@ -49,12 +49,30 @@ identity, stable worker name, and full spawn input. Tokens are never stored. The current fleet placement SDK has no caller-provided idempotency key for the placement invocation itself. Flows forwards the durable key to the selected -node in the spawn action input, and still sends at most one placement spawn -for a newly created claim. A recorded -invocation is reconciled with `commands.getInvocation`; an existing claim with -no recorded acknowledgement is the ambiguous crash window and fails closed. -It is never answered by a blind second spawn. The stable worker name provides -additional collision protection at the node. +node in the spawn action input. A version 3 claim uses a permanent sibling +dispatch fence: Flows writes and fsyncs a complete private marker, atomically +hard-links it into place, and fsyncs the directory immediately before local +`spawn.invoke` or remote `placement.spawn`. Competing workers may repeat +preflight, but only the fence winner may enter that irreversible call. A v3 +claim with no fence proves dispatch was not entered and may retry preflight; +once the fence exists, a missing receipt is ambiguous forever and fails +closed. Malformed, mismatched, or unreadable fence state also fails closed. +Legacy version 2 claims without a receipt remain ambiguous and are never +blindly retried. A recorded invocation is reconciled with +`commands.getInvocation`. The stable worker name provides additional +collision protection at the node. + +Custom `NodeSpawnPort` implementations must declare `dispatchFenceVersion: 1` +and await the supplied `beforeDispatch` callback immediately before their +first irreversible spawn call. Preflight that fails before entering that +callback may throw the exported `NodeSpawnPreDispatchError`; this is a +privileged assertion that no remote effect was attempted, not a generic error +wrapper. Ports must never use it for an error at or after dispatch. Existing +one-argument `spawn` implementations are not compatible even though JavaScript +can ignore extra arguments: the required protocol field makes them a type +error, and the runtime refuses a missing or unknown version before creating a +durable claim. A falsely declared port that omits the callback is fenced and +fails closed after its first uncertain call. The claim also fixes the task's original start time. Resume never grants a new execution deadline. Once a correlated result was durably observed before that diff --git a/docs/MIGRATION-NODE-SPAWN.md b/docs/MIGRATION-NODE-SPAWN.md index 3663ff38a..9fb2e8b9d 100644 --- a/docs/MIGRATION-NODE-SPAWN.md +++ b/docs/MIGRATION-NODE-SPAWN.md @@ -47,6 +47,19 @@ Setting both fails closed. A caller file whose publisher or workspace differs from the authenticated core identity is also refused; rotate or reprovision it in Desktop rather than editing the file. +## Custom `NodeSpawnPort` implementations + +Ports must now declare `dispatchFenceVersion: 1` and await the supplied +`beforeDispatch` callback immediately before the first irreversible spawn +call. Preflight that fails before that callback may throw the public +`NodeSpawnPreDispatchError` to preserve safe retry; using that marker after an +effect may have started is a protocol violation. Existing one-argument +`spawn` methods must be migrated even though JavaScript permits them to ignore +the callback. The required version field makes an unchanged port a type error, +and runtime use fails before claim creation. A port that falsely declares the +version but omits the callback is fenced and refused after its first uncertain +call. + ## Hosted and self-hosted execution Hosted workers use the agent-token fleet path. Self-hosted workers use the diff --git a/packages/sdk/src/agent-relay-transport.ts b/packages/sdk/src/agent-relay-transport.ts index bb7c68304..bb8d1d311 100644 --- a/packages/sdk/src/agent-relay-transport.ts +++ b/packages/sdk/src/agent-relay-transport.ts @@ -1,9 +1,11 @@ import { createHash } from 'node:crypto'; -import { claimNodeSpawnAtPath, nodeSpawnClaimPath, recordNodeSpawn, +import { claimNodeSpawnAtPath, hasNodeSpawnDispatchFence, + nodeSpawnClaimPath, recordNodeSpawn, type NodeSpawnClaim } from './node-spawn-state.js'; +import { dispatchFencedNodeSpawn } from './node-spawn-dispatch.js'; import { nodeResultInstruction, waitForNodeResult } from './node-spawn-result.js'; import { durableNodeResultDelivery } from './node-spawn-settlement.js'; -import { waitForNodeOperation, waitForNodeSpawn } from './node-spawn-deadline.js'; +import { waitForNodeOperation } from './node-spawn-deadline.js'; import { createNodeSpawnRuntime, type NodeSpawnRuntime } from './node-spawn-runtime.js'; import { NodeSpawnError, nonempty, validIdempotencyKey, validNodeSpawnIdentity, validNodeSpawnReceipt, @@ -182,6 +184,8 @@ async function executeAgentRelayTask( taskStartedAt: number, timeoutMs: number, ): Promise { + if (runtime.port.dispatchFenceVersion !== 1) throw new AgentRelayTransportError( + 'The node-spawn port does not implement durable dispatch-fence protocol version 1'); const correlation = { runId: request.runId, stepId: request.stepId, dispatchId: request.idempotencyKey }; const agentName = stableAgentName(request); const task = [ @@ -191,7 +195,7 @@ async function executeAgentRelayTask( nodeResultInstruction(correlation), ].filter(nonempty).join('\n\n'); const proposed: NodeSpawnClaim = { - version: 2, + version: 3, baseUrl: runtime.baseUrl, publisherId: runtime.publisher.id, publisherName: runtime.publisher.name, @@ -216,11 +220,14 @@ async function executeAgentRelayTask( let releaseAttempted = false; const deadline = claim.startedAt + claim.timeoutMs; try { - if (!claimed.created && claim.spawn === undefined) { - if (claim.cleanup === undefined) { - throw new AgentRelayTransportError( - 'A prior node-spawn dispatch ended in the ambiguous pre-ack window; refusing a second spawn'); - } + const dispatchFenced = claim.version === 3 && await hasNodeSpawnDispatchFence(claimed.path, claim); + if (claim.spawn === undefined && claim.cleanup === undefined + && (dispatchFenced || (!claimed.created && claim.version === 2))) { + throw new AgentRelayTransportError('A prior node-spawn dispatch ended in the ambiguous pre-ack window; refusing a second spawn'); + } + // A v3 claim without a dispatch fence proves no irreversible call was + // entered, so a replacement attempt may safely repeat preflight. + if (!claimed.created && claim.spawn === undefined && claim.cleanup !== undefined) { if (claim.release === undefined) { releaseAttempted = true; const cleanup = await tryRelease(runtime, claimed.path, claim, request.idempotencyKey); @@ -237,29 +244,22 @@ async function executeAgentRelayTask( // setup and durable claiming can take time or lose the worker lease. signal?.throwIfAborted(); const remaining = remainingDeadline(deadline); - const pendingSpawn = runtime.port.spawn({ - name: agentName, - harness: request.cli, - task, - ...(request.model === undefined ? {} : { model: request.model }), - ...(request.worker_cwd === undefined ? {} : { directory: request.worker_cwd }), - idempotencyKey: request.idempotencyKey, - ...(runtime.node === undefined ? {} : { node: runtime.node }), - ...(runtime.repo === undefined ? {} : { repo: runtime.repo }), - ...(runtime.sandboxOnly === undefined ? {} : { sandboxOnly: runtime.sandboxOnly }), - timeoutMs: remaining, - }); - const spawnTimeout = AbortSignal.timeout(remaining); - const spawnSignal = signal === undefined - ? spawnTimeout : AbortSignal.any([signal, spawnTimeout]); - const spawn = await waitForNodeSpawn(pendingSpawn, spawnSignal, outcome => { - void settleLateSpawn(outcome).catch(() => undefined); + const spawn = await dispatchFencedNodeSpawn({ + port: runtime.port, + request: { + name: agentName, + harness: request.cli, + task, + ...(request.model === undefined ? {} : { model: request.model }), + ...(request.worker_cwd === undefined ? {} : { directory: request.worker_cwd }), + idempotencyKey: request.idempotencyKey, + ...(runtime.node === undefined ? {} : { node: runtime.node }), + ...(runtime.repo === undefined ? {} : { repo: runtime.repo }), + ...(runtime.sandboxOnly === undefined ? {} : { sandboxOnly: runtime.sandboxOnly }), + timeoutMs: remaining, + }, + claimPath: claimed.path, claim, signal, settleLate: settleLateSpawn, }); - if (!validNodeSpawnReceipt(spawn) || spawn.name !== agentName) { - throw new NodeSpawnError('The node spawn did not return a valid ready identity', - 'spawn_identity_mismatch', undefined, - validNodeSpawnIdentity(spawn) ? spawn : undefined); - } claim = { ...claim, spawn }; await recordNodeSpawn(claimed.path, claim); signal?.throwIfAborted(); diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 1918883d1..b7a0e5b91 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -243,6 +243,8 @@ export { JournalClient, JournalRequestTimeoutError, type JournalClientOptions } export { AgentWorker, type AgentWorkerOptions } from './worker.js'; export { NodeSpawnError, + NodeSpawnPreDispatchError, + type BeforeNodeDispatch, type NodeReleaseRequest, type NodeSpawnAction, type NodeSpawnInvocation, diff --git a/packages/sdk/src/node-spawn-dispatch.ts b/packages/sdk/src/node-spawn-dispatch.ts new file mode 100644 index 000000000..ab97a6561 --- /dev/null +++ b/packages/sdk/src/node-spawn-dispatch.ts @@ -0,0 +1,78 @@ +import { waitForNodeSpawn } from './node-spawn-deadline.js'; +import { claimNodeSpawnDispatch, hasNodeSpawnDispatchFence, + type NodeSpawnClaim } from './node-spawn-state.js'; +import { NodeSpawnError, NodeSpawnPreDispatchError, + validNodeSpawnIdentity, validNodeSpawnReceipt, + type NodeSpawnPort, type NodeSpawnReceipt, type NodeSpawnRequest } from './node-spawn.js'; + +interface FencedNodeSpawnOptions { + port: NodeSpawnPort; + request: NodeSpawnRequest & { timeoutMs: number }; + claimPath: string; + claim: NodeSpawnClaim; + signal?: AbortSignal; + settleLate(outcome: { spawn?: NodeSpawnReceipt; error?: unknown }): Promise; +} + +/** Enters exactly one irreversible spawn behind a durable, one-way fence. */ +export async function dispatchFencedNodeSpawn( + options: FencedNodeSpawnOptions, +): Promise { + let dispatchPublished = false; + let dispatchHookEntered = false; + let pendingSpawn: Promise; + try { + pendingSpawn = options.port.spawn(options.request, async () => { + dispatchHookEntered = true; + options.signal?.throwIfAborted(); + await claimNodeSpawnDispatch(options.claimPath, options.claim); + dispatchPublished = true; + }); + } catch (error) { + pendingSpawn = Promise.reject(error); + } + const spawnTimeout = AbortSignal.timeout(options.request.timeoutMs); + const spawnSignal = options.signal === undefined + ? spawnTimeout : AbortSignal.any([options.signal, spawnTimeout]); + let spawn: NodeSpawnReceipt; + try { + spawn = await waitForNodeSpawn(pendingSpawn, spawnSignal, outcome => { + void options.settleLate(outcome).catch(() => undefined); + }); + } catch (error) { + if (!dispatchPublished && dispatchHookEntered) { + // Publication may fail before the atomic link (safe to retry) or after + // it (permanently ambiguous). Observe once without creating a fence. + dispatchPublished = await hasNodeSpawnDispatchFence(options.claimPath, options.claim); + } else if (!dispatchPublished && !(error instanceof NodeSpawnPreDispatchError)) { + // An untrusted port did not prove that it stopped before dispatch. + await claimNodeSpawnDispatch(options.claimPath, options.claim); + dispatchPublished = true; + } + throw error; + } + if (!dispatchPublished) { + await claimNodeSpawnDispatch(options.claimPath, options.claim); + dispatchPublished = true; + throw new NodeSpawnError( + 'The node-spawn port returned without publishing its required dispatch fence', + 'spawn_dispatch_fence_missing', + undefined, + validNodeSpawnIdentity(spawn) ? spawn : undefined, + ); + } + if (!await hasNodeSpawnDispatchFence(options.claimPath, options.claim)) { + throw new NodeSpawnError( + 'The node-spawn port did not leave its required dispatch fence durable', + 'spawn_dispatch_fence_missing', + undefined, + validNodeSpawnIdentity(spawn) ? spawn : undefined, + ); + } + if (!validNodeSpawnReceipt(spawn) || spawn.name !== options.request.name) { + throw new NodeSpawnError('The node spawn did not return a valid ready identity', + 'spawn_identity_mismatch', undefined, + validNodeSpawnIdentity(spawn) ? spawn : undefined); + } + return spawn; +} diff --git a/packages/sdk/src/node-spawn-local.ts b/packages/sdk/src/node-spawn-local.ts index 64dd92eab..4dc21e81e 100644 --- a/packages/sdk/src/node-spawn-local.ts +++ b/packages/sdk/src/node-spawn-local.ts @@ -2,6 +2,7 @@ import { isAbsolute, relative, sep } from 'node:path'; import { NodeSpawnError, + NodeSpawnPreDispatchError, normalizeNodeSpawnStatus, terminalStatus, type NodeReleaseRequest, @@ -13,6 +14,7 @@ import { type NodeSpawnRequest, type NodeSpawnTarget, validIdempotencyKey, + type BeforeNodeDispatch, } from './node-spawn.js'; /** The structural core-bridge surface Flows consumes. Kept here so the package stays optional. */ @@ -206,6 +208,8 @@ export async function loadCoreBridge(): Promise { /** Native local spawn adapter over one already authenticated schema-v2 bridge connection. */ export class LocalNodeSpawnPort implements NodeSpawnPort { + readonly dispatchFenceVersion = 1 as const; + constructor( private readonly core: CoreBridgeClient, private readonly bridge: Pick, @@ -239,36 +243,45 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { }); } - async spawn(request: NodeSpawnRequest): Promise { - if (!request.directory || !isAbsolute(request.directory)) { - throw new NodeSpawnError('Local node spawn requires an absolute directory', 'spawn_invalid_input'); - } - if (!this.scope.harnesses.includes(request.harness) - || !this.scope.directoryScope.some(root => withinDirectory(root, request.directory!))) { - throw new NodeSpawnError( - 'The Desktop service caller does not authorize this harness and directory', - 'outside_scope', - ); - } - if (!validIdempotencyKey(request.idempotencyKey)) { - throw new NodeSpawnError('Invalid node-spawn idempotency key', 'spawn_invalid_input'); - } - if (request.model !== undefined || request.repo !== undefined || request.sandboxOnly !== undefined) { - throw new NodeSpawnError( - 'The Desktop core bridge does not accept model, repository, or sandbox placement overrides', - 'spawn_invalid_input', - ); - } - const allTargets = await this.targets(); - if (request.node !== undefined && request.node !== this.exactNode.name) { - throw new NodeSpawnError('The requested node differs from the authenticated Desktop node', - 'spawn_identity_mismatch'); + async spawn(request: NodeSpawnRequest, beforeDispatch: BeforeNodeDispatch): Promise { + let allTargets: NodeSpawnTarget[]; + let target: NodeSpawnTarget | undefined; + try { + if (!request.directory || !isAbsolute(request.directory)) { + throw new NodeSpawnError('Local node spawn requires an absolute directory', 'spawn_invalid_input'); + } + if (!this.scope.harnesses.includes(request.harness) + || !this.scope.directoryScope.some(root => withinDirectory(root, request.directory!))) { + throw new NodeSpawnError( + 'The Desktop service caller does not authorize this harness and directory', + 'outside_scope', + ); + } + if (!validIdempotencyKey(request.idempotencyKey)) { + throw new NodeSpawnError('Invalid node-spawn idempotency key', 'spawn_invalid_input'); + } + if (request.model !== undefined || request.repo !== undefined || request.sandboxOnly !== undefined) { + throw new NodeSpawnError( + 'The Desktop core bridge does not accept model, repository, or sandbox placement overrides', + 'spawn_invalid_input', + ); + } + allTargets = await this.targets(); + if (request.node !== undefined && request.node !== this.exactNode.name) { + throw new NodeSpawnError('The requested node differs from the authenticated Desktop node', + 'spawn_identity_mismatch'); + } + target = allTargets.find(candidate => candidate.id === this.exactNode.id + && candidate.name === this.exactNode.name + && candidate.live && candidate.handlersLive && !candidate.stale + && candidate.capabilities.includes(`spawn:${request.harness}`)); + if (!target?.id) { + throw new NodeSpawnError('No eligible local Agent Relay spawn target', 'no_eligible_node'); + } + } catch (error) { + throw new NodeSpawnPreDispatchError(error); } - const target = allTargets.find(candidate => candidate.id === this.exactNode.id - && candidate.name === this.exactNode.name - && candidate.live && candidate.handlersLive && !candidate.stale - && candidate.capabilities.includes(`spawn:${request.harness}`)); - if (!target?.id) throw new NodeSpawnError('No eligible local Agent Relay spawn target', 'no_eligible_node'); + await beforeDispatch(); const raw = await this.bridge.spawnInvoke(this.core, { node_id: target.id, node_name: target.name, diff --git a/packages/sdk/src/node-spawn-remote.ts b/packages/sdk/src/node-spawn-remote.ts index 6f9fe11b9..bc8ec39e9 100644 --- a/packages/sdk/src/node-spawn-remote.ts +++ b/packages/sdk/src/node-spawn-remote.ts @@ -1,11 +1,13 @@ import { setTimeout as delay } from 'node:timers/promises'; import { NodeSpawnError, + NodeSpawnPreDispatchError, isRecord, normalizeNodeSpawnStatus, nonempty, terminalStatus, validIdempotencyKey, + type BeforeNodeDispatch, type NodeReleaseRequest, type NodeSpawnAction, type NodeSpawnIdentity, @@ -48,6 +50,8 @@ export interface RemoteNodeSpawnClient { /** Fleet node-spawn adapter backed only by the supported Agent Relay SDK. */ export class RemoteNodeSpawnPort implements NodeSpawnPort { + readonly dispatchFenceVersion = 1 as const; + constructor(private readonly client: RemoteNodeSpawnClient) {} async targets(): Promise { @@ -62,13 +66,15 @@ export class RemoteNodeSpawnPort implements NodeSpawnPort { })); } - async spawn(request: NodeSpawnRequest): Promise { + async spawn(request: NodeSpawnRequest, beforeDispatch: BeforeNodeDispatch): Promise { if (!validIdempotencyKey(request.idempotencyKey)) { - throw new NodeSpawnError('Invalid node-spawn idempotency key', 'spawn_invalid_input'); + throw new NodeSpawnPreDispatchError( + new NodeSpawnError('Invalid node-spawn idempotency key', 'spawn_invalid_input')); } const capability = `spawn:${request.harness}`; let ack: PlacementAck; try { + await beforeDispatch(); ack = await this.client.placement.spawn({ capability, ...(request.node === undefined ? {} : { node: request.node }), diff --git a/packages/sdk/src/node-spawn-state.ts b/packages/sdk/src/node-spawn-state.ts index 4f5c9c7c7..8127d6ec4 100644 --- a/packages/sdk/src/node-spawn-state.ts +++ b/packages/sdk/src/node-spawn-state.ts @@ -7,7 +7,7 @@ import { isRecord, nonempty, validIdempotencyKey, validNodeSpawnIdentity, validN type NodeSpawnIdentity, type NodeSpawnReceipt } from './node-spawn.js'; export interface NodeSpawnClaim { - version: 2; + version: 2 | 3; baseUrl: string; publisherId: string; publisherName: string; @@ -29,6 +29,19 @@ export interface NodeSpawnClaim { release?: { invocationId: string; status: string }; } +interface NodeSpawnDispatchFence { + version: 1; + claimDigest: string; +} + +/** A durable write-ahead fence already owns the only permitted dispatch. */ +export class NodeSpawnDispatchFencedError extends Error { + constructor() { + super('A prior node-spawn dispatch entered the ambiguous pre-ack window; refusing a second spawn'); + this.name = 'NodeSpawnDispatchFencedError'; + } +} + export async function claimNodeSpawn(dataDir: string, proposed: NodeSpawnClaim): Promise<{ claim: NodeSpawnClaim; created: boolean; path: string; }> { @@ -78,6 +91,63 @@ export async function nodeSpawnClaimPath( return join(canonicalDataDir, 'node-spawns', `${digest}.json`); } +export function nodeSpawnDispatchFencePath(claimPath: string): string { + return `${claimPath}.dispatch`; +} + +/** Reads and validates the one-way dispatch fence for a v3 claim. */ +export async function hasNodeSpawnDispatchFence( + claimPath: string, + claim: NodeSpawnClaim, +): Promise { + const fencePath = nodeSpawnDispatchFencePath(claimPath); + let text: string; + try { text = await readFile(fencePath, 'utf8'); } + catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; + throw invalidDispatchFence(fencePath, error); + } + let value: unknown; + try { value = JSON.parse(text) as unknown; } + catch (error) { throw invalidDispatchFence(fencePath, error); } + decodeDispatchFence(value, fencePath, claim); + return true; +} + +/** + * Atomically publishes the write-ahead fact immediately before dispatch. + * The marker is permanent: a crash after it is indistinguishable from a sent + * request, so no later worker may publish or dispatch again. + */ +export async function claimNodeSpawnDispatch( + claimPath: string, + claim: NodeSpawnClaim, +): Promise { + if (claim.version !== 3) throw new Error('Legacy node-spawn claims cannot enter the fenced dispatch protocol'); + const fencePath = nodeSpawnDispatchFencePath(claimPath); + const directory = dirname(fencePath); + const temporary = `${fencePath}.${randomUUID()}.tmp`; + const fence: NodeSpawnDispatchFence = { + version: 1, + claimDigest: dispatchClaimDigest(claim), + }; + const file = await open(temporary, 'wx', 0o600); + try { + try { await file.writeFile(canonicalize(fence)); await file.sync(); } + finally { await file.close(); } + try { + await link(temporary, fencePath); + await syncDirectory(directory); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + await hasNodeSpawnDispatchFence(claimPath, claim); + throw new NodeSpawnDispatchFencedError(); + } + } finally { + await unlink(temporary).catch(() => undefined); + } +} + export async function recordNodeSpawn(path: string, claim: NodeSpawnClaim): Promise { const temporary = `${path}.${randomUUID()}.tmp`; const file = await open(temporary, 'wx', 0o600); @@ -116,23 +186,25 @@ export async function readNodeSpawnClaim(path: string): Promise } function sameClaim(prior: NodeSpawnClaim, proposed: NodeSpawnClaim): boolean { - const { spawn: _priorSpawn, cleanup: _priorCleanup, failure: _priorFailure, + const { version: _priorVersion, + spawn: _priorSpawn, cleanup: _priorCleanup, failure: _priorFailure, result: _priorResult, cleanupPending: _priorCleanupPending, release: _priorRelease, ...priorIdentity } = prior; - const { spawn: _spawn, cleanup: _cleanup, failure: _failure, result: _result, + const { version: _version, + spawn: _spawn, cleanup: _cleanup, failure: _failure, result: _result, cleanupPending: _cleanupPending, release: _release, ...proposedIdentity } = proposed; const { task: _priorRenderedTask, ...priorRequestIdentity } = priorIdentity.request; const { task: _renderedTask, ...proposedRequestIdentity } = proposedIdentity.request; - return canonicalize({ ...priorIdentity, request: priorRequestIdentity, + return canonicalize({ ...priorIdentity, version: proposed.version, request: priorRequestIdentity, startedAt: proposed.startedAt }) === canonicalize({ - ...proposedIdentity, request: proposedRequestIdentity, + ...proposedIdentity, version: proposed.version, request: proposedRequestIdentity, }); } function decodeNodeSpawnClaim(value: unknown, label: string): NodeSpawnClaim { const invalid = (): never => { throw new Error(`Invalid durable node-spawn claim: ${label}`); }; const record = isRecord(value) ? value : invalid(); - if (record.version !== 2 + if ((record.version !== 2 && record.version !== 3) || ![record.baseUrl, record.publisherId, record.publisherName, record.runId, record.stepId, record.agentName] .every(nonempty) || !validIdempotencyKey(record.idempotencyKey) @@ -185,6 +257,28 @@ function decodeNodeSpawnClaim(value: unknown, label: string): NodeSpawnClaim { return claim; } +function dispatchClaimDigest(claim: NodeSpawnClaim): string { + const { spawn: _spawn, cleanup: _cleanup, failure: _failure, + result: _result, cleanupPending: _cleanupPending, release: _release, + ...identity } = claim; + return createHash('sha256').update(canonicalize(identity)).digest('hex'); +} + +function decodeDispatchFence(value: unknown, label: string, claim: NodeSpawnClaim): NodeSpawnDispatchFence { + const record = isRecord(value) ? value : undefined; + if (record?.version !== 1 + || typeof record.claimDigest !== 'string' + || !/^[a-f0-9]{64}$/.test(record.claimDigest) + || record.claimDigest !== dispatchClaimDigest(claim)) { + throw invalidDispatchFence(label); + } + return record as unknown as NodeSpawnDispatchFence; +} + +function invalidDispatchFence(label: string, cause?: unknown): Error { + return new Error(`Invalid durable node-spawn dispatch fence: ${label}`, { cause }); +} + const safeTime = (value: unknown): value is number => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; diff --git a/packages/sdk/src/node-spawn.ts b/packages/sdk/src/node-spawn.ts index c6568b777..6fabd4920 100644 --- a/packages/sdk/src/node-spawn.ts +++ b/packages/sdk/src/node-spawn.ts @@ -67,10 +67,15 @@ export interface NodeReleaseRequest { reason?: string; } +/** Publishes the durable write-ahead fact immediately before an irreversible spawn call. */ +export type BeforeNodeDispatch = () => Promise; + /** Credentials and transport stay inside an implementation of this port. */ export interface NodeSpawnPort { + /** Proves this port implements the required durable pre-dispatch callback protocol. */ + readonly dispatchFenceVersion: 1; targets(): Promise; - spawn(request: NodeSpawnRequest): Promise; + spawn(request: NodeSpawnRequest, beforeDispatch: BeforeNodeDispatch): Promise; status(action: NodeSpawnAction, invocationId: string): Promise; release(request: NodeReleaseRequest): Promise; } @@ -88,6 +93,18 @@ export class NodeSpawnError extends Error { } } +/** An official port proved that it failed before entering its dispatch hook. */ +export class NodeSpawnPreDispatchError extends NodeSpawnError { + constructor(error: unknown) { + super( + error instanceof Error ? error.message : 'Node-spawn preflight failed', + error instanceof NodeSpawnError ? error.code : 'node_spawn_preflight_failed', + error, + ); + this.name = 'NodeSpawnPreDispatchError'; + } +} + export const isRecord = (value: unknown): value is Record => typeof value === 'object' && value !== null && !Array.isArray(value); diff --git a/packages/sdk/tests/agent-relay-transport.test.ts b/packages/sdk/tests/agent-relay-transport.test.ts index ffabfd233..7713db9bf 100644 --- a/packages/sdk/tests/agent-relay-transport.test.ts +++ b/packages/sdk/tests/agent-relay-transport.test.ts @@ -33,14 +33,17 @@ function fixture(result: 'completed' | 'failed' = 'completed'): { runtime: NodeS id: 'inv_spawn_1', action: 'spawn', status: 'completed', nodeId: 'node_1', nodeName: 'desktop-1', name: 'placeholder', agentId: 'agent_1', spawned: true, ready: true, }; - const spawn = vi.fn(async (input: { name: string }) => ({ ...receipt, name: input.name })); + const spawn = vi.fn(async (input, beforeDispatch) => { + await beforeDispatch(); + return { ...receipt, name: input.name }; + }); const release = vi.fn(async () => ({ id: 'inv_release_1', action: 'release' as const, status: 'completed' as const, spawned: false, ready: false, released: true })); const status = vi.fn(async () => ({ ...receipt, name: receipt.name })); const ack = vi.fn(async () => ({})); const fail = vi.fn(async () => ({})); const port: NodeSpawnPort = { - targets: async () => [], spawn, status, release, + dispatchFenceVersion: 1, targets: async () => [], spawn, status, release, }; const runtime: NodeSpawnRuntime = { port, @@ -176,7 +179,10 @@ describe('native node-spawn task transport', () => { it('fails closed across an ambiguous pre-ack crash instead of spawning twice', async () => { const dataDir = await dataDirectory('flows-node-ambiguous-'); const f = fixture(); - f.spawn.mockRejectedValueOnce(new Error('connection dropped')); + f.spawn.mockImplementationOnce(async (_input, beforeDispatch) => { + await beforeDispatch(); + throw new Error('connection dropped'); + }); await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })).rejects.toThrow('connection dropped'); await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })).rejects.toThrow('ambiguous pre-ack'); expect(f.spawn).toHaveBeenCalledTimes(1); @@ -185,11 +191,14 @@ describe('native node-spawn task transport', () => { it('durably releases exact identity attached to a post-spawn verification failure', async () => { const dataDir = await dataDirectory('flows-node-postspawn-failure-'); const f = fixture(); - f.spawn.mockRejectedValueOnce(new NodeSpawnError('task delivery unconfirmed', - 'spawn_task_unconfirmed', undefined, { + f.spawn.mockImplementationOnce(async (_input, beforeDispatch) => { + await beforeDispatch(); + throw new NodeSpawnError('task delivery unconfirmed', + 'spawn_task_unconfirmed', undefined, { id: 'inv_spawn_1', nodeId: 'node_1', nodeName: 'desktop-1', name: 'worker_1', agentId: 'agent_1', - })); + }); + }); await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) .rejects.toThrow('task delivery unconfirmed'); expect(f.release).toHaveBeenCalledWith(expect.objectContaining({ @@ -205,7 +214,8 @@ describe('native node-spawn task transport', () => { const dataDir = await dataDirectory('flows-node-cancel-'); const f = fixture(); const controller = new AbortController(); - f.spawn.mockImplementationOnce(async (input: { name: string }) => { + f.spawn.mockImplementationOnce(async (input, beforeDispatch) => { + await beforeDispatch(); queueMicrotask(() => controller.abort(new Error('lease lost'))); return { id: 'inv_spawn_1', action: 'spawn' as const, status: 'completed' as const, nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, agentId: 'agent_1', @@ -240,7 +250,8 @@ describe('native node-spawn task transport', () => { const dataDir = await dataDirectory('flows-node-deadline-identity-'); const f = fixture(); const controller = new AbortController(); - f.spawn.mockImplementationOnce(async (input: { name: string }) => { + f.spawn.mockImplementationOnce(async (input, beforeDispatch) => { + await beforeDispatch(); queueMicrotask(() => controller.abort(new Error('worker stopped'))); return { id: 'inv_spawn_1', action: 'spawn' as const, status: 'completed' as const, nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, agentId: 'agent_1', @@ -284,9 +295,12 @@ describe('native node-spawn task transport', () => { const f = fixture(); const controller = new AbortController(); let confirm!: (receipt: NodeSpawnReceipt) => void; - f.spawn.mockImplementationOnce((input: { name: string }) => new Promise(resolve => { + f.spawn.mockImplementationOnce(async (input, beforeDispatch) => { + await beforeDispatch(); + return new Promise(resolve => { confirm = receipt => resolve({ ...receipt, name: input.name }); - })); + }); + }); const pending = runAgentRelayTask(request(dataDir), { runtime: f.runtime, signal: controller.signal }); await vi.waitFor(() => expect(f.spawn).toHaveBeenCalledOnce()); controller.abort(new Error('lease lost during confirmation')); diff --git a/packages/sdk/tests/node-spawn-dispatch-fence.test.ts b/packages/sdk/tests/node-spawn-dispatch-fence.test.ts new file mode 100644 index 000000000..18b147aa1 --- /dev/null +++ b/packages/sdk/tests/node-spawn-dispatch-fence.test.ts @@ -0,0 +1,291 @@ +import { chmod, mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { runAgentRelayTask, type AgentRelayTaskRequest } from '../src/agent-relay-transport.js'; +import { NODE_RESULT_PREFIX } from '../src/node-spawn-result.js'; +import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; +import { claimNodeSpawnDispatch, nodeSpawnDispatchFencePath, readNodeSpawnClaims, + recordNodeSpawn } from '../src/node-spawn-state.js'; +import { NodeSpawnError, NodeSpawnPreDispatchError, + type NodeSpawnPort, type NodeSpawnReceipt } from '../src/index.js'; + +const roots: string[] = []; + +afterEach(async () => { + vi.useRealTimers(); + await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); +}); + +async function dataDirectory(label: string): Promise { + const directory = await mkdtemp(join(tmpdir(), label)); + roots.push(directory); + return directory; +} + +function request(dataDir: string, timeoutMs = 10_000): AgentRelayTaskRequest { + return { + cli: 'codex', task: 'Implement the change', runId: 'run_fence', stepId: 'edit', + idempotencyKey: 'dispatch_fence123', attempt: 1, dataDir, timeoutMs, + }; +} + +function readyReceipt(name: string): NodeSpawnReceipt { + return { + id: 'inv_spawn_1', action: 'spawn', status: 'completed', nodeId: 'node_1', + nodeName: 'desktop-1', name, agentId: 'agent_1', spawned: true, ready: true, + }; +} + +function runtime(spawn: NodeSpawnPort['spawn']): NodeSpawnRuntime { + let spawnedName = ''; + const wrapped = vi.fn(async (input, beforeDispatch) => { + const receipt = await spawn(input, beforeDispatch); + spawnedName = receipt.name; + return receipt; + }); + return { + authority: 'desktop', + publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + port: { + dispatchFenceVersion: 1, + targets: async () => [], + spawn: wrapped, + status: vi.fn(), + release: vi.fn(async () => ({ + id: 'inv_release_1', action: 'release' as const, status: 'completed' as const, + spawned: false, ready: false, released: true, + })), + }, + inbox: { inbox: { + subscribe: async function* () { + yield { id: 'delivery_1', message: { id: 'message_1', + from: { id: 'agent_1', name: spawnedName }, + text: NODE_RESULT_PREFIX + JSON.stringify({ + version: 1, run_id: 'run_fence', step_id: 'edit', + dispatch_id: 'dispatch_fence123', status: 'completed', output: { ok: true }, + }) } }; + }, + ack: vi.fn(async () => ({})), + fail: vi.fn(async () => ({})), + } }, + }; +} + +const preDispatchFailure = (message: string, code = 'no_eligible_node'): NodeSpawnPreDispatchError => + new NodeSpawnPreDispatchError(new NodeSpawnError(message, code)); + +describe('durable node-spawn dispatch fence', () => { + it('lets a public custom port attest safe preflight and retry once when eligible', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-retry-'); + let eligible = false; + let invokes = 0; + const worker = runtime(async (input, beforeDispatch) => { + if (!eligible) throw preDispatchFailure('no eligible node'); + await beforeDispatch(); + invokes += 1; + return readyReceipt(input.name); + }); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('no eligible node'); + const [claim] = await readNodeSpawnClaims(dataDir); + expect(claim!.claim.version).toBe(3); + await expect(readFile(nodeSpawnDispatchFencePath(claim!.path), 'utf8')) + .rejects.toMatchObject({ code: 'ENOENT' }); + + eligible = true; + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .resolves.toMatchObject({ receipt: { status: 'completed' } }); + expect(invokes).toBe(1); + }); + + it('rejects an old public custom port before it creates a claim or invokes', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-old-port-'); + const spawn = vi.fn(async (input: Parameters[0]) => readyReceipt(input.name)); + const legacyPort = { + targets: async () => [], + spawn, + status: vi.fn(), + release: vi.fn(), + }; + // The public contract makes the migration compile-time visible too. + // @ts-expect-error dispatchFenceVersion is required for the fenced protocol. + const incompatible: NodeSpawnPort = legacyPort; + const worker = runtime(async () => readyReceipt('unused')); + for (const incompatiblePort of [ + incompatible, + { ...legacyPort, dispatchFenceVersion: 2 } as unknown as NodeSpawnPort, + ]) { + worker.port = incompatiblePort; + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('does not implement durable dispatch-fence protocol version 1'); + } + expect(spawn).not.toHaveBeenCalled(); + await expect(readNodeSpawnClaims(dataDir)).resolves.toEqual([]); + }); + + it('permanently refuses retry after the port crossed the fence and then failed', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-post-'); + let invokes = 0; + const worker = runtime(async (_input, beforeDispatch) => { + await beforeDispatch(); + invokes += 1; + throw new Error('connection dropped after invoke'); + }); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('connection dropped after invoke'); + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('ambiguous pre-ack'); + expect(invokes).toBe(1); + }); + + it('lets exactly one independent fence contender enter its irreversible call', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-race-'); + const worker = runtime(async () => { throw preDispatchFailure('prepare claim'); }); + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })).rejects.toThrow('prepare claim'); + const [entry] = await readNodeSpawnClaims(dataDir); + let invokes = 0; + const contender = async (): Promise => { + await claimNodeSpawnDispatch(entry!.path, entry!.claim); + invokes += 1; + }; + + const results = await Promise.allSettled([contender(), contender()]); + expect(results.filter(result => result.status === 'fulfilled')).toHaveLength(1); + expect(results.filter(result => result.status === 'rejected')).toHaveLength(1); + expect(invokes).toBe(1); + }); + + it('keeps a receipt-less legacy v2 claim ambiguous', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-legacy-'); + let calls = 0; + const worker = runtime(async () => { + calls += 1; + throw preDispatchFailure('prepare legacy claim'); + }); + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('prepare legacy claim'); + const [entry] = await readNodeSpawnClaims(dataDir); + await recordNodeSpawn(entry!.path, { ...entry!.claim, version: 2 }); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('ambiguous pre-ack'); + expect(calls).toBe(1); + }); + + it('rejects a string claim protocol version instead of coercing it', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-string-version-'); + let calls = 0; + const worker = runtime(async () => { + calls += 1; + throw preDispatchFailure('prepare malformed claim'); + }); + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('prepare malformed claim'); + const [entry] = await readNodeSpawnClaims(dataDir); + await writeFile(entry!.path, JSON.stringify({ ...entry!.claim, version: '3' })); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('Invalid durable node-spawn claim'); + expect(calls).toBe(1); + }); + + it.each([ + { label: 'malformed JSON', content: '{' }, + { label: 'wrong claim digest', content: JSON.stringify({ version: 1, claimDigest: '0'.repeat(64) }) }, + ])('fails closed on a $label dispatch fence', async ({ content }) => { + const dataDir = await dataDirectory('flows-dispatch-fence-invalid-'); + let calls = 0; + const worker = runtime(async () => { + calls += 1; + throw preDispatchFailure('prepare invalid fence'); + }); + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('prepare invalid fence'); + const [entry] = await readNodeSpawnClaims(dataDir); + await writeFile(nodeSpawnDispatchFencePath(entry!.path), content); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('Invalid durable node-spawn dispatch fence'); + expect(calls).toBe(1); + }); + + it('treats a non-ENOENT fence read failure as invalid durable state', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-read-failure-'); + let calls = 0; + const worker = runtime(async () => { + calls += 1; + throw preDispatchFailure('prepare unreadable fence'); + }); + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('prepare unreadable fence'); + const [entry] = await readNodeSpawnClaims(dataDir); + await mkdir(nodeSpawnDispatchFencePath(entry!.path)); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('Invalid durable node-spawn dispatch fence'); + expect(calls).toBe(1); + }); + + it('does not call the remote effect when dispatch-fence publication fails', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-write-failure-'); + let prepare = true; + let remoteCalls = 0; + const worker = runtime(async (input, beforeDispatch) => { + if (prepare) throw preDispatchFailure('prepare fence failure'); + await beforeDispatch(); + remoteCalls += 1; + return readyReceipt(input.name); + }); + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('prepare fence failure'); + const [entry] = await readNodeSpawnClaims(dataDir); + prepare = false; + await chmod(dirname(entry!.path), 0o500); + try { + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toMatchObject({ code: 'EACCES' }); + } finally { + await chmod(dirname(entry!.path), 0o700); + } + expect(remoteCalls).toBe(0); + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .resolves.toMatchObject({ receipt: { status: 'completed' } }); + expect(remoteCalls).toBe(1); + }); + + it('retains the original claim deadline across a safe pre-dispatch retry', async () => { + vi.useFakeTimers(); + const dataDir = await dataDirectory('flows-dispatch-fence-deadline-'); + let calls = 0; + const worker = runtime(async () => { + calls += 1; + throw preDispatchFailure('not ready yet'); + }); + await expect(runAgentRelayTask(request(dataDir, 1_000), { runtime: worker })) + .rejects.toThrow('not ready yet'); + await vi.advanceTimersByTimeAsync(1_001); + + await expect(runAgentRelayTask(request(dataDir, 1_000), { runtime: worker })) + .rejects.toThrow('deadline elapsed'); + expect(calls).toBe(1); + }); + + it('fences a custom port that returns a receipt without awaiting the required hook', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-bypass-'); + let calls = 0; + const worker = runtime(async input => { + calls += 1; + return readyReceipt(input.name); + }); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('required dispatch fence'); + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('required dispatch fence'); + expect(calls).toBe(1); + }); +}); diff --git a/packages/sdk/tests/node-spawn-live.test.ts b/packages/sdk/tests/node-spawn-live.test.ts index bb86cf784..3f2458a76 100644 --- a/packages/sdk/tests/node-spawn-live.test.ts +++ b/packages/sdk/tests/node-spawn-live.test.ts @@ -78,9 +78,11 @@ describe('node-spawn transport through the real journal', () => { baseUrl: 'https://cast.agentrelay.com', publisher: { id: 'publisher_1', name: 'flows-publisher' }, port: { + dispatchFenceVersion: 1, targets: async () => [], - spawn: async (input) => { + spawn: async (input, beforeDispatch) => { await runReady; + await beforeDispatch(); dispatchId = input.idempotencyKey; const receipt: NodeSpawnReceipt = { id: 'spawn_1', diff --git a/packages/sdk/tests/node-spawn-ports.test.ts b/packages/sdk/tests/node-spawn-ports.test.ts index 2ed2ae101..2c8404a90 100644 --- a/packages/sdk/tests/node-spawn-ports.test.ts +++ b/packages/sdk/tests/node-spawn-ports.test.ts @@ -11,6 +11,8 @@ interface TestCore { call(method: string, params?: Record): Promise; } +const beforeDispatch = async (): Promise => undefined; + function localPort(core: TestCore): LocalNodeSpawnPort { const client = { hello: { bridge_version: 2, daemon_version: 'test', pid: 1, methods: [], topics: [] }, @@ -31,9 +33,11 @@ function localPort(core: TestCore): LocalNodeSpawnPort { describe('local node-spawn port', () => { it('uses bound bridge v2 spawn methods with no raw caller credential and exact release identity', async () => { const calls: Array<[string, Record | undefined]> = []; + const dispatchOrder: string[] = []; const core: TestCore = { async call(method: string, params?: Record): Promise { calls.push([method, params]); + if (method === 'spawn.invoke') dispatchOrder.push('spawn.invoke'); if (method === 'spawn.targets') return { targets: [{ id: 'node_1', name: 'desktop-1', live: true, handlers_live: true, stale: false, capabilities: ['spawn:codex'], active_agents: 0, reserved_agents: 0, max_agents: 2 }] } as T; @@ -45,13 +49,16 @@ describe('local node-spawn port', () => { }; const port = localPort(core); const receipt = await port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', - directory: '/workspace', idempotencyKey: 'dispatch_1234567890' }); + directory: '/workspace', idempotencyKey: 'dispatch_1234567890' }, async () => { + dispatchOrder.push('dispatch.fence'); + }); await port.release({ receipt, idempotencyKey: 'release_1234567890' }); expect(calls[1]).toEqual(['spawn.invoke', expect.objectContaining({ node_id: 'node_1', name: 'worker_1' })]); expect(calls[2]).toEqual(['spawn.status', expect.objectContaining({ invocation_id: 'inv_1' })]); expect(calls[3]).toEqual(['spawn.release', expect.objectContaining({ expected_agent_id: 'agent_1', node_id: 'node_1', node_name: 'desktop-1', name: 'worker_1', idempotency_key: 'release_1234567890' })]); + expect(dispatchOrder).toEqual(['dispatch.fence', 'spawn.invoke']); for (const [, params] of calls) { if (params === undefined) continue; expect(params).not.toHaveProperty('caller_name'); @@ -64,7 +71,7 @@ describe('local node-spawn port', () => { const core: TestCore = { async call(): Promise { calls(); throw new Error('unused'); } }; const port = localPort(core); await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', directory: '/workspace', - model: 'codex-medium', idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ + model: 'codex-medium', idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)).rejects.toMatchObject({ code: 'spawn_invalid_input', }); expect(calls).not.toHaveBeenCalled(); @@ -78,11 +85,33 @@ describe('local node-spawn port', () => { const calls = vi.fn(); const port = localPort({ async call(): Promise { calls(); throw new Error('unused'); } }); await expect(port.spawn({ name: 'worker_1', harness: request.harness, task: 'work', - directory: request.directory, idempotencyKey: 'dispatch_1234567890' })) + directory: request.directory, idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)) .rejects.toMatchObject({ code: 'outside_scope' }); expect(calls).not.toHaveBeenCalled(); }); + it.each([ + { label: 'invalid input', request: { directory: undefined }, targetReadFails: false }, + { label: 'invalid scope', request: { directory: '/workspace', harness: 'claude' }, targetReadFails: false }, + { label: 'target read failure', request: { directory: '/workspace' }, targetReadFails: true }, + { label: 'target mismatch', request: { directory: '/workspace', node: 'desktop-other' }, targetReadFails: false }, + ])('does not enter the dispatch hook for $label', async testCase => { + const entered = vi.fn(async () => undefined); + const core: TestCore = { async call(method: string): Promise { + if (method !== 'spawn.targets') throw new Error('spawn must not be invoked'); + if (testCase.targetReadFails) throw new Error('target read failed'); + return { targets: [{ id: 'node_1', name: 'desktop-1', live: true, + handlers_live: true, stale: false, capabilities: ['spawn:codex'] }] } as T; + } }; + await expect(localPort(core).spawn({ + name: 'worker_1', harness: testCase.request.harness ?? 'codex', task: 'work', + idempotencyKey: 'dispatch_1234567890', + ...(testCase.request.directory === undefined ? {} : { directory: testCase.request.directory }), + ...(testCase.request.node === undefined ? {} : { node: testCase.request.node }), + }, entered)).rejects.toBeInstanceOf(Error); + expect(entered).not.toHaveBeenCalled(); + }); + it('refuses local targets without an exact string id instead of fabricating one', async () => { const core: TestCore = { async call(): Promise { return { targets: [{ id: undefined, name: 'desktop-1', live: true, @@ -105,7 +134,7 @@ describe('local node-spawn port', () => { } }; const port = localPort(core); await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', node: 'desktop-1', - directory: '/workspace', idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ + directory: '/workspace', idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)).rejects.toMatchObject({ code: 'no_eligible_node', }); expect(calls).toHaveBeenCalledTimes(1); @@ -120,7 +149,7 @@ describe('local node-spawn port', () => { throw new Error('spawn must not be invoked'); } }; await expect(localPort(core).spawn({ name: 'worker_1', harness: 'codex', task: 'work', - node: 'desktop-1', directory: '/workspace', idempotencyKey: 'dispatch_1234567890' })) + node: 'desktop-1', directory: '/workspace', idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)) .rejects.toMatchObject({ code: 'no_eligible_node' }); expect(calls).toHaveBeenCalledTimes(1); }); @@ -141,7 +170,7 @@ describe('local node-spawn port', () => { } }; const port = localPort(core); await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', directory: '/workspace', - idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ code: 'outside_scope' }); + idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)).rejects.toMatchObject({ code: 'outside_scope' }); expect(status).toHaveBeenCalledOnce(); }); @@ -160,7 +189,7 @@ describe('local node-spawn port', () => { } }; const port = localPort(core); await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', directory: '/workspace', - idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ + idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)).rejects.toMatchObject({ code: 'spawn_identity_mismatch', cleanup: { nodeId: 'node_other', nodeName: 'desktop-2', name: 'worker_1', agentId: 'agent_1' }, }); @@ -178,7 +207,7 @@ describe('local node-spawn port', () => { } }; const port = localPort(core); await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', directory: '/workspace', - idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ + idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)).rejects.toMatchObject({ code: 'spawn_task_unconfirmed', cleanup: { nodeId: 'node_1', nodeName: 'desktop-1', name: 'worker_1', agentId: 'agent_1' }, }); @@ -201,7 +230,7 @@ describe('local node-spawn port', () => { throw new Error(`unexpected core method: ${method}`); } }; const failure = expect(localPort(core).spawn({ name: 'worker_1', harness: 'codex', task: 'work', - directory: '/workspace', idempotencyKey: 'dispatch_1234567890', timeoutMs: 750 })) + directory: '/workspace', idempotencyKey: 'dispatch_1234567890', timeoutMs: 750 }, beforeDispatch)) .rejects.toMatchObject({ code: 'spawn_unconfirmed', cleanup: { id: 'inv_1', nodeId: 'node_1', nodeName: 'desktop-1', name: 'worker_1', agentId: 'agent_1', } }); @@ -230,7 +259,7 @@ describe('local node-spawn port', () => { throw new Error(`unexpected core method: ${method}`); } }; const failure = expect(localPort(core).spawn({ name: 'worker_1', harness: 'codex', task: 'work', - directory: '/workspace', idempotencyKey: 'dispatch_1234567890' })) + directory: '/workspace', idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)) .rejects.toMatchObject({ code: 'spawn_unconfirmed', cause: pollFailure, cleanup: { id: 'inv_1', nodeId: 'node_1', nodeName: 'desktop-1', name: 'worker_1', agentId: 'agent_1', } }); @@ -258,7 +287,7 @@ describe('local node-spawn port', () => { throw new Error(`unexpected core method: ${method}`); } }; const pending = localPort(core).spawn({ name: 'worker_1', harness: 'codex', task: 'work', - directory: '/workspace', idempotencyKey: 'dispatch_1234567890' }); + directory: '/workspace', idempotencyKey: 'dispatch_1234567890' }, beforeDispatch); await expect(pending).rejects.toMatchObject({ code: 'spawn_identity_mismatch', cleanup: { id: 'inv_1', nodeId: 'node_1', nodeName: 'desktop-1', name: 'worker_1', agentId: 'agent_1', } }); @@ -267,12 +296,16 @@ describe('local node-spawn port', () => { describe('remote node-spawn port', () => { it('confirms readiness, resolves exact identity, and targets release to the same node', async () => { + const dispatchOrder: string[] = []; const placement = vi.fn() - .mockResolvedValueOnce({ invocationId: 'ack_1', actionName: 'spawn', status: 'pending', + .mockImplementationOnce(async () => { + dispatchOrder.push('placement.spawn'); + return { invocationId: 'ack_1', actionName: 'spawn', status: 'pending', node: { id: 'node_1', name: 'desktop-1', status: 'online', capabilities: [] }, placement: { node: 'desktop-1', confirmed: true, state: 'ready' }, confirmation: { invocationId: 'inv_1', actionName: 'spawn', status: 'completed', - output: { spawned: true, ready: true, name: 'worker_1' } } }) + output: { spawned: true, ready: true, name: 'worker_1' } } }; + }) .mockResolvedValueOnce({ invocationId: 'rel_1', actionName: 'release', status: 'pending', placement: { node: 'desktop-1', confirmed: false, state: 'accepted' } }); const getInvocation = vi.fn().mockResolvedValue({ invocationId: 'rel_1', actionName: 'release', @@ -282,7 +315,9 @@ describe('remote node-spawn port', () => { nodes: { list: async () => [] }, placement: { spawn: placement }, commands: { getInvocation }, }); const receipt = await port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', - idempotencyKey: 'dispatch_1234567890' }); + idempotencyKey: 'dispatch_1234567890' }, async () => { + dispatchOrder.push('dispatch.fence'); + }); await port.release({ receipt, idempotencyKey: 'release_1234567890' }); expect(receipt).toMatchObject({ agentId: 'agent_1', nodeName: 'desktop-1', ready: true }); expect(placement.mock.calls[0]![0]).toMatchObject({ confirm: true, verifyReady: true, @@ -291,6 +326,7 @@ describe('remote node-spawn port', () => { node: 'desktop-1', input: { name: 'worker_1', expected_agent_id: 'agent_1', idempotency_key: 'release_1234567890' } }); expect(getInvocation).toHaveBeenCalledWith('release', 'rel_1'); + expect(dispatchOrder).toEqual(['dispatch.fence', 'placement.spawn']); }); it('returns exact cleanup identity when task delivery verification fails after spawn', async () => { @@ -305,7 +341,7 @@ describe('remote node-spawn port', () => { commands: { getInvocation: vi.fn() }, }); await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', - idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ + idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)).rejects.toMatchObject({ code: 'spawn_task_unconfirmed', cleanup: { id: 'inv_1', nodeId: 'node_1', nodeName: 'desktop-1', name: 'worker_1', agentId: 'agent_1' }, @@ -326,7 +362,7 @@ describe('remote node-spawn port', () => { commands: { getInvocation: vi.fn() }, }); await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', - idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ + idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)).rejects.toMatchObject({ code: 'spawn_task_unconfirmed', cleanup: { id: 'inv_1', nodeId: 'node_1', nodeName: 'desktop-1', name: 'worker_1', agentId: 'agent_1' }, @@ -345,7 +381,7 @@ describe('remote node-spawn port', () => { commands: { getInvocation: vi.fn() }, }); await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', - idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ + idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)).rejects.toMatchObject({ code: 'spawn_identity_mismatch', cleanup: { agentId: 'agent_spawned', name: 'worker_1', nodeName: 'desktop-1' }, }); @@ -365,7 +401,7 @@ describe('remote node-spawn port', () => { commands: { getInvocation: vi.fn() }, }); const pending = port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', - idempotencyKey: 'dispatch_1234567890' }); + idempotencyKey: 'dispatch_1234567890' }, beforeDispatch); await vi.runAllTimersAsync(); await expect(pending).rejects.toMatchObject({ code: 'spawn_identity_unavailable', @@ -389,7 +425,7 @@ describe('remote node-spawn port', () => { commands: { getInvocation: vi.fn() }, }); await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', - idempotencyKey: 'dispatch_1234567890' })).resolves.toMatchObject({ + idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)).resolves.toMatchObject({ nodeId: 'node_1', nodeName: 'desktop-1', agentId: 'agent_1', }); expect(list).toHaveBeenCalledOnce(); @@ -407,7 +443,7 @@ describe('remote node-spawn port', () => { commands: { getInvocation: vi.fn() }, }); await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', - idempotencyKey: 'dispatch_1234567890' })).rejects.toMatchObject({ + idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)).rejects.toMatchObject({ code: 'spawn_placement_unconfirmed', cleanup: expect.objectContaining({ agentId: 'agent_1', nodeName: 'desktop-1' }), }); diff --git a/packages/sdk/tests/node-spawn-settlement.test.ts b/packages/sdk/tests/node-spawn-settlement.test.ts index b78dcfd0f..c02ef3e1a 100644 --- a/packages/sdk/tests/node-spawn-settlement.test.ts +++ b/packages/sdk/tests/node-spawn-settlement.test.ts @@ -45,13 +45,16 @@ function fixture(): { runtime: NodeSpawnRuntime; spawn: ReturnType id: 'inv_spawn_1', action: 'spawn', status: 'completed', nodeId: 'node_1', nodeName: 'desktop-1', name: 'placeholder', agentId: 'agent_1', spawned: true, ready: true, }; - const spawn = vi.fn(async (input: { name: string }) => ({ ...receipt, name: input.name })); + const spawn = vi.fn(async (input, beforeDispatch) => { + await beforeDispatch(); + return { ...receipt, name: input.name }; + }); const release = vi.fn(async () => ({ id: 'inv_release_1', action: 'release' as const, status: 'completed' as const, spawned: false, ready: false, released: true })); const ack = vi.fn(async () => ({})); const fail = vi.fn(async () => ({})); const port: NodeSpawnPort = { - targets: async () => [], spawn, + dispatchFenceVersion: 1, targets: async () => [], spawn, status: vi.fn(async () => ({ ...receipt })), release, }; @@ -356,10 +359,13 @@ describe('durable node-result settlement', () => { it('rejects an invalid resolved spawn receipt before reading the inbox and releases exact identity', async () => { const dataDir = await dataDirectory('flows-node-invalid-resolved-spawn-'); const f = fixture(); - f.spawn.mockImplementationOnce(async (input: { name: string }) => ({ - id: 'inv_spawn_1', action: 'spawn', status: 'completed', nodeId: 'node_1', nodeName: 'desktop-1', - name: input.name, agentId: 'agent_1', spawned: true, ready: false, - }) as never); + f.spawn.mockImplementationOnce(async (input, beforeDispatch) => { + await beforeDispatch(); + return { + id: 'inv_spawn_1', action: 'spawn', status: 'completed', nodeId: 'node_1', nodeName: 'desktop-1', + name: input.name, agentId: 'agent_1', spawned: true, ready: false, + } as never; + }); f.runtime.inbox.inbox.subscribe = async function* () { throw new Error('inbox must not be read'); }; await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) .rejects.toThrow('valid ready identity'); @@ -369,8 +375,11 @@ describe('durable node-result settlement', () => { it('refuses malformed cleanup identity from a custom port without issuing a release', async () => { const dataDir = await dataDirectory('flows-node-malformed-cleanup-'); const f = fixture(); - f.spawn.mockRejectedValueOnce(new NodeSpawnError('custom port failed', - 'spawn_unconfirmed', undefined, { id: 'inv_spawn_1' } as never)); + f.spawn.mockImplementationOnce(async (_input, beforeDispatch) => { + await beforeDispatch(); + throw new NodeSpawnError('custom port failed', + 'spawn_unconfirmed', undefined, { id: 'inv_spawn_1' } as never); + }); await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) .rejects.toThrow('custom port failed'); await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) diff --git a/packages/sdk/tests/worker-node-runtime-sharing.test.ts b/packages/sdk/tests/worker-node-runtime-sharing.test.ts index fffbbb51b..fad9d5d09 100644 --- a/packages/sdk/tests/worker-node-runtime-sharing.test.ts +++ b/packages/sdk/tests/worker-node-runtime-sharing.test.ts @@ -97,7 +97,8 @@ describe('AgentWorker node-spawn runtime ownership', () => { await mkdir(declaredDirectory); const exactDeclaredDirectory = await realpath(declaredDirectory); let spawnedName = ''; - const spawn = vi.fn(async input => { + const spawn = vi.fn(async (input, beforeDispatch) => { + await beforeDispatch(); spawnedName = input.name; return { id: 'spawn_worker_path', action: 'spawn', status: 'completed', nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, agentId: 'agent_worker_path', @@ -107,7 +108,7 @@ describe('AgentWorker node-spawn runtime ownership', () => { publisher: { id: 'publisher_1', name: 'flows-publisher' }, baseUrl: 'https://cast.agentrelay.com', ...(testCase.authority === undefined ? {} : { authority: testCase.authority }), - port: { targets: async () => [], spawn, status: vi.fn(), + port: { dispatchFenceVersion: 1, targets: async () => [], spawn, status: vi.fn(), release: vi.fn(async () => ({ id: 'release_worker_path', action: 'release' as const, status: 'completed' as const, spawned: false, ready: false, released: true })) }, inbox: { inbox: { @@ -177,8 +178,10 @@ describe('AgentWorker node-spawn runtime ownership', () => { authority: 'desktop', publisher: { id: 'publisher_1', name: 'flows-publisher' }, baseUrl: 'https://cast.agentrelay.com', port: { + dispatchFenceVersion: 1, targets: async () => [], - spawn: async input => { + spawn: async (input, beforeDispatch) => { + await beforeDispatch(); spawnedName = input.name; return { id: 'spawn_live_teardown', action: 'spawn', status: 'completed', nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, @@ -267,8 +270,10 @@ describe('AgentWorker node-spawn runtime ownership', () => { publisher: { id: 'publisher_1', name: 'flows-publisher' }, baseUrl: 'https://cast.agentrelay.com', port: { + dispatchFenceVersion: 1, targets: async () => [], - spawn: async input => { + spawn: async (input, beforeDispatch) => { + await beforeDispatch(); spawnedName = input.name; return { id: 'spawn_1', action: 'spawn', status: 'completed', nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, agentId: 'agent_1', @@ -338,7 +343,8 @@ describe('AgentWorker node-spawn runtime ownership', () => { order.push(`ack:${inboxItemId}`); return {}; }); - const spawn = vi.fn(async input => { + const spawn = vi.fn(async (input, beforeDispatch) => { + await beforeDispatch(); const agentId = `agent_${spawned.length + 1}`; spawned.push({ name: input.name, agentId, key: input.idempotencyKey }); if (spawned.length === 2) bothSpawned(); @@ -355,6 +361,7 @@ describe('AgentWorker node-spawn runtime ownership', () => { publisher: { id: 'publisher_1', name: 'flows-publisher' }, baseUrl: 'https://cast.agentrelay.com', port: { + dispatchFenceVersion: 1, targets: async () => [], spawn, status: vi.fn(), @@ -457,7 +464,8 @@ describe('AgentWorker node-spawn runtime ownership', () => { roots.push(dataDir); const pending = dispatch('edit-after-drain', 'dispatch_afterdrain'); let spawnedName = ''; - const spawn = vi.fn(async input => { + const spawn = vi.fn(async (input, beforeDispatch) => { + await beforeDispatch(); spawnedName = input.name; return { id: 'spawn_after_drain', action: 'spawn', status: 'completed', nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, agentId: 'agent_after_drain', @@ -467,6 +475,7 @@ describe('AgentWorker node-spawn runtime ownership', () => { publisher: { id: 'publisher_1', name: 'flows-publisher' }, baseUrl: 'https://cast.agentrelay.com', port: { + dispatchFenceVersion: 1, targets: async () => [], spawn, status: vi.fn(), @@ -545,7 +554,8 @@ describe('AgentWorker node-spawn runtime ownership', () => { order.push(`ack:${inboxItemId}`); return {}; }); - const spawn = vi.fn(async input => { + const spawn = vi.fn(async (input, beforeDispatch) => { + await beforeDispatch(); const agentId = `agent_${spawned.length}`; spawned.push({ name: input.name, agentId, key: input.idempotencyKey }); if (spawned.length === dispatches.length) allSpawned(); @@ -561,7 +571,7 @@ describe('AgentWorker node-spawn runtime ownership', () => { const runtime: NodeSpawnRuntime = { publisher: { id: 'publisher_1', name: 'flows-publisher' }, baseUrl: 'https://cast.agentrelay.com', - port: { targets: async () => [], spawn, status: vi.fn(), release }, + port: { dispatchFenceVersion: 1, targets: async () => [], spawn, status: vi.fn(), release }, inbox: { inbox: { subscribe: async function* () { await spawnedGate; diff --git a/packages/sdk/tests/worker-node-runtime-timeout.test.ts b/packages/sdk/tests/worker-node-runtime-timeout.test.ts index 62453ba03..e7fa21b47 100644 --- a/packages/sdk/tests/worker-node-runtime-timeout.test.ts +++ b/packages/sdk/tests/worker-node-runtime-timeout.test.ts @@ -37,7 +37,7 @@ function runtime(spawn: NodeSpawnPort['spawn']): NodeSpawnRuntime { return { authority: 'remote', publisher: { id: 'publisher_1', name: 'flows-publisher' }, baseUrl: 'https://cast.agentrelay.com', - port: { targets: async () => [], spawn, status: vi.fn(), release: vi.fn() }, + port: { dispatchFenceVersion: 1, targets: async () => [], spawn, status: vi.fn(), release: vi.fn() }, inbox: { inbox: { subscribe: async function* () { return; }, ack: vi.fn(async () => ({})), fail: vi.fn(async () => ({})), @@ -74,8 +74,10 @@ describe('AgentWorker relay deadlines', () => { it('journals a native node-spawn deadline as timeout rather than worker_error', async () => { const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-worker-timeout-reason-')); roots.push(dataDir); - const spawn = vi.fn( - () => new Promise(() => undefined)); + const spawn = vi.fn(async (_input, beforeDispatch) => { + await beforeDispatch(); + return new Promise(() => undefined); + }); const stepComplete = vi.fn(async () => ( {} as Awaited> )); diff --git a/packages/sdk/tsconfig.tests.json b/packages/sdk/tsconfig.tests.json index bc59c4dc8..bb165423a 100644 --- a/packages/sdk/tsconfig.tests.json +++ b/packages/sdk/tsconfig.tests.json @@ -75,6 +75,7 @@ "tests/broker-retirement.test.ts", "tests/agent-relay-hardening.test.ts", "tests/node-spawn-core-bridge.test.ts", + "tests/node-spawn-dispatch-fence.test.ts", "tests/node-spawn-ports.test.ts", "tests/node-spawn-runtime.test.ts", "tests/node-spawn-settlement.test.ts", From 5c7bde039cfad5eee971b119fc2543a17c80a8a5 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Mon, 9 Nov 2026 20:22:54 -0800 Subject: [PATCH 54/75] fix(sdk): reject malformed node result status --- packages/sdk/src/node-spawn-result.ts | 2 +- .../sdk/tests/agent-relay-hardening.test.ts | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/sdk/src/node-spawn-result.ts b/packages/sdk/src/node-spawn-result.ts index b46d0d59d..9ef63949b 100644 --- a/packages/sdk/src/node-spawn-result.ts +++ b/packages/sdk/src/node-spawn-result.ts @@ -185,7 +185,7 @@ async function abortableNext( export function readNodeResult(value: unknown, expected: NodeResultCorrelation): NodeResultEnvelope { if (!isRecord(value) || value.version !== 1 || value.run_id !== expected.runId || value.step_id !== expected.stepId || value.dispatch_id !== expected.dispatchId - || !['completed', 'failed'].includes(String(value.status))) { + || (value.status !== 'completed' && value.status !== 'failed')) { throw new Error('Spawned agent result has invalid attempt correlation'); } if (value.status === 'completed' && !Object.hasOwn(value, 'output')) { diff --git a/packages/sdk/tests/agent-relay-hardening.test.ts b/packages/sdk/tests/agent-relay-hardening.test.ts index 96f81ba91..0fedd0a7f 100644 --- a/packages/sdk/tests/agent-relay-hardening.test.ts +++ b/packages/sdk/tests/agent-relay-hardening.test.ts @@ -50,6 +50,25 @@ describe('node-spawn result hardening', () => { expect(fail).toHaveBeenCalledWith(expect.objectContaining({ inboxItemId: 'delivery_bad', retry: false })); }); + it('dead-letters an array-valued result status instead of accepting a malformed claim value', async () => { + const fail = vi.fn(async () => ({})); + const delivery = await waitForNodeResult({ inbox: { + subscribe: async function* () { yield { id: 'delivery_array_status', message: { + id: 'message_array_status', from: { id: 'agent_1', name: 'worker' }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_1', step_id: 'step_1', + dispatch_id: correlation.dispatchId, status: ['completed'], output: { ok: true } }), + } }; }, + ack: async () => ({}), fail, + } }, expected); + expect(delivery.envelope).toMatchObject({ status: 'failed', error: expect.stringContaining('correlation') }); + expect(delivery.settlementAction).toBe('fail'); + expect(fail).not.toHaveBeenCalled(); + await delivery.ack(); + expect(fail).toHaveBeenCalledWith(expect.objectContaining({ + inboxItemId: 'delivery_array_status', retry: false, + })); + }); + it('closes only the normal inbox subscription without acknowledging the delivery', async () => { const returned = vi.fn(async () => { throw new Error('iterator close failed'); }); let delivered = false; From 4f74d69f8eff0569d5ffb3262a995483ccc5f016 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Mon, 9 Nov 2026 22:38:55 -0800 Subject: [PATCH 55/75] fix(sdk): reconcile late node cleanup --- packages/sdk/src/agent-relay-transport.ts | 96 ++++------------- packages/sdk/src/node-spawn-cleanup.ts | 83 ++++++++++++++ packages/sdk/src/node-spawn-settlement.ts | 16 ++- packages/sdk/src/worker-cli.ts | 4 +- packages/sdk/src/worker.ts | 9 +- .../sdk/tests/node-spawn-settlement.test.ts | 94 ++++++++++++++++ .../tests/worker-node-runtime-timeout.test.ts | 101 ++++++++++++++++++ 7 files changed, 320 insertions(+), 83 deletions(-) create mode 100644 packages/sdk/src/node-spawn-cleanup.ts diff --git a/packages/sdk/src/agent-relay-transport.ts b/packages/sdk/src/agent-relay-transport.ts index bb8d1d311..ebf3b17be 100644 --- a/packages/sdk/src/agent-relay-transport.ts +++ b/packages/sdk/src/agent-relay-transport.ts @@ -3,13 +3,14 @@ import { claimNodeSpawnAtPath, hasNodeSpawnDispatchFence, nodeSpawnClaimPath, recordNodeSpawn, type NodeSpawnClaim } from './node-spawn-state.js'; import { dispatchFencedNodeSpawn } from './node-spawn-dispatch.js'; +import { releaseNodeSpawnClaim } from './node-spawn-cleanup.js'; import { nodeResultInstruction, waitForNodeResult } from './node-spawn-result.js'; import { durableNodeResultDelivery } from './node-spawn-settlement.js'; import { waitForNodeOperation } from './node-spawn-deadline.js'; import { createNodeSpawnRuntime, type NodeSpawnRuntime } from './node-spawn-runtime.js'; import { NodeSpawnError, nonempty, validIdempotencyKey, validNodeSpawnIdentity, validNodeSpawnReceipt, - type NodeSpawnIdentity, type NodeSpawnReceipt } from './node-spawn.js'; + type NodeSpawnReceipt } from './node-spawn.js'; import type { RelayTaskResult } from './agent-relay-receipt.js'; export type AgentTransport = 'direct' | 'relay'; @@ -50,9 +51,16 @@ export class AgentRelayTransportError extends Error { } } -const CLEANUP_ATTEMPT_TIMEOUT_MS = 15_000; const claimExecutions = new Map>(); +interface AgentRelayTaskOptions extends AgentRelayEnv { + runtime?: NodeSpawnRuntime; + runtimeProvider?: (signal?: AbortSignal) => Promise; + signal?: AbortSignal; + /** Wakes durable reconciliation after a late exact release remains pending. */ + onCleanupPending?: () => void; +} + /** * Dispatches exactly once through an Agent Relay node and waits on the * publisher's durable inbox. AgentWorker acks the returned delivery only @@ -60,8 +68,7 @@ const claimExecutions = new Map>(); */ export async function runAgentRelayTask( request: AgentRelayTaskRequest, - options: AgentRelayEnv & { runtime?: NodeSpawnRuntime; - runtimeProvider?: (signal?: AbortSignal) => Promise; signal?: AbortSignal } = {}, + options: AgentRelayTaskOptions = {}, ): Promise { const taskStartedAt = Date.now(); options.signal?.throwIfAborted(); @@ -84,8 +91,7 @@ export async function runAgentRelayTask( async function runAgentRelayTaskExclusive( request: AgentRelayTaskRequest, claimPath: string, - options: AgentRelayEnv & { runtime?: NodeSpawnRuntime; - runtimeProvider?: (signal?: AbortSignal) => Promise; signal?: AbortSignal }, + options: AgentRelayTaskOptions, taskStartedAt: number, timeoutMs: number, ): Promise { @@ -128,7 +134,8 @@ async function runAgentRelayTaskExclusive( } try { return await executeAgentRelayTask( - request, claimPath, runtime, options.signal, taskStartedAt, timeoutMs); + request, claimPath, runtime, options.signal, taskStartedAt, timeoutMs, + options.onCleanupPending); } finally { if (ownsRuntime) await runtime.disconnect?.().catch(() => undefined); } @@ -183,6 +190,7 @@ async function executeAgentRelayTask( signal: AbortSignal | undefined, taskStartedAt: number, timeoutMs: number, + onCleanupPending: (() => void) | undefined, ): Promise { if (runtime.port.dispatchFenceVersion !== 1) throw new AgentRelayTransportError( 'The node-spawn port does not implement durable dispatch-fence protocol version 1'); @@ -230,7 +238,7 @@ async function executeAgentRelayTask( if (!claimed.created && claim.spawn === undefined && claim.cleanup !== undefined) { if (claim.release === undefined) { releaseAttempted = true; - const cleanup = await tryRelease(runtime, claimed.path, claim, request.idempotencyKey); + const cleanup = await releaseNodeSpawnClaim(runtime, claimed.path, claim); claim = cleanup.claim; if (cleanup.error !== undefined) throw cleanup.error; } @@ -271,7 +279,7 @@ async function executeAgentRelayTask( if ((claim.cleanupPending !== undefined || claim.failure !== undefined) && claim.release === undefined) { releaseAttempted = true; - const cleanup = await tryRelease(runtime, claimed.path, claim, request.idempotencyKey); + const cleanup = await releaseNodeSpawnClaim(runtime, claimed.path, claim); claim = cleanup.claim; if (cleanup.error !== undefined) throw cleanup.error; throw new NodeSpawnError( @@ -300,7 +308,7 @@ async function executeAgentRelayTask( if (spawn === undefined) throw new AgentRelayTransportError('Node-spawn claim lost its confirmed receipt'); if (claim.result !== undefined && claim.release === undefined) { releaseAttempted = true; - const cleanup = await tryRelease(runtime, claimed.path, claim, request.idempotencyKey); + const cleanup = await releaseNodeSpawnClaim(runtime, claimed.path, claim); claim = cleanup.claim; if (cleanup.error !== undefined) throw cleanup.error; } @@ -338,7 +346,7 @@ async function executeAgentRelayTask( const delivery = durableNodeResultDelivery(runtime, claimed.path, claim, observed); if (claim.release === undefined) { releaseAttempted = true; - const cleanup = await tryRelease(runtime, claimed.path, claim, request.idempotencyKey); + const cleanup = await releaseNodeSpawnClaim(runtime, claimed.path, claim); claim = cleanup.claim; if (cleanup.error !== undefined) throw cleanup.error; } @@ -392,7 +400,7 @@ async function executeAgentRelayTask( if ((claim.spawn !== undefined || claim.cleanup !== undefined) && claim.release === undefined && !releaseAttempted) { releaseAttempted = true; - const cleanup = await tryRelease(runtime, claimed.path, claim, request.idempotencyKey); + const cleanup = await releaseNodeSpawnClaim(runtime, claimed.path, claim); claim = cleanup.claim; cleanupError ??= cleanup.error; } @@ -426,65 +434,9 @@ async function executeAgentRelayTask( } try { await recordNodeSpawn(claimed.path, claim); } catch { /* Exact in-memory identity is still sufficient for this cleanup attempt. */ } - const cleanup = await tryRelease(runtime, claimed.path, claim, request.idempotencyKey); + const cleanup = await releaseNodeSpawnClaim(runtime, claimed.path, claim); claim = cleanup.claim; - } -} - -async function tryRelease( - runtime: NodeSpawnRuntime, - path: string, - claim: NodeSpawnClaim, - idempotencyKey: string, -): Promise<{ claim: NodeSpawnClaim; error?: unknown }> { - const receipt: NodeSpawnIdentity | undefined = claim.spawn ?? claim.cleanup; - if (receipt === undefined || claim.release !== undefined) return { claim }; - try { - const released = await boundedCleanup(runtime.port.release({ receipt, - idempotencyKey: releaseKey(idempotencyKey) })); - if (released.action !== 'release' || released.status !== 'completed' - || released.released !== true || !nonempty(released.id)) { - throw new NodeSpawnError('The node did not confirm exact agent release', - released.errorCode ?? 'release_unconfirmed'); - } - const { cleanupPending: _cleanupPending, ...withoutPending } = claim; - const updated: NodeSpawnClaim = { ...withoutPending, - release: { invocationId: released.id, status: released.status } }; - await recordNodeSpawn(path, updated); - return { claim: updated }; - } catch (error) { - const updated: NodeSpawnClaim = { ...claim, cleanupPending: { - attempts: (claim.cleanupPending?.attempts ?? 0) + 1, - lastError: safeError(error), - updatedAt: Date.now(), - } }; - try { await recordNodeSpawn(path, updated); } - catch (stateError) { - return { claim: updated, - error: new AgentRelayTransportError(`${safeError(error)}; cleanup state was not durable: ${safeError(stateError)}`, error) }; - } - return { claim: updated, error }; - } -} - -async function boundedCleanup(pending: Promise): Promise { - let timer: NodeJS.Timeout | undefined; - const deadline = new Promise((_resolve, reject) => { - timer = setTimeout(() => reject(new NodeSpawnError( - 'Exact node cleanup did not confirm within its independent deadline', 'release_unconfirmed')), - CLEANUP_ATTEMPT_TIMEOUT_MS); - }); - // A transport that ignores cancellation may settle after this attempt has - // durably recorded cleanupPending. Fold that late outcome into this promise - // so it cannot become an unhandled rejection; the next attempt reuses the - // same idempotency key and reconciles the exact release. - const outcome = pending.then(value => ({ ok: true as const, value }), error => ({ ok: false as const, error })); - try { - const settled = await Promise.race([outcome, deadline]); - if (!settled.ok) throw settled.error; - return settled.value; - } finally { - if (timer !== undefined) clearTimeout(timer); + if (cleanup.error !== undefined) onCleanupPending?.(); } } @@ -508,10 +460,6 @@ function stableAgentName(request: AgentRelayTaskRequest): string { return `rf-${label.slice(0, 30)}-${digest}`; } -function releaseKey(key: string): string { - return `release_${createHash('sha256').update(key).digest('hex').slice(0, 32)}`; -} - function safeError(error: unknown): string { return error instanceof Error && error.message ? error.message : 'Node-spawn task failed'; } diff --git a/packages/sdk/src/node-spawn-cleanup.ts b/packages/sdk/src/node-spawn-cleanup.ts new file mode 100644 index 000000000..a660363e6 --- /dev/null +++ b/packages/sdk/src/node-spawn-cleanup.ts @@ -0,0 +1,83 @@ +import { createHash } from 'node:crypto'; +import { recordNodeSpawn, type NodeSpawnClaim } from './node-spawn-state.js'; +import type { NodeSpawnRuntime } from './node-spawn-runtime.js'; +import { NodeSpawnError, nonempty, type NodeSpawnIdentity } from './node-spawn.js'; + +const CLEANUP_ATTEMPT_TIMEOUT_MS = 15_000; + +/** True only when durable terminal evidence makes exact release recoverable. */ +export function nodeSpawnCleanupRequired(claim: NodeSpawnClaim): boolean { + return claim.release === undefined + && (claim.spawn !== undefined || claim.cleanup !== undefined) + && (claim.cleanupPending !== undefined || claim.failure !== undefined || claim.result !== undefined); +} + +/** Attempts the claim's exact, idempotent release and persists either outcome. */ +export async function releaseNodeSpawnClaim( + runtime: Pick, + path: string, + claim: NodeSpawnClaim, +): Promise<{ claim: NodeSpawnClaim; error?: unknown }> { + const receipt: NodeSpawnIdentity | undefined = claim.spawn ?? claim.cleanup; + if (receipt === undefined || claim.release !== undefined) return { claim }; + try { + const released = await boundedCleanup(runtime.port.release({ + receipt, + idempotencyKey: releaseKey(claim.idempotencyKey), + })); + if (released.action !== 'release' || released.status !== 'completed' + || released.released !== true || !nonempty(released.id)) { + throw new NodeSpawnError('The node did not confirm exact agent release', + released.errorCode ?? 'release_unconfirmed'); + } + const { cleanupPending: _cleanupPending, ...withoutPending } = claim; + const updated: NodeSpawnClaim = { ...withoutPending, + release: { invocationId: released.id, status: released.status } }; + await recordNodeSpawn(path, updated); + return { claim: updated }; + } catch (error) { + const updated: NodeSpawnClaim = { ...claim, cleanupPending: { + attempts: (claim.cleanupPending?.attempts ?? 0) + 1, + lastError: safeError(error), + updatedAt: Date.now(), + } }; + try { await recordNodeSpawn(path, updated); } + catch (stateError) { + return { claim: updated, + error: new NodeSpawnError( + `${safeError(error)}; cleanup state was not durable: ${safeError(stateError)}`, + 'release_unconfirmed', error) }; + } + return { claim: updated, error }; + } +} + +async function boundedCleanup(pending: Promise): Promise { + let timer: NodeJS.Timeout | undefined; + const deadline = new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new NodeSpawnError( + 'Exact node cleanup did not confirm within its independent deadline', 'release_unconfirmed')), + CLEANUP_ATTEMPT_TIMEOUT_MS); + }); + // A transport that ignores cancellation may settle after this attempt has + // durably recorded cleanupPending. Fold that late outcome into this promise + // so it cannot become an unhandled rejection; reconciliation reuses the + // same idempotency key for the exact release. + const outcome = pending.then(value => ({ ok: true as const, value }), + error => ({ ok: false as const, error })); + try { + const settled = await Promise.race([outcome, deadline]); + if (!settled.ok) throw settled.error; + return settled.value; + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} + +function releaseKey(key: string): string { + return `release_${createHash('sha256').update(key).digest('hex').slice(0, 32)}`; +} + +function safeError(error: unknown): string { + return error instanceof Error && error.message ? error.message : 'Node-spawn task failed'; +} diff --git a/packages/sdk/src/node-spawn-settlement.ts b/packages/sdk/src/node-spawn-settlement.ts index 7636aa865..7b0e817ef 100644 --- a/packages/sdk/src/node-spawn-settlement.ts +++ b/packages/sdk/src/node-spawn-settlement.ts @@ -1,6 +1,7 @@ import type { JournalClient } from './journal-client.js'; import type { JournalEvent } from './journal-reader.js'; import type { NodeResultDelivery } from './node-spawn-result.js'; +import { nodeSpawnCleanupRequired, releaseNodeSpawnClaim } from './node-spawn-cleanup.js'; import { readNodeSpawnClaim, readNodeSpawnClaims, recordNodeSpawn, type NodeSpawnClaim } from './node-spawn-state.js'; import type { NodeSpawnRuntime } from './node-spawn-runtime.js'; @@ -165,7 +166,7 @@ async function serializeSettlementMutation(path: string, mutate: () => Promis } } -/** Reconciles journaled result settlements left by a crash or failed inbox call. */ +/** Reconciles exact cleanup and journaled result settlement left by a crash or failed call. */ export async function reconcileNodeResultSettlements( dataDir: string, runtime: NodeSpawnRuntime, @@ -185,15 +186,20 @@ async function reconcileNodeResultSettlement( runtime: NodeSpawnRuntime, journal: Pick, ): Promise { - const claim = await readNodeSpawnClaim(path); - const result = claim.result; - const settlement = result?.settlement; - if (result === undefined || settlement === undefined || settlement.settledAt !== undefined) return 'complete'; + let claim = await readNodeSpawnClaim(path); if (claim.baseUrl !== runtime.baseUrl || claim.publisherId !== runtime.publisher.id || claim.publisherName !== runtime.publisher.name) { return 'foreign'; } + if (nodeSpawnCleanupRequired(claim)) { + const cleanup = await releaseNodeSpawnClaim(runtime, path, claim); + claim = cleanup.claim; + if (cleanup.error !== undefined) throw cleanup.error; + } + const result = claim.result; + const settlement = result?.settlement; + if (result === undefined || settlement === undefined || settlement.settledAt !== undefined) return 'complete'; const live = liveTeardowns.get(path); if (live !== undefined) { const failed = [...live.barriers].find(item => item.settled && item.failure !== undefined); diff --git a/packages/sdk/src/worker-cli.ts b/packages/sdk/src/worker-cli.ts index 23d67622f..258e9f922 100644 --- a/packages/sdk/src/worker-cli.ts +++ b/packages/sdk/src/worker-cli.ts @@ -133,6 +133,7 @@ export interface AgentRelayContext { resultSchema?: unknown; nodeSpawnRuntime?: NodeSpawnRuntime; nodeSpawnRuntimeProvider?: (signal?: AbortSignal) => Promise; + nodeSpawnCleanupPending?: () => void; } export async function runAgentCli( @@ -423,7 +424,8 @@ async function runViaAgentRelay( runId: context.runId, stepId: context.stepId, idempotencyKey: context.idempotencyKey, attempt: context.attempt, dataDir: context.dataDir, ...(timeoutMs === undefined ? {} : { timeoutMs }), - }, { signal, runtime: context.nodeSpawnRuntime, runtimeProvider: context.nodeSpawnRuntimeProvider }); + }, { signal, runtime: context.nodeSpawnRuntime, runtimeProvider: context.nodeSpawnRuntimeProvider, + onCleanupPending: context.nodeSpawnCleanupPending }); const receipt = { ...received.receipt, error: received.receipt.error === null ? null : redactRelayError(received.receipt.error) }; const accounting = receipt.task_execution.accounting; diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index 11f457f99..e90ed5eb9 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -24,6 +24,7 @@ import { } from './agent-transcript.js'; import { createNodeSpawnRuntime, createNodeSpawnRuntimeOwner, explicitServiceCallerFile, mergeNodeSpawnEnvironment, type NodeSpawnRuntime, type NodeSpawnRuntimeOwner } from './node-spawn-runtime.js'; +import { nodeSpawnCleanupRequired } from './node-spawn-cleanup.js'; import { reconcileNodeResultSettlements } from './node-spawn-settlement.js'; import { readNodeSpawnClaims } from './node-spawn-state.js'; @@ -263,7 +264,8 @@ export class AgentWorker extends EventEmitter { attempt: dispatch.attempt, taskIdentity: workerInstructionIdentity(spec.instruction, dispatch), dataDir: this.options.dataDir, resultSchema: spec.verification?.json_schema, nodeSpawnRuntime: this.options.nodeSpawnRuntime, - nodeSpawnRuntimeProvider: signal => this.nodeSpawnRuntimeOwner.get(signal) }, + nodeSpawnRuntimeProvider: signal => this.nodeSpawnRuntimeOwner.get(signal), + nodeSpawnCleanupPending: () => this.scheduleSettlementReconcile() }, this.options.environment, spec.cli_identity, spec.timeout_ms, spec.resume, spec.secrets, this.client.supportsFeature?.('step_secret_scoping') ?? true) : Promise.resolve({ exit_code: null, stdout_tail: '', stderr_tail: 'agent step has no declared CLI' })); @@ -369,8 +371,9 @@ export class AgentWorker extends EventEmitter { private async reconcileRelaySettlementsOnce(): Promise { if (this.options.dataDir === undefined || (this.closing && !this.attached)) return; const claims = await readNodeSpawnClaims(this.options.dataDir); - const hasPending = claims.some(entry => entry.claim.result?.settlement !== undefined - && entry.claim.result.settlement.settledAt === undefined); + const hasPending = claims.some(entry => nodeSpawnCleanupRequired(entry.claim) + || (entry.claim.result?.settlement !== undefined + && entry.claim.result.settlement.settledAt === undefined)); if (!hasPending) return; const runtime = await this.nodeSpawnRuntimeOwner.get(); let pending: number; diff --git a/packages/sdk/tests/node-spawn-settlement.test.ts b/packages/sdk/tests/node-spawn-settlement.test.ts index c02ef3e1a..4a8ea3e23 100644 --- a/packages/sdk/tests/node-spawn-settlement.test.ts +++ b/packages/sdk/tests/node-spawn-settlement.test.ts @@ -462,6 +462,100 @@ describe('durable node-result settlement', () => { await worker.close(); }); + it('reconciles failure plus exact identity when a crash precedes cleanupPending', async () => { + const dataDir = await dataDirectory('flows-node-cleanup-only-crash-'); + const f = fixture(); + f.runtime.inbox.inbox.subscribe = async function* () { return; }; + f.release.mockRejectedValueOnce(new Error('release response lost during crash')); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toMatchObject({ cleanupPending: true }); + const [pending] = await readNodeSpawnClaims(dataDir); + const { cleanupPending: _cleanupPending, ...crashedBeforeCleanupPending } = pending!.claim; + await recordNodeSpawn(pending!.path, crashedBeforeCleanupPending); + const [crashed] = await readNodeSpawnClaims(dataDir); + expect(crashed!.claim.cleanupPending).toBeUndefined(); + expect(crashed!.claim.failure).toBeDefined(); + expect(crashed!.claim.result).toBeUndefined(); + expect(crashed!.claim.release).toBeUndefined(); + + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + journalRead: vi.fn(async () => { + throw new Error('cleanup-only reconciliation must not read the journal'); + }), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_cleanup_recovery', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + await worker.attach(); + expect(f.release).toHaveBeenCalledTimes(2); + expect(client.journalRead).not.toHaveBeenCalled(); + const [reconciled] = await readNodeSpawnClaims(dataDir); + expect(reconciled!.claim.release).toEqual({ + invocationId: 'inv_release_1', status: 'completed', + }); + expect(reconciled!.claim.cleanupPending).toBeUndefined(); + expect(reconciled!.claim.result).toBeUndefined(); + await worker.close(); + }); + + it('releases an unreleased durable result before reading its terminal journal settlement', async () => { + const dataDir = await dataDirectory('flows-node-result-before-release-crash-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + const [persisted] = await readNodeSpawnClaims(dataDir); + const { release: _release, ...crashedBeforeRelease } = persisted!.claim; + await recordNodeSpawn(persisted!.path, crashedBeforeRelease); + + const order: string[] = []; + f.release.mockClear(); + f.release.mockImplementation(async () => { + order.push('release'); + return { id: 'inv_release_reconciled', action: 'release', status: 'completed', + spawned: false, ready: false, released: true }; + }); + f.ack.mockImplementation(async () => { order.push('ack'); return {}; }); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + journalRead: vi.fn(async () => { + order.push('journal'); + expect(order).toEqual(['release', 'journal']); + return { entries: [terminalEntry()] }; + }), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_result_release_recovery', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + await worker.attach(); + expect(order).toEqual(['release', 'journal', 'ack']); + const [reconciled] = await readNodeSpawnClaims(dataDir); + expect(reconciled!.claim.release).toEqual({ + invocationId: 'inv_release_reconciled', status: 'completed', + }); + expect(reconciled!.claim.result?.settlement).toMatchObject({ settledAs: 'ack' }); + await worker.close(); + }); + + it('does not release an active spawn without durable terminal evidence', async () => { + const dataDir = await dataDirectory('flows-node-active-spawn-no-cleanup-'); + const f = fixture(); + await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + const [persisted] = await readNodeSpawnClaims(dataDir); + const { result: _result, release: _release, failure: _failure, + cleanupPending: _cleanupPending, ...active } = persisted!.claim; + await recordNodeSpawn(persisted!.path, active); + f.release.mockClear(); + const journalRead = vi.fn(async () => { + throw new Error('active spawn reconciliation must not read the journal'); + }); + + await expect(reconcileNodeResultSettlements(dataDir, f.runtime, { journalRead })) + .resolves.toBe(0); + expect(f.release).not.toHaveBeenCalled(); + expect(journalRead).not.toHaveBeenCalled(); + }); + it('skips a foreign pending settlement without blocking or rearming this runtime', async () => { const dataDir = await dataDirectory('flows-node-settlement-foreign-'); const f = fixture(); diff --git a/packages/sdk/tests/worker-node-runtime-timeout.test.ts b/packages/sdk/tests/worker-node-runtime-timeout.test.ts index e7fa21b47..4a07cc9c9 100644 --- a/packages/sdk/tests/worker-node-runtime-timeout.test.ts +++ b/packages/sdk/tests/worker-node-runtime-timeout.test.ts @@ -5,6 +5,7 @@ import { join } from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; import type { JournalClient } from '../src/journal-client.js'; import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; +import { readNodeSpawnClaims } from '../src/node-spawn-state.js'; import type { NodeSpawnPort, NodeSpawnReceipt } from '../src/node-spawn.js'; import type { StepDispatchEvent } from '../src/protocol.js'; import { AgentWorker } from '../src/worker.js'; @@ -45,6 +46,20 @@ function runtime(spawn: NodeSpawnPort['spawn']): NodeSpawnRuntime { }; } +async function waitForImmediate(assertion: () => void | Promise): Promise { + let failure: unknown; + for (let attempt = 0; attempt < 100; attempt += 1) { + try { + await assertion(); + return; + } catch (error) { + failure = error; + await new Promise(resolve => setImmediate(resolve)); + } + } + throw failure; +} + describe('AgentWorker relay deadlines', () => { it('carries an admitted journal timeout into the native node-spawn deadline', async () => { const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-worker-timeout-')); @@ -97,4 +112,90 @@ describe('AgentWorker relay deadlines', () => { }) }), ); }); + + it('durably retries failed exact cleanup after a late confirmation and terminal timeout', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-worker-late-cleanup-')); + roots.push(dataDir); + let confirm!: (receipt: NodeSpawnReceipt) => void; + const spawn = vi.fn(async (input, beforeDispatch) => { + await beforeDispatch(); + return new Promise(resolve => { + confirm = receipt => resolve({ ...receipt, name: input.name }); + }); + }); + const release = vi.fn() + .mockRejectedValueOnce(new Error('release response temporarily unavailable')) + .mockResolvedValueOnce({ + id: 'inv_release_late', action: 'release', status: 'completed', + spawned: false, ready: false, released: true, + }); + const nodeRuntime: NodeSpawnRuntime = { + authority: 'remote', publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + port: { + dispatchFenceVersion: 1, targets: async () => [], spawn, + status: vi.fn(), release, + }, + inbox: { inbox: { + subscribe: async function* () { return; }, + ack: vi.fn(async () => ({})), fail: vi.fn(async () => ({})), + } }, + }; + const stepComplete = vi.fn(async () => ( + {} as Awaited> + )); + const journal = client(stepComplete); + journal.journalRead = vi.fn(async () => { + throw new Error('cleanup-only reconciliation must not read the journal'); + }); + const worker = new AgentWorker(journal, { + workerId: 'worker_late_cleanup', pins: {}, dataDir, nodeSpawnRuntime: nodeRuntime, + }); + const errors: unknown[] = []; + worker.on('error', error => errors.push(error)); + await worker.attach(); + journal.emit('step.dispatch', dispatch('edit-late-cleanup', 'dispatch_latecleanup1', 50)); + await vi.waitFor(() => expect(stepComplete).toHaveBeenCalledOnce()); + expect(stepComplete).toHaveBeenCalledWith( + 'run_timeout', 'edit-late-cleanup', 1, 'dispatch_latecleanup1', 'timeout', expect.any(Object), + ); + + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + try { + confirm({ + id: 'inv_spawn_late', action: 'spawn', status: 'completed', + nodeId: 'node_1', nodeName: 'desktop-1', name: 'placeholder', + agentId: 'agent_late', spawned: true, ready: true, + }); + let pending: Awaited>[number] | undefined; + await waitForImmediate(async () => { + expect(release).toHaveBeenCalledOnce(); + [pending] = await readNodeSpawnClaims(dataDir); + expect(pending!.claim).toMatchObject({ + cleanupPending: { attempts: 1, lastError: 'release response temporarily unavailable' }, + }); + }); + expect(pending!.claim.result).toBeUndefined(); + expect(pending!.claim.release).toBeUndefined(); + + await waitForImmediate(() => expect(vi.getTimerCount()).toBeGreaterThan(0)); + await vi.advanceTimersByTimeAsync(5_001); + let reconciled: Awaited>[number] | undefined; + await waitForImmediate(async () => { + expect(release).toHaveBeenCalledTimes(2); + [reconciled] = await readNodeSpawnClaims(dataDir); + expect(reconciled!.claim.release).toEqual({ + invocationId: 'inv_release_late', status: 'completed', + }); + }); + expect(reconciled!.claim.cleanupPending).toBeUndefined(); + expect(stepComplete).toHaveBeenCalledOnce(); + expect(spawn).toHaveBeenCalledOnce(); + expect(journal.journalRead).not.toHaveBeenCalled(); + expect(errors).toEqual([]); + } finally { + vi.useRealTimers(); + await worker.close(); + } + }); }); From 5818230b5d87766091a0b1f1ece1f30dd2e00a79 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Mon, 9 Nov 2026 22:53:59 -0800 Subject: [PATCH 56/75] fix(sdk): preserve timely node results --- packages/sdk/src/agent-relay-transport.ts | 2 +- .../sdk/tests/agent-relay-transport.test.ts | 34 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/packages/sdk/src/agent-relay-transport.ts b/packages/sdk/src/agent-relay-transport.ts index ebf3b17be..0289e0141 100644 --- a/packages/sdk/src/agent-relay-transport.ts +++ b/packages/sdk/src/agent-relay-transport.ts @@ -350,7 +350,7 @@ async function executeAgentRelayTask( claim = cleanup.claim; if (cleanup.error !== undefined) throw cleanup.error; } - resultSignal?.throwIfAborted(); + signal?.throwIfAborted(); const envelope = delivery.envelope; const error = envelope.status === 'failed' ? envelope.error! : null; const status = error === null ? 'completed' : 'failed'; diff --git a/packages/sdk/tests/agent-relay-transport.test.ts b/packages/sdk/tests/agent-relay-transport.test.ts index 7713db9bf..b53f9ed6d 100644 --- a/packages/sdk/tests/agent-relay-transport.test.ts +++ b/packages/sdk/tests/agent-relay-transport.test.ts @@ -493,6 +493,40 @@ describe('native node-spawn task transport', () => { } }); + it('preserves a timely durable result when exact release crosses the task deadline', async () => { + const taskDeadline = new AbortController(); + const nativeTimeout = AbortSignal.timeout.bind(AbortSignal); + const timeout = vi.spyOn(AbortSignal, 'timeout').mockImplementation(milliseconds => + milliseconds < 2_000 ? taskDeadline.signal : nativeTimeout(milliseconds)); + try { + const dataDir = await dataDirectory('flows-node-result-before-deadline-'); + const f = fixture(); + let confirmRelease!: (receipt: Awaited>) => void; + f.release.mockImplementationOnce(() => new Promise(resolve => { confirmRelease = resolve; })); + const task = runAgentRelayTask({ ...request(dataDir), timeoutMs: 1_000 }, { runtime: f.runtime }); + await vi.waitFor(() => expect(f.release).toHaveBeenCalledOnce()); + + taskDeadline.abort(new DOMException('The operation was aborted due to timeout', 'TimeoutError')); + confirmRelease({ id: 'inv_release_after_deadline', action: 'release', status: 'completed', + spawned: false, ready: false, released: true }); + + const completed = await task; + expect(completed.receipt).toMatchObject({ + status: 'completed', + task_execution: { delivery_id: 'delivery_1', release_invocation_id: 'inv_release_after_deadline' }, + }); + const [persisted] = await readNodeSpawnClaims(dataDir); + expect(persisted!.claim.result).toMatchObject({ inboxItemId: 'delivery_1', messageId: 'message_1' }); + expect(persisted!.claim.release).toMatchObject({ invocationId: 'inv_release_after_deadline' }); + expect(f.ack).not.toHaveBeenCalled(); + await completed.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + await completed.delivery.ack(); + expect(f.ack).toHaveBeenCalledOnce(); + } finally { + timeout.mockRestore(); + } + }); + it('journals a failed agent envelope before acknowledging its delivery', async () => { const dataDir = await dataDirectory('flows-node-failed-result-'); const f = fixture('failed'); From 801f37d9280db9399b2565a99d14e6a674248368 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Tue, 10 Nov 2026 03:26:48 -0800 Subject: [PATCH 57/75] fix(sdk): keep foreign claims from creating authority --- docs/AGENT-RELAY-TRANSPORT.md | 5 +- packages/sdk/src/node-spawn-runtime.ts | 12 +- packages/sdk/src/node-spawn-settlement.ts | 14 +- packages/sdk/src/worker.ts | 16 +- packages/sdk/tests/node-spawn-runtime.test.ts | 2 + .../tests/worker-node-runtime-sharing.test.ts | 145 +++++++++++++++++- 6 files changed, 180 insertions(+), 14 deletions(-) diff --git a/docs/AGENT-RELAY-TRANSPORT.md b/docs/AGENT-RELAY-TRANSPORT.md index 5b81f1485..b7f5c62a6 100644 --- a/docs/AGENT-RELAY-TRANSPORT.md +++ b/docs/AGENT-RELAY-TRANSPORT.md @@ -30,7 +30,10 @@ kernel attempt and expected completion reason. The journal's terminal fact must contain that reason and the exact Relay delivery and message ids before a restart can acknowledge the inbox item. Inbox acknowledgement and rejection have independent deadlines; an unconfirmed operation remains in the durable -outbox and is retried on worker attach, after dispatch drain, and on shutdown. +outbox. It is retried on worker attach only when admission already established +the exact matching publisher authority, then after dispatch drain and on +shutdown while that authority remains established. A durable claim never +creates authority or triggers credential discovery by itself. One worker uses one lazy publisher connection across all of its concurrent relay steps. Each step closes only its inbox iterator. The default runtime is diff --git a/packages/sdk/src/node-spawn-runtime.ts b/packages/sdk/src/node-spawn-runtime.ts index 98edc2242..c696b216e 100644 --- a/packages/sdk/src/node-spawn-runtime.ts +++ b/packages/sdk/src/node-spawn-runtime.ts @@ -30,6 +30,8 @@ export interface NodeSpawnRuntime { export interface NodeSpawnRuntimeOwner { get(signal?: AbortSignal): Promise; + /** Returns only authority already established by configuration or a successful get(). */ + peek(): NodeSpawnRuntime | undefined; close(): Promise; } @@ -177,14 +179,18 @@ export function createNodeSpawnRuntimeOwner( ): NodeSpawnRuntimeOwner { const controller = new AbortController(); let pending: Promise | undefined; + let established = provided; let closed = false; let closePromise: Promise | undefined; const get = async (signal?: AbortSignal): Promise => { signal?.throwIfAborted(); if (closed) throw new NodeSpawnError('Node-spawn runtime owner is closed', 'task_cancelled'); - if (provided !== undefined) return provided; + if (established !== undefined) return established; if (pending === undefined) { - const created = factory(controller.signal); + const created = factory(controller.signal).then(runtime => { + established = runtime; + return runtime; + }); pending = created; void created.catch(() => { if (pending === created) pending = undefined; }); } @@ -197,7 +203,7 @@ export function createNodeSpawnRuntimeOwner( const runtime = await pending?.catch(() => undefined); await runtime?.disconnect?.().catch(() => undefined); }; - return { get, close: () => closePromise ??= closeOnce() }; + return { get, peek: () => established, close: () => closePromise ??= closeOnce() }; } async function bindCore( diff --git a/packages/sdk/src/node-spawn-settlement.ts b/packages/sdk/src/node-spawn-settlement.ts index 7b0e817ef..1443e553f 100644 --- a/packages/sdk/src/node-spawn-settlement.ts +++ b/packages/sdk/src/node-spawn-settlement.ts @@ -27,6 +27,16 @@ type Reconciliation = 'complete' | 'pending' | 'foreign'; const liveTeardowns = new Map(); const settlementMutations = new Map>(); +/** True only when a durable claim belongs to this already-established publisher authority. */ +export function nodeSpawnClaimBelongsToRuntime( + claim: Pick, + runtime: Pick, +): boolean { + return claim.baseUrl === runtime.baseUrl + && claim.publisherId === runtime.publisher.id + && claim.publisherName === runtime.publisher.name; +} + export function durableNodeResultDelivery( runtime: NodeSpawnRuntime, path: string, @@ -187,9 +197,7 @@ async function reconcileNodeResultSettlement( journal: Pick, ): Promise { let claim = await readNodeSpawnClaim(path); - if (claim.baseUrl !== runtime.baseUrl - || claim.publisherId !== runtime.publisher.id - || claim.publisherName !== runtime.publisher.name) { + if (!nodeSpawnClaimBelongsToRuntime(claim, runtime)) { return 'foreign'; } if (nodeSpawnCleanupRequired(claim)) { diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index e90ed5eb9..6efabe542 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -25,7 +25,7 @@ import { import { createNodeSpawnRuntime, createNodeSpawnRuntimeOwner, explicitServiceCallerFile, mergeNodeSpawnEnvironment, type NodeSpawnRuntime, type NodeSpawnRuntimeOwner } from './node-spawn-runtime.js'; import { nodeSpawnCleanupRequired } from './node-spawn-cleanup.js'; -import { reconcileNodeResultSettlements } from './node-spawn-settlement.js'; +import { nodeSpawnClaimBelongsToRuntime, reconcileNodeResultSettlements } from './node-spawn-settlement.js'; import { readNodeSpawnClaims } from './node-spawn-state.js'; export { MODEL_ENV, WAKE_CONTEXT_ENV } from './worker-cli.js'; @@ -370,12 +370,18 @@ export class AgentWorker extends EventEmitter { private async reconcileRelaySettlementsOnce(): Promise { if (this.options.dataDir === undefined || (this.closing && !this.attached)) return; + // Durable claims are not authority. A direct-only worker must not read + // ambient credentials or contact Relay merely because another publisher + // left state in the shared data directory. Explicit Desktop admission or + // a real Relay dispatch establishes the only runtime this worker may use. + const runtime = this.nodeSpawnRuntimeOwner.peek(); + if (runtime === undefined) return; const claims = await readNodeSpawnClaims(this.options.dataDir); - const hasPending = claims.some(entry => nodeSpawnCleanupRequired(entry.claim) - || (entry.claim.result?.settlement !== undefined - && entry.claim.result.settlement.settledAt === undefined)); + const hasPending = claims.some(entry => nodeSpawnClaimBelongsToRuntime(entry.claim, runtime) + && (nodeSpawnCleanupRequired(entry.claim) + || (entry.claim.result?.settlement !== undefined + && entry.claim.result.settlement.settledAt === undefined))); if (!hasPending) return; - const runtime = await this.nodeSpawnRuntimeOwner.get(); let pending: number; try { pending = await reconcileNodeResultSettlements(this.options.dataDir, runtime, this.client); diff --git a/packages/sdk/tests/node-spawn-runtime.test.ts b/packages/sdk/tests/node-spawn-runtime.test.ts index f6d900723..45a41b515 100644 --- a/packages/sdk/tests/node-spawn-runtime.test.ts +++ b/packages/sdk/tests/node-spawn-runtime.test.ts @@ -68,6 +68,7 @@ describe('remote node-spawn runtime', () => { disconnect } satisfies NodeSpawnRuntime; const factory = vi.fn((_signal: AbortSignal) => new Promise(done => { resolve = done; })); const owner = createNodeSpawnRuntimeOwner(undefined, factory); + expect(owner.peek()).toBeUndefined(); const cancelled = new AbortController(); const first = owner.get(cancelled.signal); const second = owner.get(); @@ -77,6 +78,7 @@ describe('remote node-spawn runtime', () => { expect(factory.mock.calls[0]![0].aborted).toBe(false); resolve(runtime); await expect(second).resolves.toBe(runtime); + expect(owner.peek()).toBe(runtime); await expect(owner.get()).resolves.toBe(runtime); expect(disconnect).not.toHaveBeenCalled(); await expect(owner.close()).resolves.toBeUndefined(); diff --git a/packages/sdk/tests/worker-node-runtime-sharing.test.ts b/packages/sdk/tests/worker-node-runtime-sharing.test.ts index fad9d5d09..da6e361cc 100644 --- a/packages/sdk/tests/worker-node-runtime-sharing.test.ts +++ b/packages/sdk/tests/worker-node-runtime-sharing.test.ts @@ -7,14 +7,23 @@ import { NODE_RESULT_PREFIX } from '../src/node-spawn-result.js'; import { runAgentRelayTask } from '../src/agent-relay-transport.js'; import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; import type { NodeSpawnPort } from '../src/node-spawn.js'; +import { claimNodeSpawn, readNodeSpawnClaims, type NodeSpawnClaim } from '../src/node-spawn-state.js'; import type { JournalClient } from '../src/journal-client.js'; import type { StepDispatchEvent } from '../src/protocol.js'; import { WorkerLeaseLostError } from '../src/worker-lease.js'; -const owner = vi.hoisted(() => ({ get: vi.fn(), close: vi.fn(async () => {}) })); +const owner = vi.hoisted(() => ({ get: vi.fn(), peek: vi.fn(), close: vi.fn(async () => {}) })); vi.mock('../src/node-spawn-runtime.js', async importOriginal => ({ ...await importOriginal(), - createNodeSpawnRuntimeOwner: () => owner, + createNodeSpawnRuntimeOwner: () => ({ + get: async (signal?: AbortSignal) => { + const runtime = await owner.get(signal); + owner.peek.mockReturnValue(runtime); + return runtime; + }, + peek: () => owner.peek(), + close: () => owner.close(), + }), })); import { AgentWorker } from '../src/worker.js'; @@ -23,11 +32,113 @@ const roots: string[] = []; afterEach(async () => { vi.unstubAllEnvs(); owner.get.mockReset(); + owner.peek.mockReset(); owner.close.mockClear(); await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); describe('AgentWorker node-spawn runtime ownership', () => { + it('does not acquire authority or block attach for a foreign pending claim', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-foreign-claim-')); + roots.push(dataDir); + await claimNodeSpawn(dataDir, pendingCleanupClaim( + 'publisher_foreign', 'foreign-publisher', 'dispatch_foreign_claim', 'agent_foreign')); + owner.get.mockRejectedValue(new Error('publisher_token_required')); + const journalRead = vi.fn(async () => { + throw new Error('foreign claim must not read this worker journal'); + }); + const workerAttach = vi.fn(async () => ({})); + const client = Object.assign(new EventEmitter(), { workerAttach, journalRead }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_foreign_claim', pins: {}, dataDir, + }); + + await expect(worker.attach()).resolves.toBeUndefined(); + await expect(worker.close()).resolves.toBeUndefined(); + expect(owner.peek).toHaveBeenCalled(); + expect(owner.get).not.toHaveBeenCalled(); + expect(journalRead).not.toHaveBeenCalled(); + expect(workerAttach).toHaveBeenCalledOnce(); + const [unchanged] = await readNodeSpawnClaims(dataDir); + expect(unchanged?.claim).toMatchObject({ + publisherId: 'publisher_foreign', cleanup: { agentId: 'agent_foreign' }, + }); + expect(unchanged?.claim.release).toBeUndefined(); + }); + + it('reconciles only matching old claims after a real dispatch establishes authority', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-lazy-authority-')); + roots.push(dataDir); + await claimNodeSpawn(dataDir, pendingCleanupClaim( + 'publisher_1', 'flows-publisher', 'dispatch_matching_old', 'agent_matching_old')); + await claimNodeSpawn(dataDir, pendingCleanupClaim( + 'publisher_foreign', 'foreign-publisher', 'dispatch_foreign_old', 'agent_foreign_old')); + let spawnedName = ''; + const release = vi.fn(async input => ({ + id: `release_${input.receipt.agentId}`, action: 'release', status: 'completed', + spawned: false, ready: false, released: true, + })); + const runtime: NodeSpawnRuntime = { + publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + port: { + dispatchFenceVersion: 1, + targets: async () => [], + spawn: async (input, beforeDispatch) => { + await beforeDispatch(); + spawnedName = input.name; + return { id: 'spawn_lazy_authority', action: 'spawn', status: 'completed', + nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, + agentId: 'agent_lazy_authority', spawned: true, ready: true }; + }, + status: vi.fn(), + release, + }, + inbox: { inbox: { + subscribe: async function* () { + yield { id: 'delivery_lazy_authority', message: { id: 'message_lazy_authority', + from: { id: 'agent_lazy_authority', name: spawnedName }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_parallel', + step_id: 'edit-lazy-authority', dispatch_id: 'dispatch_lazyauthority', + status: 'completed', output: { summary: 'done' } }) } }; + }, + ack: vi.fn(async () => ({})), + fail: vi.fn(async () => ({})), + } }, + }; + owner.get.mockResolvedValue(runtime); + const stepComplete = vi.fn(async () => + ({} as Awaited>)); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), + stepComplete, + journalRead: vi.fn(async () => ({ entries: [] })), + supportsFeature: vi.fn(() => true), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_lazy_authority', pins: {}, dataDir, + }); + const errors: unknown[] = []; + worker.on('error', error => errors.push(error)); + + await worker.attach(); + expect(owner.get).not.toHaveBeenCalled(); + client.emit('step.dispatch', dispatch('edit-lazy-authority', 'dispatch_lazyauthority')); + await worker.close(); + + expect(errors).toEqual([]); + expect(owner.get).toHaveBeenCalledOnce(); + expect(stepComplete).toHaveBeenCalledOnce(); + expect(release.mock.calls.map(call => call[0].receipt.agentId)).toContain('agent_matching_old'); + expect(release.mock.calls.map(call => call[0].receipt.agentId)).not.toContain('agent_foreign_old'); + const claims = await readNodeSpawnClaims(dataDir); + expect(claims.find(entry => entry.claim.publisherId === 'publisher_1' + && entry.claim.cleanup?.agentId === 'agent_matching_old')?.claim.release).toBeDefined(); + expect(claims.find(entry => entry.claim.publisherId === 'publisher_foreign')?.claim.release) + .toBeUndefined(); + }); + it('validates an explicit Desktop caller before registering the worker', async () => { const callerFailure = new Error('Desktop caller is expired'); owner.get.mockRejectedValue(callerFailure); @@ -301,6 +412,7 @@ describe('AgentWorker node-spawn runtime ownership', () => { }, { runtime }); await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); owner.get.mockResolvedValue(runtime); + owner.peek.mockReturnValue(runtime); const client = Object.assign(new EventEmitter(), { workerAttach: vi.fn(async () => ({})), journalRead: vi.fn(async () => ({ entries: [{ @@ -680,3 +792,32 @@ function dispatch(stepId: string, idempotencyKey: string, leaseTtlMs?: number): pins: {}, }; } + +function pendingCleanupClaim( + publisherId: string, + publisherName: string, + idempotencyKey: string, + agentId: string, +): NodeSpawnClaim { + return { + version: 3, + baseUrl: 'https://cast.agentrelay.com', + publisherId, + publisherName, + runId: 'run_parallel', + stepId: `step_${agentId}`, + idempotencyKey, + agentName: `relayflow-${agentId}`, + timeoutMs: 30_000, + request: { cli: 'codex', task: 'Recover the prior task', task_identity: `identity_${agentId}` }, + startedAt: Date.now(), + cleanup: { + id: `spawn_${agentId}`, + nodeId: 'node_1', + nodeName: 'desktop-1', + name: `relayflow-${agentId}`, + agentId, + }, + failure: { code: 'result_timeout', message: 'worker crashed before cleanup' }, + }; +} From 333d5100b6878ae1ddd56532edb17235bda827e9 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Tue, 10 Nov 2026 10:04:00 -0800 Subject: [PATCH 58/75] fix(sdk): fence custom ports before dispatch --- packages/sdk/src/node-spawn-dispatch.ts | 28 +++ packages/sdk/src/node-spawn-local.ts | 5 +- packages/sdk/src/node-spawn-remote.ts | 5 +- .../tests/node-spawn-dispatch-fence.test.ts | 182 ++++++++++++++---- 4 files changed, 185 insertions(+), 35 deletions(-) diff --git a/packages/sdk/src/node-spawn-dispatch.ts b/packages/sdk/src/node-spawn-dispatch.ts index ab97a6561..841de0a96 100644 --- a/packages/sdk/src/node-spawn-dispatch.ts +++ b/packages/sdk/src/node-spawn-dispatch.ts @@ -14,17 +14,45 @@ interface FencedNodeSpawnOptions { settleLate(outcome: { spawn?: NodeSpawnReceipt; error?: unknown }): Promise; } +const officialNodeSpawnPorts = new WeakMap(); + +/** + * Registers the exact built-in spawn implementation that may perform proven + * side-effect-free preflight before publishing the durable dispatch fence. + * Replaced or overridden methods remain untrusted and are fenced before entry. + */ +export function registerOfficialNodeSpawnPort( + port: NodeSpawnPort, + implementation: NodeSpawnPort['spawn'], +): void { + if (port.spawn === implementation) officialNodeSpawnPorts.set(port, implementation); +} + +function isOfficialNodeSpawnPort(port: NodeSpawnPort): boolean { + return officialNodeSpawnPorts.get(port) === port.spawn; +} + /** Enters exactly one irreversible spawn behind a durable, one-way fence. */ export async function dispatchFencedNodeSpawn( options: FencedNodeSpawnOptions, ): Promise { + const officialPort = isOfficialNodeSpawnPort(options.port); let dispatchPublished = false; let dispatchHookEntered = false; let pendingSpawn: Promise; + if (!officialPort) { + // Arbitrary public ports cannot prove that their own preflight is free of + // side effects. Publish before entering any user code, so a process crash + // can never leave a provider-contacted claim looking retryable. + options.signal?.throwIfAborted(); + await claimNodeSpawnDispatch(options.claimPath, options.claim); + dispatchPublished = true; + } try { pendingSpawn = options.port.spawn(options.request, async () => { dispatchHookEntered = true; options.signal?.throwIfAborted(); + if (dispatchPublished) return; await claimNodeSpawnDispatch(options.claimPath, options.claim); dispatchPublished = true; }); diff --git a/packages/sdk/src/node-spawn-local.ts b/packages/sdk/src/node-spawn-local.ts index 4dc21e81e..6c04f7fe0 100644 --- a/packages/sdk/src/node-spawn-local.ts +++ b/packages/sdk/src/node-spawn-local.ts @@ -16,6 +16,7 @@ import { validIdempotencyKey, type BeforeNodeDispatch, } from './node-spawn.js'; +import { registerOfficialNodeSpawnPort } from './node-spawn-dispatch.js'; /** The structural core-bridge surface Flows consumes. Kept here so the package stays optional. */ export interface CoreBridgeClient { @@ -215,7 +216,9 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { private readonly bridge: Pick, private readonly scope: Pick, private readonly exactNode: { readonly id: string; readonly name: string }, - ) {} + ) { + registerOfficialNodeSpawnPort(this, LocalNodeSpawnPort.prototype.spawn); + } async targets(): Promise { const result = await this.bridge.spawnTargets(this.core); diff --git a/packages/sdk/src/node-spawn-remote.ts b/packages/sdk/src/node-spawn-remote.ts index bc8ec39e9..e4b39a54d 100644 --- a/packages/sdk/src/node-spawn-remote.ts +++ b/packages/sdk/src/node-spawn-remote.ts @@ -17,6 +17,7 @@ import { type NodeSpawnRequest, type NodeSpawnTarget, } from './node-spawn.js'; +import { registerOfficialNodeSpawnPort } from './node-spawn-dispatch.js'; interface RelayAgent { id: string; name: string } interface RelayNode { @@ -52,7 +53,9 @@ export interface RemoteNodeSpawnClient { export class RemoteNodeSpawnPort implements NodeSpawnPort { readonly dispatchFenceVersion = 1 as const; - constructor(private readonly client: RemoteNodeSpawnClient) {} + constructor(private readonly client: RemoteNodeSpawnClient) { + registerOfficialNodeSpawnPort(this, RemoteNodeSpawnPort.prototype.spawn); + } async targets(): Promise { return (await this.client.nodes.list()).map(node => ({ diff --git a/packages/sdk/tests/node-spawn-dispatch-fence.test.ts b/packages/sdk/tests/node-spawn-dispatch-fence.test.ts index 18b147aa1..7e6dff415 100644 --- a/packages/sdk/tests/node-spawn-dispatch-fence.test.ts +++ b/packages/sdk/tests/node-spawn-dispatch-fence.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { runAgentRelayTask, type AgentRelayTaskRequest } from '../src/agent-relay-transport.js'; +import { LocalNodeSpawnPort, type CoreBridgeClient } from '../src/node-spawn-local.js'; import { NODE_RESULT_PREFIX } from '../src/node-spawn-result.js'; import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; import { claimNodeSpawnDispatch, nodeSpawnDispatchFencePath, readNodeSpawnClaims, @@ -27,6 +28,84 @@ function request(dataDir: string, timeoutMs = 10_000): AgentRelayTaskRequest { return { cli: 'codex', task: 'Implement the change', runId: 'run_fence', stepId: 'edit', idempotencyKey: 'dispatch_fence123', attempt: 1, dataDir, timeoutMs, + worker_cwd: '/tmp', + }; +} + +function officialLocalRuntime(options: { + eligible(): boolean; + invoked(name: string): void; +}): NodeSpawnRuntime { + const core: CoreBridgeClient = { + hello: { + bridge_version: 1, + daemon_version: 'test', + pid: process.pid, + methods: [], + topics: [], + }, + close() {}, + }; + const invocation = (name: string) => ({ + id: 'inv_spawn_1', + agent_id: 'agent_1', + status: 'completed', + node_id: 'node_1', + name, + spawned: true, + ready: true, + released: false, + task_unconfirmed: false, + error_code: null, + }); + let spawnedName = ''; + const bridge: ConstructorParameters[1] = { + spawnTargets: vi.fn(async () => ({ + targets: options.eligible() ? [{ + id: 'node_1', + name: 'desktop-1', + live: true, + handlers_live: true, + stale: false, + capabilities: ['spawn:codex'], + active_agents: 0, + reserved_agents: 0, + max_agents: 4, + }] : [], + })), + spawnInvoke: vi.fn(async (_core, input) => { + spawnedName = input.name; + options.invoked(input.name); + return invocation(input.name); + }), + spawnStatus: vi.fn(async () => invocation(spawnedName)), + spawnRelease: vi.fn(async () => ({ + ...invocation(spawnedName), + spawned: false, + ready: false, + released: true, + })), + }; + return { + authority: 'desktop', + publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + port: new LocalNodeSpawnPort(core, bridge, { + harnesses: ['codex'], + directoryScope: ['/tmp'], + }, { id: 'node_1', name: 'desktop-1' }), + inbox: { inbox: { + subscribe: async function* () { + yield { id: 'delivery_1', message: { id: 'message_1', + from: { id: 'agent_1', name: spawnedName }, + text: NODE_RESULT_PREFIX + JSON.stringify({ + version: 1, run_id: 'run_fence', step_id: 'edit', + dispatch_id: 'dispatch_fence123', status: 'completed', output: { ok: true }, + }) } }; + }, + ack: vi.fn(async () => ({})), + fail: vi.fn(async () => ({})), + } }, }; } @@ -77,19 +156,17 @@ const preDispatchFailure = (message: string, code = 'no_eligible_node'): NodeSpa new NodeSpawnPreDispatchError(new NodeSpawnError(message, code)); describe('durable node-spawn dispatch fence', () => { - it('lets a public custom port attest safe preflight and retry once when eligible', async () => { + it('lets an official port retry proven preflight once when eligible', async () => { const dataDir = await dataDirectory('flows-dispatch-fence-retry-'); let eligible = false; let invokes = 0; - const worker = runtime(async (input, beforeDispatch) => { - if (!eligible) throw preDispatchFailure('no eligible node'); - await beforeDispatch(); - invokes += 1; - return readyReceipt(input.name); + const worker = officialLocalRuntime({ + eligible: () => eligible, + invoked() { invokes += 1; }, }); await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) - .rejects.toThrow('no eligible node'); + .rejects.toThrow('No eligible local Agent Relay spawn target'); const [claim] = await readNodeSpawnClaims(dataDir); expect(claim!.claim.version).toBe(3); await expect(readFile(nodeSpawnDispatchFencePath(claim!.path), 'utf8')) @@ -101,6 +178,26 @@ describe('durable node-spawn dispatch fence', () => { expect(invokes).toBe(1); }); + it('fences an arbitrary public port before entering any of its code', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-public-port-'); + let calls = 0; + let fenceAtEntry = false; + const worker = runtime(async () => { + calls += 1; + const [entry] = await readNodeSpawnClaims(dataDir); + await readFile(nodeSpawnDispatchFencePath(entry!.path), 'utf8'); + fenceAtEntry = true; + throw preDispatchFailure('custom preflight cannot be trusted'); + }); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('custom preflight cannot be trusted'); + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('ambiguous pre-ack'); + expect(fenceAtEntry).toBe(true); + expect(calls).toBe(1); + }); + it('rejects an old public custom port before it creates a claim or invokes', async () => { const dataDir = await dataDirectory('flows-dispatch-fence-old-port-'); const spawn = vi.fn(async (input: Parameters[0]) => readyReceipt(input.name)); @@ -144,8 +241,9 @@ describe('durable node-spawn dispatch fence', () => { it('lets exactly one independent fence contender enter its irreversible call', async () => { const dataDir = await dataDirectory('flows-dispatch-fence-race-'); - const worker = runtime(async () => { throw preDispatchFailure('prepare claim'); }); - await expect(runAgentRelayTask(request(dataDir), { runtime: worker })).rejects.toThrow('prepare claim'); + const worker = officialLocalRuntime({ eligible: () => false, invoked() {} }); + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('No eligible local Agent Relay spawn target'); const [entry] = await readNodeSpawnClaims(dataDir); let invokes = 0; const contender = async (): Promise => { @@ -215,35 +313,33 @@ describe('durable node-spawn dispatch fence', () => { it('treats a non-ENOENT fence read failure as invalid durable state', async () => { const dataDir = await dataDirectory('flows-dispatch-fence-read-failure-'); - let calls = 0; - const worker = runtime(async () => { - calls += 1; - throw preDispatchFailure('prepare unreadable fence'); + let invokes = 0; + const worker = officialLocalRuntime({ + eligible: () => false, + invoked() { invokes += 1; }, }); await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) - .rejects.toThrow('prepare unreadable fence'); + .rejects.toThrow('No eligible local Agent Relay spawn target'); const [entry] = await readNodeSpawnClaims(dataDir); await mkdir(nodeSpawnDispatchFencePath(entry!.path)); await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) .rejects.toThrow('Invalid durable node-spawn dispatch fence'); - expect(calls).toBe(1); + expect(invokes).toBe(0); }); it('does not call the remote effect when dispatch-fence publication fails', async () => { const dataDir = await dataDirectory('flows-dispatch-fence-write-failure-'); - let prepare = true; + let eligible = false; let remoteCalls = 0; - const worker = runtime(async (input, beforeDispatch) => { - if (prepare) throw preDispatchFailure('prepare fence failure'); - await beforeDispatch(); - remoteCalls += 1; - return readyReceipt(input.name); + const worker = officialLocalRuntime({ + eligible: () => eligible, + invoked() { remoteCalls += 1; }, }); await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) - .rejects.toThrow('prepare fence failure'); + .rejects.toThrow('No eligible local Agent Relay spawn target'); const [entry] = await readNodeSpawnClaims(dataDir); - prepare = false; + eligible = true; await chmod(dirname(entry!.path), 0o500); try { await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) @@ -260,32 +356,52 @@ describe('durable node-spawn dispatch fence', () => { it('retains the original claim deadline across a safe pre-dispatch retry', async () => { vi.useFakeTimers(); const dataDir = await dataDirectory('flows-dispatch-fence-deadline-'); - let calls = 0; - const worker = runtime(async () => { - calls += 1; - throw preDispatchFailure('not ready yet'); + let invokes = 0; + const worker = officialLocalRuntime({ + eligible: () => false, + invoked() { invokes += 1; }, }); await expect(runAgentRelayTask(request(dataDir, 1_000), { runtime: worker })) - .rejects.toThrow('not ready yet'); + .rejects.toThrow('No eligible local Agent Relay spawn target'); await vi.advanceTimersByTimeAsync(1_001); await expect(runAgentRelayTask(request(dataDir, 1_000), { runtime: worker })) .rejects.toThrow('deadline elapsed'); - expect(calls).toBe(1); + expect(invokes).toBe(0); }); - it('fences a custom port that returns a receipt without awaiting the required hook', async () => { + it('treats an overridden built-in spawn method as an arbitrary port', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-overridden-port-'); + const worker = officialLocalRuntime({ eligible: () => true, invoked() {} }); + let fenceAtEntry = false; + const originalSpawn = worker.port.spawn.bind(worker.port); + worker.port.spawn = vi.fn(async (input, beforeDispatch) => { + const [entry] = await readNodeSpawnClaims(dataDir); + await readFile(nodeSpawnDispatchFencePath(entry!.path), 'utf8'); + fenceAtEntry = true; + return originalSpawn(input, beforeDispatch); + }); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .resolves.toMatchObject({ receipt: { status: 'completed' } }); + expect(fenceAtEntry).toBe(true); + }); + + it('keeps an arbitrary port fenced when it returns without calling the hook', async () => { const dataDir = await dataDirectory('flows-dispatch-fence-bypass-'); let calls = 0; + let fenceAtEntry = false; const worker = runtime(async input => { calls += 1; + const [entry] = await readNodeSpawnClaims(dataDir); + await readFile(nodeSpawnDispatchFencePath(entry!.path), 'utf8'); + fenceAtEntry = true; return readyReceipt(input.name); }); await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) - .rejects.toThrow('required dispatch fence'); - await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) - .rejects.toThrow('required dispatch fence'); + .resolves.toMatchObject({ receipt: { status: 'completed' } }); + expect(fenceAtEntry).toBe(true); expect(calls).toBe(1); }); }); From 6de14c2eb729cc8b21cd46c55ece73143d76a41f Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Tue, 10 Nov 2026 10:06:00 -0800 Subject: [PATCH 59/75] docs(sdk): clarify custom port fence semantics --- docs/AGENT-RELAY-TRANSPORT.md | 21 +++++++++++---------- docs/MIGRATION-NODE-SPAWN.md | 22 ++++++++++++---------- packages/sdk/src/node-spawn.ts | 4 ++-- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/docs/AGENT-RELAY-TRANSPORT.md b/docs/AGENT-RELAY-TRANSPORT.md index b7f5c62a6..882658ecd 100644 --- a/docs/AGENT-RELAY-TRANSPORT.md +++ b/docs/AGENT-RELAY-TRANSPORT.md @@ -66,16 +66,17 @@ blindly retried. A recorded invocation is reconciled with collision protection at the node. Custom `NodeSpawnPort` implementations must declare `dispatchFenceVersion: 1` -and await the supplied `beforeDispatch` callback immediately before their -first irreversible spawn call. Preflight that fails before entering that -callback may throw the exported `NodeSpawnPreDispatchError`; this is a -privileged assertion that no remote effect was attempted, not a generic error -wrapper. Ports must never use it for an error at or after dispatch. Existing -one-argument `spawn` implementations are not compatible even though JavaScript -can ignore extra arguments: the required protocol field makes them a type -error, and the runtime refuses a missing or unknown version before creating a -durable claim. A falsely declared port that omits the callback is fenced and -fails closed after its first uncertain call. +and accept the supplied `beforeDispatch` callback. Because Flows cannot audit +arbitrary port code, it publishes and fsyncs the fence before entering a custom +`spawn`; an error from that call is permanently ambiguous and cannot be retried. +The callback remains part of the protocol and is idempotent after publication. +Only the SDK's exact built-in local and remote port methods may use +`NodeSpawnPreDispatchError` to retain retryable, side-effect-free preflight. +Replacing or overriding either built-in method makes it custom and fences it +before entry. Existing one-argument `spawn` implementations are not compatible +even though JavaScript can ignore extra arguments: the required protocol field +makes them a type error, and the runtime refuses a missing or unknown version +before creating a durable claim. The claim also fixes the task's original start time. Resume never grants a new execution deadline. Once a correlated result was durably observed before that diff --git a/docs/MIGRATION-NODE-SPAWN.md b/docs/MIGRATION-NODE-SPAWN.md index 9fb2e8b9d..8e2597d01 100644 --- a/docs/MIGRATION-NODE-SPAWN.md +++ b/docs/MIGRATION-NODE-SPAWN.md @@ -49,16 +49,18 @@ in Desktop rather than editing the file. ## Custom `NodeSpawnPort` implementations -Ports must now declare `dispatchFenceVersion: 1` and await the supplied -`beforeDispatch` callback immediately before the first irreversible spawn -call. Preflight that fails before that callback may throw the public -`NodeSpawnPreDispatchError` to preserve safe retry; using that marker after an -effect may have started is a protocol violation. Existing one-argument -`spawn` methods must be migrated even though JavaScript permits them to ignore -the callback. The required version field makes an unchanged port a type error, -and runtime use fails before claim creation. A port that falsely declares the -version but omits the callback is fenced and refused after its first uncertain -call. +Ports must declare `dispatchFenceVersion: 1` and accept the supplied +`beforeDispatch` callback. Flows cannot prove that arbitrary port preflight is +side-effect-free, so it durably fences a custom port before entering `spawn`; +any error from that call is permanently ambiguous and is not retried. Calling +`beforeDispatch` remains required for forward compatibility and is idempotent +after that fence. `NodeSpawnPreDispatchError` preserves retry only for the SDK's +exact built-in local and remote implementations, whose preflight is audited; +it does not reopen a custom port's fenced claim. Existing one-argument `spawn` +methods must be migrated even though JavaScript permits them to ignore the +callback. The required version field makes an unchanged port a type error, and +runtime use fails before claim creation. Overriding a built-in port's `spawn` +method also makes that instance custom and therefore fenced before entry. ## Hosted and self-hosted execution diff --git a/packages/sdk/src/node-spawn.ts b/packages/sdk/src/node-spawn.ts index 6fabd4920..174a1c15e 100644 --- a/packages/sdk/src/node-spawn.ts +++ b/packages/sdk/src/node-spawn.ts @@ -67,12 +67,12 @@ export interface NodeReleaseRequest { reason?: string; } -/** Publishes the durable write-ahead fact immediately before an irreversible spawn call. */ +/** Requests the durable write-ahead fact before an irreversible spawn call. */ export type BeforeNodeDispatch = () => Promise; /** Credentials and transport stay inside an implementation of this port. */ export interface NodeSpawnPort { - /** Proves this port implements the required durable pre-dispatch callback protocol. */ + /** Declares participation in the durable dispatch-fence protocol. */ readonly dispatchFenceVersion: 1; targets(): Promise; spawn(request: NodeSpawnRequest, beforeDispatch: BeforeNodeDispatch): Promise; From 3df4bf487431560583085b1c0eae915515e641ee Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Tue, 10 Nov 2026 10:12:53 -0800 Subject: [PATCH 60/75] refactor(sdk): narrow retryable port preflight --- docs/AGENT-RELAY-TRANSPORT.md | 8 ++++---- docs/MIGRATION-NODE-SPAWN.md | 8 ++++---- packages/sdk/src/node-spawn-dispatch.ts | 14 +++++++------- packages/sdk/src/node-spawn-local.ts | 4 ++-- packages/sdk/src/node-spawn-remote.ts | 5 +---- 5 files changed, 18 insertions(+), 21 deletions(-) diff --git a/docs/AGENT-RELAY-TRANSPORT.md b/docs/AGENT-RELAY-TRANSPORT.md index 882658ecd..5ff23bb84 100644 --- a/docs/AGENT-RELAY-TRANSPORT.md +++ b/docs/AGENT-RELAY-TRANSPORT.md @@ -70,10 +70,10 @@ and accept the supplied `beforeDispatch` callback. Because Flows cannot audit arbitrary port code, it publishes and fsyncs the fence before entering a custom `spawn`; an error from that call is permanently ambiguous and cannot be retried. The callback remains part of the protocol and is idempotent after publication. -Only the SDK's exact built-in local and remote port methods may use -`NodeSpawnPreDispatchError` to retain retryable, side-effect-free preflight. -Replacing or overriding either built-in method makes it custom and fences it -before entry. Existing one-argument `spawn` implementations are not compatible +Only the SDK's exact built-in local port method may use +`NodeSpawnPreDispatchError` to retain retryable target preflight. Replacing or +overriding that method removes the narrow trust and fences it before entry. +Existing one-argument `spawn` implementations are not compatible even though JavaScript can ignore extra arguments: the required protocol field makes them a type error, and the runtime refuses a missing or unknown version before creating a durable claim. diff --git a/docs/MIGRATION-NODE-SPAWN.md b/docs/MIGRATION-NODE-SPAWN.md index 8e2597d01..38850cc4a 100644 --- a/docs/MIGRATION-NODE-SPAWN.md +++ b/docs/MIGRATION-NODE-SPAWN.md @@ -55,12 +55,12 @@ side-effect-free, so it durably fences a custom port before entering `spawn`; any error from that call is permanently ambiguous and is not retried. Calling `beforeDispatch` remains required for forward compatibility and is idempotent after that fence. `NodeSpawnPreDispatchError` preserves retry only for the SDK's -exact built-in local and remote implementations, whose preflight is audited; -it does not reopen a custom port's fenced claim. Existing one-argument `spawn` +exact built-in local implementation, whose target preflight is audited; +it does not reopen any other port's fenced claim. Existing one-argument `spawn` methods must be migrated even though JavaScript permits them to ignore the callback. The required version field makes an unchanged port a type error, and -runtime use fails before claim creation. Overriding a built-in port's `spawn` -method also makes that instance custom and therefore fenced before entry. +runtime use fails before claim creation. Overriding the built-in local port's +`spawn` method also removes that narrow trust and fences it before entry. ## Hosted and self-hosted execution diff --git a/packages/sdk/src/node-spawn-dispatch.ts b/packages/sdk/src/node-spawn-dispatch.ts index 841de0a96..9787e18ec 100644 --- a/packages/sdk/src/node-spawn-dispatch.ts +++ b/packages/sdk/src/node-spawn-dispatch.ts @@ -14,33 +14,33 @@ interface FencedNodeSpawnOptions { settleLate(outcome: { spawn?: NodeSpawnReceipt; error?: unknown }): Promise; } -const officialNodeSpawnPorts = new WeakMap(); +const retryablePreflightPorts = new WeakMap(); /** * Registers the exact built-in spawn implementation that may perform proven * side-effect-free preflight before publishing the durable dispatch fence. * Replaced or overridden methods remain untrusted and are fenced before entry. */ -export function registerOfficialNodeSpawnPort( +export function registerRetryablePreflightPort( port: NodeSpawnPort, implementation: NodeSpawnPort['spawn'], ): void { - if (port.spawn === implementation) officialNodeSpawnPorts.set(port, implementation); + if (port.spawn === implementation) retryablePreflightPorts.set(port, implementation); } -function isOfficialNodeSpawnPort(port: NodeSpawnPort): boolean { - return officialNodeSpawnPorts.get(port) === port.spawn; +function hasRetryablePreflight(port: NodeSpawnPort): boolean { + return retryablePreflightPorts.get(port) === port.spawn; } /** Enters exactly one irreversible spawn behind a durable, one-way fence. */ export async function dispatchFencedNodeSpawn( options: FencedNodeSpawnOptions, ): Promise { - const officialPort = isOfficialNodeSpawnPort(options.port); + const retryablePreflight = hasRetryablePreflight(options.port); let dispatchPublished = false; let dispatchHookEntered = false; let pendingSpawn: Promise; - if (!officialPort) { + if (!retryablePreflight) { // Arbitrary public ports cannot prove that their own preflight is free of // side effects. Publish before entering any user code, so a process crash // can never leave a provider-contacted claim looking retryable. diff --git a/packages/sdk/src/node-spawn-local.ts b/packages/sdk/src/node-spawn-local.ts index 6c04f7fe0..fa377971b 100644 --- a/packages/sdk/src/node-spawn-local.ts +++ b/packages/sdk/src/node-spawn-local.ts @@ -16,7 +16,7 @@ import { validIdempotencyKey, type BeforeNodeDispatch, } from './node-spawn.js'; -import { registerOfficialNodeSpawnPort } from './node-spawn-dispatch.js'; +import { registerRetryablePreflightPort } from './node-spawn-dispatch.js'; /** The structural core-bridge surface Flows consumes. Kept here so the package stays optional. */ export interface CoreBridgeClient { @@ -217,7 +217,7 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { private readonly scope: Pick, private readonly exactNode: { readonly id: string; readonly name: string }, ) { - registerOfficialNodeSpawnPort(this, LocalNodeSpawnPort.prototype.spawn); + registerRetryablePreflightPort(this, LocalNodeSpawnPort.prototype.spawn); } async targets(): Promise { diff --git a/packages/sdk/src/node-spawn-remote.ts b/packages/sdk/src/node-spawn-remote.ts index e4b39a54d..bc8ec39e9 100644 --- a/packages/sdk/src/node-spawn-remote.ts +++ b/packages/sdk/src/node-spawn-remote.ts @@ -17,7 +17,6 @@ import { type NodeSpawnRequest, type NodeSpawnTarget, } from './node-spawn.js'; -import { registerOfficialNodeSpawnPort } from './node-spawn-dispatch.js'; interface RelayAgent { id: string; name: string } interface RelayNode { @@ -53,9 +52,7 @@ export interface RemoteNodeSpawnClient { export class RemoteNodeSpawnPort implements NodeSpawnPort { readonly dispatchFenceVersion = 1 as const; - constructor(private readonly client: RemoteNodeSpawnClient) { - registerOfficialNodeSpawnPort(this, RemoteNodeSpawnPort.prototype.spawn); - } + constructor(private readonly client: RemoteNodeSpawnClient) {} async targets(): Promise { return (await this.client.nodes.list()).map(node => ({ From f7af0e843569795d9dfeee0657caed02481d8871 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Tue, 10 Nov 2026 10:22:03 -0800 Subject: [PATCH 61/75] fix(sdk): fingerprint retryable preflight methods --- docs/AGENT-RELAY-TRANSPORT.md | 4 +- docs/MIGRATION-NODE-SPAWN.md | 3 +- packages/sdk/src/node-spawn-dispatch.ts | 22 +++++++---- packages/sdk/src/node-spawn-local.ts | 7 +++- .../tests/node-spawn-dispatch-fence.test.ts | 37 +++++++++++++++++++ 5 files changed, 62 insertions(+), 11 deletions(-) diff --git a/docs/AGENT-RELAY-TRANSPORT.md b/docs/AGENT-RELAY-TRANSPORT.md index 5ff23bb84..d05f19c9f 100644 --- a/docs/AGENT-RELAY-TRANSPORT.md +++ b/docs/AGENT-RELAY-TRANSPORT.md @@ -70,9 +70,9 @@ and accept the supplied `beforeDispatch` callback. Because Flows cannot audit arbitrary port code, it publishes and fsyncs the fence before entering a custom `spawn`; an error from that call is permanently ambiguous and cannot be retried. The callback remains part of the protocol and is idempotent after publication. -Only the SDK's exact built-in local port method may use +Only the SDK's exact built-in local `spawn` and `targets` method set may use `NodeSpawnPreDispatchError` to retain retryable target preflight. Replacing or -overriding that method removes the narrow trust and fences it before entry. +overriding either method removes the narrow trust and fences it before entry. Existing one-argument `spawn` implementations are not compatible even though JavaScript can ignore extra arguments: the required protocol field makes them a type error, and the runtime refuses a missing or unknown version diff --git a/docs/MIGRATION-NODE-SPAWN.md b/docs/MIGRATION-NODE-SPAWN.md index 38850cc4a..78a79d1ec 100644 --- a/docs/MIGRATION-NODE-SPAWN.md +++ b/docs/MIGRATION-NODE-SPAWN.md @@ -60,7 +60,8 @@ it does not reopen any other port's fenced claim. Existing one-argument `spawn` methods must be migrated even though JavaScript permits them to ignore the callback. The required version field makes an unchanged port a type error, and runtime use fails before claim creation. Overriding the built-in local port's -`spawn` method also removes that narrow trust and fences it before entry. +`spawn` or `targets` method also removes that narrow trust and fences it before +entry. ## Hosted and self-hosted execution diff --git a/packages/sdk/src/node-spawn-dispatch.ts b/packages/sdk/src/node-spawn-dispatch.ts index 9787e18ec..03d18e48c 100644 --- a/packages/sdk/src/node-spawn-dispatch.ts +++ b/packages/sdk/src/node-spawn-dispatch.ts @@ -14,22 +14,30 @@ interface FencedNodeSpawnOptions { settleLate(outcome: { spawn?: NodeSpawnReceipt; error?: unknown }): Promise; } -const retryablePreflightPorts = new WeakMap(); +interface RetryablePreflightIdentity { + spawn: NodeSpawnPort['spawn']; + targets: NodeSpawnPort['targets']; +} + +const retryablePreflightPorts = new WeakMap(); /** - * Registers the exact built-in spawn implementation that may perform proven - * side-effect-free preflight before publishing the durable dispatch fence. - * Replaced or overridden methods remain untrusted and are fenced before entry. + * Registers the exact built-in method set that may perform proven side-effect- + * free preflight before publishing the durable dispatch fence. Replaced or + * overridden methods remain untrusted and are fenced before entry. */ export function registerRetryablePreflightPort( port: NodeSpawnPort, - implementation: NodeSpawnPort['spawn'], + identity: RetryablePreflightIdentity, ): void { - if (port.spawn === implementation) retryablePreflightPorts.set(port, implementation); + if (port.spawn === identity.spawn && port.targets === identity.targets) { + retryablePreflightPorts.set(port, identity); + } } function hasRetryablePreflight(port: NodeSpawnPort): boolean { - return retryablePreflightPorts.get(port) === port.spawn; + const identity = retryablePreflightPorts.get(port); + return identity?.spawn === port.spawn && identity.targets === port.targets; } /** Enters exactly one irreversible spawn behind a durable, one-way fence. */ diff --git a/packages/sdk/src/node-spawn-local.ts b/packages/sdk/src/node-spawn-local.ts index fa377971b..e89c0a0fc 100644 --- a/packages/sdk/src/node-spawn-local.ts +++ b/packages/sdk/src/node-spawn-local.ts @@ -217,7 +217,7 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { private readonly scope: Pick, private readonly exactNode: { readonly id: string; readonly name: string }, ) { - registerRetryablePreflightPort(this, LocalNodeSpawnPort.prototype.spawn); + registerRetryablePreflightPort(this, LOCAL_NODE_SPAWN_IDENTITY); } async targets(): Promise { @@ -399,6 +399,11 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { } } +const LOCAL_NODE_SPAWN_IDENTITY = Object.freeze({ + spawn: LocalNodeSpawnPort.prototype.spawn, + targets: LocalNodeSpawnPort.prototype.targets, +}); + function withinDirectory(root: string, candidate: string): boolean { const path = relative(root, candidate); return path === '' || (path !== '..' && !path.startsWith(`..${sep}`) diff --git a/packages/sdk/tests/node-spawn-dispatch-fence.test.ts b/packages/sdk/tests/node-spawn-dispatch-fence.test.ts index 7e6dff415..1aeab8cba 100644 --- a/packages/sdk/tests/node-spawn-dispatch-fence.test.ts +++ b/packages/sdk/tests/node-spawn-dispatch-fence.test.ts @@ -387,6 +387,43 @@ describe('durable node-spawn dispatch fence', () => { expect(fenceAtEntry).toBe(true); }); + it('treats an overridden built-in target preflight as arbitrary code', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-overridden-targets-'); + const worker = officialLocalRuntime({ eligible: () => true, invoked() {} }); + let fenceAtEntry = false; + const originalTargets = worker.port.targets.bind(worker.port); + worker.port.targets = vi.fn(async () => { + const [entry] = await readNodeSpawnClaims(dataDir); + await readFile(nodeSpawnDispatchFencePath(entry!.path), 'utf8'); + fenceAtEntry = true; + return originalTargets(); + }); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .resolves.toMatchObject({ receipt: { status: 'completed' } }); + expect(fenceAtEntry).toBe(true); + }); + + it('does not trust a local spawn prototype replaced before construction', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-replaced-prototype-'); + const originalSpawn = LocalNodeSpawnPort.prototype.spawn; + let fenceAtEntry = false; + LocalNodeSpawnPort.prototype.spawn = async function (input, beforeDispatch) { + const [entry] = await readNodeSpawnClaims(dataDir); + await readFile(nodeSpawnDispatchFencePath(entry!.path), 'utf8'); + fenceAtEntry = true; + return originalSpawn.call(this, input, beforeDispatch); + }; + try { + const worker = officialLocalRuntime({ eligible: () => true, invoked() {} }); + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .resolves.toMatchObject({ receipt: { status: 'completed' } }); + expect(fenceAtEntry).toBe(true); + } finally { + LocalNodeSpawnPort.prototype.spawn = originalSpawn; + } + }); + it('keeps an arbitrary port fenced when it returns without calling the hook', async () => { const dataDir = await dataDirectory('flows-dispatch-fence-bypass-'); let calls = 0; From b5084e11084d67b126174e16e003295ab9d71b57 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Tue, 10 Nov 2026 10:27:29 -0800 Subject: [PATCH 62/75] fix(sdk): avoid pre-fence port accessors --- packages/sdk/src/node-spawn-dispatch.ts | 25 ++++++++--- packages/sdk/src/node-spawn-local.ts | 1 + .../tests/node-spawn-dispatch-fence.test.ts | 45 ++++++++++++++++++- 3 files changed, 62 insertions(+), 9 deletions(-) diff --git a/packages/sdk/src/node-spawn-dispatch.ts b/packages/sdk/src/node-spawn-dispatch.ts index 03d18e48c..1ea06712f 100644 --- a/packages/sdk/src/node-spawn-dispatch.ts +++ b/packages/sdk/src/node-spawn-dispatch.ts @@ -15,6 +15,7 @@ interface FencedNodeSpawnOptions { } interface RetryablePreflightIdentity { + prototype: object; spawn: NodeSpawnPort['spawn']; targets: NodeSpawnPort['targets']; } @@ -30,14 +31,24 @@ export function registerRetryablePreflightPort( port: NodeSpawnPort, identity: RetryablePreflightIdentity, ): void { - if (port.spawn === identity.spawn && port.targets === identity.targets) { - retryablePreflightPorts.set(port, identity); - } + if (hasExactPreflightMethods(port, identity)) retryablePreflightPorts.set(port, identity); } function hasRetryablePreflight(port: NodeSpawnPort): boolean { const identity = retryablePreflightPorts.get(port); - return identity?.spawn === port.spawn && identity.targets === port.targets; + return identity !== undefined && hasExactPreflightMethods(port, identity); +} + +function hasExactPreflightMethods( + port: NodeSpawnPort, + identity: RetryablePreflightIdentity, +): boolean { + if (Object.getPrototypeOf(port) !== identity.prototype + || Object.getOwnPropertyDescriptor(port, 'spawn') !== undefined + || Object.getOwnPropertyDescriptor(port, 'targets') !== undefined) return false; + const spawn = Object.getOwnPropertyDescriptor(identity.prototype, 'spawn'); + const targets = Object.getOwnPropertyDescriptor(identity.prototype, 'targets'); + return spawn?.value === identity.spawn && targets?.value === identity.targets; } /** Enters exactly one irreversible spawn behind a durable, one-way fence. */ @@ -49,9 +60,9 @@ export async function dispatchFencedNodeSpawn( let dispatchHookEntered = false; let pendingSpawn: Promise; if (!retryablePreflight) { - // Arbitrary public ports cannot prove that their own preflight is free of - // side effects. Publish before entering any user code, so a process crash - // can never leave a provider-contacted claim looking retryable. + // Arbitrary public ports cannot prove that their own spawn preflight is + // free of side effects. Publish before resolving or entering spawn, so a + // process crash cannot leave a provider-contacted claim looking retryable. options.signal?.throwIfAborted(); await claimNodeSpawnDispatch(options.claimPath, options.claim); dispatchPublished = true; diff --git a/packages/sdk/src/node-spawn-local.ts b/packages/sdk/src/node-spawn-local.ts index e89c0a0fc..0f0e0bff5 100644 --- a/packages/sdk/src/node-spawn-local.ts +++ b/packages/sdk/src/node-spawn-local.ts @@ -400,6 +400,7 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { } const LOCAL_NODE_SPAWN_IDENTITY = Object.freeze({ + prototype: LocalNodeSpawnPort.prototype, spawn: LocalNodeSpawnPort.prototype.spawn, targets: LocalNodeSpawnPort.prototype.targets, }); diff --git a/packages/sdk/tests/node-spawn-dispatch-fence.test.ts b/packages/sdk/tests/node-spawn-dispatch-fence.test.ts index 1aeab8cba..fe7b8f831 100644 --- a/packages/sdk/tests/node-spawn-dispatch-fence.test.ts +++ b/packages/sdk/tests/node-spawn-dispatch-fence.test.ts @@ -1,3 +1,4 @@ +import { existsSync } from 'node:fs'; import { chmod, mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; @@ -6,8 +7,8 @@ import { runAgentRelayTask, type AgentRelayTaskRequest } from '../src/agent-rela import { LocalNodeSpawnPort, type CoreBridgeClient } from '../src/node-spawn-local.js'; import { NODE_RESULT_PREFIX } from '../src/node-spawn-result.js'; import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; -import { claimNodeSpawnDispatch, nodeSpawnDispatchFencePath, readNodeSpawnClaims, - recordNodeSpawn } from '../src/node-spawn-state.js'; +import { claimNodeSpawnDispatch, nodeSpawnClaimPath, nodeSpawnDispatchFencePath, + readNodeSpawnClaims, recordNodeSpawn } from '../src/node-spawn-state.js'; import { NodeSpawnError, NodeSpawnPreDispatchError, type NodeSpawnPort, type NodeSpawnReceipt } from '../src/index.js'; @@ -387,6 +388,46 @@ describe('durable node-spawn dispatch fence', () => { expect(fenceAtEntry).toBe(true); }); + it('does not read a custom spawn accessor before publishing the fence', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-custom-accessor-'); + const worker = runtime(async input => readyReceipt(input.name)); + const spawn = worker.port.spawn; + const observations: boolean[] = []; + const claimPath = await nodeSpawnClaimPath( + dataDir, 'run_fence', 'edit', 'dispatch_fence123'); + Object.defineProperty(worker.port, 'spawn', { + configurable: true, + get() { + observations.push(existsSync(nodeSpawnDispatchFencePath(claimPath))); + return spawn; + }, + }); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .resolves.toMatchObject({ receipt: { status: 'completed' } }); + expect(observations).toEqual([true]); + }); + + it('does not read a registered local spawn accessor before publishing the fence', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-local-accessor-'); + const worker = officialLocalRuntime({ eligible: () => true, invoked() {} }); + const spawn = worker.port.spawn; + const observations: boolean[] = []; + const claimPath = await nodeSpawnClaimPath( + dataDir, 'run_fence', 'edit', 'dispatch_fence123'); + Object.defineProperty(worker.port, 'spawn', { + configurable: true, + get() { + observations.push(existsSync(nodeSpawnDispatchFencePath(claimPath))); + return spawn; + }, + }); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .resolves.toMatchObject({ receipt: { status: 'completed' } }); + expect(observations).toEqual([true]); + }); + it('treats an overridden built-in target preflight as arbitrary code', async () => { const dataDir = await dataDirectory('flows-dispatch-fence-overridden-targets-'); const worker = officialLocalRuntime({ eligible: () => true, invoked() {} }); From af36c143c8110c33d4aef6d5429665f367670d80 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Tue, 10 Nov 2026 17:11:44 -0800 Subject: [PATCH 63/75] test: pin public core bridge contract --- packages/sdk/package-lock.json | 10 ++++++++++ packages/sdk/package.json | 1 + .../sdk/tests/node-spawn-core-bridge-public.test.ts | 10 ++++++++++ 3 files changed, 21 insertions(+) create mode 100644 packages/sdk/tests/node-spawn-core-bridge-public.test.ts diff --git a/packages/sdk/package-lock.json b/packages/sdk/package-lock.json index a64303179..e567d4d78 100644 --- a/packages/sdk/package-lock.json +++ b/packages/sdk/package-lock.json @@ -29,6 +29,7 @@ }, "devDependencies": { "@agent-relay/cli-surface": "^12.2.4", + "@agent-relay/core-bridge": "18.1.0", "@agent-relay/sdk": "17.2.1", "@types/node": "^22.7.0", "bluebird": "^3.7.2", @@ -57,6 +58,15 @@ "node": ">=22.0.0" } }, + "node_modules/@agent-relay/core-bridge": { + "version": "18.1.0", + "resolved": "https://registry.npmjs.org/@agent-relay/core-bridge/-/core-bridge-18.1.0.tgz", + "integrity": "sha512-GXSrmuPaPG5iP7hdZc8QMiNc1r2HjqQeHI/RV3lVDTtbq7z16BX+aIGZHiYMhVEn7D7Ttho/kqRakKSDP/bq6Q==", + "dev": true, + "engines": { + "node": ">=22.0.0" + } + }, "node_modules/@agent-relay/sdk": { "version": "17.2.1", "resolved": "https://registry.npmjs.org/@agent-relay/sdk/-/sdk-17.2.1.tgz", diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 2947b3036..a8dffb35b 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -64,6 +64,7 @@ }, "devDependencies": { "@agent-relay/cli-surface": "^12.2.4", + "@agent-relay/core-bridge": "18.1.0", "@agent-relay/sdk": "17.2.1", "@types/node": "^22.7.0", "bluebird": "^3.7.2", diff --git a/packages/sdk/tests/node-spawn-core-bridge-public.test.ts b/packages/sdk/tests/node-spawn-core-bridge-public.test.ts new file mode 100644 index 000000000..a5f729b08 --- /dev/null +++ b/packages/sdk/tests/node-spawn-core-bridge-public.test.ts @@ -0,0 +1,10 @@ +import * as publicCoreBridge181 from '@agent-relay/core-bridge'; +import { describe, expect, it } from 'vitest'; + +import { assertCoreBridgeApi } from '../src/node-spawn-local.js'; + +describe('public core-bridge release contract', () => { + it('accepts the exact installed 18.1.0 schema-v2 lifecycle surface', () => { + expect(() => assertCoreBridgeApi(publicCoreBridge181)).not.toThrow(); + }); +}); From d1799fb8fa251c87f26408d2265a70476b74672d Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Tue, 10 Nov 2026 17:57:47 -0800 Subject: [PATCH 64/75] fix(sdk): replay strict local spawn after lost receipt --- docs/AGENT-RELAY-TRANSPORT.md | 15 +- docs/MIGRATION-NODE-SPAWN.md | 7 + packages/sdk/src/agent-relay-transport.ts | 7 +- packages/sdk/src/node-spawn-dispatch.ts | 26 +++ packages/sdk/src/node-spawn-local.ts | 4 +- .../tests/node-spawn-local-replay-gap.test.ts | 168 ++++++++++++++++++ 6 files changed, 219 insertions(+), 8 deletions(-) create mode 100644 packages/sdk/tests/node-spawn-local-replay-gap.test.ts diff --git a/docs/AGENT-RELAY-TRANSPORT.md b/docs/AGENT-RELAY-TRANSPORT.md index d05f19c9f..221074e26 100644 --- a/docs/AGENT-RELAY-TRANSPORT.md +++ b/docs/AGENT-RELAY-TRANSPORT.md @@ -57,11 +57,16 @@ dispatch fence: Flows writes and fsyncs a complete private marker, atomically hard-links it into place, and fsyncs the directory immediately before local `spawn.invoke` or remote `placement.spawn`. Competing workers may repeat preflight, but only the fence winner may enter that irreversible call. A v3 -claim with no fence proves dispatch was not entered and may retry preflight; -once the fence exists, a missing receipt is ambiguous forever and fails -closed. Malformed, mismatched, or unreadable fence state also fails closed. -Legacy version 2 claims without a receipt remain ambiguous and are never -blindly retried. A recorded invocation is reconciled with +claim with no fence proves dispatch was not entered and may retry preflight. +The exact built-in local Desktop port is the sole fenced-replay exception: +Core Bridge 18.1 or newer durably deduplicates `spawn.invoke` by the forwarded +idempotency key, so a restart may repeat that identical request and recover the +original invocation after its reply was lost. Flows admits that replay only for +the registered, unchanged `LocalNodeSpawnPort` prototype and re-verifies the +existing fence before entry. A missing receipt from remote placement, a custom +or overridden port, or a legacy version 2 claim remains ambiguous and fails +closed without another dispatch. Malformed, mismatched, or unreadable fence +state also fails closed. A recorded invocation is reconciled with `commands.getInvocation`. The stable worker name provides additional collision protection at the node. diff --git a/docs/MIGRATION-NODE-SPAWN.md b/docs/MIGRATION-NODE-SPAWN.md index 78a79d1ec..c9665ebd8 100644 --- a/docs/MIGRATION-NODE-SPAWN.md +++ b/docs/MIGRATION-NODE-SPAWN.md @@ -63,6 +63,13 @@ runtime use fails before claim creation. Overriding the built-in local port's `spawn` or `targets` method also removes that narrow trust and fences it before entry. +The exact built-in local Desktop port has one additional recovery contract. +Core Bridge 18.1 or newer persists `spawn.invoke` idempotency keys, so after a +durable fence and a lost reply Flows may repeat the identical local request and +recover the original invocation. That exemption is registered by exact +prototype and method identity; wrappers, subclasses, overrides, accessors, +custom ports, remote placement and legacy version 2 claims remain fail-closed. + ## Hosted and self-hosted execution Hosted workers use the agent-token fleet path. Self-hosted workers use the diff --git a/packages/sdk/src/agent-relay-transport.ts b/packages/sdk/src/agent-relay-transport.ts index 0289e0141..dc46489c1 100644 --- a/packages/sdk/src/agent-relay-transport.ts +++ b/packages/sdk/src/agent-relay-transport.ts @@ -2,7 +2,8 @@ import { createHash } from 'node:crypto'; import { claimNodeSpawnAtPath, hasNodeSpawnDispatchFence, nodeSpawnClaimPath, recordNodeSpawn, type NodeSpawnClaim } from './node-spawn-state.js'; -import { dispatchFencedNodeSpawn } from './node-spawn-dispatch.js'; +import { dispatchFencedNodeSpawn, + hasReplayableFencedDispatch } from './node-spawn-dispatch.js'; import { releaseNodeSpawnClaim } from './node-spawn-cleanup.js'; import { nodeResultInstruction, waitForNodeResult } from './node-spawn-result.js'; import { durableNodeResultDelivery } from './node-spawn-settlement.js'; @@ -229,8 +230,9 @@ async function executeAgentRelayTask( const deadline = claim.startedAt + claim.timeoutMs; try { const dispatchFenced = claim.version === 3 && await hasNodeSpawnDispatchFence(claimed.path, claim); + const replayExistingFence = dispatchFenced && hasReplayableFencedDispatch(runtime.port); if (claim.spawn === undefined && claim.cleanup === undefined - && (dispatchFenced || (!claimed.created && claim.version === 2))) { + && ((dispatchFenced && !replayExistingFence) || (!claimed.created && claim.version === 2))) { throw new AgentRelayTransportError('A prior node-spawn dispatch ended in the ambiguous pre-ack window; refusing a second spawn'); } // A v3 claim without a dispatch fence proves no irreversible call was @@ -267,6 +269,7 @@ async function executeAgentRelayTask( timeoutMs: remaining, }, claimPath: claimed.path, claim, signal, settleLate: settleLateSpawn, + replayExistingFence, }); claim = { ...claim, spawn }; await recordNodeSpawn(claimed.path, claim); diff --git a/packages/sdk/src/node-spawn-dispatch.ts b/packages/sdk/src/node-spawn-dispatch.ts index 1ea06712f..d12ff908a 100644 --- a/packages/sdk/src/node-spawn-dispatch.ts +++ b/packages/sdk/src/node-spawn-dispatch.ts @@ -12,6 +12,7 @@ interface FencedNodeSpawnOptions { claim: NodeSpawnClaim; signal?: AbortSignal; settleLate(outcome: { spawn?: NodeSpawnReceipt; error?: unknown }): Promise; + replayExistingFence?: boolean; } interface RetryablePreflightIdentity { @@ -21,6 +22,7 @@ interface RetryablePreflightIdentity { } const retryablePreflightPorts = new WeakMap(); +const replayableFencedPorts = new WeakMap(); /** * Registers the exact built-in method set that may perform proven side-effect- @@ -34,6 +36,20 @@ export function registerRetryablePreflightPort( if (hasExactPreflightMethods(port, identity)) retryablePreflightPorts.set(port, identity); } +/** Registers an exact built-in port whose server contract replays a fenced dispatch by idempotency key. */ +export function registerReplayableFencedPort( + port: NodeSpawnPort, + identity: RetryablePreflightIdentity, +): void { + if (hasExactPreflightMethods(port, identity)) replayableFencedPorts.set(port, identity); +} + +/** True only while the exact registered built-in method set remains installed. */ +export function hasReplayableFencedDispatch(port: NodeSpawnPort): boolean { + const identity = replayableFencedPorts.get(port); + return identity !== undefined && hasExactPreflightMethods(port, identity); +} + function hasRetryablePreflight(port: NodeSpawnPort): boolean { const identity = retryablePreflightPorts.get(port); return identity !== undefined && hasExactPreflightMethods(port, identity); @@ -57,6 +73,16 @@ export async function dispatchFencedNodeSpawn( ): Promise { const retryablePreflight = hasRetryablePreflight(options.port); let dispatchPublished = false; + if (options.replayExistingFence === true) { + if (!hasReplayableFencedDispatch(options.port) + || !await hasNodeSpawnDispatchFence(options.claimPath, options.claim)) { + throw new NodeSpawnError( + 'A fenced node-spawn replay requires the exact built-in replay contract and durable fence', + 'spawn_dispatch_fence_missing', + ); + } + dispatchPublished = true; + } let dispatchHookEntered = false; let pendingSpawn: Promise; if (!retryablePreflight) { diff --git a/packages/sdk/src/node-spawn-local.ts b/packages/sdk/src/node-spawn-local.ts index 0f0e0bff5..4f0728d96 100644 --- a/packages/sdk/src/node-spawn-local.ts +++ b/packages/sdk/src/node-spawn-local.ts @@ -16,7 +16,8 @@ import { validIdempotencyKey, type BeforeNodeDispatch, } from './node-spawn.js'; -import { registerRetryablePreflightPort } from './node-spawn-dispatch.js'; +import { registerReplayableFencedPort, + registerRetryablePreflightPort } from './node-spawn-dispatch.js'; /** The structural core-bridge surface Flows consumes. Kept here so the package stays optional. */ export interface CoreBridgeClient { @@ -218,6 +219,7 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { private readonly exactNode: { readonly id: string; readonly name: string }, ) { registerRetryablePreflightPort(this, LOCAL_NODE_SPAWN_IDENTITY); + registerReplayableFencedPort(this, LOCAL_NODE_SPAWN_IDENTITY); } async targets(): Promise { diff --git a/packages/sdk/tests/node-spawn-local-replay-gap.test.ts b/packages/sdk/tests/node-spawn-local-replay-gap.test.ts new file mode 100644 index 000000000..d9a255a51 --- /dev/null +++ b/packages/sdk/tests/node-spawn-local-replay-gap.test.ts @@ -0,0 +1,168 @@ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { runAgentRelayTask, type AgentRelayTaskRequest } from '../src/agent-relay-transport.js'; +import { LocalNodeSpawnPort, type CoreBridgeClient } from '../src/node-spawn-local.js'; +import { NODE_RESULT_PREFIX } from '../src/node-spawn-result.js'; +import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; + +const roots: string[] = []; + +afterEach(async () => { + await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); +}); + +async function dataDirectory(): Promise { + const directory = await mkdtemp(join(tmpdir(), 'flows-local-replay-gap-')); + roots.push(directory); + return directory; +} + +function request(dataDir: string): AgentRelayTaskRequest { + return { + cli: 'codex', + task: 'Implement the exact change', + taskIdentity: 'task_sha256_exact', + runId: 'run_local_replay', + stepId: 'edit', + idempotencyKey: 'local_replay_key_123', + attempt: 1, + dataDir, + timeoutMs: 10_000, + worker_cwd: '/tmp', + }; +} + +function strictReplayRuntime(observed: { + invokeCalls: Array<{ key: string; task: string }>; + commits: Set; + dropFirstReply: boolean; + releases: number; + subscriptions: number; +}): NodeSpawnRuntime { + const core: CoreBridgeClient = { + hello: { + bridge_version: 1, + daemon_version: 'test', + pid: process.pid, + methods: [], + topics: [], + }, + close() {}, + }; + let spawnedName = ''; + const invocation = () => ({ + id: 'inv_spawn_replayed', + agent_id: 'agent_replayed', + status: 'completed', + node_id: 'node_local', + name: spawnedName, + spawned: true, + ready: true, + released: false, + task_unconfirmed: false, + error_code: null, + }); + const bridge: ConstructorParameters[1] = { + spawnTargets: vi.fn(async () => ({ + targets: [{ + id: 'node_local', + name: 'desktop-local', + live: true, + handlers_live: true, + stale: false, + capabilities: ['spawn:codex'], + active_agents: 0, + reserved_agents: 0, + max_agents: 4, + }], + })), + spawnInvoke: vi.fn(async (_core, input) => { + spawnedName = input.name; + observed.invokeCalls.push({ key: input.idempotency_key, task: input.task }); + observed.commits.add(input.idempotency_key); + if (observed.dropFirstReply) { + observed.dropFirstReply = false; + throw new Error('bridge reply lost after strict server commit'); + } + return invocation(); + }), + spawnStatus: vi.fn(async () => invocation()), + spawnRelease: vi.fn(async () => { + observed.releases += 1; + return { + ...invocation(), + spawned: false, + ready: false, + released: true, + }; + }), + }; + return { + authority: 'desktop', + publisher: { id: 'publisher_local', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + port: new LocalNodeSpawnPort(core, bridge, { + harnesses: ['codex'], + directoryScope: ['/tmp'], + }, { id: 'node_local', name: 'desktop-local' }), + inbox: { inbox: { + subscribe: async function* () { + observed.subscriptions += 1; + yield { + id: 'delivery_replayed', + message: { + id: 'message_replayed', + from: { id: 'agent_replayed', name: spawnedName }, + text: NODE_RESULT_PREFIX + JSON.stringify({ + version: 1, + run_id: 'run_local_replay', + step_id: 'edit', + dispatch_id: 'local_replay_key_123', + status: 'completed', + output: { ok: true }, + }), + }, + }; + }, + ack: vi.fn(async () => ({})), + fail: vi.fn(async () => ({})), + } }, + }; +} + +describe('official local strict replay', () => { + it('replays the same idempotency key after the first committed reply is lost', async () => { + const dataDir = await dataDirectory(); + const observed = { + invokeCalls: [] as Array<{ key: string; task: string }>, + commits: new Set(), + dropFirstReply: true, + releases: 0, + subscriptions: 0, + }; + const crashedRuntime = strictReplayRuntime(observed); + + await expect(runAgentRelayTask(request(dataDir), { runtime: crashedRuntime })) + .rejects.toThrow('bridge reply lost after strict server commit'); + const restartedRuntime = strictReplayRuntime(observed); + const recovered = await Promise.all([ + runAgentRelayTask(request(dataDir), { runtime: restartedRuntime }), + runAgentRelayTask(request(dataDir), { runtime: restartedRuntime }), + ]); + expect(recovered).toEqual([ + expect.objectContaining({ receipt: expect.objectContaining({ status: 'completed' }) }), + expect.objectContaining({ receipt: expect.objectContaining({ status: 'completed' }) }), + ]); + + expect(observed.invokeCalls).toHaveLength(2); + expect(observed.invokeCalls[0]).toEqual(observed.invokeCalls[1]); + expect(observed.invokeCalls[0]).toMatchObject({ key: 'local_replay_key_123' }); + expect(observed.invokeCalls[0]!.task).toContain('Implement the exact change'); + expect([...observed.commits]).toEqual(['local_replay_key_123']); + expect(observed.subscriptions).toBe(1); + expect(observed.releases).toBe(1); + }); +}); From e915701b98130fe629905a72323061aaf1f3304e Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Tue, 10 Nov 2026 18:15:08 -0800 Subject: [PATCH 65/75] test(sdk): add spawn response-loss proof kit --- docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md | 138 +++++++ .../scripts/node-spawn-response-loss-e2e.mjs | 259 ++++++++++++ .../node-spawn-response-loss-proxy.mjs | 230 +++++++++++ .../node-spawn-response-loss-proxy.test.ts | 380 ++++++++++++++++++ 4 files changed, 1007 insertions(+) create mode 100644 docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md create mode 100644 packages/sdk/scripts/node-spawn-response-loss-e2e.mjs create mode 100644 packages/sdk/scripts/node-spawn-response-loss-proxy.mjs create mode 100644 packages/sdk/tests/node-spawn-response-loss-proxy.test.ts diff --git a/docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md b/docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md new file mode 100644 index 000000000..9bc96e909 --- /dev/null +++ b/docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md @@ -0,0 +1,138 @@ +# Node-spawn response-loss acceptance kit + +This is an acceptance-only proof for Flows #707. It changes no production +runtime behavior. It proves recovery when upstream commits a strict spawn but +Flows loses the complete `spawn.invoke` response. + +The kit is Unix-only and must run from a plain terminal. Never run it from an +agent session tree. Use a disposable workspace, a dedicated target node, a +mode-0600 schema-v2 caller, and the packed SDK under review. Do not put caller +contents, publisher tokens, or workspace credentials in the config or logs. + +## Files + +- `packages/sdk/scripts/node-spawn-response-loss-proxy.mjs` — transparent Core + Bridge socket proxy and fail-closed evidence verifier. +- `packages/sdk/scripts/node-spawn-response-loss-e2e.mjs` — lifecycle + orchestrator. +- `packages/sdk/tests/node-spawn-response-loss-proxy.test.ts` — deterministic + fake-upstream proof of zero response bytes, same-key concurrent replays, one + upstream frame, exact task bytes, and fail-closed evidence validation. + +The proxy forwards newline-framed JSON-RPC until it reads the first complete, +successful `spawn.invoke` reply from the real core. It never writes that line +downstream. It synchronously sends `SIGKILL` to the foreground Flows process, +then closes that connection. Later connections pass through unchanged. + +## Preconditions + +1. Install dependencies from the exact #707 head and build the SDK. +2. Pack that SDK and install it with public + `@agent-relay/core-bridge@18.1.0` in a fresh mode-0700 consumer directory. +3. Provision a dedicated mode-0600 schema-v2 caller from a plain terminal. +4. Prepare fresh mode-0700 HOME and Flows data directories; the data directory + must contain no `runs/*.sqlite3`. +5. Reserve one unrelated capacity slot and record its stable identity. The + audit command must show the same reservation before and after the proof. +6. Provide a fixture audit command that reads the real Relaycast/Relay node, + Desktop caller ledger, Flows journal, and inbox settlement record. It must + print only the JSON object below on stdout and must not print secrets. + +## Config + +Create a mode-0600 JSON file outside the repository. Paths are absolute. The +start command runs the packed SDK in the foreground. The resume command must +contain `{RUN_ID}` exactly where the durable run id belongs. Do not pass +`--service-caller-file`; the kit securely copies the caller into a temporary +mode-0700 directory outside the evidence tree, changes only its socket to the proxy, and injects +`RELAYFLOW_SERVICE_CALLER_FILE`. + +```json +{ + "callerFile": "/private/proof/caller.json", + "dataDir": "/private/proof/flows-data", + "evidenceDir": "/private/proof/evidence", + "homeDir": "/private/proof/home", + "workDir": "/private/proof/work", + "timeoutMs": 600000, + "start": { + "command": "/usr/bin/node", + "args": ["/private/consumer/node_modules/@relayflows/sdk/dist/cli.js", "run", "/private/proof/work/proof.flow.ts", "--input", "{}", "--local-agent", "--json", "--no-observer-link", "--data-dir", "/private/proof/flows-data"] + }, + "resume": { + "command": "/usr/bin/node", + "args": ["/private/consumer/node_modules/@relayflows/sdk/dist/cli.js", "resume", "{RUN_ID}", "--local-agent", "--json", "--no-observer-link", "--data-dir", "/private/proof/flows-data"] + }, + "audit": { + "command": "/private/proof/bin/audit-response-loss", + "args": ["--run", "{RUN_ID}", "--json"] + } +} +``` + +The audit command stdout contract is: + +```json +{ + "schema": "relayflows-node-spawn-response-loss/1", + "strict_contract": "relay.spawn.invoke.logical-replay-resume/1", + "strict_replay_capability": "relay.spawn.replay-resume/1", + "strict_invocation_committed": true, + "upstream_spawn_frames": 1, + "workers_created": 1, + "correlated_results": 1, + "releases": 1, + "target_capacity": { "active_agents": 0, "reserved_agents": 0 }, + "journal_terminal_sequence": 41, + "inbox_settlement_sequence": 42, + "unrelated_reservation": { + "before": { "identity": "stable-unrelated-reservation-id", "count": 1 }, + "after": { "identity": "stable-unrelated-reservation-id", "count": 1 } + } +} +``` + +Sequence values are fixture-defined monotonic positions, but the terminal +journal position must be strictly less than the inbox settlement position. +The audit must count real provider frames and real created workers, not bridge +requests. The strict contract and capability fields must come from the stored +Relaycast invocation, not from the request or from a hard-coded client claim. + +## Commands + +From the exact Flows checkout: + +```sh +npm --prefix packages/sdk ci --ignore-scripts --no-audit --no-fund +npm --prefix packages/sdk run typecheck +npm --prefix packages/sdk run typecheck:tests +npm --prefix packages/sdk run build +npm --prefix packages/sdk exec -- vitest run tests/node-spawn-response-loss-proxy.test.ts --reporter=verbose +node packages/sdk/scripts/node-spawn-response-loss-e2e.mjs --config /private/proof/response-loss-config.json +``` + +The final command prints only `{ "ok": true, "evidence": ..., "run_id": ... }`. +It writes mode-0600 logs plus `response-loss-proof.json` under `evidenceDir`. +Before writing evidence it scans every captured log for caller-secret, token, +publisher-key, and Relayfile-bearer shapes. The evidence contains hashes and +counts, never the caller secret or task text. The runner requires the public +`@agent-relay/core-bridge@18.1.0` package, runs children with the fresh HOME, +and removes inherited publisher, workspace, broker, node, and API credentials. + +## Required result + +The proof passes only when all of these are true: + +- the proxy received the committed upstream reply and forwarded zero bytes of + it before `SIGKILL`; +- two concurrent resumes used the same run, task bytes, and idempotency key; +- the real provider saw one spawn frame and created one worker; +- one correlated result was journaled before one inbox settlement; +- exactly one release completed; +- target active and reserved capacity returned to zero; and +- the unrelated reservation identity and count stayed unchanged. + +Keep #707 draft after the run. Attach the exact source head, packed SDK receipt, +public Core Bridge receipt, config with private paths redacted, literal commands +and output, and the secret-free `response-loss-proof.json` for independent +review. A human decides merge readiness; this kit never merges or publishes. diff --git a/packages/sdk/scripts/node-spawn-response-loss-e2e.mjs b/packages/sdk/scripts/node-spawn-response-loss-e2e.mjs new file mode 100644 index 000000000..0e601832b --- /dev/null +++ b/packages/sdk/scripts/node-spawn-response-loss-e2e.mjs @@ -0,0 +1,259 @@ +#!/usr/bin/env node + +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { closeSync, openSync } from 'node:fs'; +import { + chmod, mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, isAbsolute, join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { setTimeout as delay } from 'node:timers/promises'; + +import { + createBridgeResponseLossProxy, + verifyResponseLossEvidence, +} from './node-spawn-response-loss-proxy.mjs'; + +function assertCommandSpec(spec, label) { + assert.equal(typeof spec?.command, 'string', `${label}.command is required`); + assert(isAbsolute(spec.command), `${label}.command must be absolute`); + assert(Array.isArray(spec.args) && spec.args.every(arg => typeof arg === 'string'), + `${label}.args must be strings`); + assert(!spec.args.some(arg => arg === '--service-caller-file' || arg.startsWith('--service-caller-file=')), + `${label} must use the proxy caller injected by the proof kit`); +} + +function commandWithRunId(spec, runId) { + return { + command: spec.command, + args: spec.args.map(arg => arg.replaceAll('{RUN_ID}', runId)), + }; +} + +async function runProcess(spec, { cwd, env, logPath, timeoutMs }) { + const fd = openSync(logPath, 'wx', 0o600); + const child = spawn(spec.command, spec.args, { + cwd, + env, + stdio: ['ignore', fd, fd], + }); + closeSync(fd); + const result = await Promise.race([ + new Promise((resolveExit, rejectExit) => { + child.once('error', rejectExit); + child.once('exit', (code, signal) => resolveExit({ code, signal })); + }), + delay(timeoutMs, undefined, { ref: false }).then(() => { + child.kill('SIGKILL'); + throw new Error(`command timed out; see ${logPath}`); + }), + ]); + return { child, result }; +} + +async function waitForRunId(dataDir, timeoutMs) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + let files = []; + try { + files = (await readdir(join(dataDir, 'runs'))).filter(file => file.endsWith('.sqlite3')); + } catch (error) { + if (error?.code !== 'ENOENT') throw error; + } + if (files.length === 1) return files[0].slice(0, -'.sqlite3'.length); + assert(files.length <= 1, 'proof data dir contains more than one run'); + await delay(25); + } + throw new Error('timed out waiting for the single proof run id'); +} + +async function assertSecretFree(paths) { + const forbidden = [ + /rsc_[A-Za-z0-9_-]{20,}/, + /(?:at|rk|nt|ot|br|arr)_(?:live_)?[A-Za-z0-9_-]{8,}/, + /(?:relay_pa_|sh_fr_)[A-Za-z0-9_-]+/, + /caller_secret/i, + /(?:RELAY|AGENT_RELAY|RELAYCAST)_(?:AGENT_TOKEN|API_KEY|WORKSPACE_KEY|NODE_TOKEN|WORKSPACES_JSON)/, + ]; + for (const path of paths) { + const text = await readFile(path, 'utf8'); + assert(!forbidden.some(pattern => pattern.test(text)), `${path} contains credential-shaped material`); + } +} + +async function installedPackageVersion(name) { + let directory = dirname(fileURLToPath(import.meta.resolve(name))); + for (;;) { + try { + const document = JSON.parse(await readFile(join(directory, 'package.json'), 'utf8')); + if (document.name === name && typeof document.version === 'string') return document.version; + } catch (error) { + if (error?.code !== 'ENOENT') throw error; + } + const parent = dirname(directory); + if (parent === directory) break; + directory = parent; + } + throw new Error(`could not resolve installed ${name} package version`); +} + +async function main() { + const configIndex = process.argv.indexOf('--config'); + assert(configIndex >= 0 && process.argv[configIndex + 1], 'usage: node node-spawn-response-loss-e2e.mjs --config /absolute/config.json'); + const configPath = resolve(process.argv[configIndex + 1]); + const config = JSON.parse(await readFile(configPath, 'utf8')); + for (const field of ['callerFile', 'dataDir', 'evidenceDir', 'homeDir', 'workDir']) { + assert.equal(typeof config[field], 'string', `${field} is required`); + assert(isAbsolute(config[field]), `${field} must be absolute`); + } + assertCommandSpec(config.start, 'start'); + assertCommandSpec(config.resume, 'resume'); + assertCommandSpec(config.audit, 'audit'); + assert(config.resume.args.some(arg => arg.includes('{RUN_ID}')), 'resume args must contain {RUN_ID}'); + const timeoutMs = config.timeoutMs ?? DEFAULT_TIMEOUT_MS; + assert(Number.isSafeInteger(timeoutMs) && timeoutMs > 0, 'timeoutMs must be positive'); + assert.equal((await stat(configPath)).mode & 0o777, 0o600, 'config must be mode 0600'); + await mkdir(config.homeDir, { recursive: true, mode: 0o700 }); + await mkdir(config.dataDir, { recursive: true, mode: 0o700 }); + await mkdir(config.evidenceDir, { recursive: true, mode: 0o700 }); + for (const [label, path] of [ + ['homeDir', config.homeDir], + ['dataDir', config.dataDir], + ['evidenceDir', config.evidenceDir], + ]) { + assert.equal((await stat(path)).mode & 0o777, 0o700, `${label} must be mode 0700`); + } + let initialRuns = []; + try { + initialRuns = (await readdir(join(config.dataDir, 'runs'))).filter(file => file.endsWith('.sqlite3')); + } catch (error) { + if (error?.code !== 'ENOENT') throw error; + } + assert.deepEqual(initialRuns, [], 'proof dataDir must not contain an earlier run'); + + const coreBridgeVersion = await installedPackageVersion('@agent-relay/core-bridge'); + assert.equal(coreBridgeVersion, '18.1.0', 'proof kit requires public @agent-relay/core-bridge@18.1.0'); + const { readCallerDescriptor } = await import('@agent-relay/core-bridge'); + const descriptor = await readCallerDescriptor(config.callerFile); + const privateDir = await mkdtemp(join(tmpdir(), 'flows-response-loss-private-')); + await chmod(privateDir, 0o700); + const proxySocket = join(privateDir, 'core.sock'); + const proxyCaller = join(privateDir, 'caller.json'); + const callerDocument = JSON.parse(await readFile(config.callerFile, 'utf8')); + callerDocument.socket = proxySocket; + await writeFile(proxyCaller, `${JSON.stringify(callerDocument)}\n`, { mode: 0o600, flag: 'wx' }); + await chmod(proxyCaller, 0o600); + + const childEnv = { ...process.env, + HOME: config.homeDir, + RELAYFLOW_SERVICE_CALLER_FILE: proxyCaller, + FLOWS_CLOUD_MIRROR: '0', + TMPDIR: privateDir, + }; + for (const name of [ + 'AGENT_RELAY_WORKSPACE_KEY', + 'RELAYCAST_WORKSPACE_KEY', + 'RELAY_AGENT_TOKEN', + 'RELAY_API_KEY', + 'RELAY_BROKER_API_KEY', + 'RELAY_NODE_TOKEN', + 'RELAY_WORKSPACE_KEY', + 'RELAY_WORKSPACES_JSON', + ]) delete childEnv[name]; + let firstChild; + const proxy = await createBridgeResponseLossProxy({ + listenSocket: proxySocket, + upstreamSocket: descriptor.socket, + onIntercept: () => { + if (firstChild?.pid) process.kill(firstChild.pid, 'SIGKILL'); + }, + }); + try { + const startLog = join(config.evidenceDir, 'start.log'); + const startFd = openSync(startLog, 'wx', 0o600); + firstChild = spawn(config.start.command, config.start.args, { + cwd: config.workDir, + env: childEnv, + stdio: ['ignore', startFd, startFd], + }); + closeSync(startFd); + const startExit = new Promise((resolveExit, rejectExit) => { + firstChild.once('error', rejectExit); + firstChild.once('exit', (code, signal) => resolveExit({ code, signal })); + }); + await Promise.race([ + proxy.intercepted, + delay(timeoutMs, undefined, { ref: false }) + .then(() => { throw new Error(`spawn reply was not intercepted; see ${startLog}`); }), + ]); + const firstResult = await startExit; + assert.equal(firstResult.signal, 'SIGKILL', 'initial Flows process must die by SIGKILL'); + const runId = await waitForRunId(config.dataDir, timeoutMs); + + const resumeSpec = commandWithRunId(config.resume, runId); + const resumeLog1 = join(config.evidenceDir, 'resume-1.log'); + const resumeLog2 = join(config.evidenceDir, 'resume-2.log'); + const [left, right] = await Promise.all([ + runProcess(resumeSpec, { + cwd: config.workDir, env: childEnv, + logPath: resumeLog1, timeoutMs, + }), + runProcess(resumeSpec, { + cwd: config.workDir, env: childEnv, + logPath: resumeLog2, timeoutMs, + }), + ]); + const resumeResults = [left.result, right.result]; + assert(resumeResults.some(result => result.code === 0), + 'at least one raced resume must complete successfully'); + assert(resumeResults.every(result => result.signal === null), + 'neither raced resume may die by signal'); + + const auditSpec = commandWithRunId(config.audit, runId); + const auditLog = join(config.evidenceDir, 'audit.stderr.log'); + const fd = openSync(auditLog, 'wx', 0o600); + const auditChild = spawn(auditSpec.command, auditSpec.args, { + cwd: config.workDir, + env: childEnv, + stdio: ['ignore', 'pipe', fd], + }); + closeSync(fd); + let auditStdout = ''; + auditChild.stdout.on('data', chunk => { auditStdout += String(chunk); }); + const auditExit = await Promise.race([ + new Promise((resolveExit, rejectExit) => { + auditChild.once('error', rejectExit); + auditChild.once('exit', (code, signal) => resolveExit({ code, signal })); + }), + delay(timeoutMs, undefined, { ref: false }).then(() => { + auditChild.kill('SIGKILL'); + throw new Error(`audit command timed out; see ${auditLog}`); + }), + ]); + assert.deepEqual(auditExit, { code: 0, signal: null }, `audit command failed; see ${auditLog}`); + const audit = JSON.parse(auditStdout); + await assertSecretFree([startLog, resumeLog1, resumeLog2, auditLog]); + const summary = verifyResponseLossEvidence(proxy.stats, audit); + const evidence = { + ...summary, + run_id: runId, + resume_exit_codes: resumeResults.map(result => result.code), + core_bridge_version: coreBridgeVersion, + }; + const evidencePath = join(config.evidenceDir, 'response-loss-proof.json'); + await writeFile(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`, { mode: 0o600, flag: 'wx' }); + console.log(JSON.stringify({ ok: true, evidence: evidencePath, run_id: runId })); + } finally { + await proxy.close(); + await rm(privateDir, { recursive: true, force: true }); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + main().catch(error => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/packages/sdk/scripts/node-spawn-response-loss-proxy.mjs b/packages/sdk/scripts/node-spawn-response-loss-proxy.mjs new file mode 100644 index 000000000..95160a886 --- /dev/null +++ b/packages/sdk/scripts/node-spawn-response-loss-proxy.mjs @@ -0,0 +1,230 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { chmod, mkdir, rm } from 'node:fs/promises'; +import net from 'node:net'; +import { dirname, isAbsolute } from 'node:path'; + +const NEWLINE = Buffer.from('\n'); +const MAX_FRAME_BYTES = 17 * 1024 * 1024; +const SPAWN_INVOKE = 'spawn.invoke'; +const SPAWN_RELEASE = 'spawn.release'; +const INBOX_ACK = 'spawn_callers.inbox.ack'; +const INBOX_FAIL = 'spawn_callers.inbox.fail'; +const STRICT_CONTRACT = 'relay.spawn.invoke.logical-replay-resume/1'; +const STRICT_REPLAY_CAPABILITY = 'relay.spawn.replay-resume/1'; + +function sha256(value) { + return createHash('sha256').update(value).digest('hex'); +} + +function safeJson(line) { + try { + const value = JSON.parse(line.toString('utf8')); + return typeof value === 'object' && value !== null ? value : undefined; + } catch { + return undefined; + } +} + +function lineReader(onLine, onOverflow) { + let buffered = Buffer.alloc(0); + return chunk => { + buffered = buffered.length === 0 ? chunk : Buffer.concat([buffered, chunk]); + if (buffered.length > MAX_FRAME_BYTES) { + onOverflow(); + buffered = Buffer.alloc(0); + return; + } + for (;;) { + const newline = buffered.indexOf(NEWLINE); + if (newline < 0) return; + const line = buffered.subarray(0, newline); + buffered = buffered.subarray(newline + 1); + onLine(line); + } + }; +} + +/** + * Transparently proxies Core Bridge JSON-RPC, except for the first successful + * spawn.invoke reply. That complete reply is read from upstream, recorded by + * digest, never written downstream, and only then handed to onIntercept so the + * Flows process can be SIGKILLed before any byte of that reply is delivered. + */ +export async function createBridgeResponseLossProxy({ + listenSocket, + upstreamSocket, + onIntercept = () => {}, +}) { + assert(process.platform !== 'win32', 'response-loss proof proxy requires a Unix socket'); + assert(isAbsolute(listenSocket), 'listenSocket must be absolute'); + assert(isAbsolute(upstreamSocket), 'upstreamSocket must be absolute'); + await mkdir(dirname(listenSocket), { recursive: true, mode: 0o700 }); + await chmod(dirname(listenSocket), 0o700); + await rm(listenSocket, { force: true }); + + const sockets = new Set(); + const stats = { + methodCounts: Object.create(null), + spawnInvokeTaskDigests: [], + spawnInvokeKeys: [], + droppedSpawnReply: undefined, + droppedSpawnReplyForwardedBytes: 0, + }; + let interceptedResolve; + let interceptedReject; + const intercepted = new Promise((resolveIntercepted, rejectIntercepted) => { + interceptedResolve = resolveIntercepted; + interceptedReject = rejectIntercepted; + }); + + const server = net.createServer(downstream => { + const upstream = net.createConnection(upstreamSocket); + sockets.add(downstream); + sockets.add(upstream); + const pending = new Map(); + + const fail = error => { + if (!stats.droppedSpawnReply) interceptedReject(error); + downstream.destroy(error); + upstream.destroy(error); + }; + const readRequest = lineReader(line => { + const request = safeJson(line); + if (typeof request?.id !== 'number' || typeof request.method !== 'string') return; + const params = typeof request.params === 'object' && request.params !== null + ? request.params + : {}; + pending.set(request.id, { method: request.method, params }); + stats.methodCounts[request.method] = (stats.methodCounts[request.method] ?? 0) + 1; + if (request.method === SPAWN_INVOKE) { + assert.equal(typeof params.task, 'string', 'spawn.invoke task must be a string'); + assert.equal(typeof params.idempotency_key, 'string', 'spawn.invoke key must be a string'); + stats.spawnInvokeTaskDigests.push(sha256(Buffer.from(params.task, 'utf8'))); + stats.spawnInvokeKeys.push(params.idempotency_key); + } + }, () => fail(new Error('Core Bridge request frame exceeded proof proxy limit'))); + + downstream.on('data', chunk => { + try { + readRequest(chunk); + upstream.write(chunk); + } catch (error) { + fail(error); + } + }); + + const readReply = lineReader(line => { + try { + const reply = safeJson(line); + const request = typeof reply?.id === 'number' ? pending.get(reply.id) : undefined; + if (typeof reply?.id === 'number') pending.delete(reply.id); + if (!stats.droppedSpawnReply && request?.method === SPAWN_INVOKE && !reply?.error) { + const result = typeof reply?.result === 'object' && reply.result !== null ? reply.result : {}; + const event = { + invocationId: typeof result.id === 'string' ? result.id : undefined, + responseDigest: sha256(Buffer.concat([line, NEWLINE])), + responseBytes: line.length + 1, + }; + stats.droppedSpawnReply = event; + // The callback issues SIGKILL synchronously. The response remains in + // this proxy and the downstream is closed without a write. + onIntercept(event); + interceptedResolve(event); + downstream.destroy(); + upstream.destroy(); + return; + } + downstream.write(Buffer.concat([line, NEWLINE])); + } catch (error) { + fail(error); + } + }, () => fail(new Error('Core Bridge reply frame exceeded proof proxy limit'))); + + upstream.on('data', readReply); + upstream.on('error', fail); + downstream.on('error', error => upstream.destroy(error)); + downstream.on('close', () => { + sockets.delete(downstream); + upstream.destroy(); + }); + upstream.on('close', () => { + sockets.delete(upstream); + downstream.destroy(); + }); + }); + + await new Promise((resolveListen, rejectListen) => { + server.once('error', rejectListen); + server.listen(listenSocket, () => { + server.off('error', rejectListen); + resolveListen(); + }); + }); + await chmod(listenSocket, 0o600); + + return { + stats, + intercepted, + async close() { + for (const socket of sockets) socket.destroy(); + await new Promise(resolveClose => server.close(() => resolveClose())); + await rm(listenSocket, { force: true }); + }, + }; +} + +export function verifyResponseLossEvidence(stats, audit) { + assert.equal(stats.droppedSpawnReplyForwardedBytes, 0, + 'no byte of the committed spawn.invoke reply may reach Flows'); + assert(stats.droppedSpawnReply, 'the proxy must drop one successful spawn.invoke reply'); + assert.equal(stats.spawnInvokeTaskDigests.length, 3, + 'the committed invoke and exactly two same-key replays must cross the proxy'); + assert.equal(stats.spawnInvokeTaskDigests.length, stats.spawnInvokeKeys.length); + assert.equal(new Set(stats.spawnInvokeTaskDigests).size, 1, + 'every same-key retry must preserve the original task bytes'); + assert.equal(new Set(stats.spawnInvokeKeys).size, 1, + 'every retry must use the original idempotency key'); + assert.equal(stats.methodCounts[SPAWN_RELEASE] ?? 0, 1, + 'Flows must issue exactly one release through the bridge'); + assert.equal((stats.methodCounts[INBOX_ACK] ?? 0) + (stats.methodCounts[INBOX_FAIL] ?? 0), 1, + 'Flows must settle the correlated inbox item exactly once'); + + assert.equal(audit.schema, 'relayflows-node-spawn-response-loss/1'); + assert.equal(audit.strict_contract, STRICT_CONTRACT); + assert.equal(audit.strict_replay_capability, STRICT_REPLAY_CAPABILITY); + assert.equal(audit.strict_invocation_committed, true); + assert.equal(audit.upstream_spawn_frames, 1); + assert.equal(audit.workers_created, 1); + assert.equal(audit.correlated_results, 1); + assert.equal(audit.releases, 1); + assert.equal(audit.target_capacity?.active_agents, 0); + assert.equal(audit.target_capacity?.reserved_agents, 0); + assert.equal(typeof audit.journal_terminal_sequence, 'number'); + assert.equal(typeof audit.inbox_settlement_sequence, 'number'); + assert(audit.journal_terminal_sequence < audit.inbox_settlement_sequence, + 'terminal journal evidence must precede inbox settlement'); + assert.equal(typeof audit.unrelated_reservation?.before?.identity, 'string'); + assert.equal(typeof audit.unrelated_reservation?.before?.count, 'number'); + assert.deepEqual(audit.unrelated_reservation.after, audit.unrelated_reservation.before, + 'the unrelated reservation must remain unchanged'); + return { + schema: audit.schema, + strict_contract: audit.strict_contract, + strict_replay_capability: audit.strict_replay_capability, + strict_invocation_committed: audit.strict_invocation_committed, + dropped_response_bytes: stats.droppedSpawnReply.responseBytes, + dropped_response_forwarded_bytes: stats.droppedSpawnReplyForwardedBytes, + spawn_invoke_calls: stats.methodCounts[SPAWN_INVOKE] ?? 0, + task_sha256: stats.spawnInvokeTaskDigests[0], + idempotency_key_sha256: sha256(stats.spawnInvokeKeys[0]), + upstream_spawn_frames: audit.upstream_spawn_frames, + workers_created: audit.workers_created, + correlated_results: audit.correlated_results, + releases: audit.releases, + target_capacity: audit.target_capacity, + unrelated_reservation: audit.unrelated_reservation, + journal_terminal_sequence: audit.journal_terminal_sequence, + inbox_settlement_sequence: audit.inbox_settlement_sequence, + }; +} diff --git a/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts b/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts new file mode 100644 index 000000000..0ca631cb0 --- /dev/null +++ b/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts @@ -0,0 +1,380 @@ +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import net from 'node:net'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +interface ProxyStats { + methodCounts: Record; + spawnInvokeTaskDigests: string[]; + spawnInvokeKeys: string[]; + droppedSpawnReply?: { invocationId?: string; responseDigest: string; responseBytes: number }; + droppedSpawnReplyForwardedBytes: number; +} + +interface ProofProxy { + stats: ProxyStats; + intercepted: Promise; + close(): Promise; +} + +interface ProxyModule { + createBridgeResponseLossProxy(options: { + listenSocket: string; + upstreamSocket: string; + onIntercept?: () => void; + }): Promise; + verifyResponseLossEvidence(stats: ProxyStats, audit: unknown): unknown; +} + +const moduleUrl = new URL('../scripts/node-spawn-response-loss-proxy.mjs', import.meta.url).href; +const roots: string[] = []; +const servers: net.Server[] = []; +const proxies: ProofProxy[] = []; + +afterEach(async () => { + await Promise.all(proxies.splice(0).map(proxy => proxy.close())); + await Promise.all(servers.splice(0).map(server => new Promise(resolveClose => server.close(() => resolveClose())))); + roots.splice(0).forEach(root => rmSync(root, { recursive: true, force: true })); +}); + +function lineServer(socket: string) { + const keys = new Set(); + let upstreamSpawnFrames = 0; + const server = net.createServer(connection => { + let buffered = ''; + connection.on('data', chunk => { + buffered += String(chunk); + for (;;) { + const newline = buffered.indexOf('\n'); + if (newline < 0) return; + const line = buffered.slice(0, newline); + buffered = buffered.slice(newline + 1); + const request = JSON.parse(line) as { + id: number; + method: string; + params?: Record; + }; + if (request.method === 'spawn.invoke') { + const key = String(request.params?.['idempotency_key']); + if (!keys.has(key)) { + keys.add(key); + upstreamSpawnFrames += 1; + } + connection.write(`${JSON.stringify({ + jsonrpc: '2.0', id: request.id, + result: { id: 'invocation-1', status: 'completed' }, + })}\n`); + } else { + connection.write(`${JSON.stringify({ jsonrpc: '2.0', id: request.id, result: { ok: true } })}\n`); + } + } + }); + }); + server.listen(socket); + servers.push(server); + return { server, upstreamSpawnFrames: () => upstreamSpawnFrames }; +} + +async function request(socket: string, value: unknown): Promise<{ bytes: number; reply?: unknown }> { + const client = net.createConnection(socket); + await once(client, 'connect'); + let buffered = ''; + let bytes = 0; + const reply = new Promise(resolveReply => { + client.on('data', chunk => { + bytes += chunk.length; + buffered += String(chunk); + const newline = buffered.indexOf('\n'); + if (newline >= 0) resolveReply(JSON.parse(buffered.slice(0, newline))); + }); + client.once('close', () => resolveReply(undefined)); + }); + client.write(`${JSON.stringify(value)}\n`); + const answer = await reply; + client.destroy(); + return { bytes, reply: answer }; +} + +async function run(command: string, args: string[], cwd: string) { + const child = spawn(command, args, { cwd, stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', chunk => { stdout += String(chunk); }); + child.stderr.on('data', chunk => { stderr += String(chunk); }); + const [code, signal] = await once(child, 'exit') as [number | null, NodeJS.Signals | null]; + return { code, signal, stdout, stderr }; +} + +describe('Core Bridge response-loss proof proxy', () => { + it('drops every byte of the committed reply and lets two same-key replays converge on one frame', async () => { + const root = mkdtempSync(join(tmpdir(), 'flows-response-loss-proxy-')); + roots.push(root); + const upstreamSocket = join(root, 'upstream.sock'); + const proxySocket = join(root, 'proxy.sock'); + const upstream = lineServer(upstreamSocket); + await once(upstream.server, 'listening'); + let intercepted = 0; + const module = await import(moduleUrl) as ProxyModule; + const proxy = await module.createBridgeResponseLossProxy({ + listenSocket: proxySocket, + upstreamSocket, + onIntercept: () => { intercepted += 1; }, + }); + proxies.push(proxy); + const spawn = { + jsonrpc: '2.0', id: 1, method: 'spawn.invoke', + params: { idempotency_key: 'a'.repeat(32), task: 'exact original task bytes' }, + }; + + const first = await request(proxySocket, spawn); + await proxy.intercepted; + expect(first).toEqual({ bytes: 0, reply: undefined }); + expect(intercepted).toBe(1); + + const [left, right] = await Promise.all([ + request(proxySocket, spawn), + request(proxySocket, spawn), + ]); + expect(left.reply).toMatchObject({ result: { id: 'invocation-1' } }); + expect(right.reply).toMatchObject({ result: { id: 'invocation-1' } }); + expect(upstream.upstreamSpawnFrames()).toBe(1); + + await request(proxySocket, { jsonrpc: '2.0', id: 2, method: 'spawn.release', params: {} }); + await request(proxySocket, { jsonrpc: '2.0', id: 3, method: 'spawn_callers.inbox.ack', params: {} }); + expect(() => module.verifyResponseLossEvidence(proxy.stats, { + schema: 'relayflows-node-spawn-response-loss/1', + strict_contract: 'relay.spawn.invoke.logical-replay-resume/1', + strict_replay_capability: 'relay.spawn.replay-resume/1', + strict_invocation_committed: true, + upstream_spawn_frames: 1, + workers_created: 1, + correlated_results: 1, + releases: 1, + target_capacity: { active_agents: 0, reserved_agents: 0 }, + journal_terminal_sequence: 10, + inbox_settlement_sequence: 11, + unrelated_reservation: { + before: { identity: 'unrelated-1', count: 1 }, + after: { identity: 'unrelated-1', count: 1 }, + }, + })).not.toThrow(); + }); + + it('rejects changed task bytes even when the fixture reports a clean lifecycle', async () => { + const module = await import(moduleUrl) as ProxyModule; + const stats: ProxyStats = { + methodCounts: { 'spawn.invoke': 3, 'spawn.release': 1, 'spawn_callers.inbox.ack': 1 }, + spawnInvokeTaskDigests: ['one', 'two', 'two'], + spawnInvokeKeys: ['key', 'key', 'key'], + droppedSpawnReply: { responseDigest: 'reply', responseBytes: 12 }, + droppedSpawnReplyForwardedBytes: 0, + }; + expect(() => module.verifyResponseLossEvidence(stats, { + schema: 'relayflows-node-spawn-response-loss/1', + strict_contract: 'relay.spawn.invoke.logical-replay-resume/1', + strict_replay_capability: 'relay.spawn.replay-resume/1', + strict_invocation_committed: true, + upstream_spawn_frames: 1, + workers_created: 1, + correlated_results: 1, + releases: 1, + target_capacity: { active_agents: 0, reserved_agents: 0 }, + journal_terminal_sequence: 10, + inbox_settlement_sequence: 11, + unrelated_reservation: { + before: { identity: 'unrelated-1', count: 1 }, + after: { identity: 'unrelated-1', count: 1 }, + }, + })).toThrow('original task bytes'); + }); + + it('requires stored strict proofs and an unchanged unrelated reservation', async () => { + const module = await import(moduleUrl) as ProxyModule; + const stats: ProxyStats = { + methodCounts: { 'spawn.invoke': 3, 'spawn.release': 1, 'spawn_callers.inbox.ack': 1 }, + spawnInvokeTaskDigests: ['same', 'same', 'same'], + spawnInvokeKeys: ['key', 'key', 'key'], + droppedSpawnReply: { responseDigest: 'reply', responseBytes: 12 }, + droppedSpawnReplyForwardedBytes: 0, + }; + const audit = { + schema: 'relayflows-node-spawn-response-loss/1', + strict_contract: 'relay.spawn.invoke.logical-replay-resume/1', + strict_replay_capability: 'relay.spawn.replay-resume/1', + strict_invocation_committed: true, + upstream_spawn_frames: 1, + workers_created: 1, + correlated_results: 1, + releases: 1, + target_capacity: { active_agents: 0, reserved_agents: 0 }, + journal_terminal_sequence: 10, + inbox_settlement_sequence: 11, + unrelated_reservation: { + before: { identity: 'unrelated-1', count: 1 }, + after: { identity: 'unrelated-1', count: 1 }, + }, + }; + expect(() => module.verifyResponseLossEvidence(stats, { + ...audit, + strict_replay_capability: 'legacy', + })).toThrow(); + expect(() => module.verifyResponseLossEvidence(stats, { + ...audit, + unrelated_reservation: { + ...audit.unrelated_reservation, + after: { identity: 'unrelated-1', count: 0 }, + }, + })).toThrow('unrelated reservation'); + }); + + it('runs the killed-child and two-resume acceptance orchestration against a local fixture', async () => { + const root = mkdtempSync(join(tmpdir(), 'flows-response-loss-runner-')); + roots.push(root); + const upstreamSocket = join(root, 'upstream.sock'); + const upstream = lineServer(upstreamSocket); + await once(upstream.server, 'listening'); + const dataDir = join(root, 'data'); + const evidenceDir = join(root, 'evidence'); + const homeDir = join(root, 'home'); + const workDir = join(root, 'work'); + mkdirSync(workDir, { recursive: true }); + const callerFile = join(root, 'caller.json'); + writeFileSync(callerFile, `${JSON.stringify({ + schema: 'agent-relay-service-caller/2', + auth: { + scheme: 'challenge-hmac-sha256/v1', + challenge: 'spawn_callers.challenge', + authenticate: 'spawn_callers.authenticate', + }, + caller_id: 'sc_0123456789abcdef', + caller_name: 'flows_e2e', + secret: `rsc_${'A'.repeat(43)}`, + publisher: { id: 'publisher_1', name: 'flows_e2e' }, + workspace: { + site_url: 'https://agentrelay.com', + base_url: 'https://cast.agentrelay.com', + workspace_id: 'workspace_1', + relaycast_workspace_id: 'relaycast_1', + }, + account: { user_id: 'user_1' }, + created_at: '2026-11-01T00:00:00.000Z', + expires_at: '2026-12-01T00:00:00.000Z', + harnesses: ['codex'], + directory_scope: [workDir], + owns_only: true, + socket: upstreamSocket, + inbox: { + transport: 'core-bridge', + open: 'spawn_callers.inbox.open', + next: 'spawn_callers.inbox.next', + ack: 'spawn_callers.inbox.ack', + fail: 'spawn_callers.inbox.fail', + close: 'spawn_callers.inbox.close', + disconnect: 'spawn_callers.disconnect', + }, + desktop_version: 'fixture', + })}\n`, { mode: 0o600 }); + chmodSync(callerFile, 0o600); + + const childScript = join(root, 'fake-flows.mjs'); + writeFileSync(childScript, ` +import { closeSync, mkdirSync, openSync, readFileSync } from 'node:fs'; +import net from 'node:net'; +import { join } from 'node:path'; +const [mode, dataDir, winner] = process.argv.slice(2); +const caller = JSON.parse(readFileSync(process.env.RELAYFLOW_SERVICE_CALLER_FILE, 'utf8')); +const rpc = (method, id) => new Promise((resolve, reject) => { + const socket = net.createConnection(caller.socket); + let buffered = ''; + socket.once('error', reject); + socket.once('connect', () => socket.write(JSON.stringify({ + jsonrpc: '2.0', id, method, params: method === 'spawn.invoke' + ? { idempotency_key: 'a'.repeat(32), task: 'exact original task bytes' } + : {}, + }) + '\\n')); + socket.on('data', chunk => { + buffered += String(chunk); + const newline = buffered.indexOf('\\n'); + if (newline < 0) return; + socket.destroy(); + resolve(JSON.parse(buffered.slice(0, newline))); + }); +}); +if (mode === 'start') { + mkdirSync(join(dataDir, 'runs'), { recursive: true }); + closeSync(openSync(join(dataDir, 'runs', 'run_fixture.sqlite3'), 'w')); + await rpc('spawn.invoke', 1); + process.exit(99); +} +await rpc('spawn.invoke', 1); +let won = false; +try { closeSync(openSync(winner, 'wx')); won = true; } catch (error) { + if (error.code !== 'EEXIST') throw error; +} +if (!won) process.exit(3); +await rpc('spawn.release', 2); +await rpc('spawn_callers.inbox.ack', 3); +`); + + const auditScript = join(root, 'audit.mjs'); + writeFileSync(auditScript, `console.log(JSON.stringify({ + schema: 'relayflows-node-spawn-response-loss/1', + strict_contract: 'relay.spawn.invoke.logical-replay-resume/1', + strict_replay_capability: 'relay.spawn.replay-resume/1', + strict_invocation_committed: true, + upstream_spawn_frames: 1, + workers_created: 1, + correlated_results: 1, + releases: 1, + target_capacity: { active_agents: 0, reserved_agents: 0 }, + journal_terminal_sequence: 10, + inbox_settlement_sequence: 11, + unrelated_reservation: { + before: { identity: 'unrelated-1', count: 1 }, + after: { identity: 'unrelated-1', count: 1 }, + }, +}));\n`); + const config = join(root, 'config.json'); + writeFileSync(config, `${JSON.stringify({ + callerFile, + dataDir, + evidenceDir, + homeDir, + workDir, + timeoutMs: 10_000, + start: { command: process.execPath, args: [childScript, 'start', dataDir, join(root, 'winner')] }, + resume: { command: process.execPath, args: [childScript, 'resume', dataDir, join(root, 'winner'), '{RUN_ID}'] }, + audit: { command: process.execPath, args: [auditScript, '{RUN_ID}'] }, + })}\n`, { mode: 0o600 }); + + const result = await run(process.execPath, [ + new URL('../scripts/node-spawn-response-loss-e2e.mjs', import.meta.url).pathname, + '--config', config, + ], resolveRoot()); + expect(result, result.stderr).toMatchObject({ code: 0, signal: null }); + expect(JSON.parse(result.stdout)).toMatchObject({ ok: true, run_id: 'run_fixture' }); + const evidence = JSON.parse(readFileSync(join(evidenceDir, 'response-loss-proof.json'), 'utf8')); + expect(evidence).toMatchObject({ + dropped_response_forwarded_bytes: 0, + spawn_invoke_calls: 3, + upstream_spawn_frames: 1, + workers_created: 1, + correlated_results: 1, + releases: 1, + target_capacity: { active_agents: 0, reserved_agents: 0 }, + unrelated_reservation: { + before: { identity: 'unrelated-1', count: 1 }, + after: { identity: 'unrelated-1', count: 1 }, + }, + }); + expect(upstream.upstreamSpawnFrames()).toBe(1); + }, 20_000); +}); + +function resolveRoot() { + return new URL('../../..', import.meta.url).pathname; +} From 8ac91bb901009967165fe8c5bf6798ce343df080 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Tue, 10 Nov 2026 18:20:48 -0800 Subject: [PATCH 66/75] test(sdk): allow coalesced spawn replay racers --- docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md | 5 +++++ packages/sdk/scripts/node-spawn-response-loss-proxy.mjs | 4 ++-- packages/sdk/tests/node-spawn-response-loss-proxy.test.ts | 4 ++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md b/docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md index 9bc96e909..fc7bf8e73 100644 --- a/docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md +++ b/docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md @@ -23,6 +23,11 @@ The proxy forwards newline-framed JSON-RPC until it reads the first complete, successful `spawn.invoke` reply from the real core. It never writes that line downstream. It synchronously sends `SIGKILL` to the foreground Flows process, then closes that connection. Later connections pass through unchanged. +The orchestrator races two resume processes. Correct durable claim ownership +may coalesce those racers into one physical bridge replay; the verifier requires +at least one exact same-key replay. The deterministic proxy regression also +sends two physical same-key replay requests and proves they converge on one +upstream provider frame. ## Preconditions diff --git a/packages/sdk/scripts/node-spawn-response-loss-proxy.mjs b/packages/sdk/scripts/node-spawn-response-loss-proxy.mjs index 95160a886..bcaed9f74 100644 --- a/packages/sdk/scripts/node-spawn-response-loss-proxy.mjs +++ b/packages/sdk/scripts/node-spawn-response-loss-proxy.mjs @@ -178,8 +178,8 @@ export function verifyResponseLossEvidence(stats, audit) { assert.equal(stats.droppedSpawnReplyForwardedBytes, 0, 'no byte of the committed spawn.invoke reply may reach Flows'); assert(stats.droppedSpawnReply, 'the proxy must drop one successful spawn.invoke reply'); - assert.equal(stats.spawnInvokeTaskDigests.length, 3, - 'the committed invoke and exactly two same-key replays must cross the proxy'); + assert(stats.spawnInvokeTaskDigests.length >= 2, + 'the committed invoke and at least one same-key replay must cross the proxy'); assert.equal(stats.spawnInvokeTaskDigests.length, stats.spawnInvokeKeys.length); assert.equal(new Set(stats.spawnInvokeTaskDigests).size, 1, 'every same-key retry must preserve the original task bytes'); diff --git a/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts b/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts index 0ca631cb0..1c6f2b171 100644 --- a/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts +++ b/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts @@ -310,12 +310,12 @@ if (mode === 'start') { await rpc('spawn.invoke', 1); process.exit(99); } -await rpc('spawn.invoke', 1); let won = false; try { closeSync(openSync(winner, 'wx')); won = true; } catch (error) { if (error.code !== 'EEXIST') throw error; } if (!won) process.exit(3); +await rpc('spawn.invoke', 1); await rpc('spawn.release', 2); await rpc('spawn_callers.inbox.ack', 3); `); @@ -360,7 +360,7 @@ await rpc('spawn_callers.inbox.ack', 3); const evidence = JSON.parse(readFileSync(join(evidenceDir, 'response-loss-proof.json'), 'utf8')); expect(evidence).toMatchObject({ dropped_response_forwarded_bytes: 0, - spawn_invoke_calls: 3, + spawn_invoke_calls: 2, upstream_spawn_frames: 1, workers_created: 1, correlated_results: 1, From 4a8307c676bc79d6f33101089e5ffb146038120b Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Tue, 10 Nov 2026 18:24:10 -0800 Subject: [PATCH 67/75] test(sdk): cover proof runner defaults and redaction --- .../sdk/scripts/node-spawn-response-loss-e2e.mjs | 6 ++++-- .../tests/node-spawn-response-loss-proxy.test.ts | 15 +++++++++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/packages/sdk/scripts/node-spawn-response-loss-e2e.mjs b/packages/sdk/scripts/node-spawn-response-loss-e2e.mjs index 0e601832b..78cef3ac2 100644 --- a/packages/sdk/scripts/node-spawn-response-loss-e2e.mjs +++ b/packages/sdk/scripts/node-spawn-response-loss-e2e.mjs @@ -16,6 +16,8 @@ import { verifyResponseLossEvidence, } from './node-spawn-response-loss-proxy.mjs'; +const DEFAULT_TIMEOUT_MS = 10 * 60_000; + function assertCommandSpec(spec, label) { assert.equal(typeof spec?.command, 'string', `${label}.command is required`); assert(isAbsolute(spec.command), `${label}.command must be absolute`); @@ -69,10 +71,10 @@ async function waitForRunId(dataDir, timeoutMs) { throw new Error('timed out waiting for the single proof run id'); } -async function assertSecretFree(paths) { +export async function assertSecretFree(paths) { const forbidden = [ /rsc_[A-Za-z0-9_-]{20,}/, - /(?:at|rk|nt|ot|br|arr)_(?:live_)?[A-Za-z0-9_-]{8,}/, + /(?:ar|arr|at|br|nt|ot|rk)_(?:live_)?[A-Za-z0-9_-]{8,}/, /(?:relay_pa_|sh_fr_)[A-Za-z0-9_-]+/, /caller_secret/i, /(?:RELAY|AGENT_RELAY|RELAYCAST)_(?:AGENT_TOKEN|API_KEY|WORKSPACE_KEY|NODE_TOKEN|WORKSPACES_JSON)/, diff --git a/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts b/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts index 1c6f2b171..723af8f34 100644 --- a/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts +++ b/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts @@ -31,6 +31,7 @@ interface ProxyModule { } const moduleUrl = new URL('../scripts/node-spawn-response-loss-proxy.mjs', import.meta.url).href; +const runnerModuleUrl = new URL('../scripts/node-spawn-response-loss-e2e.mjs', import.meta.url).href; const roots: string[] = []; const servers: net.Server[] = []; const proxies: ProofProxy[] = []; @@ -231,7 +232,18 @@ describe('Core Bridge response-loss proof proxy', () => { })).toThrow('unrelated reservation'); }); - it('runs the killed-child and two-resume acceptance orchestration against a local fixture', async () => { + it('rejects ar_ credentials in captured proof logs', async () => { + const root = mkdtempSync(join(tmpdir(), 'flows-response-loss-secret-')); + roots.push(root); + const log = join(root, 'resume.log'); + writeFileSync(log, `unexpected credential ar_${'A'.repeat(24)}\n`, { mode: 0o600 }); + const runner = await import(runnerModuleUrl) as { + assertSecretFree(paths: string[]): Promise; + }; + await expect(runner.assertSecretFree([log])).rejects.toThrow('credential-shaped material'); + }); + + it('uses the default timeout for killed-child and two-resume acceptance orchestration', async () => { const root = mkdtempSync(join(tmpdir(), 'flows-response-loss-runner-')); roots.push(root); const upstreamSocket = join(root, 'upstream.sock'); @@ -345,7 +357,6 @@ await rpc('spawn_callers.inbox.ack', 3); evidenceDir, homeDir, workDir, - timeoutMs: 10_000, start: { command: process.execPath, args: [childScript, 'start', dataDir, join(root, 'winner')] }, resume: { command: process.execPath, args: [childScript, 'resume', dataDir, join(root, 'winner'), '{RUN_ID}'] }, audit: { command: process.execPath, args: [auditScript, '{RUN_ID}'] }, From 0b68f37a05bd595f2c49854a512693592492eb34 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Tue, 10 Nov 2026 18:27:43 -0800 Subject: [PATCH 68/75] test(sdk): prove cross-process replay claim retention --- docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md | 10 +++++++++- .../scripts/node-spawn-response-loss-e2e.mjs | 11 +++++++++++ .../scripts/node-spawn-response-loss-proxy.mjs | 7 +++++++ .../node-spawn-response-loss-proxy.test.ts | 17 +++++++++++++++++ 4 files changed, 44 insertions(+), 1 deletion(-) diff --git a/docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md b/docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md index fc7bf8e73..6eed363f1 100644 --- a/docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md +++ b/docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md @@ -23,7 +23,8 @@ The proxy forwards newline-framed JSON-RPC until it reads the first complete, successful `spawn.invoke` reply from the real core. It never writes that line downstream. It synchronously sends `SIGKILL` to the foreground Flows process, then closes that connection. Later connections pass through unchanged. -The orchestrator races two resume processes. Correct durable claim ownership +The orchestrator races two independent OS process generations after the initial +generation is killed. Correct durable claim ownership may coalesce those racers into one physical bridge replay; the verifier requires at least one exact same-key replay. The deterministic proxy regression also sends two physical same-key replay requests and proves they converge on one @@ -87,6 +88,11 @@ The audit command stdout contract is: "workers_created": 1, "correlated_results": 1, "releases": 1, + "final_claim": { + "result_present": true, + "release_confirmed": true, + "cleanup_pending": false + }, "target_capacity": { "active_agents": 0, "reserved_agents": 0 }, "journal_terminal_sequence": 41, "inbox_settlement_sequence": 42, @@ -134,6 +140,8 @@ The proof passes only when all of these are true: - the real provider saw one spawn frame and created one worker; - one correlated result was journaled before one inbox settlement; - exactly one release completed; +- after both independent replay processes settled, the final durable claim + still retained that result and confirmed release with no cleanup pending; - target active and reserved capacity returned to zero; and - the unrelated reservation identity and count stayed unchanged. diff --git a/packages/sdk/scripts/node-spawn-response-loss-e2e.mjs b/packages/sdk/scripts/node-spawn-response-loss-e2e.mjs index 78cef3ac2..18fa7a40c 100644 --- a/packages/sdk/scripts/node-spawn-response-loss-e2e.mjs +++ b/packages/sdk/scripts/node-spawn-response-loss-e2e.mjs @@ -208,6 +208,13 @@ async function main() { }), ]); const resumeResults = [left.result, right.result]; + const replayPids = [left.child.pid, right.child.pid]; + assert(replayPids.every(pid => Number.isSafeInteger(pid) && pid > 0), + 'both replay process ids must be available'); + assert.equal(new Set(replayPids).size, 2, + 'the replay race must use two independent process generations'); + assert(!replayPids.includes(firstChild.pid), + 'neither replay may reuse the killed foreground process generation'); assert(resumeResults.some(result => result.code === 0), 'at least one raced resume must complete successfully'); assert(resumeResults.every(result => result.signal === null), @@ -241,6 +248,10 @@ async function main() { const evidence = { ...summary, run_id: runId, + process_generations: { + initial_pid: firstChild.pid, + replay_pids: replayPids, + }, resume_exit_codes: resumeResults.map(result => result.code), core_bridge_version: coreBridgeVersion, }; diff --git a/packages/sdk/scripts/node-spawn-response-loss-proxy.mjs b/packages/sdk/scripts/node-spawn-response-loss-proxy.mjs index bcaed9f74..8115ef37c 100644 --- a/packages/sdk/scripts/node-spawn-response-loss-proxy.mjs +++ b/packages/sdk/scripts/node-spawn-response-loss-proxy.mjs @@ -198,6 +198,12 @@ export function verifyResponseLossEvidence(stats, audit) { assert.equal(audit.workers_created, 1); assert.equal(audit.correlated_results, 1); assert.equal(audit.releases, 1); + assert.equal(audit.final_claim?.result_present, true, + 'the final durable claim must retain its correlated result'); + assert.equal(audit.final_claim?.release_confirmed, true, + 'the final durable claim must retain its confirmed release'); + assert.equal(audit.final_claim?.cleanup_pending, false, + 'the final durable claim must not retain pending cleanup'); assert.equal(audit.target_capacity?.active_agents, 0); assert.equal(audit.target_capacity?.reserved_agents, 0); assert.equal(typeof audit.journal_terminal_sequence, 'number'); @@ -222,6 +228,7 @@ export function verifyResponseLossEvidence(stats, audit) { workers_created: audit.workers_created, correlated_results: audit.correlated_results, releases: audit.releases, + final_claim: audit.final_claim, target_capacity: audit.target_capacity, unrelated_reservation: audit.unrelated_reservation, journal_terminal_sequence: audit.journal_terminal_sequence, diff --git a/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts b/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts index 723af8f34..cb7306612 100644 --- a/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts +++ b/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts @@ -155,6 +155,7 @@ describe('Core Bridge response-loss proof proxy', () => { workers_created: 1, correlated_results: 1, releases: 1, + final_claim: { result_present: true, release_confirmed: true, cleanup_pending: false }, target_capacity: { active_agents: 0, reserved_agents: 0 }, journal_terminal_sequence: 10, inbox_settlement_sequence: 11, @@ -183,6 +184,7 @@ describe('Core Bridge response-loss proof proxy', () => { workers_created: 1, correlated_results: 1, releases: 1, + final_claim: { result_present: true, release_confirmed: true, cleanup_pending: false }, target_capacity: { active_agents: 0, reserved_agents: 0 }, journal_terminal_sequence: 10, inbox_settlement_sequence: 11, @@ -211,6 +213,7 @@ describe('Core Bridge response-loss proof proxy', () => { workers_created: 1, correlated_results: 1, releases: 1, + final_claim: { result_present: true, release_confirmed: true, cleanup_pending: false }, target_capacity: { active_agents: 0, reserved_agents: 0 }, journal_terminal_sequence: 10, inbox_settlement_sequence: 11, @@ -230,6 +233,10 @@ describe('Core Bridge response-loss proof proxy', () => { after: { identity: 'unrelated-1', count: 0 }, }, })).toThrow('unrelated reservation'); + expect(() => module.verifyResponseLossEvidence(stats, { + ...audit, + final_claim: { ...audit.final_claim, result_present: false }, + })).toThrow('retain its correlated result'); }); it('rejects ar_ credentials in captured proof logs', async () => { @@ -342,6 +349,7 @@ await rpc('spawn_callers.inbox.ack', 3); workers_created: 1, correlated_results: 1, releases: 1, + final_claim: { result_present: true, release_confirmed: true, cleanup_pending: false }, target_capacity: { active_agents: 0, reserved_agents: 0 }, journal_terminal_sequence: 10, inbox_settlement_sequence: 11, @@ -376,12 +384,21 @@ await rpc('spawn_callers.inbox.ack', 3); workers_created: 1, correlated_results: 1, releases: 1, + final_claim: { result_present: true, release_confirmed: true, cleanup_pending: false }, target_capacity: { active_agents: 0, reserved_agents: 0 }, unrelated_reservation: { before: { identity: 'unrelated-1', count: 1 }, after: { identity: 'unrelated-1', count: 1 }, }, + process_generations: { + initial_pid: expect.any(Number), + replay_pids: [expect.any(Number), expect.any(Number)], + }, }); + expect(new Set(evidence.process_generations.replay_pids).size).toBe(2); + expect(evidence.process_generations.replay_pids).not.toContain( + evidence.process_generations.initial_pid, + ); expect(upstream.upstreamSpawnFrames()).toBe(1); }, 20_000); }); From de5164477a22f3d3cec39648dccb436d2784ef7f Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Tue, 10 Nov 2026 18:58:57 -0800 Subject: [PATCH 69/75] test(sdk): audit local spawn response loss --- docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md | 144 +++++---- .../node-spawn-response-loss-audit.mjs | 214 +++++++++++++ .../scripts/node-spawn-response-loss-e2e.mjs | 38 ++- .../node-spawn-response-loss-proxy.mjs | 69 ++--- .../node-spawn-response-loss-proxy.test.ts | 284 +++++++++++------- 5 files changed, 504 insertions(+), 245 deletions(-) create mode 100755 packages/sdk/scripts/node-spawn-response-loss-audit.mjs diff --git a/docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md b/docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md index 6eed363f1..dd36cb3ba 100644 --- a/docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md +++ b/docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md @@ -1,34 +1,65 @@ -# Node-spawn response-loss acceptance kit +# Node-spawn local response-loss acceptance kit This is an acceptance-only proof for Flows #707. It changes no production -runtime behavior. It proves recovery when upstream commits a strict spawn but -Flows loses the complete `spawn.invoke` response. +runtime behavior. It proves the part of recovery that the exact Flows data +directory and Core Bridge proxy can establish when upstream commits a strict +local spawn but Flows loses the complete `spawn.invoke` response. The kit is Unix-only and must run from a plain terminal. Never run it from an -agent session tree. Use a disposable workspace, a dedicated target node, a -mode-0600 schema-v2 caller, and the packed SDK under review. Do not put caller +agent session tree. Use a disposable workspace, a dedicated local target node, +a mode-0600 schema-v2 caller, and the packed SDK under review. Do not put caller contents, publisher tokens, or workspace credentials in the config or logs. ## Files - `packages/sdk/scripts/node-spawn-response-loss-proxy.mjs` — transparent Core - Bridge socket proxy and fail-closed evidence verifier. + Bridge socket proxy and response/key evidence collector. +- `packages/sdk/scripts/node-spawn-response-loss-audit.mjs` — executable local + verifier for the final durable claim, journal ordering, process generations, + and proxy evidence. - `packages/sdk/scripts/node-spawn-response-loss-e2e.mjs` — lifecycle - orchestrator. + orchestrator. It invokes the local verifier directly; no fixture-supplied + audit hook is accepted. - `packages/sdk/tests/node-spawn-response-loss-proxy.test.ts` — deterministic - fake-upstream proof of zero response bytes, same-key concurrent replays, one - upstream frame, exact task bytes, and fail-closed evidence validation. + proof of zero response bytes, same-key replay, exact durable identity, + terminal-journal-before-settlement ordering, and fail-closed validation. The proxy forwards newline-framed JSON-RPC until it reads the first complete, successful `spawn.invoke` reply from the real core. It never writes that line downstream. It synchronously sends `SIGKILL` to the foreground Flows process, -then closes that connection. Later connections pass through unchanged. -The orchestrator races two independent OS process generations after the initial -generation is killed. Correct durable claim ownership -may coalesce those racers into one physical bridge replay; the verifier requires -at least one exact same-key replay. The deterministic proxy regression also -sends two physical same-key replay requests and proves they converge on one -upstream provider frame. +then closes that connection. Later connections pass through unchanged. The +orchestrator races two independent OS process generations after the initial +generation is killed. Correct durable claim ownership may coalesce those +racers into one physical bridge replay; the verifier requires at least one +exact same-key replay. + +## Proven boundary + +The local verifier proves only these facts: + +1. the final v3 claim retains the exact spawn identity from the intercepted + reply, a correlated result, `release.status === "completed"`, and no + `cleanupPending`; +2. exactly one matching terminal `step.completed` journal entry predates the + claim's durable inbox settlement; +3. the killed process and two replay processes are distinct generations, the + initial process died by `SIGKILL`, neither replay died by signal, and at + least one replay exited zero; and +4. the proxy forwarded zero response bytes, observed the original task bytes + and one idempotency key on every invoke, at least one replay, one release + call, and one inbox settlement call. + +The following remain explicitly deferred until a strict self-host artifact +digest and an authorized server-side audit are available: + +- stored Relaycast contract and replay-capability proofs; +- provider-frame and worker-creation counts; +- target fleet capacity; +- preservation of an unrelated server reservation; and +- remote/fleet lifecycle acceptance. + +Bridge request counts are not provider-frame counts. This kit never presents +them as such and makes no remote/fleet or server-database claim. ## Preconditions @@ -36,13 +67,8 @@ upstream provider frame. 2. Pack that SDK and install it with public `@agent-relay/core-bridge@18.1.0` in a fresh mode-0700 consumer directory. 3. Provision a dedicated mode-0600 schema-v2 caller from a plain terminal. -4. Prepare fresh mode-0700 HOME and Flows data directories; the data directory - must contain no `runs/*.sqlite3`. -5. Reserve one unrelated capacity slot and record its stable identity. The - audit command must show the same reservation before and after the proof. -6. Provide a fixture audit command that reads the real Relaycast/Relay node, - Desktop caller ledger, Flows journal, and inbox settlement record. It must - print only the JSON object below on stdout and must not print secrets. +4. Prepare fresh mode-0700 HOME, evidence, and Flows data directories; the data + directory must contain no `runs/*.sqlite3`. ## Config @@ -50,8 +76,8 @@ Create a mode-0600 JSON file outside the repository. Paths are absolute. The start command runs the packed SDK in the foreground. The resume command must contain `{RUN_ID}` exactly where the durable run id belongs. Do not pass `--service-caller-file`; the kit securely copies the caller into a temporary -mode-0700 directory outside the evidence tree, changes only its socket to the proxy, and injects -`RELAYFLOW_SERVICE_CALLER_FILE`. +mode-0700 directory outside the evidence tree, changes only its socket to the +proxy, and injects `RELAYFLOW_SERVICE_CALLER_FILE`. ```json { @@ -68,46 +94,24 @@ mode-0700 directory outside the evidence tree, changes only its socket to the pr "resume": { "command": "/usr/bin/node", "args": ["/private/consumer/node_modules/@relayflows/sdk/dist/cli.js", "resume", "{RUN_ID}", "--local-agent", "--json", "--no-observer-link", "--data-dir", "/private/proof/flows-data"] - }, - "audit": { - "command": "/private/proof/bin/audit-response-loss", - "args": ["--run", "{RUN_ID}", "--json"] } } ``` -The audit command stdout contract is: +The orchestrator writes private proxy/process inputs and invokes the executable +verifier equivalently to: -```json -{ - "schema": "relayflows-node-spawn-response-loss/1", - "strict_contract": "relay.spawn.invoke.logical-replay-resume/1", - "strict_replay_capability": "relay.spawn.replay-resume/1", - "strict_invocation_committed": true, - "upstream_spawn_frames": 1, - "workers_created": 1, - "correlated_results": 1, - "releases": 1, - "final_claim": { - "result_present": true, - "release_confirmed": true, - "cleanup_pending": false - }, - "target_capacity": { "active_agents": 0, "reserved_agents": 0 }, - "journal_terminal_sequence": 41, - "inbox_settlement_sequence": 42, - "unrelated_reservation": { - "before": { "identity": "stable-unrelated-reservation-id", "count": 1 }, - "after": { "identity": "stable-unrelated-reservation-id", "count": 1 } - } -} +```sh +node packages/sdk/scripts/node-spawn-response-loss-audit.mjs \ + --run "$RUN_ID" \ + --data-dir /private/proof/flows-data \ + --proxy-stats /private/mode-0700/proxy-stats.json \ + --process-evidence /private/mode-0700/process-evidence.json \ + --json ``` -Sequence values are fixture-defined monotonic positions, but the terminal -journal position must be strictly less than the inbox settlement position. -The audit must count real provider frames and real created workers, not bridge -requests. The strict contract and capability fields must come from the stored -Relaycast invocation, not from the request or from a hard-coded client claim. +Those two input files are mode 0600 and live only in the runner's temporary +mode-0700 directory. The runner deletes that directory on exit. ## Commands @@ -125,25 +129,11 @@ node packages/sdk/scripts/node-spawn-response-loss-e2e.mjs --config /private/pro The final command prints only `{ "ok": true, "evidence": ..., "run_id": ... }`. It writes mode-0600 logs plus `response-loss-proof.json` under `evidenceDir`. Before writing evidence it scans every captured log for caller-secret, token, -publisher-key, and Relayfile-bearer shapes. The evidence contains hashes and -counts, never the caller secret or task text. The runner requires the public -`@agent-relay/core-bridge@18.1.0` package, runs children with the fresh HOME, -and removes inherited publisher, workspace, broker, node, and API credentials. - -## Required result - -The proof passes only when all of these are true: - -- the proxy received the committed upstream reply and forwarded zero bytes of - it before `SIGKILL`; -- two concurrent resumes used the same run, task bytes, and idempotency key; -- the real provider saw one spawn frame and created one worker; -- one correlated result was journaled before one inbox settlement; -- exactly one release completed; -- after both independent replay processes settled, the final durable claim - still retained that result and confirmed release with no cleanup pending; -- target active and reserved capacity returned to zero; and -- the unrelated reservation identity and count stayed unchanged. +publisher-key, and Relayfile-bearer shapes. Evidence stores spawn identity, +task, and idempotency-key digests rather than their source values. The runner requires +the public `@agent-relay/core-bridge@18.1.0` package, runs children with the +fresh HOME, and removes inherited publisher, workspace, broker, node, and API +credentials. Keep #707 draft after the run. Attach the exact source head, packed SDK receipt, public Core Bridge receipt, config with private paths redacted, literal commands diff --git a/packages/sdk/scripts/node-spawn-response-loss-audit.mjs b/packages/sdk/scripts/node-spawn-response-loss-audit.mjs new file mode 100755 index 000000000..8c60a534b --- /dev/null +++ b/packages/sdk/scripts/node-spawn-response-loss-audit.mjs @@ -0,0 +1,214 @@ +#!/usr/bin/env node + +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { readFile, stat } from 'node:fs/promises'; +import { isAbsolute, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { walkJournal } from '../dist/journal-reader.js'; +import { readNodeSpawnClaims } from '../dist/node-spawn-state.js'; +import { + spawnIdentityDigest, + verifyResponseLossProxyEvidence, +} from './node-spawn-response-loss-proxy.mjs'; + +const LOCAL_SCHEMA = 'relayflows-node-spawn-response-loss-local/1'; +const DEFERRED_ASSERTIONS = [ + 'stored_relaycast_contract_and_capability', + 'provider_frame_and_worker_counts', + 'target_capacity', + 'unrelated_reservation_preservation', + 'remote_fleet_lifecycle', +]; + +function sha256(value) { + return createHash('sha256').update(value).digest('hex'); +} + +function record(value) { + return typeof value === 'object' && value !== null ? value : {}; +} + +function terminalMatchesResult(entry, claim) { + const result = claim.result; + const settlement = result?.settlement; + if (result === undefined || settlement === undefined) return false; + const payload = record(entry.payload); + const tail = record(payload.trajectory_tail); + const relayTask = record(tail.relay_task); + const execution = record(relayTask.task_execution); + return entry.entry_type === 'step.completed' + && entry.step_id === claim.stepId + && entry.attempt === settlement.attempt + && payload.completionReason === settlement.completionReason + && execution.run_id === claim.runId + && execution.step_id === claim.stepId + && execution.dispatch_id === claim.idempotencyKey + && execution.delivery_id === result.inboxItemId + && execution.message_id === result.messageId; +} + +function verifyProcesses(processEvidence) { + const initial = record(processEvidence?.initial); + const replays = Array.isArray(processEvidence?.replays) ? processEvidence.replays : []; + assert(Number.isSafeInteger(initial.pid) && initial.pid > 0, + 'initial process id must be a positive integer'); + assert.equal(initial.code, null, 'the intercepted initial process must not exit normally'); + assert.equal(initial.signal, 'SIGKILL', 'the intercepted initial process must die by SIGKILL'); + assert.equal(replays.length, 2, 'the proof must race exactly two replay processes'); + const replayPids = replays.map(item => record(item).pid); + assert(replayPids.every(pid => Number.isSafeInteger(pid) && pid > 0), + 'both replay process ids must be positive integers'); + assert.equal(new Set([initial.pid, ...replayPids]).size, 3, + 'initial and replay process generations must be distinct'); + assert(replays.every(item => record(item).signal === null), + 'neither replay process may die by signal'); + const replayExitCodes = replays.map(item => record(item).code); + assert(replayExitCodes.every(code => Number.isSafeInteger(code) && code >= 0), + 'replay exit codes must be non-negative integers'); + assert(replayExitCodes.includes(0), 'at least one raced replay must complete successfully'); + return { + initial_pid: initial.pid, + initial_exit: { code: initial.code, signal: initial.signal }, + replay_pids: replayPids, + replay_exits: replays.map(item => ({ code: record(item).code, signal: record(item).signal })), + }; +} + +/** + * Verifies only evidence held by the exact Flows data directory and proxy. + * Server-side dispatch counts, stored Relaycast proofs and fleet state are + * deliberately outside this local contract. + */ +export function verifyLocalResponseLossEvidence({ + runId, + claims, + journalEntries, + proxyStats, + processEvidence, +}) { + assert.equal(typeof runId, 'string'); + const matching = claims.filter(entry => entry.claim?.runId === runId); + assert.equal(matching.length, 1, 'proof run must have exactly one durable node-spawn claim'); + const claim = matching[0].claim; + assert.equal(claim.version, 3, 'response-loss proof requires a fenced v3 claim'); + assert(claim.spawn, 'final durable claim must retain its exact spawn identity'); + assert.equal(claim.spawn.action, 'spawn'); + assert.equal(claim.spawn.status, 'completed'); + assert.equal(claim.spawn.spawned, true); + assert.equal(claim.spawn.ready, true); + assert.equal(claim.spawn.name, claim.agentName, + 'durable spawn receipt must retain the claimed agent name'); + assert(claim.result, 'final durable claim must retain its correlated result'); + assert.equal(claim.release?.status, 'completed', + 'final durable claim must retain its completed release'); + assert(!Object.hasOwn(claim, 'cleanupPending'), + 'final durable claim must not retain pending cleanup'); + assert(claim.result.settlement, 'final result must retain its journal settlement identity'); + assert.equal(claim.result.settlement.settledAs, claim.result.settlementAction, + 'final result must retain its exact inbox settlement action'); + assert(Number.isSafeInteger(claim.result.settlement.settledAt), + 'final result must retain its settlement time'); + + const proxy = verifyResponseLossProxyEvidence(proxyStats); + const identity = spawnIdentityDigest({ + invocationId: claim.spawn.id, + nodeId: claim.spawn.nodeId, + nodeName: claim.spawn.nodeName, + name: claim.spawn.name, + agentId: claim.spawn.agentId, + }); + assert.equal(identity, proxy.spawn_identity_sha256, + 'final durable claim must match the exact intercepted spawn identity'); + + const terminals = journalEntries.filter(entry => terminalMatchesResult(entry, claim)); + assert.equal(terminals.length, 1, + 'journal must contain exactly one terminal entry for the correlated result'); + const terminal = terminals[0]; + assert(terminal.at_ms < claim.result.settlement.settledAt, + 'terminal journal entry must precede claim settlement'); + + return { + schema: LOCAL_SCHEMA, + run_id_sha256: sha256(runId), + ...proxy, + final_claim: { + version: claim.version, + spawn_identity_sha256: identity, + result_present: true, + result_status: claim.result.status, + release_status: claim.release.status, + cleanup_pending: false, + settlement_action: claim.result.settlementAction, + }, + journal: { + terminal_sequence: terminal.seq, + terminal_at_ms: terminal.at_ms, + settlement_at_ms: claim.result.settlement.settledAt, + terminal_before_settlement: true, + }, + process_generations: verifyProcesses(processEvidence), + deferred_assertions: DEFERRED_ASSERTIONS, + }; +} + +export async function auditLocalResponseLoss({ + runId, + dataDir, + proxyStatsPath, + processEvidencePath, +}) { + for (const [label, path] of [ + ['dataDir', dataDir], + ['proxyStatsPath', proxyStatsPath], + ['processEvidencePath', processEvidencePath], + ]) { + assert.equal(typeof path, 'string', `${label} is required`); + assert(isAbsolute(path), `${label} must be absolute`); + } + for (const path of [proxyStatsPath, processEvidencePath]) { + assert.equal((await stat(path)).mode & 0o777, 0o600, + `${path} must be mode 0600`); + } + const claims = await readNodeSpawnClaims(dataDir); + const journalEntries = []; + for await (const entry of walkJournal(runId, dataDir)) journalEntries.push(entry); + const proxyStats = JSON.parse(await readFile(proxyStatsPath, 'utf8')); + const processEvidence = JSON.parse(await readFile(processEvidencePath, 'utf8')); + return verifyLocalResponseLossEvidence({ + runId, claims, journalEntries, proxyStats, processEvidence, + }); +} + +async function main() { + const value = name => { + const index = process.argv.indexOf(name); + return index >= 0 ? process.argv[index + 1] : undefined; + }; + const runId = value('--run'); + assert(typeof runId === 'string' && /^[A-Za-z0-9][A-Za-z0-9_-]*$/.test(runId), + '--run must be a safe run id'); + assert(process.argv.includes('--json'), 'usage requires --json'); + const dataDir = value('--data-dir'); + const proxyStatsPath = value('--proxy-stats'); + const processEvidencePath = value('--process-evidence'); + assert(typeof dataDir === 'string', '--data-dir is required'); + assert(typeof proxyStatsPath === 'string', '--proxy-stats is required'); + assert(typeof processEvidencePath === 'string', '--process-evidence is required'); + const evidence = await auditLocalResponseLoss({ + runId, + dataDir: resolve(dataDir), + proxyStatsPath: resolve(proxyStatsPath), + processEvidencePath: resolve(processEvidencePath), + }); + console.log(JSON.stringify(evidence)); +} + +if (process.argv[1] + && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + main().catch(error => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/packages/sdk/scripts/node-spawn-response-loss-e2e.mjs b/packages/sdk/scripts/node-spawn-response-loss-e2e.mjs index 18fa7a40c..757524852 100644 --- a/packages/sdk/scripts/node-spawn-response-loss-e2e.mjs +++ b/packages/sdk/scripts/node-spawn-response-loss-e2e.mjs @@ -11,10 +11,7 @@ import { dirname, isAbsolute, join, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { setTimeout as delay } from 'node:timers/promises'; -import { - createBridgeResponseLossProxy, - verifyResponseLossEvidence, -} from './node-spawn-response-loss-proxy.mjs'; +import { createBridgeResponseLossProxy } from './node-spawn-response-loss-proxy.mjs'; const DEFAULT_TIMEOUT_MS = 10 * 60_000; @@ -112,7 +109,8 @@ async function main() { } assertCommandSpec(config.start, 'start'); assertCommandSpec(config.resume, 'resume'); - assertCommandSpec(config.audit, 'audit'); + assert.equal(config.audit, undefined, + 'audit is built into the proof kit; fixture-supplied audit commands are not accepted'); assert(config.resume.args.some(arg => arg.includes('{RUN_ID}')), 'resume args must contain {RUN_ID}'); const timeoutMs = config.timeoutMs ?? DEFAULT_TIMEOUT_MS; assert(Number.isSafeInteger(timeoutMs) && timeoutMs > 0, 'timeoutMs must be positive'); @@ -220,7 +218,27 @@ async function main() { assert(resumeResults.every(result => result.signal === null), 'neither raced resume may die by signal'); - const auditSpec = commandWithRunId(config.audit, runId); + const proxyStatsPath = join(privateDir, 'proxy-stats.json'); + const processEvidencePath = join(privateDir, 'process-evidence.json'); + await writeFile(proxyStatsPath, `${JSON.stringify(proxy.stats)}\n`, { mode: 0o600, flag: 'wx' }); + await writeFile(processEvidencePath, `${JSON.stringify({ + initial: { pid: firstChild.pid, ...firstResult }, + replays: [ + { pid: left.child.pid, ...left.result }, + { pid: right.child.pid, ...right.result }, + ], + })}\n`, { mode: 0o600, flag: 'wx' }); + const auditSpec = { + command: process.execPath, + args: [ + fileURLToPath(new URL('./node-spawn-response-loss-audit.mjs', import.meta.url)), + '--run', runId, + '--data-dir', config.dataDir, + '--proxy-stats', proxyStatsPath, + '--process-evidence', processEvidencePath, + '--json', + ], + }; const auditLog = join(config.evidenceDir, 'audit.stderr.log'); const fd = openSync(auditLog, 'wx', 0o600); const auditChild = spawn(auditSpec.command, auditSpec.args, { @@ -244,15 +262,9 @@ async function main() { assert.deepEqual(auditExit, { code: 0, signal: null }, `audit command failed; see ${auditLog}`); const audit = JSON.parse(auditStdout); await assertSecretFree([startLog, resumeLog1, resumeLog2, auditLog]); - const summary = verifyResponseLossEvidence(proxy.stats, audit); const evidence = { - ...summary, + ...audit, run_id: runId, - process_generations: { - initial_pid: firstChild.pid, - replay_pids: replayPids, - }, - resume_exit_codes: resumeResults.map(result => result.code), core_bridge_version: coreBridgeVersion, }; const evidencePath = join(config.evidenceDir, 'response-loss-proof.json'); diff --git a/packages/sdk/scripts/node-spawn-response-loss-proxy.mjs b/packages/sdk/scripts/node-spawn-response-loss-proxy.mjs index 8115ef37c..88448c4ea 100644 --- a/packages/sdk/scripts/node-spawn-response-loss-proxy.mjs +++ b/packages/sdk/scripts/node-spawn-response-loss-proxy.mjs @@ -10,13 +10,25 @@ const SPAWN_INVOKE = 'spawn.invoke'; const SPAWN_RELEASE = 'spawn.release'; const INBOX_ACK = 'spawn_callers.inbox.ack'; const INBOX_FAIL = 'spawn_callers.inbox.fail'; -const STRICT_CONTRACT = 'relay.spawn.invoke.logical-replay-resume/1'; -const STRICT_REPLAY_CAPABILITY = 'relay.spawn.replay-resume/1'; function sha256(value) { return createHash('sha256').update(value).digest('hex'); } +export function spawnIdentityDigest(identity) { + for (const field of ['invocationId', 'nodeId', 'nodeName', 'name', 'agentId']) { + assert.equal(typeof identity?.[field], 'string', `spawn identity ${field} must be a string`); + assert(identity[field].length > 0, `spawn identity ${field} must not be empty`); + } + return sha256(JSON.stringify([ + identity.invocationId, + identity.nodeId, + identity.nodeName, + identity.name, + identity.agentId, + ])); +} + function safeJson(line) { try { const value = JSON.parse(line.toString('utf8')); @@ -121,8 +133,16 @@ export async function createBridgeResponseLossProxy({ if (typeof reply?.id === 'number') pending.delete(reply.id); if (!stats.droppedSpawnReply && request?.method === SPAWN_INVOKE && !reply?.error) { const result = typeof reply?.result === 'object' && reply.result !== null ? reply.result : {}; + const identity = { + invocationId: result.id, + nodeId: result.node_id, + nodeName: request.params.node_name, + name: result.name, + agentId: result.agent_id, + }; const event = { invocationId: typeof result.id === 'string' ? result.id : undefined, + spawnIdentityDigest: spawnIdentityDigest(identity), responseDigest: sha256(Buffer.concat([line, NEWLINE])), responseBytes: line.length + 1, }; @@ -174,7 +194,7 @@ export async function createBridgeResponseLossProxy({ }; } -export function verifyResponseLossEvidence(stats, audit) { +export function verifyResponseLossProxyEvidence(stats) { assert.equal(stats.droppedSpawnReplyForwardedBytes, 0, 'no byte of the committed spawn.invoke reply may reach Flows'); assert(stats.droppedSpawnReply, 'the proxy must drop one successful spawn.invoke reply'); @@ -190,48 +210,17 @@ export function verifyResponseLossEvidence(stats, audit) { assert.equal((stats.methodCounts[INBOX_ACK] ?? 0) + (stats.methodCounts[INBOX_FAIL] ?? 0), 1, 'Flows must settle the correlated inbox item exactly once'); - assert.equal(audit.schema, 'relayflows-node-spawn-response-loss/1'); - assert.equal(audit.strict_contract, STRICT_CONTRACT); - assert.equal(audit.strict_replay_capability, STRICT_REPLAY_CAPABILITY); - assert.equal(audit.strict_invocation_committed, true); - assert.equal(audit.upstream_spawn_frames, 1); - assert.equal(audit.workers_created, 1); - assert.equal(audit.correlated_results, 1); - assert.equal(audit.releases, 1); - assert.equal(audit.final_claim?.result_present, true, - 'the final durable claim must retain its correlated result'); - assert.equal(audit.final_claim?.release_confirmed, true, - 'the final durable claim must retain its confirmed release'); - assert.equal(audit.final_claim?.cleanup_pending, false, - 'the final durable claim must not retain pending cleanup'); - assert.equal(audit.target_capacity?.active_agents, 0); - assert.equal(audit.target_capacity?.reserved_agents, 0); - assert.equal(typeof audit.journal_terminal_sequence, 'number'); - assert.equal(typeof audit.inbox_settlement_sequence, 'number'); - assert(audit.journal_terminal_sequence < audit.inbox_settlement_sequence, - 'terminal journal evidence must precede inbox settlement'); - assert.equal(typeof audit.unrelated_reservation?.before?.identity, 'string'); - assert.equal(typeof audit.unrelated_reservation?.before?.count, 'number'); - assert.deepEqual(audit.unrelated_reservation.after, audit.unrelated_reservation.before, - 'the unrelated reservation must remain unchanged'); return { - schema: audit.schema, - strict_contract: audit.strict_contract, - strict_replay_capability: audit.strict_replay_capability, - strict_invocation_committed: audit.strict_invocation_committed, dropped_response_bytes: stats.droppedSpawnReply.responseBytes, dropped_response_forwarded_bytes: stats.droppedSpawnReplyForwardedBytes, spawn_invoke_calls: stats.methodCounts[SPAWN_INVOKE] ?? 0, + replay_attempts: stats.spawnInvokeKeys.length - 1, + unique_spawn_keys: new Set(stats.spawnInvokeKeys).size, task_sha256: stats.spawnInvokeTaskDigests[0], idempotency_key_sha256: sha256(stats.spawnInvokeKeys[0]), - upstream_spawn_frames: audit.upstream_spawn_frames, - workers_created: audit.workers_created, - correlated_results: audit.correlated_results, - releases: audit.releases, - final_claim: audit.final_claim, - target_capacity: audit.target_capacity, - unrelated_reservation: audit.unrelated_reservation, - journal_terminal_sequence: audit.journal_terminal_sequence, - inbox_settlement_sequence: audit.inbox_settlement_sequence, + spawn_identity_sha256: stats.droppedSpawnReply.spawnIdentityDigest, + release_calls: stats.methodCounts[SPAWN_RELEASE] ?? 0, + inbox_settlement_calls: (stats.methodCounts[INBOX_ACK] ?? 0) + + (stats.methodCounts[INBOX_FAIL] ?? 0), }; } diff --git a/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts b/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts index cb7306612..806a5d179 100644 --- a/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts +++ b/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts @@ -11,7 +11,12 @@ interface ProxyStats { methodCounts: Record; spawnInvokeTaskDigests: string[]; spawnInvokeKeys: string[]; - droppedSpawnReply?: { invocationId?: string; responseDigest: string; responseBytes: number }; + droppedSpawnReply?: { + invocationId?: string; + spawnIdentityDigest: string; + responseDigest: string; + responseBytes: number; + }; droppedSpawnReplyForwardedBytes: number; } @@ -27,11 +32,13 @@ interface ProxyModule { upstreamSocket: string; onIntercept?: () => void; }): Promise; - verifyResponseLossEvidence(stats: ProxyStats, audit: unknown): unknown; + spawnIdentityDigest(identity: Record): string; + verifyResponseLossProxyEvidence(stats: ProxyStats): Record; } const moduleUrl = new URL('../scripts/node-spawn-response-loss-proxy.mjs', import.meta.url).href; const runnerModuleUrl = new URL('../scripts/node-spawn-response-loss-e2e.mjs', import.meta.url).href; +const auditModuleUrl = new URL('../scripts/node-spawn-response-loss-audit.mjs', import.meta.url).href; const roots: string[] = []; const servers: net.Server[] = []; const proxies: ProofProxy[] = []; @@ -67,7 +74,11 @@ function lineServer(socket: string) { } connection.write(`${JSON.stringify({ jsonrpc: '2.0', id: request.id, - result: { id: 'invocation-1', status: 'completed' }, + result: { + id: 'invocation-1', agent_id: 'agent-1', status: 'completed', + node_id: 'node-1', name: 'worker-1', spawned: true, ready: true, + released: false, task_unconfirmed: false, error_code: null, + }, })}\n`); } else { connection.write(`${JSON.stringify({ jsonrpc: '2.0', id: request.id, result: { ok: true } })}\n`); @@ -110,6 +121,54 @@ async function run(command: string, args: string[], cwd: string) { return { code, signal, stdout, stderr }; } +async function localEvidenceFixture() { + const proxy = await import(moduleUrl) as ProxyModule; + const identity = { + invocationId: 'invocation-1', nodeId: 'node-1', nodeName: 'desktop-1', + name: 'worker-1', agentId: 'agent-1', + }; + const key = 'a'.repeat(32); + return { + runId: 'run_fixture', + claims: [{ path: '/private/claim.json', claim: { + version: 3, + baseUrl: 'https://cast.agentrelay.com', publisherId: 'publisher-1', + publisherName: 'flows-e2e', runId: 'run_fixture', stepId: 'step-1', + idempotencyKey: key, agentName: 'worker-1', timeoutMs: 600000, + request: { cli: 'codex', task: 'task', task_identity: 'task' }, startedAt: 1, + spawn: { id: identity.invocationId, action: 'spawn', status: 'completed', + nodeId: identity.nodeId, nodeName: identity.nodeName, name: identity.name, + agentId: identity.agentId, spawned: true, ready: true }, + result: { inboxItemId: 'item-1', messageId: 'message-1', status: 'completed', + envelope: { version: 1, run_id: 'run_fixture', step_id: 'step-1', + dispatch_id: key, status: 'completed', output: { ok: true } }, + settlementAction: 'ack', receivedAt: 150, + settlement: { attempt: 1, completionReason: 'success', preparedAt: 175, + settledAt: 200, settledAs: 'ack' } }, + release: { invocationId: 'release-1', status: 'completed' }, + } }], + journalEntries: [{ seq: 10, segment_id: 1, entry_type: 'step.completed', + run_id: 'run_fixture', step_id: 'step-1', attempt: 1, at_ms: 100, + payload: { completionReason: 'success', trajectory_tail: { relay_task: { + task_execution: { run_id: 'run_fixture', step_id: 'step-1', dispatch_id: key, + delivery_id: 'item-1', message_id: 'message-1' }, + } } } }], + proxyStats: { + methodCounts: { 'spawn.invoke': 2, 'spawn.release': 1, 'spawn_callers.inbox.ack': 1 }, + spawnInvokeTaskDigests: ['task-digest', 'task-digest'], + spawnInvokeKeys: [key, key], + droppedSpawnReply: { invocationId: identity.invocationId, + spawnIdentityDigest: proxy.spawnIdentityDigest(identity), + responseDigest: 'response-digest', responseBytes: 64 }, + droppedSpawnReplyForwardedBytes: 0, + }, + processEvidence: { + initial: { pid: 101, code: null, signal: 'SIGKILL' }, + replays: [{ pid: 102, code: 0, signal: null }, { pid: 103, code: 3, signal: null }], + }, + }; +} + describe('Core Bridge response-loss proof proxy', () => { it('drops every byte of the committed reply and lets two same-key replays converge on one frame', async () => { const root = mkdtempSync(join(tmpdir(), 'flows-response-loss-proxy-')); @@ -128,7 +187,10 @@ describe('Core Bridge response-loss proof proxy', () => { proxies.push(proxy); const spawn = { jsonrpc: '2.0', id: 1, method: 'spawn.invoke', - params: { idempotency_key: 'a'.repeat(32), task: 'exact original task bytes' }, + params: { + idempotency_key: 'a'.repeat(32), task: 'exact original task bytes', + node_id: 'node-1', node_name: 'desktop-1', name: 'worker-1', + }, }; const first = await request(proxySocket, spawn); @@ -146,24 +208,14 @@ describe('Core Bridge response-loss proof proxy', () => { await request(proxySocket, { jsonrpc: '2.0', id: 2, method: 'spawn.release', params: {} }); await request(proxySocket, { jsonrpc: '2.0', id: 3, method: 'spawn_callers.inbox.ack', params: {} }); - expect(() => module.verifyResponseLossEvidence(proxy.stats, { - schema: 'relayflows-node-spawn-response-loss/1', - strict_contract: 'relay.spawn.invoke.logical-replay-resume/1', - strict_replay_capability: 'relay.spawn.replay-resume/1', - strict_invocation_committed: true, - upstream_spawn_frames: 1, - workers_created: 1, - correlated_results: 1, - releases: 1, - final_claim: { result_present: true, release_confirmed: true, cleanup_pending: false }, - target_capacity: { active_agents: 0, reserved_agents: 0 }, - journal_terminal_sequence: 10, - inbox_settlement_sequence: 11, - unrelated_reservation: { - before: { identity: 'unrelated-1', count: 1 }, - after: { identity: 'unrelated-1', count: 1 }, - }, - })).not.toThrow(); + expect(module.verifyResponseLossProxyEvidence(proxy.stats)).toMatchObject({ + dropped_response_forwarded_bytes: 0, + spawn_invoke_calls: 3, + replay_attempts: 2, + unique_spawn_keys: 1, + release_calls: 1, + inbox_settlement_calls: 1, + }); }); it('rejects changed task bytes even when the fixture reports a clean lifecycle', async () => { @@ -172,71 +224,30 @@ describe('Core Bridge response-loss proof proxy', () => { methodCounts: { 'spawn.invoke': 3, 'spawn.release': 1, 'spawn_callers.inbox.ack': 1 }, spawnInvokeTaskDigests: ['one', 'two', 'two'], spawnInvokeKeys: ['key', 'key', 'key'], - droppedSpawnReply: { responseDigest: 'reply', responseBytes: 12 }, + droppedSpawnReply: { spawnIdentityDigest: 'identity', responseDigest: 'reply', responseBytes: 12 }, droppedSpawnReplyForwardedBytes: 0, }; - expect(() => module.verifyResponseLossEvidence(stats, { - schema: 'relayflows-node-spawn-response-loss/1', - strict_contract: 'relay.spawn.invoke.logical-replay-resume/1', - strict_replay_capability: 'relay.spawn.replay-resume/1', - strict_invocation_committed: true, - upstream_spawn_frames: 1, - workers_created: 1, - correlated_results: 1, - releases: 1, - final_claim: { result_present: true, release_confirmed: true, cleanup_pending: false }, - target_capacity: { active_agents: 0, reserved_agents: 0 }, - journal_terminal_sequence: 10, - inbox_settlement_sequence: 11, - unrelated_reservation: { - before: { identity: 'unrelated-1', count: 1 }, - after: { identity: 'unrelated-1', count: 1 }, - }, - })).toThrow('original task bytes'); + expect(() => module.verifyResponseLossProxyEvidence(stats)).toThrow('original task bytes'); }); - it('requires stored strict proofs and an unchanged unrelated reservation', async () => { - const module = await import(moduleUrl) as ProxyModule; - const stats: ProxyStats = { - methodCounts: { 'spawn.invoke': 3, 'spawn.release': 1, 'spawn_callers.inbox.ack': 1 }, - spawnInvokeTaskDigests: ['same', 'same', 'same'], - spawnInvokeKeys: ['key', 'key', 'key'], - droppedSpawnReply: { responseDigest: 'reply', responseBytes: 12 }, - droppedSpawnReplyForwardedBytes: 0, + it('fails closed when the final claim or journal ordering is not durable', async () => { + const audit = await import(auditModuleUrl) as { + verifyLocalResponseLossEvidence(input: Record): Record; }; - const audit = { - schema: 'relayflows-node-spawn-response-loss/1', - strict_contract: 'relay.spawn.invoke.logical-replay-resume/1', - strict_replay_capability: 'relay.spawn.replay-resume/1', - strict_invocation_committed: true, - upstream_spawn_frames: 1, - workers_created: 1, - correlated_results: 1, - releases: 1, - final_claim: { result_present: true, release_confirmed: true, cleanup_pending: false }, - target_capacity: { active_agents: 0, reserved_agents: 0 }, - journal_terminal_sequence: 10, - inbox_settlement_sequence: 11, - unrelated_reservation: { - before: { identity: 'unrelated-1', count: 1 }, - after: { identity: 'unrelated-1', count: 1 }, - }, - }; - expect(() => module.verifyResponseLossEvidence(stats, { - ...audit, - strict_replay_capability: 'legacy', - })).toThrow(); - expect(() => module.verifyResponseLossEvidence(stats, { - ...audit, - unrelated_reservation: { - ...audit.unrelated_reservation, - after: { identity: 'unrelated-1', count: 0 }, - }, - })).toThrow('unrelated reservation'); - expect(() => module.verifyResponseLossEvidence(stats, { - ...audit, - final_claim: { ...audit.final_claim, result_present: false }, - })).toThrow('retain its correlated result'); + const fixture = await localEvidenceFixture(); + expect(audit.verifyLocalResponseLossEvidence(fixture)).toMatchObject({ + schema: 'relayflows-node-spawn-response-loss-local/1', + final_claim: { result_present: true, release_status: 'completed', cleanup_pending: false }, + journal: { terminal_before_settlement: true }, + }); + const missingResult = structuredClone(fixture); + delete missingResult.claims[0].claim.result; + expect(() => audit.verifyLocalResponseLossEvidence(missingResult)) + .toThrow('retain its correlated result'); + const settledBeforeJournal = structuredClone(fixture); + settledBeforeJournal.claims[0].claim.result.settlement.settledAt = 50; + expect(() => audit.verifyLocalResponseLossEvidence(settledBeforeJournal)) + .toThrow('precede claim settlement'); }); it('rejects ar_ credentials in captured proof logs', async () => { @@ -301,10 +312,14 @@ describe('Core Bridge response-loss proof proxy', () => { const childScript = join(root, 'fake-flows.mjs'); writeFileSync(childScript, ` -import { closeSync, mkdirSync, openSync, readFileSync } from 'node:fs'; +import { closeSync, mkdirSync, openSync, readFileSync, writeFileSync } from 'node:fs'; import net from 'node:net'; import { join } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; const [mode, dataDir, winner] = process.argv.slice(2); +const runId = 'run_fixture'; +const stepId = 'step_fixture'; +const key = 'a'.repeat(32); const caller = JSON.parse(readFileSync(process.env.RELAYFLOW_SERVICE_CALLER_FILE, 'utf8')); const rpc = (method, id) => new Promise((resolve, reject) => { const socket = net.createConnection(caller.socket); @@ -312,7 +327,8 @@ const rpc = (method, id) => new Promise((resolve, reject) => { socket.once('error', reject); socket.once('connect', () => socket.write(JSON.stringify({ jsonrpc: '2.0', id, method, params: method === 'spawn.invoke' - ? { idempotency_key: 'a'.repeat(32), task: 'exact original task bytes' } + ? { idempotency_key: key, task: 'exact original task bytes', + node_id: 'node-1', node_name: 'desktop-1', name: 'worker-1' } : {}, }) + '\\n')); socket.on('data', chunk => { @@ -325,7 +341,16 @@ const rpc = (method, id) => new Promise((resolve, reject) => { }); if (mode === 'start') { mkdirSync(join(dataDir, 'runs'), { recursive: true }); - closeSync(openSync(join(dataDir, 'runs', 'run_fixture.sqlite3'), 'w')); + const db = new DatabaseSync(join(dataDir, 'runs', runId + '.sqlite3')); + db.exec(\` + CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID; + INSERT INTO meta VALUES ('run_id', 'run_fixture'), ('journal_version', '1'); + CREATE TABLE segments (segment_id INTEGER PRIMARY KEY, journal_version INTEGER NOT NULL, opened_seq INTEGER NOT NULL); + INSERT INTO segments VALUES (1, 1, 1); + CREATE TABLE entries (seq INTEGER PRIMARY KEY, segment_id INTEGER NOT NULL REFERENCES segments(segment_id), + entry_type TEXT NOT NULL, step_id TEXT, attempt INTEGER, at_ms INTEGER NOT NULL, payload TEXT NOT NULL); + \`); + db.close(); await rpc('spawn.invoke', 1); process.exit(99); } @@ -337,27 +362,47 @@ if (!won) process.exit(3); await rpc('spawn.invoke', 1); await rpc('spawn.release', 2); await rpc('spawn_callers.inbox.ack', 3); -`); - - const auditScript = join(root, 'audit.mjs'); - writeFileSync(auditScript, `console.log(JSON.stringify({ - schema: 'relayflows-node-spawn-response-loss/1', - strict_contract: 'relay.spawn.invoke.logical-replay-resume/1', - strict_replay_capability: 'relay.spawn.replay-resume/1', - strict_invocation_committed: true, - upstream_spawn_frames: 1, - workers_created: 1, - correlated_results: 1, - releases: 1, - final_claim: { result_present: true, release_confirmed: true, cleanup_pending: false }, - target_capacity: { active_agents: 0, reserved_agents: 0 }, - journal_terminal_sequence: 10, - inbox_settlement_sequence: 11, - unrelated_reservation: { - before: { identity: 'unrelated-1', count: 1 }, - after: { identity: 'unrelated-1', count: 1 }, +const journal = new DatabaseSync(join(dataDir, 'runs', runId + '.sqlite3')); +journal.prepare('INSERT INTO entries VALUES (?, ?, ?, ?, ?, ?, ?)').run( + 10, 1, 'step.completed', stepId, 1, 100, JSON.stringify({ + completionReason: 'success', + trajectory_tail: { relay_task: { task_execution: { + run_id: runId, step_id: stepId, dispatch_id: key, + delivery_id: 'item-1', message_id: 'message-1', + } } }, + }), +); +journal.close(); +const claim = { + version: 3, + baseUrl: 'https://cast.agentrelay.com', + publisherId: 'publisher-1', + publisherName: 'flows-e2e', + runId, + stepId, + idempotencyKey: key, + agentName: 'worker-1', + timeoutMs: 600000, + request: { cli: 'codex', task: 'exact original task bytes', task_identity: 'proof-task' }, + startedAt: 1, + spawn: { + id: 'invocation-1', action: 'spawn', status: 'completed', + nodeId: 'node-1', nodeName: 'desktop-1', name: 'worker-1', agentId: 'agent-1', + spawned: true, ready: true, }, -}));\n`); + result: { + inboxItemId: 'item-1', messageId: 'message-1', status: 'completed', + envelope: { version: 1, run_id: runId, step_id: stepId, dispatch_id: key, + status: 'completed', output: { ok: true } }, + settlementAction: 'ack', receivedAt: 150, + settlement: { attempt: 1, completionReason: 'success', preparedAt: 175, + settledAt: 200, settledAs: 'ack' }, + }, + release: { invocationId: 'release-1', status: 'completed' }, +}; +mkdirSync(join(dataDir, 'node-spawns'), { recursive: true }); +writeFileSync(join(dataDir, 'node-spawns', 'b'.repeat(64) + '.json'), JSON.stringify(claim), { mode: 0o600 }); +`); const config = join(root, 'config.json'); writeFileSync(config, `${JSON.stringify({ callerFile, @@ -367,7 +412,6 @@ await rpc('spawn_callers.inbox.ack', 3); workDir, start: { command: process.execPath, args: [childScript, 'start', dataDir, join(root, 'winner')] }, resume: { command: process.execPath, args: [childScript, 'resume', dataDir, join(root, 'winner'), '{RUN_ID}'] }, - audit: { command: process.execPath, args: [auditScript, '{RUN_ID}'] }, })}\n`, { mode: 0o600 }); const result = await run(process.execPath, [ @@ -378,22 +422,32 @@ await rpc('spawn_callers.inbox.ack', 3); expect(JSON.parse(result.stdout)).toMatchObject({ ok: true, run_id: 'run_fixture' }); const evidence = JSON.parse(readFileSync(join(evidenceDir, 'response-loss-proof.json'), 'utf8')); expect(evidence).toMatchObject({ + schema: 'relayflows-node-spawn-response-loss-local/1', dropped_response_forwarded_bytes: 0, spawn_invoke_calls: 2, - upstream_spawn_frames: 1, - workers_created: 1, - correlated_results: 1, - releases: 1, - final_claim: { result_present: true, release_confirmed: true, cleanup_pending: false }, - target_capacity: { active_agents: 0, reserved_agents: 0 }, - unrelated_reservation: { - before: { identity: 'unrelated-1', count: 1 }, - after: { identity: 'unrelated-1', count: 1 }, + replay_attempts: 1, + unique_spawn_keys: 1, + release_calls: 1, + inbox_settlement_calls: 1, + final_claim: { + version: 3, + result_present: true, + release_status: 'completed', + cleanup_pending: false, + settlement_action: 'ack', }, + journal: { terminal_sequence: 10, terminal_before_settlement: true }, process_generations: { initial_pid: expect.any(Number), replay_pids: [expect.any(Number), expect.any(Number)], }, + deferred_assertions: [ + 'stored_relaycast_contract_and_capability', + 'provider_frame_and_worker_counts', + 'target_capacity', + 'unrelated_reservation_preservation', + 'remote_fleet_lifecycle', + ], }); expect(new Set(evidence.process_generations.replay_pids).size).toBe(2); expect(evidence.process_generations.replay_pids).not.toContain( From 5a5de5805a269f77ce673c96cb779bb7074113c4 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Tue, 10 Nov 2026 19:10:16 -0800 Subject: [PATCH 70/75] fix(sdk): bind local replay to authentic bridge --- docs/AGENT-RELAY-TRANSPORT.md | 18 +- docs/MIGRATION-NODE-SPAWN.md | 10 +- packages/sdk/src/node-spawn-dispatch.ts | 68 ++-- packages/sdk/src/node-spawn-local.ts | 66 +++- packages/sdk/src/node-spawn-runtime.ts | 5 +- .../tests/node-spawn-local-replay-gap.test.ts | 302 +++++++++++------- packages/sdk/tests/node-spawn-runtime.test.ts | 10 + 7 files changed, 327 insertions(+), 152 deletions(-) diff --git a/docs/AGENT-RELAY-TRANSPORT.md b/docs/AGENT-RELAY-TRANSPORT.md index 221074e26..c336e4f3f 100644 --- a/docs/AGENT-RELAY-TRANSPORT.md +++ b/docs/AGENT-RELAY-TRANSPORT.md @@ -62,11 +62,15 @@ The exact built-in local Desktop port is the sole fenced-replay exception: Core Bridge 18.1 or newer durably deduplicates `spawn.invoke` by the forwarded idempotency key, so a restart may repeat that identical request and recover the original invocation after its reply was lost. Flows admits that replay only for -the registered, unchanged `LocalNodeSpawnPort` prototype and re-verifies the -existing fence before entry. A missing receipt from remote placement, a custom -or overridden port, or a legacy version 2 claim remains ambiguous and fails -closed without another dispatch. Malformed, mismatched, or unreadable fence -state also fails closed. A recorded invocation is reconciled with +the SDK runtime built from the exact module returned by `loadCoreBridge()`, with +the unchanged local port prototype and lifecycle methods, and re-verifies the +existing fence before entry. That replay repeats the already-admitted request +and exact node without applying mutable current directory-scope or live-target +preflight. A new unfenced dispatch still applies all current caller scope and +target checks. A missing receipt from remote placement, an injected bridge, a +custom or overridden port, or a legacy version 2 claim remains ambiguous and +fails closed without another dispatch. Malformed, mismatched, or unreadable +fence state also fails closed. A recorded invocation is reconciled with `commands.getInvocation`. The stable worker name provides additional collision protection at the node. @@ -75,9 +79,9 @@ and accept the supplied `beforeDispatch` callback. Because Flows cannot audit arbitrary port code, it publishes and fsyncs the fence before entering a custom `spawn`; an error from that call is permanently ambiguous and cannot be retried. The callback remains part of the protocol and is idempotent after publication. -Only the SDK's exact built-in local `spawn` and `targets` method set may use +Only the SDK's exact built-in local lifecycle method set may use `NodeSpawnPreDispatchError` to retain retryable target preflight. Replacing or -overriding either method removes the narrow trust and fences it before entry. +overriding a relevant method removes the narrow trust and fences it before entry. Existing one-argument `spawn` implementations are not compatible even though JavaScript can ignore extra arguments: the required protocol field makes them a type error, and the runtime refuses a missing or unknown version diff --git a/docs/MIGRATION-NODE-SPAWN.md b/docs/MIGRATION-NODE-SPAWN.md index c9665ebd8..0cbca00cf 100644 --- a/docs/MIGRATION-NODE-SPAWN.md +++ b/docs/MIGRATION-NODE-SPAWN.md @@ -66,9 +66,13 @@ entry. The exact built-in local Desktop port has one additional recovery contract. Core Bridge 18.1 or newer persists `spawn.invoke` idempotency keys, so after a durable fence and a lost reply Flows may repeat the identical local request and -recover the original invocation. That exemption is registered by exact -prototype and method identity; wrappers, subclasses, overrides, accessors, -custom ports, remote placement and legacy version 2 claims remain fail-closed. +recover the original invocation even when mutable current scope or target +preflight would no longer admit a new dispatch. That exemption is registered +only when the SDK runtime loaded the exact Core Bridge module and the local +prototype and lifecycle method identities remain unchanged. Injected bridges, +wrappers, subclasses, overrides, accessors, custom ports, remote placement and +legacy version 2 claims remain fail-closed. New unfenced local dispatches still +enforce current caller scope and target eligibility. ## Hosted and self-hosted execution diff --git a/packages/sdk/src/node-spawn-dispatch.ts b/packages/sdk/src/node-spawn-dispatch.ts index d12ff908a..addf767be 100644 --- a/packages/sdk/src/node-spawn-dispatch.ts +++ b/packages/sdk/src/node-spawn-dispatch.ts @@ -15,14 +15,21 @@ interface FencedNodeSpawnOptions { replayExistingFence?: boolean; } -interface RetryablePreflightIdentity { +interface ExactPortIdentity { prototype: object; spawn: NodeSpawnPort['spawn']; targets: NodeSpawnPort['targets']; + status: NodeSpawnPort['status']; + release: NodeSpawnPort['release']; } -const retryablePreflightPorts = new WeakMap(); -const replayableFencedPorts = new WeakMap(); +interface ReplayableFencedRegistration { + identity: ExactPortIdentity; + replay: NodeSpawnPort['spawn']; +} + +const retryablePreflightPorts = new WeakMap(); +const replayableFencedPorts = new WeakMap(); /** * Registers the exact built-in method set that may perform proven side-effect- @@ -31,40 +38,55 @@ const replayableFencedPorts = new WeakMap { const retryablePreflight = hasRetryablePreflight(options.port); + let replay: NodeSpawnPort['spawn'] | undefined; let dispatchPublished = false; if (options.replayExistingFence === true) { - if (!hasReplayableFencedDispatch(options.port) - || !await hasNodeSpawnDispatchFence(options.claimPath, options.claim)) { + if (!await hasNodeSpawnDispatchFence(options.claimPath, options.claim)) { + throw new NodeSpawnError( + 'A fenced node-spawn replay requires the exact built-in replay contract and durable fence', + 'spawn_dispatch_fence_missing', + ); + } + // Recheck exact port identity after the asynchronous durable-fence read and + // immediately before selecting the internal replay handler. + replay = replayableFencedRegistration(options.port)?.replay; + if (replay === undefined) { throw new NodeSpawnError( 'A fenced node-spawn replay requires the exact built-in replay contract and durable fence', 'spawn_dispatch_fence_missing', @@ -94,7 +125,8 @@ export async function dispatchFencedNodeSpawn( dispatchPublished = true; } try { - pendingSpawn = options.port.spawn(options.request, async () => { + const spawn = replay ?? options.port.spawn.bind(options.port); + pendingSpawn = spawn(options.request, async () => { dispatchHookEntered = true; options.signal?.throwIfAborted(); if (dispatchPublished) return; diff --git a/packages/sdk/src/node-spawn-local.ts b/packages/sdk/src/node-spawn-local.ts index 4f0728d96..d498949f1 100644 --- a/packages/sdk/src/node-spawn-local.ts +++ b/packages/sdk/src/node-spawn-local.ts @@ -174,6 +174,9 @@ const REQUIRED_CORE_BRIDGE_METHODS = { serviceCallerDisconnect: true, } as const satisfies Record; +const loadedCoreBridges = new WeakSet(); +const LOCAL_FENCED_REPLAY = Symbol('LocalNodeSpawnPort.fencedReplay'); + /** Fails before local admission when an installed optional bridge lacks the required contract. */ export function assertCoreBridgeApi(value: unknown): asserts value is CoreBridgeApi { const module = (typeof value === 'object' && value !== null) @@ -205,9 +208,23 @@ export async function loadCoreBridge(): Promise { ); } assertCoreBridgeApi(bridge); + loadedCoreBridges.add(bridge); return bridge; } +/** Enables replay only for the exact bridge module admitted by loadCoreBridge(). */ +export function enableLoadedCoreBridgeReplay( + port: LocalNodeSpawnPort, + bridge: CoreBridgeApi, +): void { + if (!loadedCoreBridges.has(bridge)) return; + registerReplayableFencedPort( + port, + LOCAL_NODE_SPAWN_IDENTITY, + (request, beforeDispatch) => port[LOCAL_FENCED_REPLAY](request, beforeDispatch), + ); +} + /** Native local spawn adapter over one already authenticated schema-v2 bridge connection. */ export class LocalNodeSpawnPort implements NodeSpawnPort { readonly dispatchFenceVersion = 1 as const; @@ -219,7 +236,6 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { private readonly exactNode: { readonly id: string; readonly name: string }, ) { registerRetryablePreflightPort(this, LOCAL_NODE_SPAWN_IDENTITY); - registerReplayableFencedPort(this, LOCAL_NODE_SPAWN_IDENTITY); } async targets(): Promise { @@ -286,6 +302,50 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { } catch (error) { throw new NodeSpawnPreDispatchError(error); } + return this.invokeSpawn( + { ...request, directory: request.directory }, + beforeDispatch, + { id: target.id, name: target.name }, + allTargets, + ); + } + + private async [LOCAL_FENCED_REPLAY]( + request: NodeSpawnRequest, + beforeDispatch: BeforeNodeDispatch, + ): Promise { + if (!request.directory || !isAbsolute(request.directory)) { + throw new NodeSpawnError('Local node spawn requires an absolute directory', 'spawn_invalid_input'); + } + if (!validIdempotencyKey(request.idempotencyKey)) { + throw new NodeSpawnError('Invalid node-spawn idempotency key', 'spawn_invalid_input'); + } + if (request.model !== undefined || request.repo !== undefined || request.sandboxOnly !== undefined) { + throw new NodeSpawnError( + 'The Desktop core bridge does not accept model, repository, or sandbox placement overrides', + 'spawn_invalid_input', + ); + } + if (request.node !== undefined && request.node !== this.exactNode.name) { + throw new NodeSpawnError('The requested node differs from the authenticated Desktop node', + 'spawn_identity_mismatch'); + } + // The original v3 claim was already admitted and fenced. Repeating the + // exact logical request lets Relay replay that durable claim even if the + // caller's current directory scope or live-target view has since changed. + return this.invokeSpawn( + { ...request, directory: request.directory }, + beforeDispatch, + this.exactNode, + ); + } + + private async invokeSpawn( + request: NodeSpawnRequest & { directory: string }, + beforeDispatch: BeforeNodeDispatch, + target: { readonly id: string; readonly name: string }, + knownTargets: readonly { readonly id?: string; readonly name: string }[] = [target], + ): Promise { await beforeDispatch(); const raw = await this.bridge.spawnInvoke(this.core, { node_id: target.id, @@ -301,7 +361,7 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { const confirmation = await this.wait('spawn', raw.id, request.timeoutMs, mapCoreInvocation('spawn', raw)); const settled = confirmation.invocation; - const settledTarget = allTargets.find(candidate => candidate.id === settled.nodeId); + const settledTarget = knownTargets.find(candidate => candidate.id === settled.nodeId); const cleanup = confirmation.cleanup ?? (settled.agentId && settled.name && settledTarget?.id ? { id: settled.id, nodeId: settledTarget.id, nodeName: settledTarget.name, name: settled.name, agentId: settled.agentId } @@ -405,6 +465,8 @@ const LOCAL_NODE_SPAWN_IDENTITY = Object.freeze({ prototype: LocalNodeSpawnPort.prototype, spawn: LocalNodeSpawnPort.prototype.spawn, targets: LocalNodeSpawnPort.prototype.targets, + status: LocalNodeSpawnPort.prototype.status, + release: LocalNodeSpawnPort.prototype.release, }); function withinDirectory(root: string, candidate: string): boolean { diff --git a/packages/sdk/src/node-spawn-runtime.ts b/packages/sdk/src/node-spawn-runtime.ts index c696b216e..c72517ac1 100644 --- a/packages/sdk/src/node-spawn-runtime.ts +++ b/packages/sdk/src/node-spawn-runtime.ts @@ -2,6 +2,7 @@ import { isIP } from 'node:net'; import { isAbsolute } from 'node:path'; import type { NodeResultInboxClient, NodeResultTerminalFailure } from './node-spawn-result.js'; import { + enableLoadedCoreBridgeReplay, LocalNodeSpawnPort, loadCoreBridge, type CallerDescriptor, @@ -137,8 +138,10 @@ export async function createLocalNodeSpawnRuntime( 'caller_identity_mismatch', ); } + const port = new LocalNodeSpawnPort(core, bridge, descriptor, identity.node); + enableLoadedCoreBridgeReplay(port, bridge); return { - port: new LocalNodeSpawnPort(core, bridge, descriptor, identity.node), + port, inbox: localResultInbox(bridge, descriptor), publisher: identity.publisher, baseUrl: identity.base_url, diff --git a/packages/sdk/tests/node-spawn-local-replay-gap.test.ts b/packages/sdk/tests/node-spawn-local-replay-gap.test.ts index d9a255a51..3d7cf4e0d 100644 --- a/packages/sdk/tests/node-spawn-local-replay-gap.test.ts +++ b/packages/sdk/tests/node-spawn-local-replay-gap.test.ts @@ -1,153 +1,185 @@ -import { mkdtemp, rm } from 'node:fs/promises'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { afterEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { runAgentRelayTask, type AgentRelayTaskRequest } from '../src/agent-relay-transport.js'; -import { LocalNodeSpawnPort, type CoreBridgeClient } from '../src/node-spawn-local.js'; -import { NODE_RESULT_PREFIX } from '../src/node-spawn-result.js'; -import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; +import { hasReplayableFencedDispatch } from '../src/node-spawn-dispatch.js'; +import { createLocalNodeSpawnRuntime } from '../src/node-spawn-runtime.js'; +import { nodeSpawnDispatchFencePath, readNodeSpawnClaims } from '../src/node-spawn-state.js'; -const roots: string[] = []; - -afterEach(async () => { - await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); -}); - -async function dataDirectory(): Promise { - const directory = await mkdtemp(join(tmpdir(), 'flows-local-replay-gap-')); - roots.push(directory); - return directory; -} - -function request(dataDir: string): AgentRelayTaskRequest { - return { - cli: 'codex', - task: 'Implement the exact change', - taskIdentity: 'task_sha256_exact', - runId: 'run_local_replay', - stepId: 'edit', - idempotencyKey: 'local_replay_key_123', - attempt: 1, - dataDir, - timeoutMs: 10_000, - worker_cwd: '/tmp', +const coreBridge = vi.hoisted(() => { + const state = { + directoryScope: ['/tmp'] as string[], + invokeCalls: [] as Array<{ key: string; task: string; directory: string }>, + commits: new Set(), + dropFirstReply: true, + releases: 0, + subscriptions: 0, + targetCalls: 0, + spawnedName: '', }; -} - -function strictReplayRuntime(observed: { - invokeCalls: Array<{ key: string; task: string }>; - commits: Set; - dropFirstReply: boolean; - releases: number; - subscriptions: number; -}): NodeSpawnRuntime { - const core: CoreBridgeClient = { - hello: { - bridge_version: 1, - daemon_version: 'test', - pid: process.pid, - methods: [], - topics: [], - }, - close() {}, + const core = { + hello: { bridge_version: 1, daemon_version: 'test', pid: 1, methods: [], topics: [] }, + close: vi.fn(), }; - let spawnedName = ''; const invocation = () => ({ id: 'inv_spawn_replayed', agent_id: 'agent_replayed', status: 'completed', node_id: 'node_local', - name: spawnedName, + name: state.spawnedName, spawned: true, ready: true, released: false, task_unconfirmed: false, error_code: null, }); - const bridge: ConstructorParameters[1] = { - spawnTargets: vi.fn(async () => ({ - targets: [{ - id: 'node_local', - name: 'desktop-local', - live: true, - handlers_live: true, - stale: false, - capabilities: ['spawn:codex'], - active_agents: 0, - reserved_agents: 0, - max_agents: 4, - }], + const bridge = { + connectCore: vi.fn(async () => core), + readCallerDescriptor: vi.fn(async () => ({ + schema: 'agent-relay-service-caller/2', + schemaVersion: 2, + auth: {}, + callerId: 'caller_local', + callerName: 'flows-caller', + publisher: { id: 'publisher_local', name: 'flows-publisher' }, + workspace: { + site_url: 'https://agentrelay.com', + base_url: 'https://cast.agentrelay.com', + workspace_id: 'workspace_local', + relaycast_workspace_id: 'relaycast_local', + }, + account: { user_id: 'user_local' }, + credential: {}, + createdAt: '2026-01-01T00:00:00.000Z', + expiresAt: '2027-01-01T00:00:00.000Z', + harnesses: ['codex'], + directoryScope: [...state.directoryScope], + ownsOnly: true, + socket: '/tmp/core.sock', + inbox: { + transport: 'core', open: 'open', next: 'next', ack: 'ack', fail: 'fail', + close: 'close', disconnect: 'disconnect', + }, + desktopVersion: 'test', + })), + authenticate: vi.fn(async () => ({})), + requireServiceCallerMethods: vi.fn(), + assertServiceCallerIdentity: vi.fn(), + serviceCallerIdentity: vi.fn(async () => ({ + publisher: { id: 'publisher_local', name: 'flows-publisher' }, + node: { id: 'node_local', name: 'desktop-local' }, + base_url: 'https://cast.agentrelay.com', + workspace_id: 'workspace_local', + relaycast_workspace_id: 'relaycast_local', })), - spawnInvoke: vi.fn(async (_core, input) => { - spawnedName = input.name; - observed.invokeCalls.push({ key: input.idempotency_key, task: input.task }); - observed.commits.add(input.idempotency_key); - if (observed.dropFirstReply) { - observed.dropFirstReply = false; + spawnTargets: vi.fn(async () => { + state.targetCalls += 1; + return { targets: [{ + id: 'node_local', name: 'desktop-local', live: true, handlers_live: true, + stale: false, capabilities: ['spawn:codex'], active_agents: 0, + reserved_agents: 0, max_agents: 4, + }] }; + }), + spawnInvoke: vi.fn(async (_core: unknown, input: { + name: string; task: string; directory: string; idempotency_key: string; + }) => { + state.spawnedName = input.name; + state.invokeCalls.push({ + key: input.idempotency_key, + task: input.task, + directory: input.directory, + }); + state.commits.add(input.idempotency_key); + if (state.dropFirstReply) { + state.dropFirstReply = false; throw new Error('bridge reply lost after strict server commit'); } return invocation(); }), spawnStatus: vi.fn(async () => invocation()), spawnRelease: vi.fn(async () => { - observed.releases += 1; - return { - ...invocation(), - spawned: false, - ready: false, - released: true, - }; + state.releases += 1; + return { ...invocation(), spawned: false, ready: false, released: true }; + }), + serviceCallerInboxOpen: vi.fn(async () => { + state.subscriptions += 1; + return { iterator_id: 'iterator_replayed' }; }), + serviceCallerInboxNext: vi.fn(async () => ({ delivery: { + item_id: 'delivery_replayed', + message_id: 'message_replayed', + sender: { id: 'agent_replayed', name: state.spawnedName }, + text: 'RELAYFLOWS_RESULT_V1 ' + JSON.stringify({ + version: 1, + run_id: 'run_local_replay', + step_id: 'edit', + dispatch_id: 'local_replay_key_123', + status: 'completed', + output: { ok: true }, + }), + invocation_id: 'inv_spawn_replayed', + sent_at: '2026-01-01T00:00:00.000Z', + } })), + serviceCallerInboxAck: vi.fn(async () => ({})), + serviceCallerInboxFail: vi.fn(async () => ({})), + serviceCallerInboxClose: vi.fn(async () => ({})), + serviceCallerDisconnect: vi.fn(async () => ({})), }; + return { bridge, state }; +}); + +vi.mock('@agent-relay/core-bridge', () => coreBridge.bridge); + +const roots: string[] = []; + +beforeEach(() => { + coreBridge.state.directoryScope = ['/tmp']; + coreBridge.state.invokeCalls.length = 0; + coreBridge.state.commits.clear(); + coreBridge.state.dropFirstReply = true; + coreBridge.state.releases = 0; + coreBridge.state.subscriptions = 0; + coreBridge.state.targetCalls = 0; + coreBridge.state.spawnedName = ''; + vi.clearAllMocks(); +}); + +afterEach(async () => { + await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); +}); + +async function dataDirectory(): Promise { + const directory = await mkdtemp(join(tmpdir(), 'flows-local-replay-gap-')); + roots.push(directory); + return directory; +} + +function request(dataDir: string, workerCwd = '/tmp'): AgentRelayTaskRequest { return { - authority: 'desktop', - publisher: { id: 'publisher_local', name: 'flows-publisher' }, - baseUrl: 'https://cast.agentrelay.com', - port: new LocalNodeSpawnPort(core, bridge, { - harnesses: ['codex'], - directoryScope: ['/tmp'], - }, { id: 'node_local', name: 'desktop-local' }), - inbox: { inbox: { - subscribe: async function* () { - observed.subscriptions += 1; - yield { - id: 'delivery_replayed', - message: { - id: 'message_replayed', - from: { id: 'agent_replayed', name: spawnedName }, - text: NODE_RESULT_PREFIX + JSON.stringify({ - version: 1, - run_id: 'run_local_replay', - step_id: 'edit', - dispatch_id: 'local_replay_key_123', - status: 'completed', - output: { ok: true }, - }), - }, - }; - }, - ack: vi.fn(async () => ({})), - fail: vi.fn(async () => ({})), - } }, + cli: 'codex', + task: 'Implement the exact change', + taskIdentity: 'task_sha256_exact', + runId: 'run_local_replay', + stepId: 'edit', + idempotencyKey: 'local_replay_key_123', + attempt: 1, + dataDir, + timeoutMs: 10_000, + worker_cwd: workerCwd, }; } describe('official local strict replay', () => { - it('replays the same idempotency key after the first committed reply is lost', async () => { + it('replays an accepted same-key claim after current directory scope is revoked', async () => { const dataDir = await dataDirectory(); - const observed = { - invokeCalls: [] as Array<{ key: string; task: string }>, - commits: new Set(), - dropFirstReply: true, - releases: 0, - subscriptions: 0, - }; - const crashedRuntime = strictReplayRuntime(observed); + const crashedRuntime = await createLocalNodeSpawnRuntime('/tmp/caller.json'); await expect(runAgentRelayTask(request(dataDir), { runtime: crashedRuntime })) .rejects.toThrow('bridge reply lost after strict server commit'); - const restartedRuntime = strictReplayRuntime(observed); + coreBridge.state.directoryScope = ['/revoked-root']; + const restartedRuntime = await createLocalNodeSpawnRuntime('/tmp/caller.json'); const recovered = await Promise.all([ runAgentRelayTask(request(dataDir), { runtime: restartedRuntime }), runAgentRelayTask(request(dataDir), { runtime: restartedRuntime }), @@ -157,12 +189,40 @@ describe('official local strict replay', () => { expect.objectContaining({ receipt: expect.objectContaining({ status: 'completed' }) }), ]); - expect(observed.invokeCalls).toHaveLength(2); - expect(observed.invokeCalls[0]).toEqual(observed.invokeCalls[1]); - expect(observed.invokeCalls[0]).toMatchObject({ key: 'local_replay_key_123' }); - expect(observed.invokeCalls[0]!.task).toContain('Implement the exact change'); - expect([...observed.commits]).toEqual(['local_replay_key_123']); - expect(observed.subscriptions).toBe(1); - expect(observed.releases).toBe(1); + expect(coreBridge.state.invokeCalls).toHaveLength(2); + expect(coreBridge.state.invokeCalls[0]).toEqual(coreBridge.state.invokeCalls[1]); + expect(coreBridge.state.invokeCalls[0]).toMatchObject({ + key: 'local_replay_key_123', + directory: '/tmp', + }); + expect(coreBridge.state.invokeCalls[0]!.task).toContain('Implement the exact change'); + expect([...coreBridge.state.commits]).toEqual(['local_replay_key_123']); + expect(coreBridge.state.targetCalls).toBe(1); + expect(coreBridge.state.subscriptions).toBe(1); + expect(coreBridge.state.releases).toBe(1); + }); + + it('still applies current directory scope before a new unfenced dispatch', async () => { + const dataDir = await dataDirectory(); + coreBridge.state.directoryScope = ['/revoked-root']; + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json'); + + await expect(runAgentRelayTask(request(dataDir), { runtime })) + .rejects.toThrow('does not authorize this harness and directory'); + expect(coreBridge.state.invokeCalls).toHaveLength(0); + expect(coreBridge.state.targetCalls).toBe(0); + const [claim] = await readNodeSpawnClaims(dataDir); + await expect(readFile(nodeSpawnDispatchFencePath(claim!.path), 'utf8')) + .rejects.toMatchObject({ code: 'ENOENT' }); }); + + it.each(['spawn', 'targets', 'status', 'release'] as const)( + 'revokes replay trust when authentic port method %s is overridden', async method => { + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json'); + expect(hasReplayableFencedDispatch(runtime.port)).toBe(true); + const original = runtime.port[method].bind(runtime.port); + Object.defineProperty(runtime.port, method, { value: vi.fn(original) }); + expect(hasReplayableFencedDispatch(runtime.port)).toBe(false); + }, + ); }); diff --git a/packages/sdk/tests/node-spawn-runtime.test.ts b/packages/sdk/tests/node-spawn-runtime.test.ts index 45a41b515..39655fc2c 100644 --- a/packages/sdk/tests/node-spawn-runtime.test.ts +++ b/packages/sdk/tests/node-spawn-runtime.test.ts @@ -26,6 +26,7 @@ import { createLocalNodeSpawnRuntime, createNodeSpawnRuntime, createNodeSpawnRun import type { CallerDescriptor, CoreBridgeApi, CoreBridgeClient } from '../src/node-spawn-local.js'; import { waitForNodeResult } from '../src/node-spawn-result.js'; import { runAgentRelayTask } from '../src/agent-relay-transport.js'; +import { hasReplayableFencedDispatch } from '../src/node-spawn-dispatch.js'; describe('remote node-spawn runtime', () => { beforeEach(() => { @@ -304,6 +305,15 @@ describe('Desktop service-caller runtime', () => { expect(close).toHaveBeenCalledOnce(); }); + it('never grants fenced replay authority to an injected structural bridge', async () => { + const { bridge } = fixture(); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge }); + expect(hasReplayableFencedDispatch(runtime.port)).toBe(false); + runtime.port.status = vi.fn(runtime.port.status.bind(runtime.port)); + expect(hasReplayableFencedDispatch(runtime.port)).toBe(false); + await runtime.disconnect?.(); + }); + it('fails closed when the live publisher or workspace differs from caller.json', async () => { const { bridge, close } = fixture(); vi.mocked(bridge.serviceCallerIdentity).mockResolvedValue({ From f136c1a3e367ab3798d6600400c5b734b25175aa Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Tue, 10 Nov 2026 19:18:16 -0800 Subject: [PATCH 71/75] fix(sdk): gate local preflight on official bridge --- docs/AGENT-RELAY-TRANSPORT.md | 3 +- docs/MIGRATION-NODE-SPAWN.md | 5 +- packages/sdk/src/node-spawn-dispatch.ts | 28 ++++++--- packages/sdk/src/node-spawn-local.ts | 48 ++++++++++++++- packages/sdk/src/node-spawn-runtime.ts | 4 +- .../tests/node-spawn-dispatch-fence.test.ts | 60 +++++++++++++++++-- .../tests/node-spawn-local-replay-gap.test.ts | 9 +++ .../node-spawn-response-loss-proxy.test.ts | 1 + 8 files changed, 139 insertions(+), 19 deletions(-) diff --git a/docs/AGENT-RELAY-TRANSPORT.md b/docs/AGENT-RELAY-TRANSPORT.md index c336e4f3f..6275a2e07 100644 --- a/docs/AGENT-RELAY-TRANSPORT.md +++ b/docs/AGENT-RELAY-TRANSPORT.md @@ -79,7 +79,8 @@ and accept the supplied `beforeDispatch` callback. Because Flows cannot audit arbitrary port code, it publishes and fsyncs the fence before entering a custom `spawn`; an error from that call is permanently ambiguous and cannot be retried. The callback remains part of the protocol and is idempotent after publication. -Only the SDK's exact built-in local lifecycle method set may use +Only the SDK runtime constructed from its internally loaded Core Bridge module +and exact built-in local lifecycle method/state set may use `NodeSpawnPreDispatchError` to retain retryable target preflight. Replacing or overriding a relevant method removes the narrow trust and fences it before entry. Existing one-argument `spawn` implementations are not compatible diff --git a/docs/MIGRATION-NODE-SPAWN.md b/docs/MIGRATION-NODE-SPAWN.md index 0cbca00cf..139a7e23d 100644 --- a/docs/MIGRATION-NODE-SPAWN.md +++ b/docs/MIGRATION-NODE-SPAWN.md @@ -54,8 +54,9 @@ Ports must declare `dispatchFenceVersion: 1` and accept the supplied side-effect-free, so it durably fences a custom port before entering `spawn`; any error from that call is permanently ambiguous and is not retried. Calling `beforeDispatch` remains required for forward compatibility and is idempotent -after that fence. `NodeSpawnPreDispatchError` preserves retry only for the SDK's -exact built-in local implementation, whose target preflight is audited; +after that fence. `NodeSpawnPreDispatchError` preserves retry only for the SDK +runtime built from its internally loaded Core Bridge module and exact local +method/state identity, whose target preflight is audited; it does not reopen any other port's fenced claim. Existing one-argument `spawn` methods must be migrated even though JavaScript permits them to ignore the callback. The required version field makes an unchanged port a type error, and diff --git a/packages/sdk/src/node-spawn-dispatch.ts b/packages/sdk/src/node-spawn-dispatch.ts index addf767be..eff7f977c 100644 --- a/packages/sdk/src/node-spawn-dispatch.ts +++ b/packages/sdk/src/node-spawn-dispatch.ts @@ -25,10 +25,16 @@ interface ExactPortIdentity { interface ReplayableFencedRegistration { identity: ExactPortIdentity; + validateState: () => boolean; replay: NodeSpawnPort['spawn']; } -const retryablePreflightPorts = new WeakMap(); +interface ExactPortRegistration { + identity: ExactPortIdentity; + validateState: () => boolean; +} + +const retryablePreflightPorts = new WeakMap(); const replayableFencedPorts = new WeakMap(); /** @@ -39,18 +45,22 @@ const replayableFencedPorts = new WeakMap boolean, ): void { - if (hasExactPortMethods(port, identity)) retryablePreflightPorts.set(port, identity); + if (hasExactPortMethods(port, identity) && validateState()) { + retryablePreflightPorts.set(port, { identity, validateState }); + } } /** Registers an internal handler whose server contract replays a fenced dispatch by idempotency key. */ export function registerReplayableFencedPort( port: NodeSpawnPort, identity: ExactPortIdentity, + validateState: () => boolean, replay: NodeSpawnPort['spawn'], ): void { - if (hasExactPortMethods(port, identity)) { - replayableFencedPorts.set(port, { identity, replay }); + if (hasExactPortMethods(port, identity) && validateState()) { + replayableFencedPorts.set(port, { identity, validateState, replay }); } } @@ -60,15 +70,19 @@ export function hasReplayableFencedDispatch(port: NodeSpawnPort): boolean { } function hasRetryablePreflight(port: NodeSpawnPort): boolean { - const identity = retryablePreflightPorts.get(port); - return identity !== undefined && hasExactPortMethods(port, identity); + const registration = retryablePreflightPorts.get(port); + return registration !== undefined + && hasExactPortMethods(port, registration.identity) + && registration.validateState(); } function replayableFencedRegistration( port: NodeSpawnPort, ): ReplayableFencedRegistration | undefined { const registration = replayableFencedPorts.get(port); - return registration !== undefined && hasExactPortMethods(port, registration.identity) + return registration !== undefined + && hasExactPortMethods(port, registration.identity) + && registration.validateState() ? registration : undefined; } diff --git a/packages/sdk/src/node-spawn-local.ts b/packages/sdk/src/node-spawn-local.ts index d498949f1..0518b981b 100644 --- a/packages/sdk/src/node-spawn-local.ts +++ b/packages/sdk/src/node-spawn-local.ts @@ -176,6 +176,19 @@ const REQUIRED_CORE_BRIDGE_METHODS = { const loadedCoreBridges = new WeakSet(); const LOCAL_FENCED_REPLAY = Symbol('LocalNodeSpawnPort.fencedReplay'); +const LOCAL_STATE_IS_EXACT = Symbol('LocalNodeSpawnPort.stateIsExact'); + +interface LocalNodeSpawnState { + core: CoreBridgeClient; + bridge: Pick; + scope: Pick; + exactNode: { readonly id: string; readonly name: string }; + bridgeMethods: readonly unknown[]; + exactNodeId: string; + exactNodeName: string; +} + +const localNodeSpawnStates = new WeakMap(); /** Fails before local admission when an installed optional bridge lacks the required contract. */ export function assertCoreBridgeApi(value: unknown): asserts value is CoreBridgeApi { @@ -213,14 +226,17 @@ export async function loadCoreBridge(): Promise { } /** Enables replay only for the exact bridge module admitted by loadCoreBridge(). */ -export function enableLoadedCoreBridgeReplay( +export function enableLoadedCoreBridgeCapabilities( port: LocalNodeSpawnPort, bridge: CoreBridgeApi, ): void { if (!loadedCoreBridges.has(bridge)) return; + const validateState = (): boolean => port[LOCAL_STATE_IS_EXACT](); + registerRetryablePreflightPort(port, LOCAL_NODE_SPAWN_IDENTITY, validateState); registerReplayableFencedPort( port, LOCAL_NODE_SPAWN_IDENTITY, + validateState, (request, beforeDispatch) => port[LOCAL_FENCED_REPLAY](request, beforeDispatch), ); } @@ -235,7 +251,35 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { private readonly scope: Pick, private readonly exactNode: { readonly id: string; readonly name: string }, ) { - registerRetryablePreflightPort(this, LOCAL_NODE_SPAWN_IDENTITY); + localNodeSpawnStates.set(this, { + core, + bridge, + scope, + exactNode, + bridgeMethods: [ + bridge.spawnTargets, + bridge.spawnInvoke, + bridge.spawnStatus, + bridge.spawnRelease, + ], + exactNodeId: exactNode.id, + exactNodeName: exactNode.name, + }); + } + + private [LOCAL_STATE_IS_EXACT](): boolean { + const state = localNodeSpawnStates.get(this); + return state !== undefined + && this.core === state.core + && this.bridge === state.bridge + && this.scope === state.scope + && this.exactNode === state.exactNode + && this.exactNode.id === state.exactNodeId + && this.exactNode.name === state.exactNodeName + && this.bridge.spawnTargets === state.bridgeMethods[0] + && this.bridge.spawnInvoke === state.bridgeMethods[1] + && this.bridge.spawnStatus === state.bridgeMethods[2] + && this.bridge.spawnRelease === state.bridgeMethods[3]; } async targets(): Promise { diff --git a/packages/sdk/src/node-spawn-runtime.ts b/packages/sdk/src/node-spawn-runtime.ts index c72517ac1..9281d8b4a 100644 --- a/packages/sdk/src/node-spawn-runtime.ts +++ b/packages/sdk/src/node-spawn-runtime.ts @@ -2,7 +2,7 @@ import { isIP } from 'node:net'; import { isAbsolute } from 'node:path'; import type { NodeResultInboxClient, NodeResultTerminalFailure } from './node-spawn-result.js'; import { - enableLoadedCoreBridgeReplay, + enableLoadedCoreBridgeCapabilities, LocalNodeSpawnPort, loadCoreBridge, type CallerDescriptor, @@ -139,7 +139,7 @@ export async function createLocalNodeSpawnRuntime( ); } const port = new LocalNodeSpawnPort(core, bridge, descriptor, identity.node); - enableLoadedCoreBridgeReplay(port, bridge); + enableLoadedCoreBridgeCapabilities(port, bridge); return { port, inbox: localResultInbox(bridge, descriptor), diff --git a/packages/sdk/tests/node-spawn-dispatch-fence.test.ts b/packages/sdk/tests/node-spawn-dispatch-fence.test.ts index fe7b8f831..505edb0a4 100644 --- a/packages/sdk/tests/node-spawn-dispatch-fence.test.ts +++ b/packages/sdk/tests/node-spawn-dispatch-fence.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { runAgentRelayTask, type AgentRelayTaskRequest } from '../src/agent-relay-transport.js'; +import { registerRetryablePreflightPort } from '../src/node-spawn-dispatch.js'; import { LocalNodeSpawnPort, type CoreBridgeClient } from '../src/node-spawn-local.js'; import { NODE_RESULT_PREFIX } from '../src/node-spawn-result.js'; import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; @@ -13,6 +14,13 @@ import { NodeSpawnError, NodeSpawnPreDispatchError, type NodeSpawnPort, type NodeSpawnReceipt } from '../src/index.js'; const roots: string[] = []; +const TEST_LOCAL_IDENTITY = { + prototype: LocalNodeSpawnPort.prototype, + spawn: LocalNodeSpawnPort.prototype.spawn, + targets: LocalNodeSpawnPort.prototype.targets, + status: LocalNodeSpawnPort.prototype.status, + release: LocalNodeSpawnPort.prototype.release, +}; afterEach(async () => { vi.useRealTimers(); @@ -36,7 +44,7 @@ function request(dataDir: string, timeoutMs = 10_000): AgentRelayTaskRequest { function officialLocalRuntime(options: { eligible(): boolean; invoked(name: string): void; -}): NodeSpawnRuntime { +}, registerInternalPreflight = true): NodeSpawnRuntime { const core: CoreBridgeClient = { hello: { bridge_version: 1, @@ -87,14 +95,18 @@ function officialLocalRuntime(options: { released: true, })), }; + const port = new LocalNodeSpawnPort(core, bridge, { + harnesses: ['codex'], + directoryScope: ['/tmp'], + }, { id: 'node_1', name: 'desktop-1' }); + if (registerInternalPreflight) { + registerRetryablePreflightPort(port, TEST_LOCAL_IDENTITY, () => true); + } return { authority: 'desktop', publisher: { id: 'publisher_1', name: 'flows-publisher' }, baseUrl: 'https://cast.agentrelay.com', - port: new LocalNodeSpawnPort(core, bridge, { - harnesses: ['codex'], - directoryScope: ['/tmp'], - }, { id: 'node_1', name: 'desktop-1' }), + port, inbox: { inbox: { subscribe: async function* () { yield { id: 'delivery_1', message: { id: 'message_1', @@ -199,6 +211,44 @@ describe('durable node-spawn dispatch fence', () => { expect(calls).toBe(1); }); + it('never replays an injected local bridge after a lost non-idempotent reply', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-injected-local-'); + let workersCreated = 0; + const worker = officialLocalRuntime({ + eligible: () => true, + invoked() { + workersCreated += 1; + throw new Error('lost reply after non-idempotent create'); + }, + }, false); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('lost reply after non-idempotent create'); + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('ambiguous pre-ack'); + expect(workersCreated).toBe(1); + }); + + it('never grants retryable target preflight to an injected local bridge', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-injected-preflight-'); + let eligible = false; + let workersCreated = 0; + const worker = officialLocalRuntime({ + eligible: () => eligible, + invoked() { workersCreated += 1; }, + }, false); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('No eligible local Agent Relay spawn target'); + const [claim] = await readNodeSpawnClaims(dataDir); + await expect(readFile(nodeSpawnDispatchFencePath(claim!.path), 'utf8')) + .resolves.toContain('claimDigest'); + eligible = true; + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('ambiguous pre-ack'); + expect(workersCreated).toBe(0); + }); + it('rejects an old public custom port before it creates a claim or invokes', async () => { const dataDir = await dataDirectory('flows-dispatch-fence-old-port-'); const spawn = vi.fn(async (input: Parameters[0]) => readyReceipt(input.name)); diff --git a/packages/sdk/tests/node-spawn-local-replay-gap.test.ts b/packages/sdk/tests/node-spawn-local-replay-gap.test.ts index 3d7cf4e0d..1c8ab14d0 100644 --- a/packages/sdk/tests/node-spawn-local-replay-gap.test.ts +++ b/packages/sdk/tests/node-spawn-local-replay-gap.test.ts @@ -225,4 +225,13 @@ describe('official local strict replay', () => { expect(hasReplayableFencedDispatch(runtime.port)).toBe(false); }, ); + + it.each(['core', 'bridge', 'scope', 'exactNode'] as const)( + 'revokes replay trust when authentic port state %s is replaced', async field => { + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json'); + expect(hasReplayableFencedDispatch(runtime.port)).toBe(true); + Object.defineProperty(runtime.port, field, { value: {}, configurable: true }); + expect(hasReplayableFencedDispatch(runtime.port)).toBe(false); + }, + ); }); diff --git a/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts b/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts index 806a5d179..cb796f84a 100644 --- a/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts +++ b/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts @@ -449,6 +449,7 @@ writeFileSync(join(dataDir, 'node-spawns', 'b'.repeat(64) + '.json'), JSON.strin 'remote_fleet_lifecycle', ], }); + expect(evidence).not.toHaveProperty('unrelated_reservation'); expect(new Set(evidence.process_generations.replay_pids).size).toBe(2); expect(evidence.process_generations.replay_pids).not.toContain( evidence.process_generations.initial_pid, From 64ccf2bb3a8fba16f4b83d012c8a932657ff1e00 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Tue, 10 Nov 2026 19:23:03 -0800 Subject: [PATCH 72/75] test(sdk): bind response-loss proof to consumer --- docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md | 21 ++- .../scripts/node-spawn-response-loss-e2e.mjs | 136 ++++++++++++++++-- .../node-spawn-response-loss-proxy.test.ts | 96 ++++++++++++- 3 files changed, 231 insertions(+), 22 deletions(-) diff --git a/docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md b/docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md index dd36cb3ba..1a8ffdb5b 100644 --- a/docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md +++ b/docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md @@ -66,6 +66,8 @@ them as such and makes no remote/fleet or server-database claim. 1. Install dependencies from the exact #707 head and build the SDK. 2. Pack that SDK and install it with public `@agent-relay/core-bridge@18.1.0` in a fresh mode-0700 consumer directory. + Keep the generated `package-lock.json`; the kit hashes it and both installed + package trees. 3. Provision a dedicated mode-0600 schema-v2 caller from a plain terminal. 4. Prepare fresh mode-0700 HOME, evidence, and Flows data directories; the data directory must contain no `runs/*.sqlite3`. @@ -82,6 +84,7 @@ proxy, and injects `RELAYFLOW_SERVICE_CALLER_FILE`. ```json { "callerFile": "/private/proof/caller.json", + "consumerRoot": "/private/consumer", "dataDir": "/private/proof/flows-data", "evidenceDir": "/private/proof/evidence", "homeDir": "/private/proof/home", @@ -98,6 +101,13 @@ proxy, and injects `RELAYFLOW_SERVICE_CALLER_FILE`. } ``` +Both `start.args[0]` and `resume.args[0]` must be the absolute `flows` CLI +resolved from `consumerRoot/node_modules/@relayflows/sdk`; both commands must +use the same Node executable that runs the proof kit. The runner rejects an +arbitrary wrapper, checkout CLI, or CLI from another consumer. It resolves and +imports Core Bridge from that same consumer root, not from the orchestrator +checkout. + The orchestrator writes private proxy/process inputs and invokes the executable verifier equivalently to: @@ -131,9 +141,14 @@ It writes mode-0600 logs plus `response-loss-proof.json` under `evidenceDir`. Before writing evidence it scans every captured log for caller-secret, token, publisher-key, and Relayfile-bearer shapes. Evidence stores spawn identity, task, and idempotency-key digests rather than their source values. The runner requires -the public `@agent-relay/core-bridge@18.1.0` package, runs children with the -fresh HOME, and removes inherited publisher, workspace, broker, node, and API -credentials. +the public `@agent-relay/core-bridge@18.1.0` package from `consumerRoot`, runs +children with the fresh HOME, and removes inherited publisher, workspace, +broker, node, and API credentials. The evidence records the consumer lockfile +SHA-256 plus package-manifest, resolved-entry, complete package-tree, and exact +SDK CLI SHA-256 receipts for the installed SDK and Core Bridge. These receipts +bind the local proof to the packages that the launched CLI actually resolves; +registry provenance and public tarball integrity remain separately reviewed +release evidence. Keep #707 draft after the run. Attach the exact source head, packed SDK receipt, public Core Bridge receipt, config with private paths redacted, literal commands diff --git a/packages/sdk/scripts/node-spawn-response-loss-e2e.mjs b/packages/sdk/scripts/node-spawn-response-loss-e2e.mjs index 757524852..a216eb541 100644 --- a/packages/sdk/scripts/node-spawn-response-loss-e2e.mjs +++ b/packages/sdk/scripts/node-spawn-response-loss-e2e.mjs @@ -2,12 +2,14 @@ import assert from 'node:assert/strict'; import { spawn } from 'node:child_process'; +import { createHash } from 'node:crypto'; import { closeSync, openSync } from 'node:fs'; import { - chmod, mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile, + chmod, mkdir, mkdtemp, readFile, readdir, realpath, rm, stat, writeFile, } from 'node:fs/promises'; +import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; -import { dirname, isAbsolute, join, resolve } from 'node:path'; +import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { setTimeout as delay } from 'node:timers/promises'; @@ -82,12 +84,66 @@ export async function assertSecretFree(paths) { } } -async function installedPackageVersion(name) { - let directory = dirname(fileURLToPath(import.meta.resolve(name))); +function sha256(value) { + return createHash('sha256').update(value).digest('hex'); +} + +function within(root, candidate) { + const path = relative(root, candidate); + return path === '' || (path !== '..' && !path.startsWith(`..${sep}`) && !isAbsolute(path)); +} + +async function packageTreeReceipt(packageDirectory) { + const hash = createHash('sha256'); + let fileCount = 0; + const walk = async (directory, prefix = '') => { + const entries = await readdir(directory, { withFileTypes: true }); + entries.sort((left, right) => left.name.localeCompare(right.name)); + for (const entry of entries) { + if (entry.name === 'node_modules') continue; + const path = join(directory, entry.name); + const name = prefix === '' ? entry.name : `${prefix}/${entry.name}`; + if (entry.isDirectory()) await walk(path, name); + else if (entry.isFile()) { + hash.update(name).update('\0').update(await readFile(path)).update('\0'); + fileCount += 1; + } else { + throw new Error(`consumer package contains unsupported filesystem entry: ${name}`); + } + } + }; + await walk(packageDirectory); + return { tree_sha256: hash.digest('hex'), file_count: fileCount }; +} + +async function installedPackageReceipt(requireFromConsumer, consumerRoot, name) { + const entryPath = await realpath(requireFromConsumer.resolve(name)); + let directory = dirname(entryPath); for (;;) { try { - const document = JSON.parse(await readFile(join(directory, 'package.json'), 'utf8')); - if (document.name === name && typeof document.version === 'string') return document.version; + const manifestPath = join(directory, 'package.json'); + const manifestBytes = await readFile(manifestPath); + const document = JSON.parse(manifestBytes.toString('utf8')); + if (document.name === name && typeof document.version === 'string') { + const packageDirectory = await realpath(directory); + assert(within(consumerRoot, packageDirectory), + `${name} must be physically installed beneath consumerRoot`); + const tree = await packageTreeReceipt(packageDirectory); + return { + document, + entryPath, + packageDirectory, + evidence: { + name, + version: document.version, + package_path: relative(consumerRoot, packageDirectory), + entry_path: relative(consumerRoot, entryPath), + package_json_sha256: sha256(manifestBytes), + entry_sha256: sha256(await readFile(entryPath)), + ...tree, + }, + }; + } } catch (error) { if (error?.code !== 'ENOENT') throw error; } @@ -95,7 +151,54 @@ async function installedPackageVersion(name) { if (parent === directory) break; directory = parent; } - throw new Error(`could not resolve installed ${name} package version`); + throw new Error(`could not resolve installed ${name} package from consumerRoot`); +} + +export async function resolveConsumerArtifacts(consumerRoot) { + assert.equal(typeof consumerRoot, 'string', 'consumerRoot is required'); + assert(isAbsolute(consumerRoot), 'consumerRoot must be absolute'); + const exactRoot = await realpath(consumerRoot); + assert.equal((await stat(exactRoot)).mode & 0o777, 0o700, + 'consumerRoot must be mode 0700'); + const requireFromConsumer = createRequire(pathToFileURL(join(exactRoot, 'package.json'))); + const sdk = await installedPackageReceipt( + requireFromConsumer, exactRoot, '@relayflows/sdk'); + const coreBridge = await installedPackageReceipt( + requireFromConsumer, exactRoot, '@agent-relay/core-bridge'); + assert.equal(coreBridge.document.version, '18.1.0', + 'proof kit requires public @agent-relay/core-bridge@18.1.0 in consumerRoot'); + const bin = typeof sdk.document.bin === 'string' + ? sdk.document.bin : sdk.document.bin?.flows; + assert.equal(typeof bin, 'string', 'consumer @relayflows/sdk must expose its flows CLI'); + const cliPath = await realpath(resolve(sdk.packageDirectory, bin)); + assert(within(sdk.packageDirectory, cliPath), + 'consumer @relayflows/sdk CLI must stay inside its package'); + const lockPath = join(exactRoot, 'package-lock.json'); + const lockBytes = await readFile(lockPath); + return { + root: exactRoot, + sdkCliPath: cliPath, + coreBridgeEntryPath: coreBridge.entryPath, + evidence: { + consumer_lock_sha256: sha256(lockBytes), + relayflows_sdk: { + ...sdk.evidence, + cli_path: relative(exactRoot, cliPath), + cli_sha256: sha256(await readFile(cliPath)), + }, + core_bridge: coreBridge.evidence, + }, + }; +} + +export async function assertConsumerCommand(spec, label, artifacts) { + assertCommandSpec(spec, label); + assert.equal(await realpath(spec.command), await realpath(process.execPath), + `${label}.command must be the Node executable running the proof kit`); + assert(spec.args.length > 0 && isAbsolute(spec.args[0]), + `${label}.args[0] must be the absolute consumer SDK CLI`); + assert.equal(await realpath(spec.args[0]), artifacts.sdkCliPath, + `${label} must launch the exact @relayflows/sdk CLI from consumerRoot`); } async function main() { @@ -103,12 +206,15 @@ async function main() { assert(configIndex >= 0 && process.argv[configIndex + 1], 'usage: node node-spawn-response-loss-e2e.mjs --config /absolute/config.json'); const configPath = resolve(process.argv[configIndex + 1]); const config = JSON.parse(await readFile(configPath, 'utf8')); - for (const field of ['callerFile', 'dataDir', 'evidenceDir', 'homeDir', 'workDir']) { + for (const field of [ + 'callerFile', 'consumerRoot', 'dataDir', 'evidenceDir', 'homeDir', 'workDir', + ]) { assert.equal(typeof config[field], 'string', `${field} is required`); assert(isAbsolute(config[field]), `${field} must be absolute`); } - assertCommandSpec(config.start, 'start'); - assertCommandSpec(config.resume, 'resume'); + const consumerArtifacts = await resolveConsumerArtifacts(config.consumerRoot); + await assertConsumerCommand(config.start, 'start', consumerArtifacts); + await assertConsumerCommand(config.resume, 'resume', consumerArtifacts); assert.equal(config.audit, undefined, 'audit is built into the proof kit; fixture-supplied audit commands are not accepted'); assert(config.resume.args.some(arg => arg.includes('{RUN_ID}')), 'resume args must contain {RUN_ID}'); @@ -133,9 +239,9 @@ async function main() { } assert.deepEqual(initialRuns, [], 'proof dataDir must not contain an earlier run'); - const coreBridgeVersion = await installedPackageVersion('@agent-relay/core-bridge'); - assert.equal(coreBridgeVersion, '18.1.0', 'proof kit requires public @agent-relay/core-bridge@18.1.0'); - const { readCallerDescriptor } = await import('@agent-relay/core-bridge'); + const { readCallerDescriptor } = await import( + pathToFileURL(consumerArtifacts.coreBridgeEntryPath).href + ); const descriptor = await readCallerDescriptor(config.callerFile); const privateDir = await mkdtemp(join(tmpdir(), 'flows-response-loss-private-')); await chmod(privateDir, 0o700); @@ -161,6 +267,8 @@ async function main() { 'RELAY_NODE_TOKEN', 'RELAY_WORKSPACE_KEY', 'RELAY_WORKSPACES_JSON', + 'NODE_OPTIONS', + 'NODE_PATH', ]) delete childEnv[name]; let firstChild; const proxy = await createBridgeResponseLossProxy({ @@ -265,7 +373,7 @@ async function main() { const evidence = { ...audit, run_id: runId, - core_bridge_version: coreBridgeVersion, + consumer_artifacts: consumerArtifacts.evidence, }; const evidencePath = join(config.evidenceDir, 'response-loss-proof.json'); await writeFile(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`, { mode: 0o600, flag: 'wx' }); diff --git a/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts b/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts index cb796f84a..e49325282 100644 --- a/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts +++ b/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts @@ -121,6 +121,39 @@ async function run(command: string, args: string[], cwd: string) { return { code, signal, stdout, stderr }; } +function writeConsumerFixture(root: string, cliSource: string) { + const consumerRoot = join(root, 'consumer'); + const sdkRoot = join(consumerRoot, 'node_modules', '@relayflows', 'sdk'); + const bridgeRoot = join(consumerRoot, 'node_modules', '@agent-relay', 'core-bridge'); + const cliPath = join(sdkRoot, 'dist', 'cli.js'); + mkdirSync(join(sdkRoot, 'dist'), { recursive: true }); + mkdirSync(bridgeRoot, { recursive: true }); + chmodSync(consumerRoot, 0o700); + writeFileSync(join(consumerRoot, 'package.json'), JSON.stringify({ + name: 'response-loss-consumer', private: true, + })); + writeFileSync(join(consumerRoot, 'package-lock.json'), JSON.stringify({ + name: 'response-loss-consumer', lockfileVersion: 3, + })); + writeFileSync(join(sdkRoot, 'package.json'), JSON.stringify({ + name: '@relayflows/sdk', version: '2.0.66', type: 'module', + main: './dist/index.js', bin: { flows: './dist/cli.js' }, + })); + writeFileSync(join(sdkRoot, 'dist', 'index.js'), 'export {};\n'); + writeFileSync(cliPath, cliSource); + writeFileSync(join(bridgeRoot, 'package.json'), JSON.stringify({ + name: '@agent-relay/core-bridge', version: '18.1.0', type: 'module', main: './index.js', + })); + writeFileSync(join(bridgeRoot, 'index.js'), ` +import { readFile } from 'node:fs/promises'; +export async function readCallerDescriptor(path) { + const value = JSON.parse(await readFile(path, 'utf8')); + return { socket: value.socket }; +} +`); + return { consumerRoot, cliPath }; +} + async function localEvidenceFixture() { const proxy = await import(moduleUrl) as ProxyModule; const identity = { @@ -261,6 +294,46 @@ describe('Core Bridge response-loss proof proxy', () => { await expect(runner.assertSecretFree([log])).rejects.toThrow('credential-shaped material'); }); + it('binds both commands to the exact SDK CLI resolved from consumerRoot', async () => { + const root = mkdtempSync(join(tmpdir(), 'flows-response-loss-consumer-')); + roots.push(root); + const { consumerRoot, cliPath } = writeConsumerFixture(root, 'export {};\n'); + const outside = join(root, 'outside.mjs'); + writeFileSync(outside, 'export {};\n'); + const runner = await import(runnerModuleUrl) as { + resolveConsumerArtifacts(root: string): Promise<{ + sdkCliPath: string; + evidence: Record; + }>; + assertConsumerCommand( + spec: { command: string; args: string[] }, + label: string, + artifacts: { sdkCliPath: string }, + ): Promise; + }; + const artifacts = await runner.resolveConsumerArtifacts(consumerRoot); + await expect(runner.assertConsumerCommand({ + command: process.execPath, args: [cliPath, 'run'], + }, 'start', artifacts)).resolves.toBeUndefined(); + await expect(runner.assertConsumerCommand({ + command: process.execPath, args: [outside, 'run'], + }, 'start', artifacts)).rejects.toThrow( + 'must launch the exact @relayflows/sdk CLI from consumerRoot', + ); + expect(artifacts.evidence).toMatchObject({ + consumer_lock_sha256: expect.stringMatching(/^[a-f0-9]{64}$/), + relayflows_sdk: { + version: '2.0.66', + tree_sha256: expect.stringMatching(/^[a-f0-9]{64}$/), + cli_sha256: expect.stringMatching(/^[a-f0-9]{64}$/), + }, + core_bridge: { + version: '18.1.0', + tree_sha256: expect.stringMatching(/^[a-f0-9]{64}$/), + }, + }); + }); + it('uses the default timeout for killed-child and two-resume acceptance orchestration', async () => { const root = mkdtempSync(join(tmpdir(), 'flows-response-loss-runner-')); roots.push(root); @@ -310,8 +383,7 @@ describe('Core Bridge response-loss proof proxy', () => { })}\n`, { mode: 0o600 }); chmodSync(callerFile, 0o600); - const childScript = join(root, 'fake-flows.mjs'); - writeFileSync(childScript, ` + const childSource = ` import { closeSync, mkdirSync, openSync, readFileSync, writeFileSync } from 'node:fs'; import net from 'node:net'; import { join } from 'node:path'; @@ -402,16 +474,18 @@ const claim = { }; mkdirSync(join(dataDir, 'node-spawns'), { recursive: true }); writeFileSync(join(dataDir, 'node-spawns', 'b'.repeat(64) + '.json'), JSON.stringify(claim), { mode: 0o600 }); -`); +`; + const { consumerRoot, cliPath } = writeConsumerFixture(root, childSource); const config = join(root, 'config.json'); writeFileSync(config, `${JSON.stringify({ callerFile, + consumerRoot, dataDir, evidenceDir, homeDir, workDir, - start: { command: process.execPath, args: [childScript, 'start', dataDir, join(root, 'winner')] }, - resume: { command: process.execPath, args: [childScript, 'resume', dataDir, join(root, 'winner'), '{RUN_ID}'] }, + start: { command: process.execPath, args: [cliPath, 'start', dataDir, join(root, 'winner')] }, + resume: { command: process.execPath, args: [cliPath, 'resume', dataDir, join(root, 'winner'), '{RUN_ID}'] }, })}\n`, { mode: 0o600 }); const result = await run(process.execPath, [ @@ -429,6 +503,18 @@ writeFileSync(join(dataDir, 'node-spawns', 'b'.repeat(64) + '.json'), JSON.strin unique_spawn_keys: 1, release_calls: 1, inbox_settlement_calls: 1, + consumer_artifacts: { + consumer_lock_sha256: expect.stringMatching(/^[a-f0-9]{64}$/), + relayflows_sdk: { + version: '2.0.66', + tree_sha256: expect.stringMatching(/^[a-f0-9]{64}$/), + cli_sha256: expect.stringMatching(/^[a-f0-9]{64}$/), + }, + core_bridge: { + version: '18.1.0', + tree_sha256: expect.stringMatching(/^[a-f0-9]{64}$/), + }, + }, final_claim: { version: 3, result_present: true, From e2dcdd0d5982ee4135cac0aa0de3e4e9614ae7a5 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Tue, 10 Nov 2026 19:25:40 -0800 Subject: [PATCH 73/75] fix(sdk): harden official local replay capability --- packages/sdk/src/node-spawn-local.ts | 16 +++++++++++++--- packages/sdk/src/node-spawn-runtime.ts | 3 ++- .../tests/node-spawn-local-replay-gap.test.ts | 16 ++++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/packages/sdk/src/node-spawn-local.ts b/packages/sdk/src/node-spawn-local.ts index 0518b981b..7a8bc69a7 100644 --- a/packages/sdk/src/node-spawn-local.ts +++ b/packages/sdk/src/node-spawn-local.ts @@ -186,6 +186,7 @@ interface LocalNodeSpawnState { bridgeMethods: readonly unknown[]; exactNodeId: string; exactNodeName: string; + internalMethods: readonly unknown[]; } const localNodeSpawnStates = new WeakMap(); @@ -231,13 +232,14 @@ export function enableLoadedCoreBridgeCapabilities( bridge: CoreBridgeApi, ): void { if (!loadedCoreBridges.has(bridge)) return; - const validateState = (): boolean => port[LOCAL_STATE_IS_EXACT](); + const validateState = port[LOCAL_STATE_IS_EXACT].bind(port); + const replay = port[LOCAL_FENCED_REPLAY].bind(port); registerRetryablePreflightPort(port, LOCAL_NODE_SPAWN_IDENTITY, validateState); registerReplayableFencedPort( port, LOCAL_NODE_SPAWN_IDENTITY, validateState, - (request, beforeDispatch) => port[LOCAL_FENCED_REPLAY](request, beforeDispatch), + replay, ); } @@ -264,6 +266,11 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { ], exactNodeId: exactNode.id, exactNodeName: exactNode.name, + internalMethods: [ + this.invokeSpawn, + this.wait, + this.cleanupIdentity, + ], }); } @@ -279,7 +286,10 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { && this.bridge.spawnTargets === state.bridgeMethods[0] && this.bridge.spawnInvoke === state.bridgeMethods[1] && this.bridge.spawnStatus === state.bridgeMethods[2] - && this.bridge.spawnRelease === state.bridgeMethods[3]; + && this.bridge.spawnRelease === state.bridgeMethods[3] + && this.invokeSpawn === state.internalMethods[0] + && this.wait === state.internalMethods[1] + && this.cleanupIdentity === state.internalMethods[2]; } async targets(): Promise { diff --git a/packages/sdk/src/node-spawn-runtime.ts b/packages/sdk/src/node-spawn-runtime.ts index 9281d8b4a..5f6d78f45 100644 --- a/packages/sdk/src/node-spawn-runtime.ts +++ b/packages/sdk/src/node-spawn-runtime.ts @@ -126,6 +126,7 @@ export async function createLocalNodeSpawnRuntime( 'caller_file_invalid', ); } + const internallyLoadedBridge = options.bridge === undefined; const bridge = options.bridge ?? await loadCoreBridge(); const descriptor = await bridge.readCallerDescriptor(serviceCallerFile); const core = await bindCore(bridge, descriptor, options.signal); @@ -139,7 +140,7 @@ export async function createLocalNodeSpawnRuntime( ); } const port = new LocalNodeSpawnPort(core, bridge, descriptor, identity.node); - enableLoadedCoreBridgeCapabilities(port, bridge); + if (internallyLoadedBridge) enableLoadedCoreBridgeCapabilities(port, bridge); return { port, inbox: localResultInbox(bridge, descriptor), diff --git a/packages/sdk/tests/node-spawn-local-replay-gap.test.ts b/packages/sdk/tests/node-spawn-local-replay-gap.test.ts index 1c8ab14d0..e862ad4ce 100644 --- a/packages/sdk/tests/node-spawn-local-replay-gap.test.ts +++ b/packages/sdk/tests/node-spawn-local-replay-gap.test.ts @@ -5,6 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { runAgentRelayTask, type AgentRelayTaskRequest } from '../src/agent-relay-transport.js'; import { hasReplayableFencedDispatch } from '../src/node-spawn-dispatch.js'; +import { loadCoreBridge } from '../src/node-spawn-local.js'; import { createLocalNodeSpawnRuntime } from '../src/node-spawn-runtime.js'; import { nodeSpawnDispatchFencePath, readNodeSpawnClaims } from '../src/node-spawn-state.js'; @@ -234,4 +235,19 @@ describe('official local strict replay', () => { expect(hasReplayableFencedDispatch(runtime.port)).toBe(false); }, ); + + it.each(['invokeSpawn', 'wait', 'cleanupIdentity'] as const)( + 'revokes replay trust when internal replay method %s is overridden', async method => { + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json'); + expect(hasReplayableFencedDispatch(runtime.port)).toBe(true); + Object.defineProperty(runtime.port, method, { value: vi.fn(), configurable: true }); + expect(hasReplayableFencedDispatch(runtime.port)).toBe(false); + }, + ); + + it('does not trust even the official module when supplied through options.bridge', async () => { + const bridge = await loadCoreBridge(); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge }); + expect(hasReplayableFencedDispatch(runtime.port)).toBe(false); + }); }); From feeba7ebf752faef42c806b515075c9973a41803 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Tue, 10 Nov 2026 19:54:33 -0800 Subject: [PATCH 74/75] fix(sdk): replay durable local spawn request --- docs/AGENT-RELAY-TRANSPORT.md | 19 ++-- docs/MIGRATION-NODE-SPAWN.md | 4 +- packages/sdk/src/agent-relay-transport.ts | 27 +++-- packages/sdk/src/node-spawn-local.ts | 10 +- packages/sdk/src/node-spawn-runtime.ts | 3 + packages/sdk/src/node-spawn-state.ts | 10 +- packages/sdk/src/node-spawn.ts | 2 + .../tests/node-spawn-local-replay-gap.test.ts | 101 ++++++++++++++++-- 8 files changed, 143 insertions(+), 33 deletions(-) diff --git a/docs/AGENT-RELAY-TRANSPORT.md b/docs/AGENT-RELAY-TRANSPORT.md index 6275a2e07..ab4065bfb 100644 --- a/docs/AGENT-RELAY-TRANSPORT.md +++ b/docs/AGENT-RELAY-TRANSPORT.md @@ -64,15 +64,16 @@ idempotency key, so a restart may repeat that identical request and recover the original invocation after its reply was lost. Flows admits that replay only for the SDK runtime built from the exact module returned by `loadCoreBridge()`, with the unchanged local port prototype and lifecycle methods, and re-verifies the -existing fence before entry. That replay repeats the already-admitted request -and exact node without applying mutable current directory-scope or live-target -preflight. A new unfenced dispatch still applies all current caller scope and -target checks. A missing receipt from remote placement, an injected bridge, a -custom or overridden port, or a legacy version 2 claim remains ambiguous and -fails closed without another dispatch. Malformed, mismatched, or unreadable -fence state also fails closed. A recorded invocation is reconciled with -`commands.getInvocation`. The stable worker name provides additional -collision protection at the node. +existing fence before entry. The durable claim retains the original rendered +task and exact local node name/id. Replay uses those stored bytes; a changed +node identity fails closed, while mutable current directory-scope or live-target +preflight is not reapplied. A new unfenced dispatch still applies all current +caller scope and target checks. A missing receipt from remote placement, an +injected bridge, a custom or overridden port, or a legacy version 2 claim +remains ambiguous and fails closed without another dispatch. Malformed, +mismatched, or unreadable fence state also fails closed. A recorded invocation +is reconciled with `commands.getInvocation`. The stable worker name provides +additional collision protection at the node. Custom `NodeSpawnPort` implementations must declare `dispatchFenceVersion: 1` and accept the supplied `beforeDispatch` callback. Because Flows cannot audit diff --git a/docs/MIGRATION-NODE-SPAWN.md b/docs/MIGRATION-NODE-SPAWN.md index 139a7e23d..837a73c26 100644 --- a/docs/MIGRATION-NODE-SPAWN.md +++ b/docs/MIGRATION-NODE-SPAWN.md @@ -73,7 +73,9 @@ only when the SDK runtime loaded the exact Core Bridge module and the local prototype and lifecycle method identities remain unchanged. Injected bridges, wrappers, subclasses, overrides, accessors, custom ports, remote placement and legacy version 2 claims remain fail-closed. New unfenced local dispatches still -enforce current caller scope and target eligibility. +enforce current caller scope and target eligibility. Fenced replay uses the +durable original rendered task and exact local node name/id; a changed node +identity is refused before replay. ## Hosted and self-hosted execution diff --git a/packages/sdk/src/agent-relay-transport.ts b/packages/sdk/src/agent-relay-transport.ts index dc46489c1..f13b26600 100644 --- a/packages/sdk/src/agent-relay-transport.ts +++ b/packages/sdk/src/agent-relay-transport.ts @@ -219,6 +219,7 @@ async function executeAgentRelayTask( ...(request.worker_cwd === undefined ? {} : { worker_cwd: request.worker_cwd }), ...(request.result_schema === undefined ? {} : { result_schema: request.result_schema }), ...(runtime.node === undefined ? {} : { node: runtime.node }), + ...(runtime.nodeId === undefined ? {} : { node_id: runtime.nodeId }), ...(runtime.repo === undefined ? {} : { repo: runtime.repo }), ...(runtime.sandboxOnly === undefined ? {} : { sandbox_only: runtime.sandboxOnly }), }, @@ -254,18 +255,26 @@ async function executeAgentRelayTask( // setup and durable claiming can take time or lose the worker lease. signal?.throwIfAborted(); const remaining = remainingDeadline(deadline); + const durableRequest = claim.request; const spawn = await dispatchFencedNodeSpawn({ port: runtime.port, request: { - name: agentName, - harness: request.cli, - task, - ...(request.model === undefined ? {} : { model: request.model }), - ...(request.worker_cwd === undefined ? {} : { directory: request.worker_cwd }), - idempotencyKey: request.idempotencyKey, - ...(runtime.node === undefined ? {} : { node: runtime.node }), - ...(runtime.repo === undefined ? {} : { repo: runtime.repo }), - ...(runtime.sandboxOnly === undefined ? {} : { sandboxOnly: runtime.sandboxOnly }), + name: claim.agentName, + harness: durableRequest.cli as string, + task: durableRequest.task as string, + ...(durableRequest.model === undefined + ? {} : { model: durableRequest.model as string }), + ...(durableRequest.worker_cwd === undefined + ? {} : { directory: durableRequest.worker_cwd as string }), + idempotencyKey: claim.idempotencyKey, + ...(durableRequest.node === undefined + ? {} : { node: durableRequest.node as string }), + ...(durableRequest.node_id === undefined + ? {} : { nodeId: durableRequest.node_id as string }), + ...(durableRequest.repo === undefined + ? {} : { repo: durableRequest.repo as string }), + ...(durableRequest.sandbox_only === undefined + ? {} : { sandboxOnly: durableRequest.sandbox_only as boolean }), timeoutMs: remaining, }, claimPath: claimed.path, claim, signal, settleLate: settleLateSpawn, diff --git a/packages/sdk/src/node-spawn-local.ts b/packages/sdk/src/node-spawn-local.ts index 7a8bc69a7..b55768ea7 100644 --- a/packages/sdk/src/node-spawn-local.ts +++ b/packages/sdk/src/node-spawn-local.ts @@ -346,6 +346,10 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { throw new NodeSpawnError('The requested node differs from the authenticated Desktop node', 'spawn_identity_mismatch'); } + if (request.nodeId !== undefined && request.nodeId !== this.exactNode.id) { + throw new NodeSpawnError('The requested node id differs from the authenticated Desktop node', + 'spawn_identity_mismatch'); + } target = allTargets.find(candidate => candidate.id === this.exactNode.id && candidate.name === this.exactNode.name && candidate.live && candidate.handlersLive && !candidate.stale @@ -380,8 +384,8 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { 'spawn_invalid_input', ); } - if (request.node !== undefined && request.node !== this.exactNode.name) { - throw new NodeSpawnError('The requested node differs from the authenticated Desktop node', + if (!request.node || !request.nodeId) { + throw new NodeSpawnError('Fenced local replay requires the durable exact node identity', 'spawn_identity_mismatch'); } // The original v3 claim was already admitted and fenced. Repeating the @@ -390,7 +394,7 @@ export class LocalNodeSpawnPort implements NodeSpawnPort { return this.invokeSpawn( { ...request, directory: request.directory }, beforeDispatch, - this.exactNode, + { id: request.nodeId, name: request.node }, ); } diff --git a/packages/sdk/src/node-spawn-runtime.ts b/packages/sdk/src/node-spawn-runtime.ts index 5f6d78f45..dbe672ac9 100644 --- a/packages/sdk/src/node-spawn-runtime.ts +++ b/packages/sdk/src/node-spawn-runtime.ts @@ -21,6 +21,8 @@ export interface NodeSpawnRuntime { publisher: NodeSpawnPublisher; baseUrl: string; node?: string; + /** Exact local node id paired with node; remote placement leaves this unset. */ + nodeId?: string; repo?: string; sandboxOnly?: boolean; /** Explicit authority family; Desktop alone may receive the worker's local run root. */ @@ -148,6 +150,7 @@ export async function createLocalNodeSpawnRuntime( baseUrl: identity.base_url, authority: 'desktop', node: identity.node.name, + nodeId: identity.node.id, disconnect: async () => { await disconnectBoundCore(bridge, core); }, diff --git a/packages/sdk/src/node-spawn-state.ts b/packages/sdk/src/node-spawn-state.ts index 8127d6ec4..a2fc9cbb1 100644 --- a/packages/sdk/src/node-spawn-state.ts +++ b/packages/sdk/src/node-spawn-state.ts @@ -204,14 +204,20 @@ function sameClaim(prior: NodeSpawnClaim, proposed: NodeSpawnClaim): boolean { function decodeNodeSpawnClaim(value: unknown, label: string): NodeSpawnClaim { const invalid = (): never => { throw new Error(`Invalid durable node-spawn claim: ${label}`); }; const record = isRecord(value) ? value : invalid(); + const requestRecord = isRecord(record.request) ? record.request : invalid(); if ((record.version !== 2 && record.version !== 3) || ![record.baseUrl, record.publisherId, record.publisherName, record.runId, record.stepId, record.agentName] .every(nonempty) || !validIdempotencyKey(record.idempotencyKey) || !positiveInteger(record.timeoutMs) || record.timeoutMs > 86_400_000 - || !isRecord(record.request) - || ![record.request.cli, record.request.task, record.request.task_identity].every(nonempty) + || ![requestRecord.cli, requestRecord.task, requestRecord.task_identity].every(nonempty) || !safeTime(record.startedAt)) invalid(); + for (const field of ['model', 'worker_cwd', 'node', 'node_id', 'repo'] as const) { + if (requestRecord[field] !== undefined && !nonempty(requestRecord[field])) invalid(); + } + if (requestRecord.node_id !== undefined && requestRecord.node === undefined) invalid(); + if (requestRecord.sandbox_only !== undefined + && typeof requestRecord.sandbox_only !== 'boolean') invalid(); const claim = record as unknown as NodeSpawnClaim; if (record.spawn !== undefined && !validNodeSpawnReceipt(record.spawn)) invalid(); if (record.cleanup !== undefined && !validNodeSpawnIdentity(record.cleanup)) invalid(); diff --git a/packages/sdk/src/node-spawn.ts b/packages/sdk/src/node-spawn.ts index 174a1c15e..566414164 100644 --- a/packages/sdk/src/node-spawn.ts +++ b/packages/sdk/src/node-spawn.ts @@ -19,6 +19,8 @@ export interface NodeSpawnRequest { directory?: string; idempotencyKey: string; node?: string; + /** Exact local node identity retained only for durable fenced replay. */ + nodeId?: string; repo?: string; timeoutMs?: number; sandboxOnly?: boolean; diff --git a/packages/sdk/tests/node-spawn-local-replay-gap.test.ts b/packages/sdk/tests/node-spawn-local-replay-gap.test.ts index e862ad4ce..fc6cdf02c 100644 --- a/packages/sdk/tests/node-spawn-local-replay-gap.test.ts +++ b/packages/sdk/tests/node-spawn-local-replay-gap.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; @@ -12,13 +12,18 @@ import { nodeSpawnDispatchFencePath, readNodeSpawnClaims } from '../src/node-spa const coreBridge = vi.hoisted(() => { const state = { directoryScope: ['/tmp'] as string[], - invokeCalls: [] as Array<{ key: string; task: string; directory: string }>, + invokeCalls: [] as Array<{ + key: string; task: string; directory: string; nodeId: string; nodeName: string; + }>, commits: new Set(), dropFirstReply: true, releases: 0, subscriptions: 0, targetCalls: 0, + preflightFailures: 0, spawnedName: '', + nodeId: 'node_local', + committedNodeId: '', }; const core = { hello: { bridge_version: 1, daemon_version: 'test', pid: 1, methods: [], topics: [] }, @@ -28,7 +33,7 @@ const coreBridge = vi.hoisted(() => { id: 'inv_spawn_replayed', agent_id: 'agent_replayed', status: 'completed', - node_id: 'node_local', + node_id: state.committedNodeId, name: state.spawnedName, spawned: true, ready: true, @@ -70,27 +75,35 @@ const coreBridge = vi.hoisted(() => { assertServiceCallerIdentity: vi.fn(), serviceCallerIdentity: vi.fn(async () => ({ publisher: { id: 'publisher_local', name: 'flows-publisher' }, - node: { id: 'node_local', name: 'desktop-local' }, + node: { id: state.nodeId, name: 'desktop-local' }, base_url: 'https://cast.agentrelay.com', workspace_id: 'workspace_local', relaycast_workspace_id: 'relaycast_local', })), spawnTargets: vi.fn(async () => { state.targetCalls += 1; + if (state.preflightFailures > 0) { + state.preflightFailures -= 1; + throw new Error('target lookup failed before dispatch'); + } return { targets: [{ - id: 'node_local', name: 'desktop-local', live: true, handlers_live: true, + id: state.nodeId, name: 'desktop-local', live: true, handlers_live: true, stale: false, capabilities: ['spawn:codex'], active_agents: 0, reserved_agents: 0, max_agents: 4, }] }; }), spawnInvoke: vi.fn(async (_core: unknown, input: { name: string; task: string; directory: string; idempotency_key: string; + node_id: string; node_name: string; }) => { state.spawnedName = input.name; + state.committedNodeId ||= input.node_id; state.invokeCalls.push({ key: input.idempotency_key, task: input.task, directory: input.directory, + nodeId: input.node_id, + nodeName: input.node_name, }); state.commits.add(input.idempotency_key); if (state.dropFirstReply) { @@ -143,7 +156,10 @@ beforeEach(() => { coreBridge.state.releases = 0; coreBridge.state.subscriptions = 0; coreBridge.state.targetCalls = 0; + coreBridge.state.preflightFailures = 0; coreBridge.state.spawnedName = ''; + coreBridge.state.nodeId = 'node_local'; + coreBridge.state.committedNodeId = ''; vi.clearAllMocks(); }); @@ -157,10 +173,14 @@ async function dataDirectory(): Promise { return directory; } -function request(dataDir: string, workerCwd = '/tmp'): AgentRelayTaskRequest { +function request( + dataDir: string, + workerCwd = '/tmp', + task = 'Implement the exact change', +): AgentRelayTaskRequest { return { cli: 'codex', - task: 'Implement the exact change', + task, taskIdentity: 'task_sha256_exact', runId: 'run_local_replay', stepId: 'edit', @@ -182,8 +202,10 @@ describe('official local strict replay', () => { coreBridge.state.directoryScope = ['/revoked-root']; const restartedRuntime = await createLocalNodeSpawnRuntime('/tmp/caller.json'); const recovered = await Promise.all([ - runAgentRelayTask(request(dataDir), { runtime: restartedRuntime }), - runAgentRelayTask(request(dataDir), { runtime: restartedRuntime }), + runAgentRelayTask(request(dataDir, '/tmp', 'Changed rendering after SDK upgrade'), + { runtime: restartedRuntime }), + runAgentRelayTask(request(dataDir, '/tmp', 'Changed rendering after SDK upgrade'), + { runtime: restartedRuntime }), ]); expect(recovered).toEqual([ expect.objectContaining({ receipt: expect.objectContaining({ status: 'completed' }) }), @@ -195,6 +217,8 @@ describe('official local strict replay', () => { expect(coreBridge.state.invokeCalls[0]).toMatchObject({ key: 'local_replay_key_123', directory: '/tmp', + nodeId: 'node_local', + nodeName: 'desktop-local', }); expect(coreBridge.state.invokeCalls[0]!.task).toContain('Implement the exact change'); expect([...coreBridge.state.commits]).toEqual(['local_replay_key_123']); @@ -217,6 +241,65 @@ describe('official local strict replay', () => { .rejects.toMatchObject({ code: 'ENOENT' }); }); + it('keeps the original request across a safe preflight retry and fenced replay', async () => { + const dataDir = await dataDirectory(); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json'); + coreBridge.state.preflightFailures = 1; + + await expect(runAgentRelayTask(request(dataDir), { runtime })) + .rejects.toThrow('target lookup failed before dispatch'); + expect(coreBridge.state.invokeCalls).toHaveLength(0); + const [claim] = await readNodeSpawnClaims(dataDir); + await expect(readFile(nodeSpawnDispatchFencePath(claim!.path), 'utf8')) + .rejects.toMatchObject({ code: 'ENOENT' }); + + await expect(runAgentRelayTask( + request(dataDir, '/tmp', 'Changed rendering before the safe retry'), { runtime }, + )).rejects.toThrow('bridge reply lost after strict server commit'); + const recovered = await runAgentRelayTask( + request(dataDir, '/tmp', 'Changed rendering again after response loss'), { runtime }, + ); + + expect(recovered.receipt.status).toBe('completed'); + expect(coreBridge.state.invokeCalls).toHaveLength(2); + expect(coreBridge.state.invokeCalls[0]).toEqual(coreBridge.state.invokeCalls[1]); + expect(coreBridge.state.invokeCalls[0]!.task).toContain('Implement the exact change'); + expect(coreBridge.state.targetCalls).toBe(2); + }); + + it('refuses fenced recovery after the exact Desktop node identity changes', async () => { + const dataDir = await dataDirectory(); + const crashedRuntime = await createLocalNodeSpawnRuntime('/tmp/caller.json'); + + await expect(runAgentRelayTask(request(dataDir), { runtime: crashedRuntime })) + .rejects.toThrow('bridge reply lost after strict server commit'); + coreBridge.state.nodeId = 'node_after_reenrolment'; + const restartedRuntime = await createLocalNodeSpawnRuntime('/tmp/caller.json'); + + await expect(runAgentRelayTask(request(dataDir), { runtime: restartedRuntime })) + .rejects.toThrow('conflicts with its durable identity or input'); + expect(coreBridge.state.invokeCalls).toHaveLength(1); + }); + + it('rejects a durable local node id without its paired node name', async () => { + const dataDir = await dataDirectory(); + coreBridge.state.directoryScope = ['/revoked-root']; + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json'); + + await expect(runAgentRelayTask(request(dataDir), { runtime })) + .rejects.toThrow('does not authorize this harness and directory'); + const [entry] = await readNodeSpawnClaims(dataDir); + const claim = JSON.parse(await readFile(entry!.path, 'utf8')) as { + request: Record; + }; + delete claim.request.node; + await writeFile(entry!.path, JSON.stringify(claim)); + + await expect(runAgentRelayTask(request(dataDir), { runtime })) + .rejects.toThrow('Invalid durable node-spawn claim'); + expect(coreBridge.state.invokeCalls).toHaveLength(0); + }); + it.each(['spawn', 'targets', 'status', 'release'] as const)( 'revokes replay trust when authentic port method %s is overridden', async method => { const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json'); From 2dc93663f92b5040e950cb57883751d50ab3151e Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Tue, 10 Nov 2026 20:29:13 -0800 Subject: [PATCH 75/75] test(sdk): make late cleanup timeout deterministic --- .../tests/worker-node-runtime-timeout.test.ts | 55 ++++++++++++------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/packages/sdk/tests/worker-node-runtime-timeout.test.ts b/packages/sdk/tests/worker-node-runtime-timeout.test.ts index 4a07cc9c9..c715e94f5 100644 --- a/packages/sdk/tests/worker-node-runtime-timeout.test.ts +++ b/packages/sdk/tests/worker-node-runtime-timeout.test.ts @@ -47,8 +47,9 @@ function runtime(spawn: NodeSpawnPort['spawn']): NodeSpawnRuntime { } async function waitForImmediate(assertion: () => void | Promise): Promise { + const deadline = process.hrtime.bigint() + 10_000_000_000n; let failure: unknown; - for (let attempt = 0; attempt < 100; attempt += 1) { + while (process.hrtime.bigint() < deadline) { try { await assertion(); return; @@ -77,7 +78,7 @@ describe('AgentWorker relay deadlines', () => { }); await worker.attach(); journal.emit('step.dispatch', dispatch('edit-timeout', 'dispatch_timeout000', declaredTimeoutMs)); - await vi.waitFor(() => expect(stepComplete).toHaveBeenCalledOnce()); + await waitForImmediate(() => expect(stepComplete).toHaveBeenCalledOnce()); await worker.close(); expect(spawn).toHaveBeenCalledOnce(); @@ -102,7 +103,7 @@ describe('AgentWorker relay deadlines', () => { }); await worker.attach(); journal.emit('step.dispatch', dispatch('edit-timeout-reason', 'dispatch_timeout001', 100)); - await vi.waitFor(() => expect(stepComplete).toHaveBeenCalledOnce()); + await waitForImmediate(() => expect(stepComplete).toHaveBeenCalledOnce()); await worker.close(); expect(stepComplete).toHaveBeenCalledWith( @@ -117,10 +118,13 @@ describe('AgentWorker relay deadlines', () => { const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-worker-late-cleanup-')); roots.push(dataDir); let confirm!: (receipt: NodeSpawnReceipt) => void; + let spawnEntered!: () => void; + const entered = new Promise(resolve => { spawnEntered = resolve; }); const spawn = vi.fn(async (input, beforeDispatch) => { await beforeDispatch(); return new Promise(resolve => { confirm = receipt => resolve({ ...receipt, name: input.name }); + spawnEntered(); }); }); const release = vi.fn() @@ -153,15 +157,22 @@ describe('AgentWorker relay deadlines', () => { }); const errors: unknown[] = []; worker.on('error', error => errors.push(error)); - await worker.attach(); - journal.emit('step.dispatch', dispatch('edit-late-cleanup', 'dispatch_latecleanup1', 50)); - await vi.waitFor(() => expect(stepComplete).toHaveBeenCalledOnce()); - expect(stepComplete).toHaveBeenCalledWith( - 'run_timeout', 'edit-late-cleanup', 1, 'dispatch_latecleanup1', 'timeout', expect.any(Object), - ); - - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const taskDeadline = new AbortController(); + const nativeTimeout = AbortSignal.timeout.bind(AbortSignal); + const timeout = vi.spyOn(AbortSignal, 'timeout').mockImplementation(milliseconds => + milliseconds < 2_000 ? taskDeadline.signal : nativeTimeout(milliseconds)); + let replacement: AgentWorker | undefined; try { + await worker.attach(); + journal.emit('step.dispatch', dispatch('edit-late-cleanup', 'dispatch_latecleanup1', 1_000)); + await entered; + taskDeadline.abort(new DOMException('The operation was aborted due to timeout', 'TimeoutError')); + await waitForImmediate(() => expect(stepComplete).toHaveBeenCalledOnce()); + expect(stepComplete).toHaveBeenCalledWith( + 'run_timeout', 'edit-late-cleanup', 1, 'dispatch_latecleanup1', 'timeout', expect.any(Object), + ); + await worker.close(); + confirm({ id: 'inv_spawn_late', action: 'spawn', status: 'completed', nodeId: 'node_1', nodeName: 'desktop-1', name: 'placeholder', @@ -178,15 +189,16 @@ describe('AgentWorker relay deadlines', () => { expect(pending!.claim.result).toBeUndefined(); expect(pending!.claim.release).toBeUndefined(); - await waitForImmediate(() => expect(vi.getTimerCount()).toBeGreaterThan(0)); - await vi.advanceTimersByTimeAsync(5_001); - let reconciled: Awaited>[number] | undefined; - await waitForImmediate(async () => { - expect(release).toHaveBeenCalledTimes(2); - [reconciled] = await readNodeSpawnClaims(dataDir); - expect(reconciled!.claim.release).toEqual({ - invocationId: 'inv_release_late', status: 'completed', - }); + replacement = new AgentWorker(journal, { + workerId: 'worker_late_cleanup_replacement', pins: {}, dataDir, nodeSpawnRuntime: nodeRuntime, + }); + replacement.on('error', error => errors.push(error)); + await replacement.attach(); + + expect(release).toHaveBeenCalledTimes(2); + const [reconciled] = await readNodeSpawnClaims(dataDir); + expect(reconciled!.claim.release).toEqual({ + invocationId: 'inv_release_late', status: 'completed', }); expect(reconciled!.claim.cleanupPending).toBeUndefined(); expect(stepComplete).toHaveBeenCalledOnce(); @@ -194,7 +206,8 @@ describe('AgentWorker relay deadlines', () => { expect(journal.journalRead).not.toHaveBeenCalled(); expect(errors).toEqual([]); } finally { - vi.useRealTimers(); + timeout.mockRestore(); + await replacement?.close(); await worker.close(); } });