diff --git a/docs/AGENT-RELAY-TRANSPORT.md b/docs/AGENT-RELAY-TRANSPORT.md index 873929b63..ab4065bfb 100644 --- a/docs/AGENT-RELAY-TRANSPORT.md +++ b/docs/AGENT-RELAY-TRANSPORT.md @@ -1,58 +1,135 @@ -# Relay agent task transport - -An agent step with `transport: 'relay'` waits for a durable final task receipt. -Spawn readiness and invocation acceptance do not complete the Flows step. - -This adapter requires the task contract in AgentWorkforce/relaycast#436 -(`c80202f2dfe7a15a6cef5b81f2452b1d7acd6add`) and its provider in -AgentWorkforce/relay#1772 (`6a06fd176df94ca1054d5b7965bd777b7d881eb6`; -subsequent proof-only commit `a8577736dd32c0bcf51a37f88ac0b04fb056da3f`). -These changes need to be admitted and deployed in that order before a live -Cloud proof can validate this adapter. The independent schema-validation fix in AgentWorkforce/flows#398 is also an -explicit validate dependency; its patch is not included here. Local wire fixtures do not establish -that the deployed service supports the contract. - -Configure a pre-provisioned `RELAY_AGENT_TOKEN` for the caller in the provider's -workspace and, if needed, `RELAY_BASE_URL` (default `https://cast.agentrelay.com`). -A workspace API key cannot substitute for the agent token. HTTPS is required -except for literal loopback IP addresses used in local development. The Relay provider -must explicitly enable `AGENT_RELAY_TASK_PROVIDER=1` with persistent state. -The worker uses its injected `agent_result` tool to submit the final output or -failure and waits for the provider's durable acknowledgment before exiting. - -Flows resolves the caller with `GET /v1/agent` and invokes -`POST /v1/actions/task.run/invoke`. The journal idempotency key identifies the -invocation; the input carries run, step, dispatch identity, model, working -directory, result schema, wake context, and a 24-hour task deadline. The adapter -uses the pinned engine's `action-invoke-v1` deterministic invocation ID so it -can reconcile a lost POST acknowledgment without another POST. - -Before dispatch, Flows exclusively creates and syncs a private claim under its -durable data directory's `relay-tasks/`. It binds the canonical engine origin, -workspace, caller, invocation, journal identity, and full input. The data -directory must survive runner restarts; it includes the task prompt and needs -the same protection as the journal. Tokens are never stored in the claim. -A corrupt claim or changed caller, endpoint, or input fails closed. Token -rotation for the same caller remains possible. - -Only the original claim creator sends a POST. Resumed or ambiguous dispatches -use `GET /v1/actions/task.run/invocations/:id` exclusively. If the task cannot be -confirmed, Flows reports an explicit failure; it never creates another task -under a replacement principal. A terminal failure is immutable for that -journal identity, including on a retry of the same dispatch. Starting a new -logical task requires a new journal identity. - -While polling, Flows renews the existing worker lease. Lease loss aborts polling -and prevents a stale journal completion. Execution identity must remain stable across every observed receipt. Generation -and acceptance time may appear together once and must then remain stable; final receipts must match the caller, invocation, -input, run, step, and dispatch. A completed receipt supplies the exact JSON -output, including arrays, scalars, and null. A failed receipt supplies an -explicit failure reason. Receipt correlation is retained in the step's -trajectory metadata. Known credential values are redacted from error details. - -Optional accounting counters `tokens_input` and `tokens_output` feed existing -model pricing. A priced task without the required usage still fails the -existing usage check. Cancellation of local polling does not invent a remote -cancellation API; the durable provider and engine task deadline govern the -remote execution. Historical terminal receipts can still be recovered after -the deadline. +# Agent Relay node-spawn transport + +An agent step with `transport: 'relay'` runs through Agent Relay's native node +spawn control. Flows no longer invokes the Relaycast `task.run` action and does +not start, connect to, or embed the retired Relay broker. + +## Runtime contract + +The worker needs a dedicated publisher `RELAY_AGENT_TOKEN`; a workspace key +cannot replace it. `RELAY_BASE_URL` defaults to +`https://cast.agentrelay.com`. Optional placement constraints are +`RELAYFLOW_NODE_NAME`, `RELAYFLOW_NODE_REPO`, and +`RELAYFLOW_NODE_SANDBOX_ONLY=1`. + +The remote path uses `@agent-relay/sdk >=17.2.1 <19`: + +1. `placement.spawn` selects a live node advertising `spawn:`. +2. Flows requests `confirm: true` and `verifyReady: true` and refuses an + unconfirmed task, identity, node, launch, or readiness result. +3. The spawned agent reports one versioned result envelope to the publisher's + durable inbox. Flows verifies the immutable sender id and name plus run, + step, and dispatch correlation. +4. Flows invokes the node's `release` capability on the same node with the + exact agent id, then records spawn, delivery, and release evidence. +5. Only after `stepComplete` is durably accepted by the journal does Flows ack + the inbox delivery. Relay is transport; the journal is execution authority. + +Before the journal append, Flows records a settlement intent containing the +kernel attempt and expected completion reason. The journal's terminal fact +must contain that reason and the exact Relay delivery and message ids before a +restart can acknowledge the inbox item. Inbox acknowledgement and rejection +have independent deadlines; an unconfirmed operation remains in the durable +outbox. It is retried on worker attach only when admission already established +the exact matching publisher authority, then after dispatch drain and on +shutdown while that authority remains established. A durable claim never +creates authority or triggers credential discovery by itself. + +One worker uses one lazy publisher connection across all of its concurrent +relay steps. Each step closes only its inbox iterator. The default runtime is +disconnected after worker shutdown drains every step; cancellation teardown is +best-effort. Injected runtimes remain caller-owned: their caller must +disconnect them after every worker sharing that runtime has drained. + +Spawn readiness is not task completion. A result without the exact sender and +attempt correlation cannot complete a step. + +## Replay and cancellation + +Before dispatch, Flows creates and fsyncs a private claim under +`/node-spawns/`. It binds publisher identity, Relay origin, journal +identity, stable worker name, and full spawn input. Tokens are never stored. + +The current fleet placement SDK has no caller-provided idempotency key for the +placement invocation itself. Flows forwards the durable key to the selected +node in the spawn action input. A version 3 claim uses a permanent sibling +dispatch fence: Flows writes and fsyncs a complete private marker, atomically +hard-links it into place, and fsyncs the directory immediately before local +`spawn.invoke` or remote `placement.spawn`. Competing workers may repeat +preflight, but only the fence winner may enter that irreversible call. A v3 +claim with no fence proves dispatch was not entered and may retry preflight. +The exact built-in local Desktop port is the sole fenced-replay exception: +Core Bridge 18.1 or newer durably deduplicates `spawn.invoke` by the forwarded +idempotency key, so a restart may repeat that identical request and recover the +original invocation after its reply was lost. Flows admits that replay only for +the SDK runtime built from the exact module returned by `loadCoreBridge()`, with +the unchanged local port prototype and lifecycle methods, and re-verifies the +existing fence before entry. The durable claim retains the original rendered +task and exact local node name/id. Replay uses those stored bytes; a changed +node identity fails closed, while mutable current directory-scope or live-target +preflight is not reapplied. A new unfenced dispatch still applies all current +caller scope and target checks. A missing receipt from remote placement, an +injected bridge, a custom or overridden port, or a legacy version 2 claim +remains ambiguous and fails closed without another dispatch. Malformed, +mismatched, or unreadable fence state also fails closed. A recorded invocation +is reconciled with `commands.getInvocation`. The stable worker name provides +additional collision protection at the node. + +Custom `NodeSpawnPort` implementations must declare `dispatchFenceVersion: 1` +and accept the supplied `beforeDispatch` callback. Because Flows cannot audit +arbitrary port code, it publishes and fsyncs the fence before entering a custom +`spawn`; an error from that call is permanently ambiguous and cannot be retried. +The callback remains part of the protocol and is idempotent after publication. +Only the SDK runtime constructed from its internally loaded Core Bridge module +and exact built-in local lifecycle method/state set may use +`NodeSpawnPreDispatchError` to retain retryable target preflight. Replacing or +overriding a relevant method removes the narrow trust and fences it before entry. +Existing one-argument `spawn` implementations are not compatible +even though JavaScript can ignore extra arguments: the required protocol field +makes them a type error, and the runtime refuses a missing or unknown version +before creating a durable claim. + +The claim also fixes the task's original start time. Resume never grants a new +execution deadline. Once a correlated result was durably observed before that +deadline, the validated envelope is part of the claim and can be replayed to +the journal after any restart delay. Recovery cannot use that fact to spawn or +extend remote execution. + +Lease loss or timeout aborts the inbox wait and attempts exact release. A +post-spawn verification failure carries any exact identity already proven into +the same cleanup path. Spawn and release action inputs carry their stable +idempotency keys. Every release attempt has an independent deadline. A failed +or unconfirmed cleanup is recorded as pending in the durable claim, +the attempt remains incomplete for kernel redispatch, and the result remains +unacked. Kernel redelivery therefore reopens the same claim and retries exact +release before it can accept a result or issue another spawn. A result delivery +stays unacked until its terminal journal append succeeds, so a worker crash +cannot turn Relay acknowledgement into fabricated execution truth. + +## Local Desktop bridge + +The SDK contains a local `NodeSpawnPort` adapter for Desktop's schema-v2 +service-caller bridge. The caller is an explicit private `caller.json` path +supplied with `--service-caller-file`, the SDK `serviceCallerFile` option, or +`RELAYFLOW_SERVICE_CALLER_FILE`; Flows never scans Desktop's app-data +directory. Optional `@agent-relay/core-bridge` `>=18.1.0 <19` validates the +handoff and local socket or named pipe, challenge-authenticates a fresh +connection, verifies the bound publisher/workspace/node identity, and then +calls `spawn.targets`, `spawn.invoke`, `spawn.status`, and `spawn.release` without sending a caller +name or secret. The same authenticated bridge provides the invocation-scoped +result inbox and its ack/fail/close/disconnect lifecycle. + +An explicit caller handoff is bound before the local worker registers with the +journal, so a missing, expired, malformed, revoked or identity-mismatched file +cannot admit a run and fail only at its first Relay step. The bridge package +owns platform-specific socket and Windows named-pipe trust checks; a bridge +version hello by itself is not authentication. +The current bridge does not accept model, repository, or sandbox placement +overrides, so the local adapter refuses those fields instead of dropping them. + +The CLI selects this adapter only when the caller file is explicit and +absolute. It is mutually exclusive with `RELAY_AGENT_TOKEN`; Flows does not +borrow another session's name and does not invent or provision authority. +Desktop creates, rotates and revokes the private handoff. + +See [the migration note](MIGRATION-NODE-SPAWN.md) for removed broker surfaces. diff --git a/docs/MIGRATION-NODE-SPAWN.md b/docs/MIGRATION-NODE-SPAWN.md new file mode 100644 index 000000000..837a73c26 --- /dev/null +++ b/docs/MIGRATION-NODE-SPAWN.md @@ -0,0 +1,98 @@ +# Migrating from Relay broker execution to node spawn + +This release removes the frozen `@agent-relay/harness-driver` dependency and +all broker/PTy launch paths from `@relayflows/sdk`. + +## One-shot remote agents + +Keep `transport: 'relay'`, install `@agent-relay/sdk` in the supported range +`>=17.2.1 <19`, and provide a dedicated publisher `RELAY_AGENT_TOKEN`. The +transport now uses fleet node placement and the durable result inbox instead +of Relaycast `task.run`. + +Custom executable paths and arbitrary injected environment variables do not +carry over. The selected node must advertise `spawn:` for the declared +harness. Use `RELAYFLOW_NODE_NAME` for an exact node or +`RELAYFLOW_NODE_REPO` for repository placement. + +## Removed surfaces + +- `transport: 'broker'`, `keepAlive`, `f.message`, and `f.release` for retained + sessions fail before child-run admission. +- YAML with non-empty `communication.links` fails preflight. An empty list + declares no linked session and remains a no-op. +- `examples/issue-swarm` is retained only as a labelled migration reference; + it is not runnable on this release. +- `RELAYFLOW_RELAY_BROKER_BIN`, broker connection files, helper sockets, and + broker health checks are no longer read. + +There is deliberately no compatibility fallback to broker 17.2.1. A local +Desktop run uses a separately approved schema-v2 service caller. Pass its +private file explicitly with +`--local-agent --service-caller-file /absolute/caller.json`, +the SDK `serviceCallerFile` option, or `RELAYFLOW_SERVICE_CALLER_FILE`. Flows +never scans Desktop's app-data directory. It securely parses and +challenge-authenticates the handoff through `@agent-relay/core-bridge` 18.1 or +newer. Install a compatible version +(`npm install @agent-relay/core-bridge@^18.1.0`) when using Desktop mode; the +peer is optional only when Desktop mode is not used. +Flows then sends no caller name or secret +on spawn or inbox calls. The handoff's harness and directory scope is enforced +before dispatch. A local relay step with no authored `cwd` spawns in that run's +root; remote relay placement still receives no local directory. + +`RELAYFLOW_SERVICE_CALLER_FILE` and `RELAY_AGENT_TOKEN` are mutually exclusive: +the former is local Desktop authority and the latter is remote fleet authority. +Setting both fails closed. A caller file whose publisher or workspace differs +from the authenticated core identity is also refused; rotate or reprovision it +in Desktop rather than editing the file. + +## Custom `NodeSpawnPort` implementations + +Ports must declare `dispatchFenceVersion: 1` and accept the supplied +`beforeDispatch` callback. Flows cannot prove that arbitrary port preflight is +side-effect-free, so it durably fences a custom port before entering `spawn`; +any error from that call is permanently ambiguous and is not retried. Calling +`beforeDispatch` remains required for forward compatibility and is idempotent +after that fence. `NodeSpawnPreDispatchError` preserves retry only for the SDK +runtime built from its internally loaded Core Bridge module and exact local +method/state identity, whose target preflight is audited; +it does not reopen any other port's fenced claim. Existing one-argument `spawn` +methods must be migrated even though JavaScript permits them to ignore the +callback. The required version field makes an unchanged port a type error, and +runtime use fails before claim creation. Overriding the built-in local port's +`spawn` or `targets` method also removes that narrow trust and fences it before +entry. + +The exact built-in local Desktop port has one additional recovery contract. +Core Bridge 18.1 or newer persists `spawn.invoke` idempotency keys, so after a +durable fence and a lost reply Flows may repeat the identical local request and +recover the original invocation even when mutable current scope or target +preflight would no longer admit a new dispatch. That exemption is registered +only when the SDK runtime loaded the exact Core Bridge module and the local +prototype and lifecycle method identities remain unchanged. Injected bridges, +wrappers, subclasses, overrides, accessors, custom ports, remote placement and +legacy version 2 claims remain fail-closed. New unfenced local dispatches still +enforce current caller scope and target eligibility. Fenced replay uses the +durable original rendered task and exact local node name/id; a changed node +identity is refused before replay. + +## Hosted and self-hosted execution + +Hosted workers use the agent-token fleet path. Self-hosted workers use the +same path for remote nodes. The local Desktop mode requires a Desktop release +whose `hello.methods` contains the complete service-caller challenge, identity, +spawn, invocation-scoped inbox, settlement, close and disconnect contract. +Each result subscription uses its own authenticated connection. Flows closes +that iterator and connection before settling the item through another fresh +authenticated connection. Older Desktop builds fail closed before dispatch. + +Self-hosted Flows licensing is owned by relaycontrol's signed offline licence +design. This SDK does not invent an entitlement store; deployments must fail +closed until a signed `flows` entitlement attestation is available. Scheduled +self-hosted submissions remain Kubernetes CronJobs, not `relaycron`. + +Relay CLI migration remains a separate, approved follow-up: `relay flows` +already consumes the v2 SDK, while legacy `node workflow run` and the cloud +worker move together in their own stacked draft after the broker-retirement +change. This package is not published as part of either draft. diff --git a/docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md b/docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md new file mode 100644 index 000000000..1a8ffdb5b --- /dev/null +++ b/docs/NODE-SPAWN-RESPONSE-LOSS-E2E.md @@ -0,0 +1,156 @@ +# Node-spawn local response-loss acceptance kit + +This is an acceptance-only proof for Flows #707. It changes no production +runtime behavior. It proves the part of recovery that the exact Flows data +directory and Core Bridge proxy can establish when upstream commits a strict +local spawn but Flows loses the complete `spawn.invoke` response. + +The kit is Unix-only and must run from a plain terminal. Never run it from an +agent session tree. Use a disposable workspace, a dedicated local target node, +a mode-0600 schema-v2 caller, and the packed SDK under review. Do not put caller +contents, publisher tokens, or workspace credentials in the config or logs. + +## Files + +- `packages/sdk/scripts/node-spawn-response-loss-proxy.mjs` — transparent Core + Bridge socket proxy and response/key evidence collector. +- `packages/sdk/scripts/node-spawn-response-loss-audit.mjs` — executable local + verifier for the final durable claim, journal ordering, process generations, + and proxy evidence. +- `packages/sdk/scripts/node-spawn-response-loss-e2e.mjs` — lifecycle + orchestrator. It invokes the local verifier directly; no fixture-supplied + audit hook is accepted. +- `packages/sdk/tests/node-spawn-response-loss-proxy.test.ts` — deterministic + proof of zero response bytes, same-key replay, exact durable identity, + terminal-journal-before-settlement ordering, and fail-closed validation. + +The proxy forwards newline-framed JSON-RPC until it reads the first complete, +successful `spawn.invoke` reply from the real core. It never writes that line +downstream. It synchronously sends `SIGKILL` to the foreground Flows process, +then closes that connection. Later connections pass through unchanged. The +orchestrator races two independent OS process generations after the initial +generation is killed. Correct durable claim ownership may coalesce those +racers into one physical bridge replay; the verifier requires at least one +exact same-key replay. + +## Proven boundary + +The local verifier proves only these facts: + +1. the final v3 claim retains the exact spawn identity from the intercepted + reply, a correlated result, `release.status === "completed"`, and no + `cleanupPending`; +2. exactly one matching terminal `step.completed` journal entry predates the + claim's durable inbox settlement; +3. the killed process and two replay processes are distinct generations, the + initial process died by `SIGKILL`, neither replay died by signal, and at + least one replay exited zero; and +4. the proxy forwarded zero response bytes, observed the original task bytes + and one idempotency key on every invoke, at least one replay, one release + call, and one inbox settlement call. + +The following remain explicitly deferred until a strict self-host artifact +digest and an authorized server-side audit are available: + +- stored Relaycast contract and replay-capability proofs; +- provider-frame and worker-creation counts; +- target fleet capacity; +- preservation of an unrelated server reservation; and +- remote/fleet lifecycle acceptance. + +Bridge request counts are not provider-frame counts. This kit never presents +them as such and makes no remote/fleet or server-database claim. + +## Preconditions + +1. Install dependencies from the exact #707 head and build the SDK. +2. Pack that SDK and install it with public + `@agent-relay/core-bridge@18.1.0` in a fresh mode-0700 consumer directory. + Keep the generated `package-lock.json`; the kit hashes it and both installed + package trees. +3. Provision a dedicated mode-0600 schema-v2 caller from a plain terminal. +4. Prepare fresh mode-0700 HOME, evidence, and Flows data directories; the data + directory must contain no `runs/*.sqlite3`. + +## Config + +Create a mode-0600 JSON file outside the repository. Paths are absolute. The +start command runs the packed SDK in the foreground. The resume command must +contain `{RUN_ID}` exactly where the durable run id belongs. Do not pass +`--service-caller-file`; the kit securely copies the caller into a temporary +mode-0700 directory outside the evidence tree, changes only its socket to the +proxy, and injects `RELAYFLOW_SERVICE_CALLER_FILE`. + +```json +{ + "callerFile": "/private/proof/caller.json", + "consumerRoot": "/private/consumer", + "dataDir": "/private/proof/flows-data", + "evidenceDir": "/private/proof/evidence", + "homeDir": "/private/proof/home", + "workDir": "/private/proof/work", + "timeoutMs": 600000, + "start": { + "command": "/usr/bin/node", + "args": ["/private/consumer/node_modules/@relayflows/sdk/dist/cli.js", "run", "/private/proof/work/proof.flow.ts", "--input", "{}", "--local-agent", "--json", "--no-observer-link", "--data-dir", "/private/proof/flows-data"] + }, + "resume": { + "command": "/usr/bin/node", + "args": ["/private/consumer/node_modules/@relayflows/sdk/dist/cli.js", "resume", "{RUN_ID}", "--local-agent", "--json", "--no-observer-link", "--data-dir", "/private/proof/flows-data"] + } +} +``` + +Both `start.args[0]` and `resume.args[0]` must be the absolute `flows` CLI +resolved from `consumerRoot/node_modules/@relayflows/sdk`; both commands must +use the same Node executable that runs the proof kit. The runner rejects an +arbitrary wrapper, checkout CLI, or CLI from another consumer. It resolves and +imports Core Bridge from that same consumer root, not from the orchestrator +checkout. + +The orchestrator writes private proxy/process inputs and invokes the executable +verifier equivalently to: + +```sh +node packages/sdk/scripts/node-spawn-response-loss-audit.mjs \ + --run "$RUN_ID" \ + --data-dir /private/proof/flows-data \ + --proxy-stats /private/mode-0700/proxy-stats.json \ + --process-evidence /private/mode-0700/process-evidence.json \ + --json +``` + +Those two input files are mode 0600 and live only in the runner's temporary +mode-0700 directory. The runner deletes that directory on exit. + +## Commands + +From the exact Flows checkout: + +```sh +npm --prefix packages/sdk ci --ignore-scripts --no-audit --no-fund +npm --prefix packages/sdk run typecheck +npm --prefix packages/sdk run typecheck:tests +npm --prefix packages/sdk run build +npm --prefix packages/sdk exec -- vitest run tests/node-spawn-response-loss-proxy.test.ts --reporter=verbose +node packages/sdk/scripts/node-spawn-response-loss-e2e.mjs --config /private/proof/response-loss-config.json +``` + +The final command prints only `{ "ok": true, "evidence": ..., "run_id": ... }`. +It writes mode-0600 logs plus `response-loss-proof.json` under `evidenceDir`. +Before writing evidence it scans every captured log for caller-secret, token, +publisher-key, and Relayfile-bearer shapes. Evidence stores spawn identity, +task, and idempotency-key digests rather than their source values. The runner requires +the public `@agent-relay/core-bridge@18.1.0` package from `consumerRoot`, runs +children with the fresh HOME, and removes inherited publisher, workspace, +broker, node, and API credentials. The evidence records the consumer lockfile +SHA-256 plus package-manifest, resolved-entry, complete package-tree, and exact +SDK CLI SHA-256 receipts for the installed SDK and Core Bridge. These receipts +bind the local proof to the packages that the launched CLI actually resolves; +registry provenance and public tarball integrity remain separately reviewed +release evidence. + +Keep #707 draft after the run. Attach the exact source head, packed SDK receipt, +public Core Bridge receipt, config with private paths redacted, literal commands +and output, and the secret-free `response-loss-proof.json` for independent +review. A human decides merge readiness; this kit never merges or publishes. diff --git a/docs/SURFACE.md b/docs/SURFACE.md index af9b2a5c1..d7768cdf6 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -276,8 +276,9 @@ No process runs between events: the handler wakes, executes to its next await, p one, while a worker that dies stops renewing and the lease expires. The run's wallclock budget is separate and stops *new* work, draining whatever is already running rather than cancelling a step mid-flight. A wrapper step and - a native step therefore get the same duration by default. Agent steps may - declare the per-step `timeout` described below, enforced on both paths. + a native step therefore get the same duration by default. Direct agent steps + may declare the per-step `timeout` described below; relay transport refuses + that authored option because the worker cannot stop a remote process. `flows check` resolves the binary (a path is relative to the declaring flow or project config; a bare name resolves via `PATH`) and caches each resolved @@ -412,64 +413,22 @@ flow-wide `FlowHeader.workspace` / `tools.fs` scopes. The chief harness above remains an aspirational example; this option does not make that entire harness executable today. -### Interactive broker sessions - -`f.agent(name, { transport: "broker", task, cli, model, cwd, skills, keepAlive, -channel, timeout })` starts an interactive PTY on the existing local Relay broker. -Install the optional `@agent-relay/harness-driver` and `@agent-relay/sdk` packages -and start the broker with `agent-relay up` in the run root. The SDK connects using -that root's connection file (or `AGENT_RELAY_STATE_DIR`). It does not start a -private broker. Preflight checks the optional packages and a read-only broker -health request before admitting the step. - -```ts -const implementer = await f.agent("implement-42", { - transport: "broker", cli: "codex", task: "Implement issue #42 and open a PR.", - cwd: "/worktrees/issue-42", skills: ["review-fix-signoff-loop"], keepAlive: true, -}); -const fixed = await f.message(implementer.agentName!, "Address the review findings.", { - timeout: "10m", -}); -await f.release(implementer.agentName!); -``` - -Broker `cwd` accepts an existing absolute directory or a path relative to the run -root, including an external git worktree. `skills` names come from the canonical -`AgentWorkforce/skills` repository. The `skills` CLI installs full directories, -including supporting files, in the agent's project before spawning. Supported -skill targets are Claude Code, Codex, Gemini CLI, Cursor, OpenCode, and Devin -(through its Windsurf-compatible skill directory); an -unsupported target or installation failure fails the step. - -The broker name contains the sanitized flow and authored agent name plus a -run/step hash. The default channel and broker `team` are `flow-`; `channel` -overrides only the channel. Relay Desktop can group `/api/spawned` entries by -team. Run, flow, and step attribution also appears in `RELAYFLOW_RUN_ID`, -`RELAYFLOW_FLOW_NAME`, and `RELAYFLOW_STEP_NAME` in the harness environment. The -broker spawn API has no arbitrary metadata field. - -The prompt supplies a scoped completion helper and a turn ID. The agent calls -`complete TURN_ID JSON_SUMMARY`; `AgentResult.summary` contains that JSON text, -`agentName` identifies the broker session, and `artifacts` is empty. Stale and -duplicate turn completions are refused. `keepAlive: true` retains the session for -`f.message`; otherwise completion releases it. `f.release` journals an explicit, -idempotent release, so a later step can safely remove the worktree. Closing the -flow's local worker releases remaining sessions and disconnects from the broker. -Retained sessions require a local worker stream and reject attached named or -predicate gates before spawning. Validate their results in a later step so the -agent name remains available for explicit cleanup. An idle PTY exit retires the -session; subsequent messages fail immediately. - -`timeout` accepts milliseconds or a duration, defaults to 60 minutes, and is -bounded at 60 minutes. `f.message` inherits the agent timeout unless overridden. -A timeout resolves with `completionReason: "timeout"` and releases the session. -Transport errors fail the step. Broker sessions report no token or dollar usage; -preflight warns that those budget ceilings cannot bound their spend, including -when the declared model has a frozen price. Broker sessions use `maxIterations: 1` and -`transportRetries: 0`; `resume`, `recoveryMode`, `workspace`, and `permissions` -are refused. Completed calls replay from the journal. A lost worker cannot -reattach a retained session's completion socket: a later message fails closed -instead of silently spawning a replacement. +### Native node-spawn sessions + +`transport: "relay"` uses Agent Relay fleet placement and a durable publisher +inbox. The node must advertise `spawn:`; spawn readiness is recorded as +evidence but does not complete the step. The exact spawned identity reports a +versioned result, the node releases that same identity, the worker appends the +terminal journal fact, and only then acknowledges the Relay delivery. + +The retired `transport: "broker"` retained-session surface (`keepAlive`, +`f.message`, and `f.release`) is no longer admitted. It has no PTY or local +broker fallback. Native retained sessions remain fail-closed until Agent Relay +Desktop exposes an authorized headless Flows caller/delegation contract; Flows +will not borrow a live coding session's registration. The local adapter carries +the Desktop-issued private handoff into a challenge-authenticated bridge; it +sends no caller name or secret on invoke, status, or release. See [the migration +note](MIGRATION-NODE-SPAWN.md). ### Agent retry and recovery controls diff --git a/docs/agent-communication.md b/docs/agent-communication.md index 8ea781aad..7d2d7ab3f 100644 --- a/docs/agent-communication.md +++ b/docs/agent-communication.md @@ -1,111 +1,20 @@ -# Agent communication - -A YAML/JSON flow opts in with explicit, directed links: - -```yaml -communication: - timeoutMs: 300000 - links: - - { from: designer, to: reviewer } - - { from: reviewer, to: designer } -``` - -See [the runnable conversation](../workflows/agent-communication.flow.yaml). -With no links, ordinary agent steps use the existing runner: no communication -worker, optional Relay package import, or broker is started. DAG dependencies -alone do not opt in. Both linked steps must be agents that can run concurrently; -ancestral dependencies and shared writable surfaces are rejected. - -Install `@agent-relay/harness-driver` and `@agent-relay/sdk` (12.3.1 or newer in 12.x) in the SDK's -runtime environment. They are optional peers, so ordinary SDK installs do not -pull them in. For workspace development, use the built packages from `../relay`. -Set `RELAY_API_KEY` to the existing workspace key and run: - -```bash -flows run workflows/agent-communication.flow.yaml --local-agent -``` - -`flows check` verifies the workspace-key format, optional package resolution, -Node availability, and executable broker before submission. These are local -checks; they do not prove the workspace key is accepted by the service. -Missing prerequisites produce a `probe_failed` refusal, rather than a daemon -protocol error. Ordinary flows do not perform these probes. - -Use the canonical workspace setup in `scripts/run-workflow.sh` when running a -watchable conversation. The run report includes a scoped observer URL; readable -conversation projections go to `wf-`. Workspace keys never belong in URLs. - -The kernel journal remains authoritative. Each directed link has one writable -channel, allowing concurrent dispatch. A small helper journals sends; the -receiving worker records a durable delivery and forwards it through Relaycast. -The existing Relay broker injects it into the declared CLI's managed PTY session. Agents -do not call a receive/poll tool. They explicitly acknowledge processing and call -complete. Injection/verification receipts do **not** acknowledge processing. -Observer channel creation and publication run independently. Their failures emit -`communication_projection_failed` diagnostics without failing or blocking the -conversation. A failed observer publication can be absent from the workspace; -the journal still holds the message. Direct injection and journal failures -remain execution failures. - -Sends use stable semantic message IDs. A reset attempt receives its conversation -history from the journal; unacknowledged messages are injected again. Injection -is at-least-once across attempts, not exactly-once model execution. History over -64 channel facts or 24 KB refuses reset rather than silently discarding context. -Use `flows resume --local-agent --data-dir ` for recovery. - -The bridge is CLI-independent: it passes the declared executable, model, working -directory, and helper environment to Relay's PTY harness. Relay owns CLI-specific -launch flags and injection; Flows does not impose a CLI allowlist or add Claude -flags to other tools. Different CLI tools can participate in the same topology. -The CLI must support an interactive session with a shell/terminal tool. Claude, -Codex, Gemini, Cursor, OpenCode, Droid, Aider, Goose, Grok, Pi, DeepAgents, and custom -interactive executables use the same path. Availability of a binary alone is -not proof of authentication or model access: for tools without a Flows auth -probe, preflight emits `managed_cli_unverified`. A login prompt, unavailable -model, or failed task does not count as successful completion. - -Current scope is local sessions with stream surfaces. This does not add -the public TypeScript `f.channel` API or hosted worker provisioning. Node must be -on PATH for the helper. Interactive sessions report unmetered usage: dollar and -token ceilings cannot bound their spend. Preflight warns; `timeoutMs` bounds the -session (default five minutes, maximum fifteen). Each attempt releases its owned -agent identity; the last session closes the run's broker and publisher. - -The broker receives a filtered environment. Managed CLIs receive their known -provider authentication variables (multi-provider tools receive the supported -provider variables), plus a unique helper session token; unrelated ambient -credentials and broker administration keys are excluded. Custom CLIs can use -their own login/configuration files. Arbitrary environment variables are not -forwarded. Helper requests without the matching token are rejected before any -journal operation. Tokens are never included in the prompt or journal. - -Local agents share the operator's OS user and home directory. This is a trusted -local execution mode, not a sandbox: environment filtering and session tokens -do not prevent a hostile same-user process from reading accessible credentials -or another process's environment. Declared permissions remain advisory until -the separate permission-enforcement work lands. - -Communication workers opt into the journal protocol's `worker.attach.required_streams` -filter using their step's receipt stream. Holding extra pins alone is not enough -to reserve them for unrelated work. Resume reattaches both the ordinary local -worker and the linked workers, so mixed flows retain capacity for each peer. -Use the matching daemon build with this SDK; an older daemon refuses the new -attach field rather than silently ignoring the eligibility constraint. - -After the example completes, verify its recorded exchange: - -```bash -python scripts/verify-agent-communication.py /runs/.sqlite3 -``` - -The verifier checks message IDs, append/delivery/ack ordering, concurrent starts, -injection receipts, both agent completions, and the run's successful completion. - -A mixed-CLI example is [Claude → Codex → Cursor](../workflows/mixed-cli-communication.flow.yaml). -Those three tools have completed a live journaled exchange. Gemini was blocked -by missing local Google authentication; OpenCode's default free model exhausted -its quota; its configured OpenAI key was also rejected on a second attempt. -Routing tests cover all names above and a custom executable; those -are not claims of live end-to-end coverage for every installed tool. - -Verify the mixed example with `python scripts/verify-mixed-cli-communication.py /runs/.sqlite3`. +# Agent communication status + +The former communication runtime launched interactive broker-managed PTYs and +injected a local helper socket. That path was removed with the Relay broker. +Flows does not fall back to it. + +Existing YAML declarations with non-empty `communication.links` are still +parsed so old bundles receive a precise admission error. Preflight refuses +them before a run is admitted; an empty list declares no links and remains a +no-op. Native node-spawn communication will require two contracts that +do not yet exist together: + +- a supported headless local service caller/delegation contract from Agent + Relay Desktop; and +- a journal-authoritative send/receive/ack protocol that works without a PTY, + helper socket, arbitrary environment injection, or direct broker events. + +Until those contracts land, split communicating work into ordinary journaled +steps or use one-shot `transport: 'relay'` agents. Relay messages may carry +task/result transport, but they do not replace journal facts. diff --git a/examples/issue-swarm/README.md b/examples/issue-swarm/README.md index 78b02348a..17bb07e04 100644 --- a/examples/issue-swarm/README.md +++ b/examples/issue-swarm/README.md @@ -1,5 +1,11 @@ # issue-swarm +> **Legacy example:** this source demonstrates the retired retained-session +> broker API (`transport: 'broker'`, `keepAlive`, `f.message`, and `f.release`). +> Current releases refuse it before child-run admission. It remains as migration +> reference until a native retained-session design is approved; do not deploy or +> run it. See [the node-spawn migration note](../../docs/MIGRATION-NODE-SPAWN.md). + Open labeled GitHub issues become independently implemented pull requests. Each issue gets a branch, worktree, and persistent broker implementer. Three reviewers use `adversarial-review` with maintainability, git history, and code structure @@ -7,28 +13,9 @@ lenses. Findings return to the same implementer. A bounded monitor sends CI, conflict, and unresolved review-thread repairs using its `pr-comments` skill. The flow never merges. -Run from a checkout of the target repository with its `origin` configured: - -```sh -flows run examples/issue-swarm/issue-swarm.flow.ts \ - --input '{"label":"agent","count":3,"monitorMinutes":30}' -``` - -The source is self-contained and deploys as-is for manual runs by name: - -```sh -flows deploy examples/issue-swarm/issue-swarm.flow.ts --repo owner/name -flows run issue-swarm --input '{"label":"agent","count":3}' -``` - The example requires the surface/SDK broker transport with `f.message` timeouts -and `f.release`. Install and authenticate `gh`, `git`, Node 22+, Claude Code and -Codex (or the selected CLIs), and start the local Agent Relay broker. The broker -installs named skills from the canonical AgentWorkforce skills package. Broker -agents are ordinary sessions visible in Relay Desktop; the returned agent name -identifies the session receiving follow-up messages. The monitor has write -authority to push branches and address PR comments, so run on repositories where -those actions are intended. +and `f.release`; those surfaces no longer have an execution backend. The +remaining description documents the former behavior for migration planning. | Input | Default | Meaning | | --- | --- | --- | diff --git a/packages/sdk/bun.lock b/packages/sdk/bun.lock index f0afee965..8dcd18060 100644 --- a/packages/sdk/bun.lock +++ b/packages/sdk/bun.lock @@ -6,41 +6,46 @@ "name": "@relayflows/sdk", "dependencies": { "@modelcontextprotocol/sdk": "^1.30.0", - "@relayfile/adapter-core": "0.6.2", - "@relayfile/relay-helpers": "0.4.12", - "@relayflows/surface": "2.0.29", + "@relayfile/adapter-core": "0.8.5", + "@relayfile/relay-helpers": "0.6.1", + "@relayflows/surface": "2.0.65", "@types/js-yaml": "^4.0.9", + "acorn": "^8.18.0", "ai-hist": "0.4.1", "ajv": "^8.17.1", "ajv-draft-04": "^1.0.0", "js-yaml": "^5.4.1", - "re2js": "^2.8.6", + "re2js": "2.8.6", + "typescript": "^5.6.0", + "undici": "^7.29.1", "yaml": "^2.5.1", }, "devDependencies": { "@agent-relay/cli-surface": "^12.2.4", + "@agent-relay/sdk": "17.2.1", "@types/node": "^22.7.0", - "typescript": "^5.6.0", + "bluebird": "^3.7.2", + "esbuild": "^0.21.5", "vitest": "^2.1.0", }, "peerDependencies": { - "@agent-relay/harness-driver": ">=12.3.1 <13", - "@agent-relay/sdk": ">=12.3.1 <13", + "@agent-relay/sdk": ">=17.2.1 <18", }, "optionalPeers": [ - "@agent-relay/harness-driver", "@agent-relay/sdk", ], }, }, "overrides": { "@relayflows/surface": { - "@relayfile/relay-helpers": "0.4.12", + "@relayfile/relay-helpers": "0.6.1", }, }, "packages": { "@agent-relay/cli-surface": ["@agent-relay/cli-surface@12.2.4", "", {}, "sha512-DqJXout26UOLNXi+yTgAD53ek9TXGoSw+5LX8wRwqWeiItiBdw5oHE54UeIzjdbVxNZyk58Dw1494goHN/+AUg=="], + "@agent-relay/sdk": ["@agent-relay/sdk@17.2.1", "", { "dependencies": { "@relaycast/sdk": "^8.0.7", "@relaycast/types": "^8.0.7", "zod": "^4.4.3" } }, "sha512-HkcrbmkqmQLF7t7pNGvl+sjHH3K4ZyqE2jl4nP0qsRLzc1J2ZS+HO+km/FurW9RhbBPFRy16Ev1G3JN1FHIVpA=="], + "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.21.5", "", { "os": "aix", "cpu": "ppc64" }, "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ=="], "@esbuild/android-arm": ["@esbuild/android-arm@0.21.5", "", { "os": "android", "cpu": "arm" }, "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg=="], @@ -97,11 +102,15 @@ "@napi-rs/lzma-linux-x64-gnu": ["@napi-rs/lzma-linux-x64-gnu@1.5.1", "", { "os": "linux", "cpu": "x64" }, "sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ=="], - "@relayfile/adapter-core": ["@relayfile/adapter-core@0.6.2", "", { "dependencies": { "@scalar/postman-to-openapi": "^0.6.0", "cheerio": "^1.2.0", "minimatch": "^10.0.3", "yaml": "^2.8.1" }, "peerDependencies": { "@relayfile/sdk": ">=0.6.0 <1" }, "bin": { "adapter-core": "dist/src/cli.js" } }, "sha512-fVBwiK1W0af4wQ4w/Jr7GA1VBH1qz9Qw1lm90MrOJN0zQfVq1YRwRu9fPk4rp50TQDveLBKE0+MoaLMUmqr5Jg=="], + "@relaycast/sdk": ["@relaycast/sdk@8.16.0", "", { "dependencies": { "@relaycast/types": "8.16.0", "zod": "^4.3.6" } }, "sha512-c3jXSDR2WAJCaiNsd92N/3sADLDHkFraw4Ogey2sKLtq48D/p/TQLnD9BE5r9Rk5zoZ0Y8Ls4mzceCeVGekFBQ=="], + + "@relaycast/types": ["@relaycast/types@8.16.0", "", { "dependencies": { "zod": "^4.3.6" } }, "sha512-yLho3/MMMCu4+GEt0L67XU/n5HUatq9mDEsNlOZUPcAjHTCbbTNjN7MRfb6y91wKeLvzha0ggHJ3DzxR4mH4GQ=="], - "@relayfile/adapter-linear": ["@relayfile/adapter-linear@0.4.13", "", { "dependencies": { "@relayfile/adapter-core": "^0.6.1" }, "peerDependencies": { "@relayfile/sdk": ">=0.6.0 <1" } }, "sha512-cJtjrmhK2dV5o7QD61N2vuvl4XnMGjrcWX5zrM9otneVVA983GHRKdHv014Hq7QSIpG5FMSuLGPziYn6vFJI+A=="], + "@relayfile/adapter-core": ["@relayfile/adapter-core@0.8.5", "", { "dependencies": { "@scalar/postman-to-openapi": "^0.6.0", "cheerio": "^1.2.0", "minimatch": "^10.0.3", "yaml": "^2.8.1" }, "peerDependencies": { "@relayfile/sdk": ">=0.6.0 <1" }, "bin": { "adapter-core": "dist/src/cli.js" } }, "sha512-J10VAyHz1P7wQDKKmVCJD+2kORQs1icP7CtqG1u8r0wO73S76UStimPsCzlGl9UhVhzqdkCZ7wf+GKH9f4N6Aw=="], - "@relayfile/adapter-reddit": ["@relayfile/adapter-reddit@0.2.10", "", { "dependencies": { "@relayfile/adapter-core": "^0.6.1" }, "peerDependencies": { "@relayfile/sdk": ">=0.6.0 <1" } }, "sha512-FBGpWqJgOAC7nSE3SUAPil+pYxpBne1higJmSqGIrycbiIvBPxbluUsTb5Ct7XSyvDB3QmMbOMog5CDaVLhnKQ=="], + "@relayfile/adapter-linear": ["@relayfile/adapter-linear@0.6.1", "", { "dependencies": { "@relayfile/adapter-core": "^0.8.5" }, "peerDependencies": { "@relayfile/sdk": ">=0.6.0 <1" } }, "sha512-BeTSUcm0+vN4krpESFiAfvjJ/WesxKAKhpdQM6uuNYWyl3efNXMj5Qe5nDGCSO2QHMyhKAZYjCyzzgqB9YnqQw=="], + + "@relayfile/adapter-reddit": ["@relayfile/adapter-reddit@0.4.1", "", { "dependencies": { "@relayfile/adapter-core": "^0.8.5" }, "peerDependencies": { "@relayfile/sdk": ">=0.6.0 <1" } }, "sha512-Mz57v1vIE5sEy5F4Z7ix550Ga6l615lgQi0QDc8kgjWiUD9QqZW+Wm+iG+96S4vIQueM70pm60/fitrGGnIGAw=="], "@relayfile/cli-darwin-arm64": ["@relayfile/cli-darwin-arm64@0.10.69", "", { "os": "darwin", "cpu": "arm64" }, "sha512-zJpbOfz4Iq2273TkQHvzOmQ06tA2jga6Pxa0WSYBHTfxMwWE7tk9gGwPrtU4ppOiyON6jvQuTQ3byBhwOwlLsQ=="], @@ -125,11 +134,11 @@ "@relayfile/mount-linux-x64": ["@relayfile/mount-linux-x64@0.10.69", "", { "os": "linux", "cpu": "x64" }, "sha512-aTy6ZiYgF+8SuSF9hGDgJU/z9K6rh+UUAl6Emp/valiw3FmM0fLOt2rAt1/KyXtuhgJE09/XoPRYBLa8UyIeUg=="], - "@relayfile/relay-helpers": ["@relayfile/relay-helpers@0.4.12", "", { "dependencies": { "@relayfile/adapter-core": "^0.6.1", "@relayfile/adapter-linear": "^0.4.13", "@relayfile/adapter-reddit": "^0.2.10" } }, "sha512-Izjl33HLCjxDFVZHK/+f+e/73IUiEg534QNBJGkRYZXjH6LvPKfGiPXViiY5zYLxFtgnzNQa+/TI6MB4X3BnEQ=="], + "@relayfile/relay-helpers": ["@relayfile/relay-helpers@0.6.1", "", { "dependencies": { "@relayfile/adapter-core": "^0.8.5", "@relayfile/adapter-linear": "^0.6.1", "@relayfile/adapter-reddit": "^0.4.1" } }, "sha512-C43jTVFwsqoiyGoS2RczvGgpSyeu9+v7q9nDKZj2NJp9WBqRHTZd6DlU2jH/OB1ovnY4cIGbH3o7TMdNLc50aQ=="], "@relayfile/sdk": ["@relayfile/sdk@0.10.69", "", { "dependencies": { "@relayfile/core": "0.10.69", "ignore": "^7.0.5", "tar": "^7.5.10" }, "optionalDependencies": { "@relayfile/cli-darwin-arm64": "0.10.69", "@relayfile/cli-darwin-x64": "0.10.69", "@relayfile/cli-linux-arm64": "0.10.69", "@relayfile/cli-linux-x64": "0.10.69", "@relayfile/cli-win32-arm64": "0.10.69", "@relayfile/cli-win32-x64": "0.10.69", "@relayfile/mount-darwin-arm64": "0.10.69", "@relayfile/mount-darwin-x64": "0.10.69", "@relayfile/mount-linux-arm64": "0.10.69", "@relayfile/mount-linux-x64": "0.10.69" } }, "sha512-yKImAt0XrR7lqJTaWXCv4NlCyZe1YFxh1qEBKoKCPYhEiF31Rrbarq+PZlQybg4r2Kbiff4XtSERMy31iHvoSw=="], - "@relayflows/surface": ["@relayflows/surface@2.0.29", "", { "dependencies": { "ai-hist": "0.4.1" }, "peerDependencies": { "@relayfile/relay-helpers": "0.4.12" } }, "sha512-DpxA5hJWkHvRYR0uxkpPbjiakYNvFRgaeMbAc5wizgC68DQwxelM3PxF6yrvkBCdndqRNjMK+OqD9tZMMJRrrQ=="], + "@relayflows/surface": ["@relayflows/surface@2.0.65", "", { "dependencies": { "ai-hist": "0.4.1" }, "peerDependencies": { "@relayfile/relay-helpers": "0.6.1" } }, "sha512-QGMnlZ+4Jvm2GsB6NyFZ7Wosb9/roHooYUTAqxv8nOmQ2RY9GslGuH7dOGcQN3kY87amkFrAIEY7yizOWtjetw=="], "@rollup/rollup-android-arm-eabi": ["@rollup/rollup-android-arm-eabi@4.63.0", "", { "os": "android", "cpu": "arm" }, "sha512-70TeIFezKKy65LgAVyQh+w94/gjWhvPWaLaGGeMEgVrPkQhuj/M5bAYYZzIFUj9Y69oHyTm5Um/R6gcLh4A8JA=="], @@ -209,6 +218,8 @@ "accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="], + "acorn": ["acorn@8.19.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-oJlA3XiRm7Cyk6qFD2Jn8ak9B7jSy0qy00ADO3+8dpT0LSjFihQYv4C02LFCSYJV3Q37xYwwRy5m+IpIiUzqWw=="], + "ai-hist": ["ai-hist@0.4.1", "", { "dependencies": { "@modelcontextprotocol/sdk": "^1.29.0", "sql.js": "^1.13.0", "zod": "^4.4.3" }, "bin": { "ai-hist-mcp": "dist/mcp-server.js" } }, "sha512-qn/jXFtWoY4timtzRj1DO5RgqPJA8UoDAm9Qe3bJ9wM3ph+McK+kY/6H2YDUs67MHzWjGlSc1KAe6cdmFbDfeQ=="], "ajv": ["ajv@8.17.1", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g=="], @@ -223,6 +234,8 @@ "balanced-match": ["balanced-match@4.0.4", "", {}, "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA=="], + "bluebird": ["bluebird@3.7.2", "", {}, "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="], + "body-parser": ["body-parser@2.3.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^2.0.0", "debug": "^4.4.3", "http-errors": "^2.0.1", "iconv-lite": "^0.7.2", "on-finished": "^2.4.1", "qs": "^6.15.2", "raw-body": "^3.0.2", "type-is": "^2.1.0" } }, "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw=="], "boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="], diff --git a/packages/sdk/package-lock.json b/packages/sdk/package-lock.json index 9708d84f2..e567d4d78 100644 --- a/packages/sdk/package-lock.json +++ b/packages/sdk/package-lock.json @@ -29,17 +29,19 @@ }, "devDependencies": { "@agent-relay/cli-surface": "^12.2.4", + "@agent-relay/core-bridge": "18.1.0", + "@agent-relay/sdk": "17.2.1", "@types/node": "^22.7.0", "bluebird": "^3.7.2", "esbuild": "^0.21.5", "vitest": "^2.1.0" }, "peerDependencies": { - "@agent-relay/harness-driver": ">=12.3.1 <13 || >=15.1.0 <18", - "@agent-relay/sdk": ">=12.3.1 <13 || >=15.1.0 <19" + "@agent-relay/core-bridge": ">=18.1.0 <19", + "@agent-relay/sdk": ">=17.2.1 <19" }, "peerDependenciesMeta": { - "@agent-relay/harness-driver": { + "@agent-relay/core-bridge": { "optional": true }, "@agent-relay/sdk": { @@ -56,6 +58,29 @@ "node": ">=22.0.0" } }, + "node_modules/@agent-relay/core-bridge": { + "version": "18.1.0", + "resolved": "https://registry.npmjs.org/@agent-relay/core-bridge/-/core-bridge-18.1.0.tgz", + "integrity": "sha512-GXSrmuPaPG5iP7hdZc8QMiNc1r2HjqQeHI/RV3lVDTtbq7z16BX+aIGZHiYMhVEn7D7Ttho/kqRakKSDP/bq6Q==", + "dev": true, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@agent-relay/sdk": { + "version": "17.2.1", + "resolved": "https://registry.npmjs.org/@agent-relay/sdk/-/sdk-17.2.1.tgz", + "integrity": "sha512-HkcrbmkqmQLF7t7pNGvl+sjHH3K4ZyqE2jl4nP0qsRLzc1J2ZS+HO+km/FurW9RhbBPFRy16Ev1G3JN1FHIVpA==", + "dev": true, + "dependencies": { + "@relaycast/sdk": "^8.0.7", + "@relaycast/types": "^8.0.7", + "zod": "^4.4.3" + }, + "engines": { + "node": ">=22.0.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", @@ -536,6 +561,25 @@ "node": "^22.20 || ^24.12 || >=25" } }, + "node_modules/@relaycast/sdk": { + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/@relaycast/sdk/-/sdk-8.16.0.tgz", + "integrity": "sha512-c3jXSDR2WAJCaiNsd92N/3sADLDHkFraw4Ogey2sKLtq48D/p/TQLnD9BE5r9Rk5zoZ0Y8Ls4mzceCeVGekFBQ==", + "dev": true, + "dependencies": { + "@relaycast/types": "8.16.0", + "zod": "^4.3.6" + } + }, + "node_modules/@relaycast/types": { + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/@relaycast/types/-/types-8.16.0.tgz", + "integrity": "sha512-yLho3/MMMCu4+GEt0L67XU/n5HUatq9mDEsNlOZUPcAjHTCbbTNjN7MRfb6y91wKeLvzha0ggHJ3DzxR4mH4GQ==", + "dev": true, + "dependencies": { + "zod": "^4.3.6" + } + }, "node_modules/@relayfile/adapter-core": { "version": "0.8.5", "resolved": "https://registry.npmjs.org/@relayfile/adapter-core/-/adapter-core-0.8.5.tgz", diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 1d7f14588..a8dffb35b 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -64,6 +64,8 @@ }, "devDependencies": { "@agent-relay/cli-surface": "^12.2.4", + "@agent-relay/core-bridge": "18.1.0", + "@agent-relay/sdk": "17.2.1", "@types/node": "^22.7.0", "bluebird": "^3.7.2", "esbuild": "^0.21.5", @@ -75,11 +77,11 @@ "directory": "packages/sdk" }, "peerDependencies": { - "@agent-relay/harness-driver": ">=12.3.1 <13 || >=15.1.0 <18", - "@agent-relay/sdk": ">=12.3.1 <13 || >=15.1.0 <19" + "@agent-relay/core-bridge": ">=18.1.0 <19", + "@agent-relay/sdk": ">=17.2.1 <19" }, "peerDependenciesMeta": { - "@agent-relay/harness-driver": { + "@agent-relay/core-bridge": { "optional": true }, "@agent-relay/sdk": { diff --git a/packages/sdk/scripts/node-spawn-response-loss-audit.mjs b/packages/sdk/scripts/node-spawn-response-loss-audit.mjs new file mode 100755 index 000000000..8c60a534b --- /dev/null +++ b/packages/sdk/scripts/node-spawn-response-loss-audit.mjs @@ -0,0 +1,214 @@ +#!/usr/bin/env node + +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { readFile, stat } from 'node:fs/promises'; +import { isAbsolute, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { walkJournal } from '../dist/journal-reader.js'; +import { readNodeSpawnClaims } from '../dist/node-spawn-state.js'; +import { + spawnIdentityDigest, + verifyResponseLossProxyEvidence, +} from './node-spawn-response-loss-proxy.mjs'; + +const LOCAL_SCHEMA = 'relayflows-node-spawn-response-loss-local/1'; +const DEFERRED_ASSERTIONS = [ + 'stored_relaycast_contract_and_capability', + 'provider_frame_and_worker_counts', + 'target_capacity', + 'unrelated_reservation_preservation', + 'remote_fleet_lifecycle', +]; + +function sha256(value) { + return createHash('sha256').update(value).digest('hex'); +} + +function record(value) { + return typeof value === 'object' && value !== null ? value : {}; +} + +function terminalMatchesResult(entry, claim) { + const result = claim.result; + const settlement = result?.settlement; + if (result === undefined || settlement === undefined) return false; + const payload = record(entry.payload); + const tail = record(payload.trajectory_tail); + const relayTask = record(tail.relay_task); + const execution = record(relayTask.task_execution); + return entry.entry_type === 'step.completed' + && entry.step_id === claim.stepId + && entry.attempt === settlement.attempt + && payload.completionReason === settlement.completionReason + && execution.run_id === claim.runId + && execution.step_id === claim.stepId + && execution.dispatch_id === claim.idempotencyKey + && execution.delivery_id === result.inboxItemId + && execution.message_id === result.messageId; +} + +function verifyProcesses(processEvidence) { + const initial = record(processEvidence?.initial); + const replays = Array.isArray(processEvidence?.replays) ? processEvidence.replays : []; + assert(Number.isSafeInteger(initial.pid) && initial.pid > 0, + 'initial process id must be a positive integer'); + assert.equal(initial.code, null, 'the intercepted initial process must not exit normally'); + assert.equal(initial.signal, 'SIGKILL', 'the intercepted initial process must die by SIGKILL'); + assert.equal(replays.length, 2, 'the proof must race exactly two replay processes'); + const replayPids = replays.map(item => record(item).pid); + assert(replayPids.every(pid => Number.isSafeInteger(pid) && pid > 0), + 'both replay process ids must be positive integers'); + assert.equal(new Set([initial.pid, ...replayPids]).size, 3, + 'initial and replay process generations must be distinct'); + assert(replays.every(item => record(item).signal === null), + 'neither replay process may die by signal'); + const replayExitCodes = replays.map(item => record(item).code); + assert(replayExitCodes.every(code => Number.isSafeInteger(code) && code >= 0), + 'replay exit codes must be non-negative integers'); + assert(replayExitCodes.includes(0), 'at least one raced replay must complete successfully'); + return { + initial_pid: initial.pid, + initial_exit: { code: initial.code, signal: initial.signal }, + replay_pids: replayPids, + replay_exits: replays.map(item => ({ code: record(item).code, signal: record(item).signal })), + }; +} + +/** + * Verifies only evidence held by the exact Flows data directory and proxy. + * Server-side dispatch counts, stored Relaycast proofs and fleet state are + * deliberately outside this local contract. + */ +export function verifyLocalResponseLossEvidence({ + runId, + claims, + journalEntries, + proxyStats, + processEvidence, +}) { + assert.equal(typeof runId, 'string'); + const matching = claims.filter(entry => entry.claim?.runId === runId); + assert.equal(matching.length, 1, 'proof run must have exactly one durable node-spawn claim'); + const claim = matching[0].claim; + assert.equal(claim.version, 3, 'response-loss proof requires a fenced v3 claim'); + assert(claim.spawn, 'final durable claim must retain its exact spawn identity'); + assert.equal(claim.spawn.action, 'spawn'); + assert.equal(claim.spawn.status, 'completed'); + assert.equal(claim.spawn.spawned, true); + assert.equal(claim.spawn.ready, true); + assert.equal(claim.spawn.name, claim.agentName, + 'durable spawn receipt must retain the claimed agent name'); + assert(claim.result, 'final durable claim must retain its correlated result'); + assert.equal(claim.release?.status, 'completed', + 'final durable claim must retain its completed release'); + assert(!Object.hasOwn(claim, 'cleanupPending'), + 'final durable claim must not retain pending cleanup'); + assert(claim.result.settlement, 'final result must retain its journal settlement identity'); + assert.equal(claim.result.settlement.settledAs, claim.result.settlementAction, + 'final result must retain its exact inbox settlement action'); + assert(Number.isSafeInteger(claim.result.settlement.settledAt), + 'final result must retain its settlement time'); + + const proxy = verifyResponseLossProxyEvidence(proxyStats); + const identity = spawnIdentityDigest({ + invocationId: claim.spawn.id, + nodeId: claim.spawn.nodeId, + nodeName: claim.spawn.nodeName, + name: claim.spawn.name, + agentId: claim.spawn.agentId, + }); + assert.equal(identity, proxy.spawn_identity_sha256, + 'final durable claim must match the exact intercepted spawn identity'); + + const terminals = journalEntries.filter(entry => terminalMatchesResult(entry, claim)); + assert.equal(terminals.length, 1, + 'journal must contain exactly one terminal entry for the correlated result'); + const terminal = terminals[0]; + assert(terminal.at_ms < claim.result.settlement.settledAt, + 'terminal journal entry must precede claim settlement'); + + return { + schema: LOCAL_SCHEMA, + run_id_sha256: sha256(runId), + ...proxy, + final_claim: { + version: claim.version, + spawn_identity_sha256: identity, + result_present: true, + result_status: claim.result.status, + release_status: claim.release.status, + cleanup_pending: false, + settlement_action: claim.result.settlementAction, + }, + journal: { + terminal_sequence: terminal.seq, + terminal_at_ms: terminal.at_ms, + settlement_at_ms: claim.result.settlement.settledAt, + terminal_before_settlement: true, + }, + process_generations: verifyProcesses(processEvidence), + deferred_assertions: DEFERRED_ASSERTIONS, + }; +} + +export async function auditLocalResponseLoss({ + runId, + dataDir, + proxyStatsPath, + processEvidencePath, +}) { + for (const [label, path] of [ + ['dataDir', dataDir], + ['proxyStatsPath', proxyStatsPath], + ['processEvidencePath', processEvidencePath], + ]) { + assert.equal(typeof path, 'string', `${label} is required`); + assert(isAbsolute(path), `${label} must be absolute`); + } + for (const path of [proxyStatsPath, processEvidencePath]) { + assert.equal((await stat(path)).mode & 0o777, 0o600, + `${path} must be mode 0600`); + } + const claims = await readNodeSpawnClaims(dataDir); + const journalEntries = []; + for await (const entry of walkJournal(runId, dataDir)) journalEntries.push(entry); + const proxyStats = JSON.parse(await readFile(proxyStatsPath, 'utf8')); + const processEvidence = JSON.parse(await readFile(processEvidencePath, 'utf8')); + return verifyLocalResponseLossEvidence({ + runId, claims, journalEntries, proxyStats, processEvidence, + }); +} + +async function main() { + const value = name => { + const index = process.argv.indexOf(name); + return index >= 0 ? process.argv[index + 1] : undefined; + }; + const runId = value('--run'); + assert(typeof runId === 'string' && /^[A-Za-z0-9][A-Za-z0-9_-]*$/.test(runId), + '--run must be a safe run id'); + assert(process.argv.includes('--json'), 'usage requires --json'); + const dataDir = value('--data-dir'); + const proxyStatsPath = value('--proxy-stats'); + const processEvidencePath = value('--process-evidence'); + assert(typeof dataDir === 'string', '--data-dir is required'); + assert(typeof proxyStatsPath === 'string', '--proxy-stats is required'); + assert(typeof processEvidencePath === 'string', '--process-evidence is required'); + const evidence = await auditLocalResponseLoss({ + runId, + dataDir: resolve(dataDir), + proxyStatsPath: resolve(proxyStatsPath), + processEvidencePath: resolve(processEvidencePath), + }); + console.log(JSON.stringify(evidence)); +} + +if (process.argv[1] + && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + main().catch(error => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/packages/sdk/scripts/node-spawn-response-loss-e2e.mjs b/packages/sdk/scripts/node-spawn-response-loss-e2e.mjs new file mode 100644 index 000000000..a216eb541 --- /dev/null +++ b/packages/sdk/scripts/node-spawn-response-loss-e2e.mjs @@ -0,0 +1,392 @@ +#!/usr/bin/env node + +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { closeSync, openSync } from 'node:fs'; +import { + chmod, mkdir, mkdtemp, readFile, readdir, realpath, rm, stat, writeFile, +} from 'node:fs/promises'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { setTimeout as delay } from 'node:timers/promises'; + +import { createBridgeResponseLossProxy } from './node-spawn-response-loss-proxy.mjs'; + +const DEFAULT_TIMEOUT_MS = 10 * 60_000; + +function assertCommandSpec(spec, label) { + assert.equal(typeof spec?.command, 'string', `${label}.command is required`); + assert(isAbsolute(spec.command), `${label}.command must be absolute`); + assert(Array.isArray(spec.args) && spec.args.every(arg => typeof arg === 'string'), + `${label}.args must be strings`); + assert(!spec.args.some(arg => arg === '--service-caller-file' || arg.startsWith('--service-caller-file=')), + `${label} must use the proxy caller injected by the proof kit`); +} + +function commandWithRunId(spec, runId) { + return { + command: spec.command, + args: spec.args.map(arg => arg.replaceAll('{RUN_ID}', runId)), + }; +} + +async function runProcess(spec, { cwd, env, logPath, timeoutMs }) { + const fd = openSync(logPath, 'wx', 0o600); + const child = spawn(spec.command, spec.args, { + cwd, + env, + stdio: ['ignore', fd, fd], + }); + closeSync(fd); + const result = await Promise.race([ + new Promise((resolveExit, rejectExit) => { + child.once('error', rejectExit); + child.once('exit', (code, signal) => resolveExit({ code, signal })); + }), + delay(timeoutMs, undefined, { ref: false }).then(() => { + child.kill('SIGKILL'); + throw new Error(`command timed out; see ${logPath}`); + }), + ]); + return { child, result }; +} + +async function waitForRunId(dataDir, timeoutMs) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + let files = []; + try { + files = (await readdir(join(dataDir, 'runs'))).filter(file => file.endsWith('.sqlite3')); + } catch (error) { + if (error?.code !== 'ENOENT') throw error; + } + if (files.length === 1) return files[0].slice(0, -'.sqlite3'.length); + assert(files.length <= 1, 'proof data dir contains more than one run'); + await delay(25); + } + throw new Error('timed out waiting for the single proof run id'); +} + +export async function assertSecretFree(paths) { + const forbidden = [ + /rsc_[A-Za-z0-9_-]{20,}/, + /(?:ar|arr|at|br|nt|ot|rk)_(?:live_)?[A-Za-z0-9_-]{8,}/, + /(?:relay_pa_|sh_fr_)[A-Za-z0-9_-]+/, + /caller_secret/i, + /(?:RELAY|AGENT_RELAY|RELAYCAST)_(?:AGENT_TOKEN|API_KEY|WORKSPACE_KEY|NODE_TOKEN|WORKSPACES_JSON)/, + ]; + for (const path of paths) { + const text = await readFile(path, 'utf8'); + assert(!forbidden.some(pattern => pattern.test(text)), `${path} contains credential-shaped material`); + } +} + +function sha256(value) { + return createHash('sha256').update(value).digest('hex'); +} + +function within(root, candidate) { + const path = relative(root, candidate); + return path === '' || (path !== '..' && !path.startsWith(`..${sep}`) && !isAbsolute(path)); +} + +async function packageTreeReceipt(packageDirectory) { + const hash = createHash('sha256'); + let fileCount = 0; + const walk = async (directory, prefix = '') => { + const entries = await readdir(directory, { withFileTypes: true }); + entries.sort((left, right) => left.name.localeCompare(right.name)); + for (const entry of entries) { + if (entry.name === 'node_modules') continue; + const path = join(directory, entry.name); + const name = prefix === '' ? entry.name : `${prefix}/${entry.name}`; + if (entry.isDirectory()) await walk(path, name); + else if (entry.isFile()) { + hash.update(name).update('\0').update(await readFile(path)).update('\0'); + fileCount += 1; + } else { + throw new Error(`consumer package contains unsupported filesystem entry: ${name}`); + } + } + }; + await walk(packageDirectory); + return { tree_sha256: hash.digest('hex'), file_count: fileCount }; +} + +async function installedPackageReceipt(requireFromConsumer, consumerRoot, name) { + const entryPath = await realpath(requireFromConsumer.resolve(name)); + let directory = dirname(entryPath); + for (;;) { + try { + const manifestPath = join(directory, 'package.json'); + const manifestBytes = await readFile(manifestPath); + const document = JSON.parse(manifestBytes.toString('utf8')); + if (document.name === name && typeof document.version === 'string') { + const packageDirectory = await realpath(directory); + assert(within(consumerRoot, packageDirectory), + `${name} must be physically installed beneath consumerRoot`); + const tree = await packageTreeReceipt(packageDirectory); + return { + document, + entryPath, + packageDirectory, + evidence: { + name, + version: document.version, + package_path: relative(consumerRoot, packageDirectory), + entry_path: relative(consumerRoot, entryPath), + package_json_sha256: sha256(manifestBytes), + entry_sha256: sha256(await readFile(entryPath)), + ...tree, + }, + }; + } + } catch (error) { + if (error?.code !== 'ENOENT') throw error; + } + const parent = dirname(directory); + if (parent === directory) break; + directory = parent; + } + throw new Error(`could not resolve installed ${name} package from consumerRoot`); +} + +export async function resolveConsumerArtifacts(consumerRoot) { + assert.equal(typeof consumerRoot, 'string', 'consumerRoot is required'); + assert(isAbsolute(consumerRoot), 'consumerRoot must be absolute'); + const exactRoot = await realpath(consumerRoot); + assert.equal((await stat(exactRoot)).mode & 0o777, 0o700, + 'consumerRoot must be mode 0700'); + const requireFromConsumer = createRequire(pathToFileURL(join(exactRoot, 'package.json'))); + const sdk = await installedPackageReceipt( + requireFromConsumer, exactRoot, '@relayflows/sdk'); + const coreBridge = await installedPackageReceipt( + requireFromConsumer, exactRoot, '@agent-relay/core-bridge'); + assert.equal(coreBridge.document.version, '18.1.0', + 'proof kit requires public @agent-relay/core-bridge@18.1.0 in consumerRoot'); + const bin = typeof sdk.document.bin === 'string' + ? sdk.document.bin : sdk.document.bin?.flows; + assert.equal(typeof bin, 'string', 'consumer @relayflows/sdk must expose its flows CLI'); + const cliPath = await realpath(resolve(sdk.packageDirectory, bin)); + assert(within(sdk.packageDirectory, cliPath), + 'consumer @relayflows/sdk CLI must stay inside its package'); + const lockPath = join(exactRoot, 'package-lock.json'); + const lockBytes = await readFile(lockPath); + return { + root: exactRoot, + sdkCliPath: cliPath, + coreBridgeEntryPath: coreBridge.entryPath, + evidence: { + consumer_lock_sha256: sha256(lockBytes), + relayflows_sdk: { + ...sdk.evidence, + cli_path: relative(exactRoot, cliPath), + cli_sha256: sha256(await readFile(cliPath)), + }, + core_bridge: coreBridge.evidence, + }, + }; +} + +export async function assertConsumerCommand(spec, label, artifacts) { + assertCommandSpec(spec, label); + assert.equal(await realpath(spec.command), await realpath(process.execPath), + `${label}.command must be the Node executable running the proof kit`); + assert(spec.args.length > 0 && isAbsolute(spec.args[0]), + `${label}.args[0] must be the absolute consumer SDK CLI`); + assert.equal(await realpath(spec.args[0]), artifacts.sdkCliPath, + `${label} must launch the exact @relayflows/sdk CLI from consumerRoot`); +} + +async function main() { + const configIndex = process.argv.indexOf('--config'); + assert(configIndex >= 0 && process.argv[configIndex + 1], 'usage: node node-spawn-response-loss-e2e.mjs --config /absolute/config.json'); + const configPath = resolve(process.argv[configIndex + 1]); + const config = JSON.parse(await readFile(configPath, 'utf8')); + for (const field of [ + 'callerFile', 'consumerRoot', 'dataDir', 'evidenceDir', 'homeDir', 'workDir', + ]) { + assert.equal(typeof config[field], 'string', `${field} is required`); + assert(isAbsolute(config[field]), `${field} must be absolute`); + } + const consumerArtifacts = await resolveConsumerArtifacts(config.consumerRoot); + await assertConsumerCommand(config.start, 'start', consumerArtifacts); + await assertConsumerCommand(config.resume, 'resume', consumerArtifacts); + assert.equal(config.audit, undefined, + 'audit is built into the proof kit; fixture-supplied audit commands are not accepted'); + assert(config.resume.args.some(arg => arg.includes('{RUN_ID}')), 'resume args must contain {RUN_ID}'); + const timeoutMs = config.timeoutMs ?? DEFAULT_TIMEOUT_MS; + assert(Number.isSafeInteger(timeoutMs) && timeoutMs > 0, 'timeoutMs must be positive'); + assert.equal((await stat(configPath)).mode & 0o777, 0o600, 'config must be mode 0600'); + await mkdir(config.homeDir, { recursive: true, mode: 0o700 }); + await mkdir(config.dataDir, { recursive: true, mode: 0o700 }); + await mkdir(config.evidenceDir, { recursive: true, mode: 0o700 }); + for (const [label, path] of [ + ['homeDir', config.homeDir], + ['dataDir', config.dataDir], + ['evidenceDir', config.evidenceDir], + ]) { + assert.equal((await stat(path)).mode & 0o777, 0o700, `${label} must be mode 0700`); + } + let initialRuns = []; + try { + initialRuns = (await readdir(join(config.dataDir, 'runs'))).filter(file => file.endsWith('.sqlite3')); + } catch (error) { + if (error?.code !== 'ENOENT') throw error; + } + assert.deepEqual(initialRuns, [], 'proof dataDir must not contain an earlier run'); + + const { readCallerDescriptor } = await import( + pathToFileURL(consumerArtifacts.coreBridgeEntryPath).href + ); + const descriptor = await readCallerDescriptor(config.callerFile); + const privateDir = await mkdtemp(join(tmpdir(), 'flows-response-loss-private-')); + await chmod(privateDir, 0o700); + const proxySocket = join(privateDir, 'core.sock'); + const proxyCaller = join(privateDir, 'caller.json'); + const callerDocument = JSON.parse(await readFile(config.callerFile, 'utf8')); + callerDocument.socket = proxySocket; + await writeFile(proxyCaller, `${JSON.stringify(callerDocument)}\n`, { mode: 0o600, flag: 'wx' }); + await chmod(proxyCaller, 0o600); + + const childEnv = { ...process.env, + HOME: config.homeDir, + RELAYFLOW_SERVICE_CALLER_FILE: proxyCaller, + FLOWS_CLOUD_MIRROR: '0', + TMPDIR: privateDir, + }; + for (const name of [ + 'AGENT_RELAY_WORKSPACE_KEY', + 'RELAYCAST_WORKSPACE_KEY', + 'RELAY_AGENT_TOKEN', + 'RELAY_API_KEY', + 'RELAY_BROKER_API_KEY', + 'RELAY_NODE_TOKEN', + 'RELAY_WORKSPACE_KEY', + 'RELAY_WORKSPACES_JSON', + 'NODE_OPTIONS', + 'NODE_PATH', + ]) delete childEnv[name]; + let firstChild; + const proxy = await createBridgeResponseLossProxy({ + listenSocket: proxySocket, + upstreamSocket: descriptor.socket, + onIntercept: () => { + if (firstChild?.pid) process.kill(firstChild.pid, 'SIGKILL'); + }, + }); + try { + const startLog = join(config.evidenceDir, 'start.log'); + const startFd = openSync(startLog, 'wx', 0o600); + firstChild = spawn(config.start.command, config.start.args, { + cwd: config.workDir, + env: childEnv, + stdio: ['ignore', startFd, startFd], + }); + closeSync(startFd); + const startExit = new Promise((resolveExit, rejectExit) => { + firstChild.once('error', rejectExit); + firstChild.once('exit', (code, signal) => resolveExit({ code, signal })); + }); + await Promise.race([ + proxy.intercepted, + delay(timeoutMs, undefined, { ref: false }) + .then(() => { throw new Error(`spawn reply was not intercepted; see ${startLog}`); }), + ]); + const firstResult = await startExit; + assert.equal(firstResult.signal, 'SIGKILL', 'initial Flows process must die by SIGKILL'); + const runId = await waitForRunId(config.dataDir, timeoutMs); + + const resumeSpec = commandWithRunId(config.resume, runId); + const resumeLog1 = join(config.evidenceDir, 'resume-1.log'); + const resumeLog2 = join(config.evidenceDir, 'resume-2.log'); + const [left, right] = await Promise.all([ + runProcess(resumeSpec, { + cwd: config.workDir, env: childEnv, + logPath: resumeLog1, timeoutMs, + }), + runProcess(resumeSpec, { + cwd: config.workDir, env: childEnv, + logPath: resumeLog2, timeoutMs, + }), + ]); + const resumeResults = [left.result, right.result]; + const replayPids = [left.child.pid, right.child.pid]; + assert(replayPids.every(pid => Number.isSafeInteger(pid) && pid > 0), + 'both replay process ids must be available'); + assert.equal(new Set(replayPids).size, 2, + 'the replay race must use two independent process generations'); + assert(!replayPids.includes(firstChild.pid), + 'neither replay may reuse the killed foreground process generation'); + assert(resumeResults.some(result => result.code === 0), + 'at least one raced resume must complete successfully'); + assert(resumeResults.every(result => result.signal === null), + 'neither raced resume may die by signal'); + + const proxyStatsPath = join(privateDir, 'proxy-stats.json'); + const processEvidencePath = join(privateDir, 'process-evidence.json'); + await writeFile(proxyStatsPath, `${JSON.stringify(proxy.stats)}\n`, { mode: 0o600, flag: 'wx' }); + await writeFile(processEvidencePath, `${JSON.stringify({ + initial: { pid: firstChild.pid, ...firstResult }, + replays: [ + { pid: left.child.pid, ...left.result }, + { pid: right.child.pid, ...right.result }, + ], + })}\n`, { mode: 0o600, flag: 'wx' }); + const auditSpec = { + command: process.execPath, + args: [ + fileURLToPath(new URL('./node-spawn-response-loss-audit.mjs', import.meta.url)), + '--run', runId, + '--data-dir', config.dataDir, + '--proxy-stats', proxyStatsPath, + '--process-evidence', processEvidencePath, + '--json', + ], + }; + const auditLog = join(config.evidenceDir, 'audit.stderr.log'); + const fd = openSync(auditLog, 'wx', 0o600); + const auditChild = spawn(auditSpec.command, auditSpec.args, { + cwd: config.workDir, + env: childEnv, + stdio: ['ignore', 'pipe', fd], + }); + closeSync(fd); + let auditStdout = ''; + auditChild.stdout.on('data', chunk => { auditStdout += String(chunk); }); + const auditExit = await Promise.race([ + new Promise((resolveExit, rejectExit) => { + auditChild.once('error', rejectExit); + auditChild.once('exit', (code, signal) => resolveExit({ code, signal })); + }), + delay(timeoutMs, undefined, { ref: false }).then(() => { + auditChild.kill('SIGKILL'); + throw new Error(`audit command timed out; see ${auditLog}`); + }), + ]); + assert.deepEqual(auditExit, { code: 0, signal: null }, `audit command failed; see ${auditLog}`); + const audit = JSON.parse(auditStdout); + await assertSecretFree([startLog, resumeLog1, resumeLog2, auditLog]); + const evidence = { + ...audit, + run_id: runId, + consumer_artifacts: consumerArtifacts.evidence, + }; + const evidencePath = join(config.evidenceDir, 'response-loss-proof.json'); + await writeFile(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`, { mode: 0o600, flag: 'wx' }); + console.log(JSON.stringify({ ok: true, evidence: evidencePath, run_id: runId })); + } finally { + await proxy.close(); + await rm(privateDir, { recursive: true, force: true }); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + main().catch(error => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/packages/sdk/scripts/node-spawn-response-loss-proxy.mjs b/packages/sdk/scripts/node-spawn-response-loss-proxy.mjs new file mode 100644 index 000000000..88448c4ea --- /dev/null +++ b/packages/sdk/scripts/node-spawn-response-loss-proxy.mjs @@ -0,0 +1,226 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { chmod, mkdir, rm } from 'node:fs/promises'; +import net from 'node:net'; +import { dirname, isAbsolute } from 'node:path'; + +const NEWLINE = Buffer.from('\n'); +const MAX_FRAME_BYTES = 17 * 1024 * 1024; +const SPAWN_INVOKE = 'spawn.invoke'; +const SPAWN_RELEASE = 'spawn.release'; +const INBOX_ACK = 'spawn_callers.inbox.ack'; +const INBOX_FAIL = 'spawn_callers.inbox.fail'; + +function sha256(value) { + return createHash('sha256').update(value).digest('hex'); +} + +export function spawnIdentityDigest(identity) { + for (const field of ['invocationId', 'nodeId', 'nodeName', 'name', 'agentId']) { + assert.equal(typeof identity?.[field], 'string', `spawn identity ${field} must be a string`); + assert(identity[field].length > 0, `spawn identity ${field} must not be empty`); + } + return sha256(JSON.stringify([ + identity.invocationId, + identity.nodeId, + identity.nodeName, + identity.name, + identity.agentId, + ])); +} + +function safeJson(line) { + try { + const value = JSON.parse(line.toString('utf8')); + return typeof value === 'object' && value !== null ? value : undefined; + } catch { + return undefined; + } +} + +function lineReader(onLine, onOverflow) { + let buffered = Buffer.alloc(0); + return chunk => { + buffered = buffered.length === 0 ? chunk : Buffer.concat([buffered, chunk]); + if (buffered.length > MAX_FRAME_BYTES) { + onOverflow(); + buffered = Buffer.alloc(0); + return; + } + for (;;) { + const newline = buffered.indexOf(NEWLINE); + if (newline < 0) return; + const line = buffered.subarray(0, newline); + buffered = buffered.subarray(newline + 1); + onLine(line); + } + }; +} + +/** + * Transparently proxies Core Bridge JSON-RPC, except for the first successful + * spawn.invoke reply. That complete reply is read from upstream, recorded by + * digest, never written downstream, and only then handed to onIntercept so the + * Flows process can be SIGKILLed before any byte of that reply is delivered. + */ +export async function createBridgeResponseLossProxy({ + listenSocket, + upstreamSocket, + onIntercept = () => {}, +}) { + assert(process.platform !== 'win32', 'response-loss proof proxy requires a Unix socket'); + assert(isAbsolute(listenSocket), 'listenSocket must be absolute'); + assert(isAbsolute(upstreamSocket), 'upstreamSocket must be absolute'); + await mkdir(dirname(listenSocket), { recursive: true, mode: 0o700 }); + await chmod(dirname(listenSocket), 0o700); + await rm(listenSocket, { force: true }); + + const sockets = new Set(); + const stats = { + methodCounts: Object.create(null), + spawnInvokeTaskDigests: [], + spawnInvokeKeys: [], + droppedSpawnReply: undefined, + droppedSpawnReplyForwardedBytes: 0, + }; + let interceptedResolve; + let interceptedReject; + const intercepted = new Promise((resolveIntercepted, rejectIntercepted) => { + interceptedResolve = resolveIntercepted; + interceptedReject = rejectIntercepted; + }); + + const server = net.createServer(downstream => { + const upstream = net.createConnection(upstreamSocket); + sockets.add(downstream); + sockets.add(upstream); + const pending = new Map(); + + const fail = error => { + if (!stats.droppedSpawnReply) interceptedReject(error); + downstream.destroy(error); + upstream.destroy(error); + }; + const readRequest = lineReader(line => { + const request = safeJson(line); + if (typeof request?.id !== 'number' || typeof request.method !== 'string') return; + const params = typeof request.params === 'object' && request.params !== null + ? request.params + : {}; + pending.set(request.id, { method: request.method, params }); + stats.methodCounts[request.method] = (stats.methodCounts[request.method] ?? 0) + 1; + if (request.method === SPAWN_INVOKE) { + assert.equal(typeof params.task, 'string', 'spawn.invoke task must be a string'); + assert.equal(typeof params.idempotency_key, 'string', 'spawn.invoke key must be a string'); + stats.spawnInvokeTaskDigests.push(sha256(Buffer.from(params.task, 'utf8'))); + stats.spawnInvokeKeys.push(params.idempotency_key); + } + }, () => fail(new Error('Core Bridge request frame exceeded proof proxy limit'))); + + downstream.on('data', chunk => { + try { + readRequest(chunk); + upstream.write(chunk); + } catch (error) { + fail(error); + } + }); + + const readReply = lineReader(line => { + try { + const reply = safeJson(line); + const request = typeof reply?.id === 'number' ? pending.get(reply.id) : undefined; + if (typeof reply?.id === 'number') pending.delete(reply.id); + if (!stats.droppedSpawnReply && request?.method === SPAWN_INVOKE && !reply?.error) { + const result = typeof reply?.result === 'object' && reply.result !== null ? reply.result : {}; + const identity = { + invocationId: result.id, + nodeId: result.node_id, + nodeName: request.params.node_name, + name: result.name, + agentId: result.agent_id, + }; + const event = { + invocationId: typeof result.id === 'string' ? result.id : undefined, + spawnIdentityDigest: spawnIdentityDigest(identity), + responseDigest: sha256(Buffer.concat([line, NEWLINE])), + responseBytes: line.length + 1, + }; + stats.droppedSpawnReply = event; + // The callback issues SIGKILL synchronously. The response remains in + // this proxy and the downstream is closed without a write. + onIntercept(event); + interceptedResolve(event); + downstream.destroy(); + upstream.destroy(); + return; + } + downstream.write(Buffer.concat([line, NEWLINE])); + } catch (error) { + fail(error); + } + }, () => fail(new Error('Core Bridge reply frame exceeded proof proxy limit'))); + + upstream.on('data', readReply); + upstream.on('error', fail); + downstream.on('error', error => upstream.destroy(error)); + downstream.on('close', () => { + sockets.delete(downstream); + upstream.destroy(); + }); + upstream.on('close', () => { + sockets.delete(upstream); + downstream.destroy(); + }); + }); + + await new Promise((resolveListen, rejectListen) => { + server.once('error', rejectListen); + server.listen(listenSocket, () => { + server.off('error', rejectListen); + resolveListen(); + }); + }); + await chmod(listenSocket, 0o600); + + return { + stats, + intercepted, + async close() { + for (const socket of sockets) socket.destroy(); + await new Promise(resolveClose => server.close(() => resolveClose())); + await rm(listenSocket, { force: true }); + }, + }; +} + +export function verifyResponseLossProxyEvidence(stats) { + assert.equal(stats.droppedSpawnReplyForwardedBytes, 0, + 'no byte of the committed spawn.invoke reply may reach Flows'); + assert(stats.droppedSpawnReply, 'the proxy must drop one successful spawn.invoke reply'); + assert(stats.spawnInvokeTaskDigests.length >= 2, + 'the committed invoke and at least one same-key replay must cross the proxy'); + assert.equal(stats.spawnInvokeTaskDigests.length, stats.spawnInvokeKeys.length); + assert.equal(new Set(stats.spawnInvokeTaskDigests).size, 1, + 'every same-key retry must preserve the original task bytes'); + assert.equal(new Set(stats.spawnInvokeKeys).size, 1, + 'every retry must use the original idempotency key'); + assert.equal(stats.methodCounts[SPAWN_RELEASE] ?? 0, 1, + 'Flows must issue exactly one release through the bridge'); + assert.equal((stats.methodCounts[INBOX_ACK] ?? 0) + (stats.methodCounts[INBOX_FAIL] ?? 0), 1, + 'Flows must settle the correlated inbox item exactly once'); + + return { + dropped_response_bytes: stats.droppedSpawnReply.responseBytes, + dropped_response_forwarded_bytes: stats.droppedSpawnReplyForwardedBytes, + spawn_invoke_calls: stats.methodCounts[SPAWN_INVOKE] ?? 0, + replay_attempts: stats.spawnInvokeKeys.length - 1, + unique_spawn_keys: new Set(stats.spawnInvokeKeys).size, + task_sha256: stats.spawnInvokeTaskDigests[0], + idempotency_key_sha256: sha256(stats.spawnInvokeKeys[0]), + spawn_identity_sha256: stats.droppedSpawnReply.spawnIdentityDigest, + release_calls: stats.methodCounts[SPAWN_RELEASE] ?? 0, + inbox_settlement_calls: (stats.methodCounts[INBOX_ACK] ?? 0) + + (stats.methodCounts[INBOX_FAIL] ?? 0), + }; +} diff --git a/packages/sdk/src/agent-relay-receipt.ts b/packages/sdk/src/agent-relay-receipt.ts index 958e6f30c..1909f9a27 100644 --- a/packages/sdk/src/agent-relay-receipt.ts +++ b/packages/sdk/src/agent-relay-receipt.ts @@ -1,9 +1,5 @@ -import { canonicalize } from "./canonical.js"; - -export const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value); -export const nonempty = (value: unknown): value is string => - typeof value === "string" && value.length > 0; +import type { NodeResultDelivery, NodeResultEnvelope } from './node-spawn-result.js'; +import type { NodeSpawnReceipt } from './node-spawn.js'; export interface RelayTaskExecution { execution_id: string; @@ -11,110 +7,31 @@ export interface RelayTaskExecution { step_id: string; dispatch_id: string; deadline: string; - worker_generation?: string; - accepted_at?: string; + node_id?: string; + node_name: string; + agent_id: string; + agent_name: string; + delivery_id: string; + message_id: string; + release_invocation_id?: string; accounting?: Record; } + +/** Journal evidence for a task executed through native node spawn. */ export interface RelayTaskReceipt { invocation_id: string; - action_name: "task.run"; - status: "pending" | "dispatched" | "running" | "completed" | "failed"; + action_name: 'spawn'; + status: 'completed' | 'failed'; task_execution: RelayTaskExecution; output: unknown; error: string | null; - completed_at: string | null; + completed_at: string; } -/** GET is authoritative. A spawn/POST acknowledgment is never a task result. */ -export function readTaskReceipt( - value: unknown, - invocationId: string, - input: Record, - previous?: RelayTaskReceipt, -): RelayTaskReceipt { - const context = input.task_context as Record; - if ( - !isRecord(value) || - value.invocation_id !== invocationId || - value.action_name !== "task.run" || - canonicalize(value.input) !== canonicalize(input) || - !isRecord(value.task_execution) - ) { - throw new Error("Relay task receipt has mismatched invocation or input"); - } - const execution = value.task_execution; - const validDate = (date: unknown): date is string => - nonempty(date) && Number.isFinite(Date.parse(date)); - if ( - !nonempty(execution.execution_id) || - !validDate(execution.deadline) || - ["run_id", "step_id", "dispatch_id"].some( - (key) => execution[key] !== context[key], - ) || - !["pending", "dispatched", "running", "completed", "failed"].includes( - String(value.status), - ) - ) { - throw new Error("Relay task receipt has invalid execution correlation"); - } - const accepted = - nonempty(execution.worker_generation) && validDate(execution.accepted_at); - if ( - (value.status === "running" || value.status === "completed") && - !accepted - ) { - throw new Error("Relay task receipt is missing durable acceptance"); - } - if ( - (execution.worker_generation !== undefined || - execution.accepted_at !== undefined) && - !accepted - ) { - throw new Error("Relay task receipt has incomplete durable acceptance"); - } - if ( - previous !== undefined && - (previous.task_execution.execution_id !== execution.execution_id || - previous.task_execution.deadline !== execution.deadline || - (previous.task_execution.worker_generation !== undefined && - (previous.task_execution.worker_generation !== - execution.worker_generation || - previous.task_execution.accepted_at !== execution.accepted_at))) - ) { - throw new Error( - "Relay task receipt changed its execution, accepted generation/time, or deadline", - ); - } - if ( - execution.accounting !== undefined && - (!isRecord(execution.accounting) || - Object.values(execution.accounting).some( - (n) => typeof n !== "number" || !Number.isFinite(n) || n < 0, - )) - ) { - throw new Error("Relay task receipt has invalid accounting"); - } - if ( - value.status === "completed" && - (!Object.hasOwn(value, "output") || - value.error !== null || - !validDate(value.completed_at)) - ) { - throw new Error("Relay task receipt has no authoritative final output"); - } - if ( - value.status === "failed" && - (!nonempty(value.error) || !validDate(value.completed_at)) - ) { - throw new Error("Relay task failure receipt has no reason"); - } - return { - invocation_id: invocationId, - action_name: "task.run", - status: value.status as RelayTaskReceipt["status"], - task_execution: execution as unknown as RelayTaskExecution, - output: value.output, - error: value.error as string | null, - completed_at: value.completed_at as string | null, - }; +/** The delivery callback is process-local and must never enter journal output. */ +export interface RelayTaskResult { + receipt: RelayTaskReceipt; + delivery: NodeResultDelivery; + spawn: NodeSpawnReceipt; + envelope: NodeResultEnvelope; } diff --git a/packages/sdk/src/agent-relay-transport.ts b/packages/sdk/src/agent-relay-transport.ts index 73e8ffac4..f13b26600 100644 --- a/packages/sdk/src/agent-relay-transport.ts +++ b/packages/sdk/src/agent-relay-transport.ts @@ -1,273 +1,486 @@ -import { createHash } from "node:crypto"; -import { isIP } from "node:net"; -import { setTimeout as delay } from "node:timers/promises"; -import { claimRelayTask } from "./agent-relay-state.js"; -import { canonicalize } from "./canonical.js"; -import { - isRecord, - nonempty, - readTaskReceipt, - type RelayTaskReceipt, -} from "./agent-relay-receipt.js"; +import { createHash } from 'node:crypto'; +import { claimNodeSpawnAtPath, hasNodeSpawnDispatchFence, + nodeSpawnClaimPath, recordNodeSpawn, + type NodeSpawnClaim } from './node-spawn-state.js'; +import { dispatchFencedNodeSpawn, + hasReplayableFencedDispatch } from './node-spawn-dispatch.js'; +import { releaseNodeSpawnClaim } from './node-spawn-cleanup.js'; +import { nodeResultInstruction, waitForNodeResult } from './node-spawn-result.js'; +import { durableNodeResultDelivery } from './node-spawn-settlement.js'; +import { waitForNodeOperation } from './node-spawn-deadline.js'; +import { createNodeSpawnRuntime, type NodeSpawnRuntime } from './node-spawn-runtime.js'; +import { NodeSpawnError, nonempty, validIdempotencyKey, + validNodeSpawnIdentity, validNodeSpawnReceipt, + type NodeSpawnReceipt } from './node-spawn.js'; +import type { RelayTaskResult } from './agent-relay-receipt.js'; -export type AgentTransport = "direct" | "relay"; +export type AgentTransport = 'direct' | 'relay'; export interface AgentRelayTaskRequest { cli: string; task: string; + /** Stable task input without per-attempt recovery evidence. */ + taskIdentity?: string; model?: string; worker_cwd?: string; result_schema?: unknown; runId: string; stepId: string; idempotencyKey: string; + attempt: number; dataDir: string; - /** Engine task contract ceiling; bounded independently of the renewing lease. */ + /** Engine task contract ceiling; bounded independently of the worker lease. */ timeoutMs?: number; } export interface AgentRelayEnv { baseUrl?: string; agentToken?: string; + /** Absolute Desktop schema-v2 caller.json path; mutually exclusive with agentToken. */ + serviceCallerFile?: string; + node?: string; + repo?: string; + sandboxOnly?: boolean; } export class AgentRelayTransportError extends Error { constructor( message: string, public readonly cause?: unknown, + /** Exact cleanup is durable and must be retried by a replacement attempt. */ + public readonly cleanupPending = false, ) { super(message); - this.name = "AgentRelayTransportError"; + this.name = 'AgentRelayTransportError'; } } -export function readAgentRelayEnv( - env: NodeJS.ProcessEnv = process.env, -): Required { - const agentToken = env.RELAY_AGENT_TOKEN?.trim(); - if (!agentToken) - throw new AgentRelayTransportError( - "Relay task transport requires a pre-provisioned RELAY_AGENT_TOKEN.", - ); - return { - baseUrl: env.RELAY_BASE_URL?.trim() || "https://cast.agentrelay.com", - agentToken, - }; + +const claimExecutions = new Map>(); + +interface AgentRelayTaskOptions extends AgentRelayEnv { + runtime?: NodeSpawnRuntime; + runtimeProvider?: (signal?: AbortSignal) => Promise; + signal?: AbortSignal; + /** Wakes durable reconciliation after a late exact release remains pending. */ + onCleanupPending?: () => void; } -/** Exact Relaycast #436 HTTP contract. Only terminal GET receipts can return. */ +/** + * Dispatches exactly once through an Agent Relay node and waits on the + * publisher's durable inbox. AgentWorker acks the returned delivery only + * after stepComplete has durably entered the journal. + */ export async function runAgentRelayTask( request: AgentRelayTaskRequest, - options: AgentRelayEnv & { - fetch?: typeof fetch; - signal?: AbortSignal; - pollMs?: number; - } = {}, -): Promise { - const env = options.agentToken - ? { - baseUrl: options.baseUrl || "https://cast.agentrelay.com", - agentToken: options.agentToken, - } - : { ...readAgentRelayEnv(), ...options }; - const base = new URL(env.baseUrl!); - if ( - base.username || - base.password || - base.search || - base.hash || - base.pathname !== "/" || - !["https:", "http:"].includes(base.protocol) - ) - throw new AgentRelayTransportError("Invalid Relay task base URL"); - const loopback = - base.hostname === "[::1]" || - (isIP(base.hostname) === 4 && base.hostname.startsWith("127.")); - if (base.protocol !== "https:" && !loopback) { - throw new AgentRelayTransportError( - "Relay task credentials require HTTPS outside literal loopback addresses", - ); - } - const baseUrl = base.origin; + options: AgentRelayTaskOptions = {}, +): Promise { + const taskStartedAt = Date.now(); + options.signal?.throwIfAborted(); const timeoutMs = request.timeoutMs ?? 86_400_000; - if ( - !Number.isSafeInteger(timeoutMs) || - timeoutMs < 1 || - timeoutMs > 86_400_000 || - ![ - request.runId, - request.stepId, - request.idempotencyKey, - request.dataDir, - request.cli, - ].every(nonempty) - ) { - throw new AgentRelayTransportError( - "Relay task requires durable dispatch identity and a valid deadline", - ); + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 86_400_000 + || !Number.isSafeInteger(request.attempt) || request.attempt < 1 + || ![request.runId, request.stepId, request.dataDir, request.cli].every(nonempty) + || !validIdempotencyKey(request.idempotencyKey)) { + throw new AgentRelayTransportError('Node-spawn task requires durable dispatch identity and a valid deadline'); } - const pollMs = options.pollMs ?? 1000; - if (!Number.isFinite(pollMs) || pollMs < 1) - throw new AgentRelayTransportError("Invalid Relay task polling interval"); - const doFetch = options.fetch ?? globalThis.fetch; - const outer = options.signal ?? new AbortController().signal; - let deadline = Date.now() + timeoutMs + 30_000; - let missingDeadline = Date.now() + 30_000; - async function http( - path: string, - body?: unknown, - ): Promise> { - for (;;) { - outer.throwIfAborted(); - if (Date.now() >= deadline) - throw new AgentRelayTransportError( - "Relay task status unavailable before its reconciliation deadline", - ); - const bounded = AbortSignal.any([ - outer, - AbortSignal.timeout( - Math.min(15_000, Math.max(1, deadline - Date.now())), - ), - ]); - let response: Response | undefined; - let value: unknown; - try { - response = await doFetch(new URL(path, baseUrl), { - method: body === undefined ? "GET" : "POST", - redirect: "error", - signal: bounded, - headers: { - authorization: `Bearer ${env.agentToken}`, - "content-type": "application/json", - ...(body === undefined - ? {} - : { "Idempotency-Key": request.idempotencyKey }), - }, - ...(body === undefined ? {} : { body: canonicalize(body) }), - }); - if ( - response.status === 404 && - body === undefined && - path.includes("/invocations/") - ) { - if (Date.now() >= missingDeadline) - throw new AgentRelayTransportError( - "Relay task dispatch remains unconfirmed; no repeat POST is permitted", - ); - } else if ( - !response.ok && - response.status !== 429 && - response.status < 500 - ) { - // Never copy untrusted response bodies, URLs, or tokens into diagnostics. - throw new AgentRelayTransportError( - `Relay task request refused with HTTP ${response.status}`, - ); - } - if (response.ok) value = await response.json(); - if (body !== undefined && !response.ok) return {}; // ambiguous POST: GET only below - } catch (error) { - outer.throwIfAborted(); - if (error instanceof AgentRelayTransportError) throw error; - if (body !== undefined) return {}; // response loss or invalid JSON never causes another POST - if (response?.ok && !bounded.aborted) - throw new AgentRelayTransportError( - "Relay task response was not valid JSON", - ); - } - outer.throwIfAborted(); - if (response?.ok && value !== undefined) { - if (!isRecord(value) || value.ok !== true || !isRecord(value.data)) - throw new AgentRelayTransportError( - "Relay task response has an invalid data envelope", - ); - return value.data; + const claimPath = await nodeSpawnClaimPath( + request.dataDir, request.runId, request.stepId, request.idempotencyKey); + const taskDeadline = taskStartedAt + timeoutMs; + return serializeClaimExecution(claimPath, taskDeadline, options.signal, async () => { + options.signal?.throwIfAborted(); + return runAgentRelayTaskExclusive(request, claimPath, options, taskStartedAt, timeoutMs); + }); +} + +async function runAgentRelayTaskExclusive( + request: AgentRelayTaskRequest, + claimPath: string, + options: AgentRelayTaskOptions, + taskStartedAt: number, + timeoutMs: number, +): Promise { + const taskDeadline = taskStartedAt + timeoutMs; + let acquisitionRemaining: number; + try { acquisitionRemaining = remainingDeadline(taskDeadline); } + catch (error) { + throw new AgentRelayTransportError('The node-spawn task deadline elapsed', error); + } + const ownsRuntime = options.runtime === undefined && options.runtimeProvider === undefined; + const acquisitionTimeout = AbortSignal.timeout(acquisitionRemaining); + const acquisitionSignal = options.signal === undefined + ? acquisitionTimeout : AbortSignal.any([options.signal, acquisitionTimeout]); + let runtime: NodeSpawnRuntime; + try { + if (options.runtime !== undefined) { + runtime = options.runtime; + } else if (options.runtimeProvider !== undefined) { + runtime = await waitForNodeOperation(options.runtimeProvider(acquisitionSignal), acquisitionSignal); + } else { + runtime = await createNodeSpawnRuntime({ + ...process.env, + ...(options.baseUrl === undefined ? {} : { RELAY_BASE_URL: options.baseUrl }), + ...(options.agentToken === undefined ? {} : { RELAY_AGENT_TOKEN: options.agentToken }), + ...(options.serviceCallerFile === undefined + ? {} + : { RELAYFLOW_SERVICE_CALLER_FILE: options.serviceCallerFile }), + ...(options.node === undefined ? {} : { RELAYFLOW_NODE_NAME: options.node }), + ...(options.repo === undefined ? {} : { RELAYFLOW_NODE_REPO: options.repo }), + ...(options.sandboxOnly === undefined + ? {} : { RELAYFLOW_NODE_SANDBOX_ONLY: options.sandboxOnly ? '1' : '0' }), + }, { signal: acquisitionSignal }); + } + } catch (error) { + if (options.signal?.aborted !== true && acquisitionTimeout.aborted) { + const timeout = new NodeSpawnError('The node-spawn task deadline elapsed', 'task_timeout', error); + throw new AgentRelayTransportError(timeout.message, timeout); + } + throw error; + } + try { + return await executeAgentRelayTask( + request, claimPath, runtime, options.signal, taskStartedAt, timeoutMs, + options.onCleanupPending); + } finally { + if (ownsRuntime) await runtime.disconnect?.().catch(() => undefined); + } +} + +async function serializeClaimExecution( + path: string, + deadline: number, + signal: AbortSignal | undefined, + execute: () => Promise, +): Promise { + const prior = claimExecutions.get(path) ?? Promise.resolve(); + let release!: () => void; + const occupied = new Promise(resolve => { release = resolve; }); + const queued = prior.then(() => occupied, () => occupied); + claimExecutions.set(path, queued); + let acquired = false; + try { + let remaining: number; + try { remaining = remainingDeadline(deadline); } + catch (error) { + throw new AgentRelayTransportError('The node-spawn task deadline elapsed', error); + } + const queueTimeout = AbortSignal.timeout(remaining); + const queueSignal = signal === undefined ? queueTimeout : AbortSignal.any([signal, queueTimeout]); + try { await waitForNodeOperation(prior, queueSignal); } + catch (error) { + if (signal?.aborted !== true && queueTimeout.aborted) { + const timeout = new NodeSpawnError('The node-spawn task deadline elapsed', 'task_timeout', error); + throw new AgentRelayTransportError(timeout.message, timeout); } - await delay(pollMs, undefined, { signal: outer }); + throw error; + } + acquired = true; + return await execute(); + } finally { + release(); + if (acquired && claimExecutions.get(path) === queued) { + claimExecutions.delete(path); + } else if (!acquired) { + void queued.then(() => { + if (claimExecutions.get(path) === queued) claimExecutions.delete(path); + }); } } - // Resolving the agent is read-only. Pin identity before any invocation so a - // restarted runner cannot create a second task using another caller's key. - const agent = await http("/v1/agent"); - if (!nonempty(agent.id)) - throw new AgentRelayTransportError("Relay task caller identity is missing"); - if (!nonempty(agent.workspace_id)) - throw new AgentRelayTransportError( - "Relay task workspace identity is missing", - ); - // Pinned action-invoke-v1 identity contract from Relaycast #436. - const invocationId = - "inv_idem_" + - createHash("sha256") - .update( - [ - "action-invoke-v1", - agent.workspace_id, - agent.id, - "task.run", - request.idempotencyKey, - ].join("\0"), - ) - .digest("hex"); - const input = { - cli: request.cli, - task: request.task, - ...(request.model === undefined ? {} : { model: request.model }), - ...(request.worker_cwd === undefined - ? {} - : { worker_cwd: request.worker_cwd }), - ...(request.result_schema === undefined - ? {} - : { result_schema: request.result_schema }), - task_context: { - run_id: request.runId, - step_id: request.stepId, - dispatch_id: request.idempotencyKey, - timeout_ms: timeoutMs, - }, - }; - const { claim, created } = await claimRelayTask(request.dataDir, { - version: 1, - baseUrl, - callerId: agent.id, - workspaceId: agent.workspace_id, - invocationId, +} + +async function executeAgentRelayTask( + request: AgentRelayTaskRequest, + claimPath: string, + runtime: NodeSpawnRuntime, + signal: AbortSignal | undefined, + taskStartedAt: number, + timeoutMs: number, + onCleanupPending: (() => void) | undefined, +): Promise { + if (runtime.port.dispatchFenceVersion !== 1) throw new AgentRelayTransportError( + 'The node-spawn port does not implement durable dispatch-fence protocol version 1'); + const correlation = { runId: request.runId, stepId: request.stepId, dispatchId: request.idempotencyKey }; + const agentName = stableAgentName(request); + const task = [ + request.task, + request.result_schema === undefined ? undefined + : `The final output must satisfy this JSON Schema: ${JSON.stringify(request.result_schema)}`, + nodeResultInstruction(correlation), + ].filter(nonempty).join('\n\n'); + const proposed: NodeSpawnClaim = { + version: 3, + baseUrl: runtime.baseUrl, + publisherId: runtime.publisher.id, + publisherName: runtime.publisher.name, runId: request.runId, stepId: request.stepId, idempotencyKey: request.idempotencyKey, - input, - startedAt: Date.now(), - }); - // A restarted runner may recover a terminal receipt after the task deadline. - // Give that read one bounded window; never reopen or extend the remote task. - deadline = Math.max( - claim.startedAt + timeoutMs + 30_000, - Date.now() + 15_000, - ); - missingDeadline = claim.startedAt + 30_000; - if (created) { - const ack = await http("/v1/actions/task.run/invoke", { input }); - if ( - Object.keys(ack).length && - (ack.invocation_id !== invocationId || - ack.action_name !== "task.run" || - canonicalize(ack.input) !== canonicalize(input)) - ) { - throw new AgentRelayTransportError( - "Relay task acknowledgment has mismatched invocation or input", - ); + agentName, + timeoutMs, + request: { + cli: request.cli, task, task_identity: request.taskIdentity ?? request.task, + ...(request.model === undefined ? {} : { model: request.model }), + ...(request.worker_cwd === undefined ? {} : { worker_cwd: request.worker_cwd }), + ...(request.result_schema === undefined ? {} : { result_schema: request.result_schema }), + ...(runtime.node === undefined ? {} : { node: runtime.node }), + ...(runtime.nodeId === undefined ? {} : { node_id: runtime.nodeId }), + ...(runtime.repo === undefined ? {} : { repo: runtime.repo }), + ...(runtime.sandboxOnly === undefined ? {} : { sandbox_only: runtime.sandboxOnly }), + }, + startedAt: taskStartedAt, + }; + const claimed = await claimNodeSpawnAtPath(claimPath, proposed); + let claim = claimed.claim; + let releaseAttempted = false; + const deadline = claim.startedAt + claim.timeoutMs; + try { + const dispatchFenced = claim.version === 3 && await hasNodeSpawnDispatchFence(claimed.path, claim); + const replayExistingFence = dispatchFenced && hasReplayableFencedDispatch(runtime.port); + if (claim.spawn === undefined && claim.cleanup === undefined + && ((dispatchFenced && !replayExistingFence) || (!claimed.created && claim.version === 2))) { + throw new AgentRelayTransportError('A prior node-spawn dispatch ended in the ambiguous pre-ack window; refusing a second spawn'); + } + // A v3 claim without a dispatch fence proves no irreversible call was + // entered, so a replacement attempt may safely repeat preflight. + if (!claimed.created && claim.spawn === undefined && claim.cleanup !== undefined) { + if (claim.release === undefined) { + releaseAttempted = true; + const cleanup = await releaseNodeSpawnClaim(runtime, claimed.path, claim); + claim = cleanup.claim; + if (cleanup.error !== undefined) throw cleanup.error; + } + throw new NodeSpawnError(claim.failure?.message ?? 'The prior node spawn failed after launch', + claim.failure?.code ?? 'spawn_unconfirmed'); + } + if (claim.result === undefined) assertWithinDeadline(deadline); + signal?.throwIfAborted(); + if (claim.spawn === undefined) { + // This is deliberately adjacent to the irreversible dispatch. Runtime + // setup and durable claiming can take time or lose the worker lease. + signal?.throwIfAborted(); + const remaining = remainingDeadline(deadline); + const durableRequest = claim.request; + const spawn = await dispatchFencedNodeSpawn({ + port: runtime.port, + request: { + name: claim.agentName, + harness: durableRequest.cli as string, + task: durableRequest.task as string, + ...(durableRequest.model === undefined + ? {} : { model: durableRequest.model as string }), + ...(durableRequest.worker_cwd === undefined + ? {} : { directory: durableRequest.worker_cwd as string }), + idempotencyKey: claim.idempotencyKey, + ...(durableRequest.node === undefined + ? {} : { node: durableRequest.node as string }), + ...(durableRequest.node_id === undefined + ? {} : { nodeId: durableRequest.node_id as string }), + ...(durableRequest.repo === undefined + ? {} : { repo: durableRequest.repo as string }), + ...(durableRequest.sandbox_only === undefined + ? {} : { sandboxOnly: durableRequest.sandbox_only as boolean }), + timeoutMs: remaining, + }, + claimPath: claimed.path, claim, signal, settleLate: settleLateSpawn, + replayExistingFence, + }); + claim = { ...claim, spawn }; + await recordNodeSpawn(claimed.path, claim); + signal?.throwIfAborted(); + } else { + // A durable result is the authoritative completion record. Requiring a + // live placement lookup here would make crash recovery depend on a + // remote invocation record that may expire after exact cleanup. + if (claim.result === undefined) { + if ((claim.cleanupPending !== undefined || claim.failure !== undefined) + && claim.release === undefined) { + releaseAttempted = true; + const cleanup = await releaseNodeSpawnClaim(runtime, claimed.path, claim); + claim = cleanup.claim; + if (cleanup.error !== undefined) throw cleanup.error; + throw new NodeSpawnError( + claim.failure?.message ?? 'The prior node spawn failed after launch', + claim.failure?.code ?? 'spawn_unconfirmed', + ); + } + if (claim.release !== undefined) { + throw new NodeSpawnError( + claim.failure?.message ?? 'The recorded node spawn was released before it reported a result', + claim.failure?.code ?? 'spawn_released', + ); + } + const remaining = remainingDeadline(deadline); + const statusTimeout = AbortSignal.timeout(remaining); + const statusSignal = signal === undefined + ? statusTimeout : AbortSignal.any([signal, statusTimeout]); + const status = await waitForNodeOperation(runtime.port.status('spawn', claim.spawn.id), statusSignal); + if (status.status !== 'completed') { + throw new NodeSpawnError('The recorded node spawn is no longer completed', + status.errorCode ?? `spawn_${status.status}`); + } + } + } + const spawn = claim.spawn; + if (spawn === undefined) throw new AgentRelayTransportError('Node-spawn claim lost its confirmed receipt'); + if (claim.result !== undefined && claim.release === undefined) { + releaseAttempted = true; + const cleanup = await releaseNodeSpawnClaim(runtime, claimed.path, claim); + claim = cleanup.claim; + if (cleanup.error !== undefined) throw cleanup.error; + } + const resultSignal = claim.result === undefined + ? (signal === undefined + ? AbortSignal.timeout(remainingDeadline(deadline)) + : AbortSignal.any([signal, AbortSignal.timeout(remainingDeadline(deadline))])) + : signal; + const observed = claim.result === undefined + ? await waitForNodeResult(runtime.inbox, { + agentId: spawn.agentId, + agentName: spawn.name, + invocationId: spawn.id, + ...correlation, + }, resultSignal) + : undefined; + resultSignal?.throwIfAborted(); + if (claim.result !== undefined + && observed !== undefined + && (claim.result.inboxItemId !== observed.inboxItemId || claim.result.messageId !== observed.messageId)) { + throw new NodeSpawnError('The redelivered result did not match the durable delivery identity', + 'result_identity_mismatch'); + } + if (claim.result === undefined) { + if (observed === undefined) throw new NodeSpawnError('Node result disappeared before it became durable'); + claim = { ...claim, result: { inboxItemId: observed.inboxItemId, + messageId: observed.messageId, status: observed.envelope.status, + envelope: observed.envelope, settlementAction: observed.settlementAction, + receivedAt: Date.now() } }; + await recordNodeSpawn(claimed.path, claim); + } + // Register the live inbox teardown before exact release. A failed release + // is recoverable from the durable result, but a same-process retry cannot + // recreate this process-generation barrier after the iterator is gone. + const delivery = durableNodeResultDelivery(runtime, claimed.path, claim, observed); + if (claim.release === undefined) { + releaseAttempted = true; + const cleanup = await releaseNodeSpawnClaim(runtime, claimed.path, claim); + claim = cleanup.claim; + if (cleanup.error !== undefined) throw cleanup.error; + } + signal?.throwIfAborted(); + const envelope = delivery.envelope; + const error = envelope.status === 'failed' ? envelope.error! : null; + const status = error === null ? 'completed' : 'failed'; + return { + spawn, + delivery, + envelope, + receipt: { + invocation_id: spawn.id, + action_name: 'spawn', + status, + task_execution: { + execution_id: spawn.id, + run_id: request.runId, + step_id: request.stepId, + dispatch_id: request.idempotencyKey, + deadline: new Date(deadline).toISOString(), + node_id: spawn.nodeId, + node_name: spawn.nodeName, + agent_id: spawn.agentId, + agent_name: spawn.name, + delivery_id: delivery.inboxItemId, + message_id: delivery.messageId, + ...(claim.release === undefined ? {} : { release_invocation_id: claim.release.invocationId }), + ...(envelope.accounting === undefined ? {} : { accounting: envelope.accounting }), + }, + output: envelope.output, + error, + completed_at: new Date().toISOString(), + }, + }; + } catch (error) { + let cleanupError: unknown; + if ((claim.spawn !== undefined || claim.cleanup !== undefined) + && claim.release === undefined && claim.failure === undefined) { + claim = { ...claim, failure: failureRecord(error, signal) }; + try { await recordNodeSpawn(claimed.path, claim); } + catch (stateError) { cleanupError = stateError; } + } + if (error instanceof NodeSpawnError && validNodeSpawnIdentity(error.cleanup) + && claim.spawn === undefined) { + claim = { ...claim, cleanup: error.cleanup, + failure: { code: error.code, message: error.message } }; + try { await recordNodeSpawn(claimed.path, claim); } + catch (stateError) { cleanupError = stateError; } } + if ((claim.spawn !== undefined || claim.cleanup !== undefined) + && claim.release === undefined && !releaseAttempted) { + releaseAttempted = true; + const cleanup = await releaseNodeSpawnClaim(runtime, claimed.path, claim); + claim = cleanup.claim; + cleanupError ??= cleanup.error; + } + signal?.throwIfAborted(); + const detail = cleanupError === undefined + ? safeError(error) + : `${safeError(error)}; exact cleanup remains pending: ${safeError(cleanupError)}`; + const exactCleanupPending = claim.release === undefined + && claim.cleanupPending !== undefined + && (claim.spawn !== undefined || claim.cleanup !== undefined); + throw new AgentRelayTransportError(detail, error, exactCleanupPending); + } + + async function settleLateSpawn(outcome: { spawn?: NodeSpawnReceipt; error?: unknown }): Promise { + if (outcome.spawn !== undefined) { + const identity = validNodeSpawnIdentity(outcome.spawn) ? outcome.spawn : undefined; + if (identity === undefined) return; + claim = { ...claim, + ...(validNodeSpawnReceipt(outcome.spawn) && outcome.spawn.name === agentName + ? { spawn: outcome.spawn } : { cleanup: identity }), + failure: { + code: 'spawn_cancelled', + message: 'The node spawn confirmed after its caller stopped waiting', + } }; + } else if (outcome.error instanceof NodeSpawnError + && validNodeSpawnIdentity(outcome.error.cleanup)) { + claim = { ...claim, cleanup: outcome.error.cleanup, + failure: { code: outcome.error.code, message: outcome.error.message } }; + } else { + return; + } + try { await recordNodeSpawn(claimed.path, claim); } + catch { /* Exact in-memory identity is still sufficient for this cleanup attempt. */ } + const cleanup = await releaseNodeSpawnClaim(runtime, claimed.path, claim); + claim = cleanup.claim; + if (cleanup.error !== undefined) onCleanupPending?.(); } - let previous: RelayTaskReceipt | undefined; - for (;;) { - const value = await http( - `/v1/actions/task.run/invocations/${encodeURIComponent(invocationId)}`, - ); - if (value.caller_id !== agent.id) - throw new AgentRelayTransportError( - "Relay task receipt belongs to another caller", - ); - const receipt = readTaskReceipt(value, invocationId, input, previous); - outer.throwIfAborted(); - if (receipt.status === "completed" || receipt.status === "failed") - return receipt; - previous = receipt; - await delay(pollMs, undefined, { signal: outer }); +} + +function remainingDeadline(deadline: number): number { + const remaining = deadline - Date.now(); + if (!Number.isSafeInteger(remaining) || remaining < 1) { + throw new NodeSpawnError('The node-spawn task deadline elapsed', 'task_timeout'); } + return remaining; +} + +function assertWithinDeadline(deadline: number): void { + remainingDeadline(deadline); +} + +function stableAgentName(request: AgentRelayTaskRequest): string { + const digest = createHash('sha256') + .update(JSON.stringify([request.runId, request.stepId, request.idempotencyKey])) + .digest('hex').slice(0, 24); + const label = request.stepId.toLowerCase().replace(/[^a-z0-9_-]+/g, '-').replace(/^-+|-+$/g, '') || 'step'; + return `rf-${label.slice(0, 30)}-${digest}`; +} + +function safeError(error: unknown): string { + return error instanceof Error && error.message ? error.message : 'Node-spawn task failed'; +} + +function failureRecord(error: unknown, signal?: AbortSignal): { code: string; message: string } { + return { + code: error instanceof NodeSpawnError + ? error.code + : signal?.aborted === true ? 'task_cancelled' : 'node_spawn_failed', + message: safeError(error), + }; } diff --git a/packages/sdk/src/authored-broker.ts b/packages/sdk/src/authored-broker.ts index 27578dafd..7166ebd90 100644 --- a/packages/sdk/src/authored-broker.ts +++ b/packages/sdk/src/authored-broker.ts @@ -1,73 +1,23 @@ -import { randomUUID } from 'node:crypto'; import type { AgentOptions, AgentResult, NamedGate } from '@relayflows/surface'; -import { parseAgentStepTimeout } from './compile.js'; import type { StepSpec } from './spec.js'; -import { BROKER_TYPE, brokerAgentName, brokerOptionsError, type BrokerInstruction } from './communication/broker-spec.js'; import { AuthoredFlowExecutionError } from './authored-flow-error.js'; -type Declaration = { instruction: BrokerInstruction; cli?: string; model?: string; secrets?: string[] }; +const RETIRED = 'transport: broker was removed with the Relay broker. Use transport: relay for one-shot fleet execution. ' + + 'Retained local sessions require an authorized Desktop service-caller contract and remain fail-closed.'; -/** Broker options stay in a journaled instruction; the kernel only sees agent steps. */ -export function authoredBroker(definition: { name: string }, rootRunId: string | undefined, - localStream: string | undefined, run: (step: StepSpec) => Promise) { - const owner = rootRunId ?? randomUUID(); - const declarations = new Map(); - const invalid = (message: string): never => { throw new AuthoredFlowExecutionError('agent_cli_unresolved', message); }; - const execute = async (id: string, declaration: Declaration, verification?: NamedGate): Promise => { - const { instruction, cli, model, secrets } = declaration; - const output = await run({ id, type: 'agent', instruction: JSON.stringify(instruction), cli, model, - timeoutMs: instruction.timeoutMs, - // Retained sessions must return to the same worker, including after a child completes. - surfaces: localStream === undefined ? undefined : { streams: [{ stream: localStream }] }, - maxIterations: 1, transportRetries: 0, - ...(secrets?.length ? { secrets } : {}), - ...(verification === undefined ? {} : { verification }) }); - if (!output || typeof output !== 'object' || !('completionReason' in output)) { - throw new AuthoredFlowExecutionError('journal_protocol_violation', 'Broker step produced an invalid result'); - } - return output as AgentResult; - }; - const lookup = (name: string): Declaration => declarations.get(name) ?? invalid(`No retained broker agent named ${name} in this flow`); +/** Compatibility boundary for authored flows compiled before broker retirement. */ +export function authoredBroker( + _definition: { name: string }, + _rootRunId: string | undefined, + _localStream: string | undefined, + _run: (step: StepSpec) => Promise, +) { + const refused = (): never => { throw new AuthoredFlowExecutionError('agent_cli_unresolved', RETIRED); }; return { - async agent(id: string, name: string, options: AgentOptions, verification?: NamedGate, - secrets?: string[]): Promise { - const problem = brokerOptionsError(options); - if (problem) invalid(`f.agent options: ${problem}`); - if (options.keepAlive && localStream === undefined) invalid('Retained broker agents require a local agent worker stream'); - if (options.keepAlive && verification !== undefined) invalid('Retained broker agents do not support gates; validate their result in a later step'); - if (typeof options.task !== 'string' || !options.task.trim()) invalid('f.agent task must be nonempty'); - if (options.workspace !== undefined || options.permissions !== undefined) invalid('Broker agents use cwd; workspace and permissions are not supported'); - if ((options.maxIterations ?? 1) !== 1 || (options.transportRetries ?? 0) !== 0 - || options.recoveryMode !== undefined) invalid('Broker sessions require maxIterations 1, transportRetries 0, and no recoveryMode'); - if (typeof name !== 'string' || !name.trim()) invalid('Broker agent name must be nonempty'); - const instruction: BrokerInstruction = { type: BROKER_TYPE, operation: 'spawn', owner, - flow: definition.name, step: name, agentName: brokerAgentName(definition.name, name, owner, id), - task: options.task, cwd: options.cwd, skills: options.skills ?? [], - channel: options.channel ?? `flow-${owner}`, keepAlive: options.keepAlive ?? false, - timeoutMs: options.timeout === undefined ? 3_600_000 : parseAgentStepTimeout(options.timeout) }; - const declaration = { instruction, cli: options.cli, model: options.model, - ...(secrets?.length ? { secrets } : {}) }; - const result = await execute(id, declaration, verification); - if (instruction.keepAlive && result.completionReason === 'success') declarations.set(instruction.agentName, declaration); - return result; - }, - async message(id: string, name: string, text: string, options?: { timeout?: string | number }, - verification?: NamedGate): Promise { - const declaration = lookup(name); - if (typeof text !== 'string' || !text.trim()) invalid('f.message text must be nonempty'); - const result = await execute(id, { ...declaration, instruction: { ...declaration.instruction, - operation: 'message', task: text, timeoutMs: options?.timeout === undefined - ? declaration.instruction.timeoutMs : parseAgentStepTimeout(options.timeout) } }, verification); - return result; - }, - async release(id: string, name: string): Promise { - const declaration = lookup(name); - await run({ id, type: 'agent', cli: declaration.cli, model: declaration.model, - instruction: JSON.stringify({ ...declaration.instruction, operation: 'release' }), - surfaces: localStream === undefined ? undefined : { streams: [{ stream: localStream }] }, - ...(declaration.secrets?.length ? { secrets: declaration.secrets } : {}), - maxIterations: 1, transportRetries: 0 }); - // Keep the declaration so an explicit release remains idempotent. - }, + agent: async (_id: string, _name: string, _options: AgentOptions, _verification?: NamedGate, + _secrets?: string[]): Promise => refused(), + message: async (_id: string, _name: string, _text: string, _options?: { timeout?: string | number }, + _verification?: NamedGate): Promise => refused(), + release: async (_id: string, _name: string): Promise => refused(), }; } diff --git a/packages/sdk/src/cli-commands.ts b/packages/sdk/src/cli-commands.ts index f7ea48e52..7d48525a1 100644 --- a/packages/sdk/src/cli-commands.ts +++ b/packages/sdk/src/cli-commands.ts @@ -95,6 +95,10 @@ const LOCAL_EXECUTION_OPTIONS = [ description: `With --local-agent, how many agent steps (and, separately, LLM steps) run at once (1-${MAX_LOCAL_AGENT_CAPACITY})`, defaultValue: String(DEFAULT_LOCAL_AGENT_CAPACITY), }, + { + flags: '--service-caller-file ', + description: 'With --local-agent, use this explicit Desktop schema-v2 service caller for relay agent steps', + }, { flags: '--no-spawn', description: 'Require a running relayflowd rather than starting one' }, { flags: '--no-observer-link', description: 'Do not mint an observer link for this run' }, { flags: '--cloud-mirror', description: 'Also put this run on the Cloud dashboard (also FLOWS_CLOUD_MIRROR=1)' }, diff --git a/packages/sdk/src/cli.ts b/packages/sdk/src/cli.ts index 3ba441e0b..bd6d24bf7 100644 --- a/packages/sdk/src/cli.ts +++ b/packages/sdk/src/cli.ts @@ -22,6 +22,7 @@ import { type CloudMirrorReceipt, } from './cli/cloud-mirror-session.js'; import { realpathSync } from 'node:fs'; +import { isAbsolute } from 'node:path'; import { pathToFileURL } from 'node:url'; import { checkFlow, @@ -115,8 +116,8 @@ export type ParsedArgs = | { command: 'schedules'; json: boolean } | { command: 'unschedule'; scheduleId: string; json: boolean } | { command: 'check'; json: boolean; watch: boolean; explainWarnings: boolean; daemonValidation: DaemonValidationMode; value: string } - | { command: 'run'; detach?: boolean; bucket: string | undefined; reuseFromRunId: string | undefined; localAgent: boolean; agentCapacity: number | undefined; dataDir: string; input: string | undefined; json: boolean; spawn: boolean; noObserverLink: boolean; cloudMirror: boolean; allowHumanInfluenced: boolean; value: string } - | { command: 'resume'; detach?: boolean; localAgent: boolean; agentCapacity: number | undefined; dataDir: string; json: boolean; spawn: boolean; noObserverLink: boolean; cloudMirror: boolean; allowHumanInfluenced: boolean; value: string } + | { command: 'run'; detach?: boolean; bucket: string | undefined; reuseFromRunId: string | undefined; localAgent: boolean; agentCapacity: number | undefined; serviceCallerFile: string | undefined; dataDir: string; input: string | undefined; json: boolean; spawn: boolean; noObserverLink: boolean; cloudMirror: boolean; allowHumanInfluenced: boolean; value: string } + | { command: 'resume'; detach?: boolean; localAgent: boolean; agentCapacity: number | undefined; serviceCallerFile: string | undefined; dataDir: string; json: boolean; spawn: boolean; noObserverLink: boolean; cloudMirror: boolean; allowHumanInfluenced: boolean; value: string } | { command: 'cloud-answer'; runId: string; answer: boolean; json: boolean; note?: string; source?: string } | { command: 'answer'; dataDir: string; json: boolean; spawn: boolean; note: string | undefined; by: string | undefined; runId: string; waitId: string; answer: boolean } | RunsArgs @@ -151,14 +152,14 @@ const USAGE = [ 'flows run @sha256: [--bucket ] [--data-dir ] [--json] [--detach]', 'flows check [--against-daemon|--no-daemon-check] [--watch] [--json] [--explain-warnings] ', 'flows serve-webhook --data-dir --port

[--allow [,]]', - 'flows run [--json] [--no-spawn] [--no-observer-link] [--cloud-mirror] [--detach] [--data-dir

] [--local-agent [--agent-capacity ]] [--reuse-from ] ', + 'flows run [--json] [--no-spawn] [--no-observer-link] [--cloud-mirror] [--detach] [--data-dir ] [--local-agent [--agent-capacity ] [--service-caller-file ]] [--reuse-from ] ', 'flows run --cloud [--json] [--wait] [--sync-code] [--no-connect] [--budget-minutes ] ', 'flows run --cloud [--json] [--wait] [--sync-code] [--no-connect] [--budget-minutes ] --input ', 'flows sync [--json] [--dry-run] [--dir ] ', - 'flows run [--json] [--no-spawn] [--no-observer-link] [--cloud-mirror] [--detach] [--data-dir ] [--local-agent [--agent-capacity ]] --input ', + 'flows run [--json] [--no-spawn] [--no-observer-link] [--cloud-mirror] [--detach] [--data-dir ] [--local-agent [--agent-capacity ] [--service-caller-file ]] --input ', 'flows run [--input ]', 'flows tick start --schedule-id --interval-ms [--epoch-ms ] [--max-catch-up ] [--poll-interval-ms ] [--data-dir ] ', - 'flows resume [--allow-human-influenced] [--json] [--no-spawn] [--no-observer-link] [--cloud-mirror] [--detach] [--data-dir ] [--local-agent [--agent-capacity ]] ', + 'flows resume [--allow-human-influenced] [--json] [--no-spawn] [--no-observer-link] [--cloud-mirror] [--detach] [--data-dir ] [--local-agent [--agent-capacity ] [--service-caller-file ]] ', 'flows answer --cloud [--json] [--note ] [--source ] ', 'flows answer [--json] [--no-spawn] [--data-dir ] [--note ] [--by ] ', 'flows replay [--allow-human-influenced] [--json] [--data-dir ] [--at ]', @@ -445,6 +446,7 @@ export async function runCli( ...(parsed.command === 'run' && parsed.reuseFromRunId !== undefined ? { reuseFromRunId: parsed.reuseFromRunId } : {}), localAgent: parsed.localAgent, ...(parsed.agentCapacity === undefined ? {} : { agentCapacity: parsed.agentCapacity }), + ...(parsed.serviceCallerFile === undefined ? {} : { serviceCallerFile: parsed.serviceCallerFile }), onProgress: showProgress, ...(observer === undefined && mirror === undefined && options.detachedReceipt === undefined ? {} : { onJournalEntry: (entry: JournalEvent) => { @@ -761,6 +763,7 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { let localAgent = false; let detach = false; let agentCapacity: number | undefined; + let serviceCallerFile: string | undefined; let allowHumanInfluenced = false; let dataDir = DEFAULT_DATA_DIR; let sawDataDir = false; @@ -813,6 +816,13 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { if (!isAgentCapacity(agentCapacity)) return undefined; continue; } + if (argument === '--service-caller-file') { + const value = args[++index]; + if (command === 'check' || serviceCallerFile !== undefined || value === undefined + || value.startsWith('-') || value.includes('\0') || !isAbsolute(value)) return undefined; + serviceCallerFile = value; + continue; + } if (argument === '--against-daemon' || argument === '--no-daemon-check') { if (command !== 'check' || daemonValidation !== undefined) return undefined; daemonValidation = argument === '--against-daemon' ? 'required' : 'off'; @@ -890,6 +900,7 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { if (positionals.length !== 1) return undefined; // It sizes the in-process worker, so without `--local-agent` it describes nothing. if (agentCapacity !== undefined && !localAgent) return undefined; + if (serviceCallerFile !== undefined && !localAgent) return undefined; if (bucket !== undefined && (cloud || !parseDigestReference(positionals[0]!))) return undefined; if (cloud) { @@ -898,7 +909,7 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { // observer-link opt-out -- describes nothing there and is refused rather // than ignored. `--input` is the authored body's argument and travels with // the source, so it is accepted exactly where a local run accepts it. - if (detach || allowHumanInfluenced || sawDataDir || !spawn || localAgent || noObserverLink || cloudMirror + if (detach || allowHumanInfluenced || sawDataDir || !spawn || localAgent || serviceCallerFile !== undefined || noObserverLink || cloudMirror || reuseFromRunId !== undefined) return undefined; if (sawInput && !isAuthoredFlowPath(positionals[0]!)) return undefined; return { command: 'cloud-run', value: positionals[0]!, json, wait, input, syncCode, noConnect, budgetMinutes }; @@ -912,8 +923,8 @@ function parseArgs(args: readonly string[]): ParsedArgs | undefined { daemonValidation: daemonValidation ?? (process.env['FLOWS_NO_DAEMON_CHECK'] === '1' ? 'off' : process.env['FLOWS_CHECK_AGAINST_DAEMON'] === '1' ? 'required' : 'auto') } : command === 'run' - ? { command, ...(detach ? { detach } : {}), bucket, reuseFromRunId, localAgent, agentCapacity, dataDir, input, json, spawn, noObserverLink, cloudMirror, allowHumanInfluenced, value: positionals[0]! } - : { command, ...(detach ? { detach } : {}), localAgent, agentCapacity, dataDir, json, spawn, noObserverLink, cloudMirror, allowHumanInfluenced, value: positionals[0]! }; + ? { command, ...(detach ? { detach } : {}), bucket, reuseFromRunId, localAgent, agentCapacity, serviceCallerFile, dataDir, input, json, spawn, noObserverLink, cloudMirror, allowHumanInfluenced, value: positionals[0]! } + : { command, ...(detach ? { detach } : {}), localAgent, agentCapacity, serviceCallerFile, dataDir, json, spawn, noObserverLink, cloudMirror, allowHumanInfluenced, value: positionals[0]! }; } /** diff --git a/packages/sdk/src/cli/check.ts b/packages/sdk/src/cli/check.ts index d4466bcda..20479539d 100644 --- a/packages/sdk/src/cli/check.ts +++ b/packages/sdk/src/cli/check.ts @@ -541,9 +541,17 @@ function bindResolvedCliPaths( if (executable !== undefined && isAbsolute(executable)) { identities.set(step.id, resolution.cli); } - return { ...step, cli: boundCli, - ...(resolution.modelSource === 'adapter' && resolution.model !== undefined - ? { model: resolution.model } : {}) }; + // Relay nodes own their harness default. Materializing the local CLI + // adapter's default here turns an omitted model into a placement + // override; Desktop correctly refuses that override. Explicit step + // models remain on `step`; named-agent models remain in the declaration + // map and are materialized by `toKernelSpec` at the journal boundary. + const adapterModel = resolution.modelSource === 'adapter' + && resolution.model !== undefined + && (step.type !== 'agent' || step.transport !== 'relay') + ? { model: resolution.model } + : {}; + return { ...step, cli: boundCli, ...adapterModel }; }), }; return rememberResolvedCliIdentities(bound, identities); diff --git a/packages/sdk/src/cli/direct-run.ts b/packages/sdk/src/cli/direct-run.ts index 4e28ecd4f..9b9cff50f 100644 --- a/packages/sdk/src/cli/direct-run.ts +++ b/packages/sdk/src/cli/direct-run.ts @@ -87,7 +87,7 @@ export async function runDirectFlow( if (options.localAgent) { localAgent = await attachLocalAgent( client, dataDir, options.onPtyReady, authoredLocalAgentStream(admissionIdentity), workerCapacity, - undefined, [], agentEnvironment, + undefined, [], agentEnvironment, options.serviceCallerFile, ); // A session owns one worker registration. Keep the workspace-free LLM // worker on its own connection so it cannot replace the agent worker. diff --git a/packages/sdk/src/cli/run.ts b/packages/sdk/src/cli/run.ts index 33777dbf0..297c680e1 100644 --- a/packages/sdk/src/cli/run.ts +++ b/packages/sdk/src/cli/run.ts @@ -161,6 +161,8 @@ export interface RunLifecycleOptions { localAgent?: boolean; /** `--agent-capacity`: the local workers' concurrency; the default is `DEFAULT_LOCAL_AGENT_CAPACITY`. */ agentCapacity?: number; + /** Absolute Desktop schema-v2 caller.json path for local relay agent steps. */ + serviceCallerFile?: string; signal?: AbortSignal; onWait?: (progress: RunProgress) => void; /** @@ -235,7 +237,7 @@ async function executeCheckedFlow( if (options.localAgent) { localAgent = await attachLocalAgent( client, dataDir, options.onPtyReady, undefined, options.agentCapacity, - undefined, declaredLocalAgentStreams(spec), agentEnvironment, + undefined, declaredLocalAgentStreams(spec), agentEnvironment, options.serviceCallerFile, ); } if (options.localAgent && spec.steps.some(step => step.type === 'agent' && communicationInstruction(step.instruction))) { @@ -356,7 +358,7 @@ export async function resumeFlow( if (options.localAgent) { authoredAgent = await attachLocalAgent( client, dataDir, options.onPtyReady, authoredRoot.localAgentStream, workerCapacity, - undefined, [], agentEnvironment, + undefined, [], agentEnvironment, options.serviceCallerFile, ); authoredLlmClient = new JournalClient(socketPath); await authoredLlmClient.connect(); @@ -395,7 +397,7 @@ export async function resumeFlow( if (!spec) throw new Error('Cannot attach a local worker: the journaled run spec is missing'); authoredAgent = await attachLocalAgent( client, dataDir, options.onPtyReady, undefined, options.agentCapacity, - undefined, declaredLocalAgentStreams(spec), agentEnvironment, + undefined, declaredLocalAgentStreams(spec), agentEnvironment, options.serviceCallerFile, ); if (spec?.steps.some(step => step.type === 'agent' && communicationInstruction(step.instruction))) { const { attachCommunicationWorkers } = await import('../communication/local.js'); diff --git a/packages/sdk/src/cli/running-step.ts b/packages/sdk/src/cli/running-step.ts index 2b3ca12cf..315bfccd1 100644 --- a/packages/sdk/src/cli/running-step.ts +++ b/packages/sdk/src/cli/running-step.ts @@ -1,3 +1,4 @@ +import { AsyncResource } from 'node:async_hooks'; import { JournalFrameError, JournalProtocolError, type JournalClient } from '../journal-client.js'; import { AuthoredFlowExecutionError } from '../authored-flow-error.js'; import type { JournalEvent } from '../journal-reader.js'; @@ -8,6 +9,14 @@ const LEASE_POLL_MS = 2_000; const STALE_LEASE_MS = 30_000 + LEASE_SWEEP_GRACE_MS; export interface RunningStep extends ParkedStep { leaseDeadlineMs: number } +// Completion deliberately aborts a lease snapshot that is no longer needed. +// The snapshot transport is runtime work, not an authored callback: create its +// promises in this module-load context while the enclosing wait remains visible +// to the authored lifecycle and still propagates every real read failure. +const leaseSnapshotContext = new AsyncResource('RelayflowsLeaseSnapshot'); +const readRunSnapshot = (client: JournalClient, runId: string, signal?: AbortSignal) => + leaseSnapshotContext.runInAsyncScope(() => client.runGet(runId, signal === undefined ? {} : { signal })); + /** Watch completion; snapshots only refresh the non-journaled live lease deadline. */ export async function waitForRunningStep(client: JournalClient, runId: string, runningStep: RunningStep, options: RunLifecycleOptions): Promise { @@ -82,7 +91,7 @@ export async function waitForRunningStep(client: JournalClient, runId: string, `could not establish the completion watch: ${error instanceof Error ? error.message : String(error)}`); }), completed]); if (unconfirmed !== undefined) { - const snapshot = await client.runGet(runId, options.signal === undefined ? {} : { signal: options.signal }); + const snapshot = await readRunSnapshot(client, runId, options.signal); if (snapshot.steps[runningStep.id]?.state !== 'running') return; throw new AuthoredFlowExecutionError('daemon_unresponsive', `could not establish the completion watch: ${unconfirmed instanceof Error ? unconfirmed.message : String(unconfirmed)}`); @@ -105,7 +114,7 @@ export async function waitForRunningStep(client: JournalClient, runId: string, const read = snapshotRead = new AbortController(); const signal = options.signal === undefined ? read.signal : AbortSignal.any([options.signal, read.signal]); - const snapshot = await client.runGet(runId, { signal }) + const snapshot = await readRunSnapshot(client, runId, signal) .catch(error => { if (options.signal?.aborted) throw canceled(); if (protocolFailure !== undefined) throw protocolFailure; diff --git a/packages/sdk/src/communication/broker-preflight.ts b/packages/sdk/src/communication/broker-preflight.ts index 514e0db60..51984e286 100644 --- a/packages/sdk/src/communication/broker-preflight.ts +++ b/packages/sdk/src/communication/broker-preflight.ts @@ -1,22 +1,6 @@ -import { createRequire } from 'node:module'; -import { spawnSync } from 'node:child_process'; -import { pathToFileURL } from 'node:url'; - -const require = createRequire(import.meta.url); - -/** Read-only SDK connection probe: never starts a broker or an agent harness. */ -export function checkBrokerEnvironment(cwd = process.cwd()): void { - let driver: string; - try { - require.resolve('@agent-relay/sdk'); - driver = require.resolve('@agent-relay/harness-driver'); - } catch { - throw new Error('Broker agents require optional @agent-relay/harness-driver and @agent-relay/sdk packages (>=12.3.1).'); - } - const probe = spawnSync(process.execPath, ['--input-type=module', '-e', - 'const { HarnessDriverClient } = await import(process.argv[1]); const client = HarnessDriverClient.connect({ cwd: process.argv[2] }); try { await client.healthCheck(); } finally { await client.shutdown(); }', - pathToFileURL(driver).href, cwd], { encoding: 'utf8', timeout: 10_000, maxBuffer: 64 * 1024 }); - if (probe.status !== 0) { - throw new Error(`Broker agents require a reachable local Relay broker for ${JSON.stringify(cwd)}. Run agent-relay up in the run root or set AGENT_RELAY_STATE_DIR to its connection directory.`); - } +/** Compatibility preflight for the retired `transport: broker` surface. */ +export function checkBrokerEnvironment(_cwd = process.cwd()): never { + throw new Error( + 'transport: broker was removed. Use transport: relay for one-shot fleet execution; retained local sessions remain fail-closed.', + ); } diff --git a/packages/sdk/src/communication/broker-worker.ts b/packages/sdk/src/communication/broker-worker.ts deleted file mode 100644 index 9940b9595..000000000 --- a/packages/sdk/src/communication/broker-worker.ts +++ /dev/null @@ -1,198 +0,0 @@ -import { realpath, stat } from 'node:fs/promises'; -import { resolve } from 'node:path'; -import type { JournalClient } from '../journal-client.js'; -import type { StepDispatchEvent } from '../protocol.js'; -import type { ResolvedKernelAgentStep } from '../resolved-cli-identity.js'; -import { isLeaseLost, withWorkerLease } from '../worker-lease.js'; -import { isReadInterruptionError } from '../journal-read-policy.js'; -import { workerInstruction } from '../worker-input.js'; -import { connectLocalBroker, type LocalBroker } from './relay.js'; -import { spawnFlowPty, shellQuote } from './pty.js'; -import { installBrokerSkills } from './skills.js'; -import { openCommunicationTools } from './tools.js'; -import type { BrokerInstruction } from './broker-spec.js'; - -interface Turn { - id: string; - completed?: boolean; - resolve(summary: string): void; - reject(error: unknown): void; -} -interface Session { - owner: string; - broker: LocalBroker; - pty?: Awaited>; - tools?: Awaited>; - turn?: Turn; - unsubscribe?: () => void; - exited?: string; - releasing?: Promise; -} - -/** Worker-owned live resources. Only journaled agent dispatches start a turn. */ -export class BrokerSessions { - private readonly sessions = new Map(); - constructor(private readonly runRoot: string = process.cwd(), private readonly environment?: NodeJS.ProcessEnv) {} - - async close(): Promise { - const results = await Promise.allSettled([...this.sessions.keys()].map(name => this.release(name))); - const errors = results.flatMap(result => result.status === 'rejected' ? [result.reason] : []); - if (errors.length) throw new AggregateError(errors, 'Failed to release flow broker sessions'); - } - - private async release(name: string): Promise { - const session = this.sessions.get(name); - if (!session) return; - if (session.releasing) return session.releasing; - session.releasing = Promise.resolve().then(() => this.releaseSession(name, session)); - try { await session.releasing; } - finally { session.releasing = undefined; } - } - - private async releaseSession(name: string, session: Session): Promise { - session.turn?.reject(new Error('Broker session released')); - await session.pty?.handle.release('Flow broker session ended', { deleteIdentity: true }); - session.unsubscribe?.(); - try { await session.tools?.close(); } - finally { try { await session.pty?.close(); } finally { await session.broker.shutdown(); } } - this.sessions.delete(name); - } - - async complete(client: JournalClient, dispatch: StepDispatchEvent, instruction: BrokerInstruction, dataDir: string): Promise { - let output: unknown; - let reason: 'success' | 'worker_error' | 'timeout' = 'success'; - try { - output = await withWorkerLease(client, dispatch, signal => this.run(dispatch, instruction, dataDir, signal, - client.supportsFeature?.('step_secret_scoping') ?? true)); - } catch (error) { - if (isLeaseLost(error) || isReadInterruptionError(error)) throw error; - reason = error instanceof Error && error.name === 'TimeoutError' ? 'timeout' : 'worker_error'; - output = { error: error instanceof Error ? error.message : String(error) }; - } - try { - await client.stepComplete(dispatch.run_id, dispatch.step_id, dispatch.attempt, dispatch.idempotency_key, - reason, { output, usage: { tokens_in: 0, tokens_out: 0, dollars_unmetered: true }, - started_pins: dispatch.pins, end_pins: dispatch.pins }); - } catch (error) { - await this.release(instruction.agentName); - throw error; - } - } - - private async run(dispatch: StepDispatchEvent, instruction: BrokerInstruction, dataDir: string, lease: AbortSignal, - scopeSecrets = true): Promise { - const { agentName: name, owner } = instruction; - let session = this.sessions.get(name); - if (session && session.owner !== owner) throw new Error('Broker agent belongs to another flow'); - if (session?.turn) throw new Error('Broker agent already has work in progress'); - if (instruction.operation === 'release') { - await this.release(name); - return { released: true }; - } - if (session?.exited) { - await this.release(name); - throw new Error(`Broker agent ${name}: ${session.exited}`); - } - if (instruction.operation === 'message' && !session) { - throw new Error(`Broker agent ${name} is no longer available; its worker session was lost or released`); - } - if (instruction.operation === 'spawn' && session) throw new Error(`Broker agent ${name} already exists`); - const deadline = AbortSignal.timeout(instruction.timeoutMs); - const signal = AbortSignal.any([lease, deadline]); - const turnId = `${dispatch.run_id}:${dispatch.step_id}:${dispatch.attempt}`; - let succeed!: (summary: string) => void; - let fail!: (error: unknown) => void; - const result = new Promise((yes, no) => { succeed = yes; fail = no; }); - void result.catch(() => {}); - let abortWait!: (error: unknown) => void; - const aborted = new Promise((_, reject) => { abortWait = reject; }); - void aborted.catch(() => {}); - const abort = () => { fail(signal.reason); abortWait(signal.reason); }; - signal.addEventListener('abort', abort, { once: true }); - if (signal.aborted) abort(); - let succeeded = false; - try { - if (instruction.operation === 'spawn') { - const spec = dispatch.spec as ResolvedKernelAgentStep; - if (!spec.cli) throw new Error('Broker agent requires a declared CLI'); - const cwd = await realpath(resolve(this.runRoot, instruction.cwd ?? '.')); - if (!(await stat(cwd)).isDirectory()) throw new Error(`Broker cwd is not a directory: ${cwd}`); - signal.throwIfAborted(); - await installBrokerSkills(instruction.skills, spec.cli_identity ?? spec.cli, cwd, signal); - const broker = await connectLocalBroker(this.runRoot); - session = { owner, broker, turn: { id: turnId, resolve: succeed, reject: fail } }; - this.sessions.set(name, session); - const current = session; - session.tools = await openCommunicationTools(async ({ operation, values }) => { - if (operation !== 'complete' || values.length !== 2 || values[0] !== current.turn?.id || current.turn?.completed) { - throw new Error('Use complete CURRENT_TURN_ID JSON_SUMMARY; stale and duplicate completions are refused'); - } - const summary = JSON.stringify(JSON.parse(values[1]!)); - const turn = current.turn; - turn!.completed = true; - // Allow the helper's response to flush before releasing the PTY. - setTimeout(() => turn!.resolve(summary), 100); - return { completed: true }; - }); - session.unsubscribe = broker.onEvent(event => { - if (event.name !== name || !['agent_exit', 'agent_exited', 'agent_released'].includes(event.kind)) return; - current.exited = event.kind; - if (!current.turn?.completed) current.turn?.reject(new Error(`Broker agent ${name}: ${event.kind}`)); - // An idle exit retires resources immediately. A failed release stays - // in the map and is retried (and surfaced) on the next call or close. - if (!current.turn) void this.release(name).catch(() => {}); - }); - broker.connectEvents(); - const task = `${workerInstruction(instruction.task, dispatch)}\n\n${this.completionPrompt(session, turnId)}\n${instruction.keepAlive ? 'Stay alive after completing; later flow steps can send more work.' : 'The flow releases this session after completion.'}`; - const spawned = spawnFlowPty(broker, { name, cli: spec.cli, identity: spec.cli_identity, - model: spec.model, task, cwd, dataDir, skipRelayPrompt: false, channels: [instruction.channel], team: `flow-${owner}`, - environment: this.environment, socket: session.tools.path, token: session.tools.token, - secrets: spec.secrets, scopeSecrets, - metadata: { runId: owner, flow: instruction.flow, step: instruction.step } }); - let pty: Awaited> | undefined; - try { - pty = await Promise.race([spawned, aborted]); - signal.throwIfAborted(); - session.pty = pty; - } catch (error) { - // Spawn can resolve after the timeout. The session release below - // only sees a handle that was stored in time. - if (session.pty === undefined) void spawned.then(async late => { - try { await late.handle.release('Flow broker session ended', { deleteIdentity: true }); } - catch { /* The timed-out step is already journaled. */ } - try { await late.close(); } - catch { /* A failed link removal must not reject this detached cleanup. */ } - }, () => {}); - throw error; - } - const ready = await Promise.race([session.pty.handle.waitForReady(Math.min(instruction.timeoutMs, 90_000)), - result.then(() => ({ reason: 'ready' }))]); - if (ready.reason !== 'ready') throw new Error(`Broker agent did not become ready: ${ready.reason}`); - } else { - session!.turn = { id: turnId, resolve: succeed, reject: fail }; - signal.throwIfAborted(); - const receipt = await Promise.race([session!.broker.sendMessage({ to: name, from: `flow-${owner}`, - text: `${instruction.task}\n\n${this.completionPrompt(session!, turnId)}`, mode: 'wait', - data: { runId: owner, flow: instruction.flow, step: instruction.step, turnId } }), aborted]); - if (!receipt.targets.includes(name) || receipt.event_id === 'unsupported_operation') { - throw new Error(`Broker did not accept the message for ${name}`); - } - } - const summary = await result; - signal.throwIfAborted(); - succeeded = true; - return { completionReason: 'success', summary, artifacts: [], agentName: name }; - } catch (error) { - if (signal.aborted) throw signal.reason; - throw error; - } finally { - signal.removeEventListener('abort', abort); - if (session) session.turn = undefined; - if (!succeeded || !instruction.keepAlive || session?.exited) await this.release(name); - } - } - - private completionPrompt(session: Session, turn: string): string { - return `Report completion with your shell tool: node ${shellQuote(session.tools!.helperPath)} complete ${shellQuote(turn)} 'JSON_SUMMARY'. The summary must be valid JSON (object, array, or string). Call exactly once for this turn, after the work is finished.`; - } -} diff --git a/packages/sdk/src/communication/environment.ts b/packages/sdk/src/communication/environment.ts index 7a65d450d..d1dfdb97a 100644 --- a/packages/sdk/src/communication/environment.ts +++ b/packages/sdk/src/communication/environment.ts @@ -53,7 +53,7 @@ export function agentEnvironment(cli: string, source: NodeJS.ProcessEnv = proces const platform = platformAgentNames(source, cliName); const allowed = scopeSecrets ? validateStepSecrets(grants, source) : []; if (allowed.some(name => relayNames.includes(name))) { - throw new Error('Relay broker credentials cannot be granted to a communication agent'); + throw new Error('Agent Relay control credentials cannot be granted to a communication agent'); } for (const name of names ?? []) { if (!allowed.includes(name) && !platform.has(name) diff --git a/packages/sdk/src/communication/preflight.ts b/packages/sdk/src/communication/preflight.ts index c3abfa5f6..72a7104b0 100644 --- a/packages/sdk/src/communication/preflight.ts +++ b/packages/sdk/src/communication/preflight.ts @@ -1,34 +1,12 @@ -import { accessSync, constants } from 'node:fs'; -import { createRequire } from 'node:module'; -import { spawnSync } from 'node:child_process'; -import { pathToFileURL } from 'node:url'; import type { FlowSpec, KernelRunSpec } from '../spec.js'; import { communicationInstruction } from './spec.js'; -const require = createRequire(import.meta.url); export class CommunicationEnvironmentError extends Error {} -/** Local-only probes: no broker, workspace creation, or optional runtime import in this process. */ +/** Refuse before admission; no hidden fallback may restart the retired broker. */ export function checkCommunicationEnvironment(spec: FlowSpec | KernelRunSpec): void { if (!spec.steps.some(step => step.type === 'agent' && communicationInstruction(step.instruction))) return; - if (!process.env.RELAY_API_KEY?.trim().startsWith('rk_live_')) { - throw new CommunicationEnvironmentError('Agent communication requires RELAY_API_KEY for an existing workspace.'); - } - let driverPath: string; - try { - require.resolve('@agent-relay/sdk'); - require.resolve('@agent-relay/harness-driver'); - driverPath = require.resolve('@agent-relay/harness-driver/broker-path'); - } catch { - throw new CommunicationEnvironmentError('Agent communication requires optional @agent-relay/harness-driver and @agent-relay/sdk packages (>=12.3.1).'); - } - const probe = spawnSync('node', ['--input-type=module', '-e', - 'const {getBrokerBinaryPath} = await import(process.argv[1]); const path = process.env.RELAYFLOW_RELAY_BROKER_BIN || getBrokerBinaryPath(); if (!path) process.exit(1); process.stdout.write(path);', - pathToFileURL(driverPath).href], { encoding: 'utf8', timeout: 10_000, maxBuffer: 64 * 1024 }); - try { - if (probe.status !== 0 || !probe.stdout.trim()) throw new Error(); - accessSync(probe.stdout.trim(), constants.X_OK); - } catch { - throw new CommunicationEnvironmentError('Agent communication requires Node.js and an executable Relay broker; install Agent Relay or set RELAYFLOW_RELAY_BROKER_BIN.'); - } + throw new CommunicationEnvironmentError( + 'Agent communication through broker/PTy was removed. Native node-spawn communication requires the pending authorized local service-caller contract.', + ); } diff --git a/packages/sdk/src/communication/pty.ts b/packages/sdk/src/communication/pty.ts deleted file mode 100644 index 88fb31075..000000000 --- a/packages/sdk/src/communication/pty.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { mkdir, mkdtemp, rm, symlink } from 'node:fs/promises'; -import { basename, join } from 'node:path'; -import { prepareDevinRuntime } from '../devin-runtime.js'; -import { agentEnvironment } from './environment.js'; -import { resolveCliModel } from '../cli-adapter.js'; -import type { RelayBroker } from './relay.js'; - -export const shellQuote = (value: string) => "'" + value.replaceAll("'", "'\"'\"'") + "'"; - -/** Both communication steps and retained sessions use Relay's PTY launch contract. */ -export async function spawnFlowPty(broker: Pick, options: { - name: string; cli: string; identity?: string; model?: string; task: string; - skipRelayPrompt?: boolean; - cwd: string; dataDir: string; channels: string[]; team?: string; - environment?: NodeJS.ProcessEnv; socket: string; token: string; - secrets?: string[]; scopeSecrets?: boolean; - metadata?: { runId: string; flow: string; step: string }; -}) { - const identity = options.identity ?? options.cli; - const argv0 = basename(identity); - let command = options.cli; - let link: string | undefined; - const environment = agentEnvironment(identity, options.environment ?? process.env, - options.secrets, options.scopeSecrets ?? true); - const prepared = argv0.replace(/\.exe$/i, '') === 'devin' ? prepareDevinRuntime(environment) : undefined; - try { - if (basename(command) !== argv0) { - const root = join(options.dataDir, 'communication'); - await mkdir(root, { recursive: true, mode: 0o700 }); - link = await mkdtemp(join(root, 'cli-')); - command = join(link, argv0); - await symlink(options.cli, command, 'file'); - } - const handle = await broker.spawnPty({ name: options.name, cli: argv0.replace(/\.exe$/i, ''), - task: options.task, channels: options.channels, skipRelayPrompt: options.skipRelayPrompt ?? true, - ...(options.team === undefined ? {} : { team: options.team }), - model: resolveCliModel(identity, options.model), cwd: options.cwd, - harnessConfig: { runtime: 'pty', command: shellQuote(command), args: [], cwd: options.cwd, - env: { ...(prepared?.env ?? environment), - RELAYFLOW_COMMUNICATION_SOCKET: options.socket, RELAYFLOW_COMMUNICATION_TOKEN: options.token, - ...(options.metadata === undefined ? {} : { - RELAYFLOW_RUN_ID: options.metadata.runId, RELAYFLOW_FLOW_NAME: options.metadata.flow, - RELAYFLOW_STEP_NAME: options.metadata.step, - }) }, delivery: { mode: 'pty-injection', format: 'relay-block' } } }); - return { handle, close: async () => { prepared?.close(); if (link) await rm(link, { recursive: true, force: true }); } }; - } catch (error) { - prepared?.close(); - if (link) await rm(link, { recursive: true, force: true }); - throw error; - } -} diff --git a/packages/sdk/src/communication/relay.ts b/packages/sdk/src/communication/relay.ts index 89dd6d9ce..478e56ce8 100644 --- a/packages/sdk/src/communication/relay.ts +++ b/packages/sdk/src/communication/relay.ts @@ -1,113 +1,39 @@ -import { randomUUID, createHash } from 'node:crypto'; -import { mkdir } from 'node:fs/promises'; -import { join } from 'node:path'; -import { createProjection } from './projection.js'; -import { brokerEnvironment } from './environment.js'; - -export interface RelayEvent { kind: string; name?: string; event_id?: string; delivery_id?: string; verification?: string; [key: string]: unknown } -export interface RelayHandle { - name: string; - generation?: string; - waitForReady(timeout?: number): Promise<{ reason: string }>; - release(reason?: string, options?: { deleteIdentity?: boolean }): Promise; -} -export interface RelayBroker { - spawnPty(input: Record): Promise; - onEvent(callback: (event: RelayEvent) => void): () => void; - shutdown(): Promise; -} +/** + * Compatibility types for journal communication helpers. Runtime broker + * construction was intentionally removed; no call here starts a daemon. + */ export interface RelayMessaging { - agents: { register(input: { name: string; type: string }): Promise<{ token: string }>; delete(name: string): Promise }; - channels: { create(input: { name: string }): Promise; get(name: string): Promise; join(name: string): Promise }; messages: { - direct(input: { to: string; text: string; idempotencyKey: string; metadata: Record; mode: 'wait' }): Promise<{ id: string }>; - send(input: { channel: string; text: string; idempotencyKey: string; metadata: Record }): Promise; + direct(input: Record): Promise; + send(input: Record): Promise; + }; + channels: { + create(input: Record): Promise; + get(name: string): Promise; + join(name: string): Promise; }; } -interface RelayModules { - HarnessDriverClient: { spawn(options: Record): Promise; - connect(options: { cwd: string }): LocalBroker }; - RelaycastMessagingClient: new (options: Record) => RelayMessaging; -} -export async function loadRelayModules(): Promise { - // Variable imports keep optional packages out of ordinary execution and bundles. - const driver = '@agent-relay/harness-driver'; - const sdk = '@agent-relay/sdk'; - try { - const [harness, messaging] = await Promise.all([import(driver), import(sdk)]); - return { HarnessDriverClient: harness.HarnessDriverClient, RelaycastMessagingClient: messaging.RelaycastMessagingClient }; - } catch { - throw new Error('Agent communication requires optional @agent-relay/harness-driver and @agent-relay/sdk packages (>=12.3.1).'); - } -} -export function relayWorkspaceKey(env = process.env): string { - const key = env.RELAY_API_KEY?.trim(); - if (!key?.startsWith('rk_live_')) throw new Error('Agent communication requires RELAY_API_KEY for an existing workspace.'); - return key; -} + export interface RelayRuntime { - broker: RelayBroker; - messaging: RelayMessaging; prefix: string; - channel: string; - project: ReturnType; + messaging: RelayMessaging; + project(input: Record): void; close(): Promise; } -const runtimes = new Map }>(); -export async function acquireRelayRuntime(dataDir: string, runId: string): Promise { - const key = `${dataDir}\0${runId}`; - let entry = runtimes.get(key); - if (!entry) { - entry = { references: 0, runtime: createRuntime(dataDir, runId) }; - runtimes.set(key, entry); - } - entry.references++; - try { - const runtime = await entry.runtime; - let closed = false; - return { ...runtime, close: async () => { - if (closed) return; - closed = true; - if (--entry!.references === 0) { runtimes.delete(key); await runtime.close(); } - } }; - } catch (error) { - if (--entry.references === 0) runtimes.delete(key); - throw error; - } -} -async function createRuntime(dataDir: string, runId: string): Promise { - const workspaceKey = relayWorkspaceKey(); - const { HarnessDriverClient, RelaycastMessagingClient } = await loadRelayModules(); - const hash = createHash('sha256').update(runId).digest('hex').slice(0, 12); - const prefix = `flow-${hash}-${randomUUID().slice(0, 8)}`; - const channel = `wf-${runId.toLowerCase()}`; - const options = { workspaceKey, ...(process.env.RELAY_BASE_URL ? { baseUrl: process.env.RELAY_BASE_URL } : {}) }; - const workspace = new RelaycastMessagingClient(options); - const publisher = `${prefix}-journal`; - const registration = await workspace.agents.register({ name: publisher, type: 'agent' }); - let broker: RelayBroker | undefined; + +export async function loadRelayModules(): Promise<{ RelaycastMessagingClient: unknown }> { try { - const messaging = new RelaycastMessagingClient({ ...options, agentToken: registration.token }); - const cwd = join(dataDir, 'communication', prefix); - await mkdir(cwd, { recursive: true, mode: 0o700 }); - broker = await HarnessDriverClient.spawn({ workspaceKey, brokerName: prefix, cwd, channels: [], - ...(process.env.RELAYFLOW_RELAY_BROKER_BIN ? { binaryPath: process.env.RELAYFLOW_RELAY_BROKER_BIN } : {}), - env: brokerEnvironment(process.env) }); - return { broker, messaging, prefix, channel, project: createProjection(messaging, channel, runId), close: async () => { - try { await broker!.shutdown(); } finally { await workspace.agents.delete(publisher); } - } }; + const sdk = await import('@agent-relay/sdk'); + return { RelaycastMessagingClient: sdk.RelaycastMessagingClient }; } catch (error) { - try { await broker?.shutdown(); } finally { await workspace.agents.delete(publisher); } - throw error; + throw new Error('Agent Relay node spawn requires optional @agent-relay/sdk >=17.2.1 <19.', { cause: error }); } } -/** Connected clients disconnect on shutdown; they never terminate the user's broker. */ -export interface LocalBroker extends RelayBroker { - connectEvents(): void; - sendMessage(input: { to: string; text: string; from: string; mode: 'wait'; data: Record }): Promise<{ event_id: string; targets: string[] }>; +export async function acquireRelayRuntime(_dataDir: string, _runId: string): Promise { + throw new Error('Broker-backed communication runtime was removed; native node-spawn communication is not admitted yet'); } -export async function connectLocalBroker(cwd: string): Promise { - const { HarnessDriverClient } = await loadRelayModules(); - return HarnessDriverClient.connect({ cwd }); + +export async function connectLocalBroker(_cwd: string): Promise { + throw new Error('Local Relay broker connections were removed'); } diff --git a/packages/sdk/src/communication/worker.ts b/packages/sdk/src/communication/worker.ts index f7b6a17e6..9783a9010 100644 --- a/packages/sdk/src/communication/worker.ts +++ b/packages/sdk/src/communication/worker.ts @@ -1,113 +1,5 @@ -import { communicationHistory } from './history.js'; -import { setTimeout as delay } from 'node:timers/promises'; -import type { JournalClient } from '../journal-client.js'; -import type { StepDispatchEvent } from '../protocol.js'; -import type { ResolvedKernelAgentStep } from '../resolved-cli-identity.js'; -import { isLeaseLost, withWorkerLease } from '../worker-lease.js'; -import { isReadInterruptionError } from '../journal-read-policy.js'; -import { workerInstruction } from '../worker-input.js'; -import { resolveAgentCwd } from '../agent-cwd.js'; -import { channelName, type CommunicationInstruction } from './spec.js'; -import { acquireRelayRuntime, type RelayHandle } from './relay.js'; -import { CommunicationSession } from './session.js'; -import { openCommunicationTools } from './tools.js'; -import { spawnFlowPty, shellQuote } from './pty.js'; - +/** The old communication runtime required a broker-managed PTY. */ export function requireCommunicationCli(cli: string | undefined): void { - if (!cli?.trim()) throw new Error('Agent communication requires a declared CLI executable'); -} -export async function completeCommunicationDispatch(client: JournalClient, dispatch: StepDispatchEvent, - instruction: CommunicationInstruction, dataDir: string, runRoot?: string, - environment?: NodeJS.ProcessEnv): Promise { - const spec = dispatch.spec as ResolvedKernelAgentStep; - requireCommunicationCli(spec.cli); - let output: unknown; - let completionReason: 'success' | 'worker_error' = 'success'; - try { - output = await withWorkerLease(client, dispatch, signal => { - // A lost lease stops this attempt's history reads instead of letting - // them run out the budget beside the kernel's retry. - const releaseReads = client.scopeReads?.(signal) ?? (() => {}); - return run(client, dispatch, instruction, spec, dataDir, signal, runRoot, environment).finally(releaseReads); - }); - } catch (error) { - // A lost lease or an unanswered read belongs to the kernel's sweep and - // retry: never complete the attempt here as a worker failure. - if (isLeaseLost(error) || isReadInterruptionError(error)) throw error; - completionReason = 'worker_error'; - output = { error: error instanceof Error ? error.message : String(error) }; - } - await client.stepComplete(dispatch.run_id, dispatch.step_id, dispatch.attempt, dispatch.idempotency_key, - completionReason, { output, usage: { tokens_in: 0, tokens_out: 0, dollars_unmetered: true }, - started_pins: dispatch.pins, end_pins: dispatch.pins }); -} -async function run(client: JournalClient, dispatch: StepDispatchEvent, instruction: CommunicationInstruction, - spec: ResolvedKernelAgentStep, dataDir: string, lease: AbortSignal, runRoot?: string, - environment?: NodeJS.ProcessEnv): Promise { - // Same contract as the CLI worker: a declared directory is resolved and held - // inside the same run root the CLI worker measures against, before anything - // is spawned. Thrown, not reported, because `completeCommunicationDispatch` - // already turns a throw here into a `worker_error` completion carrying the - // message. - const root = runRoot ?? process.cwd(); - const directory = resolveAgentCwd(root, spec.cwd) ?? root; - const controller = new AbortController(); - const signal = AbortSignal.any([lease, controller.signal, AbortSignal.timeout(instruction.timeoutMs)]); - const relay = await acquireRelayRuntime(dataDir, dispatch.run_id); - let handle: RelayHandle | undefined; - let tools: Awaited> | undefined; - let pumping: Promise | undefined; - let receipts = Promise.resolve(); - let unsubscribe: (() => void) | undefined; - let pty: Awaited> | undefined; - try { - let resolve!: (value: unknown) => void; - let reject!: (reason: unknown) => void; - const result = new Promise((yes, no) => { resolve = yes; reject = no; }); - // Install the rejection handler before spawn/readiness can fail. - void result.catch(() => {}); - const abort = () => reject(signal.reason); - signal.addEventListener('abort', abort, { once: true }); - if (signal.aborted) abort(); - const session = new CommunicationSession(client, dispatch, instruction, relay, summary => resolve({ summary }), reject); - tools = await openCommunicationTools(request => session.invoke(request)); - const helper = `node ${shellQuote(tools.helperPath)}`; - const history = await communicationHistory(client, dispatch, instruction); - const prompt = `${workerInstruction(instruction.instruction, dispatch)}${history}\n\nYou are flow agent ${dispatch.step_id}. Concurrent peers send messages automatically into your session through Relay. Do not poll for messages. Use your shell/terminal execution tool with ${helper} for flow communication:\n- send PEER STABLE_ID 'message text' (outgoing peers: ${instruction.outgoing.join(', ') || 'none'})\n- ack PEER DELIVERY_SEQ only after processing that message (incoming peers: ${instruction.incoming.join(', ') || 'none'})\n- complete 'summary' when your task and conversation are finished.\nUse these helpers for all peer communication; do not use Relay MCP messaging tools. Stay available for injected messages until the conversation is complete. Use stable semantic message IDs so retries deduplicate sends.`; - const name = `${relay.prefix}-${dispatch.step_id}`; - unsubscribe = relay.broker.onEvent(event => { - if (event.name !== name || !['delivery_injected', 'delivery_verified'].includes(event.kind)) return; - receipts = receipts.then(async () => { - await client.streamAppend(dispatch.run_id, channelName(dispatch.step_id, '$receipts'), { - kind: event.kind, delivery_id: event.delivery_id, verification: event.verification, - attempt: dispatch.attempt, processing_ack: false, - }); - }); - void receipts.catch(reject); - }); - pty = await spawnFlowPty(relay.broker, { name, cli: spec.cli!, identity: spec.cli_identity, - model: spec.model, task: prompt, channels: [], cwd: directory, dataDir, - environment, socket: tools.path, token: tools.token, - secrets: spec.secrets, scopeSecrets: client.supportsFeature?.('step_secret_scoping') ?? true }); - handle = pty.handle; - const ready = await handle.waitForReady(Math.min(instruction.timeoutMs, 90_000)); - if (ready.reason !== 'ready') throw new Error(`Communication agent did not become ready: ${ready.reason}`); - pumping = (async () => { - while (!signal.aborted) { await session.pump(); await delay(200, undefined, { signal }); } - })(); - void pumping.catch(error => { if (!controller.signal.aborted) reject(error); }); - try { return await result; } finally { signal.removeEventListener('abort', abort); } - } finally { - controller.abort(); - unsubscribe?.(); - try { await pumping?.catch(error => { if (error?.name !== 'AbortError') throw error; }); } - finally { - try { await receipts; } - finally { try { await handle?.release('Flow communication attempt ended', { deleteIdentity: true }); } - finally { try { await tools?.close(); } finally { - try { await relay.close(); } - finally { await pty?.close(); } - } } } - } - } + if (!cli?.trim()) throw new Error('Agent communication requires a declared CLI'); + throw new Error('Agent communication through broker/PTy was removed; native node-spawn communication is not admitted yet'); } diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index adbcc075f..b7a0e5b91 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -241,6 +241,33 @@ export { JOURNAL_WRITE_FAILED, PROTOCOL_VERSION } from './protocol.js'; export { JournalClient, JournalRequestTimeoutError, type JournalClientOptions } from './journal-client.js'; export { AgentWorker, type AgentWorkerOptions } from './worker.js'; +export { + NodeSpawnError, + NodeSpawnPreDispatchError, + type BeforeNodeDispatch, + type NodeReleaseRequest, + type NodeSpawnAction, + type NodeSpawnInvocation, + type NodeSpawnPort, + type NodeSpawnReceipt, + type NodeSpawnRequest, + type NodeSpawnTarget, +} from './node-spawn.js'; +export { + LocalNodeSpawnPort, + loadCoreBridge, + type CallerDescriptor, + type CoreBridgeApi, + type CoreBridgeClient, +} from './node-spawn-local.js'; +export { RemoteNodeSpawnPort, type RemoteNodeSpawnClient } from './node-spawn-remote.js'; +export { + createLocalNodeSpawnRuntime, + createNodeSpawnRuntime, + createRemoteNodeSpawnRuntime, + type NodeSpawnPublisher, + type NodeSpawnRuntime, +} from './node-spawn-runtime.js'; export { validateWorkPackage, diff --git a/packages/sdk/src/local-agent.ts b/packages/sdk/src/local-agent.ts index 97378c693..7ba563260 100644 --- a/packages/sdk/src/local-agent.ts +++ b/packages/sdk/src/local-agent.ts @@ -17,6 +17,7 @@ export async function attachLocalAgent( runRoot?: string, declaredStreams: readonly string[] = [], environment?: NodeJS.ProcessEnv, + serviceCallerFile?: string, ): Promise<{ stream: string; readonly failure: unknown; @@ -34,7 +35,7 @@ export async function attachLocalAgent( // second parking behind the first. Authored bodies size their admission to // this same number (worker-slots.ts), so they never ask for more. capacity, - dataDir, onPtyReady, runRoot, environment, + dataDir, onPtyReady, runRoot, environment, serviceCallerFile, pins: { workspace: [{ surface: LOCAL_AGENT_WORKSPACE, revision_id: LOCAL_AGENT_REVISION }], streams: [...new Set([stream, ...declaredStreams])].map(stream => ({ stream, read_offset: 0 })), diff --git a/packages/sdk/src/node-spawn-cleanup.ts b/packages/sdk/src/node-spawn-cleanup.ts new file mode 100644 index 000000000..a660363e6 --- /dev/null +++ b/packages/sdk/src/node-spawn-cleanup.ts @@ -0,0 +1,83 @@ +import { createHash } from 'node:crypto'; +import { recordNodeSpawn, type NodeSpawnClaim } from './node-spawn-state.js'; +import type { NodeSpawnRuntime } from './node-spawn-runtime.js'; +import { NodeSpawnError, nonempty, type NodeSpawnIdentity } from './node-spawn.js'; + +const CLEANUP_ATTEMPT_TIMEOUT_MS = 15_000; + +/** True only when durable terminal evidence makes exact release recoverable. */ +export function nodeSpawnCleanupRequired(claim: NodeSpawnClaim): boolean { + return claim.release === undefined + && (claim.spawn !== undefined || claim.cleanup !== undefined) + && (claim.cleanupPending !== undefined || claim.failure !== undefined || claim.result !== undefined); +} + +/** Attempts the claim's exact, idempotent release and persists either outcome. */ +export async function releaseNodeSpawnClaim( + runtime: Pick, + path: string, + claim: NodeSpawnClaim, +): Promise<{ claim: NodeSpawnClaim; error?: unknown }> { + const receipt: NodeSpawnIdentity | undefined = claim.spawn ?? claim.cleanup; + if (receipt === undefined || claim.release !== undefined) return { claim }; + try { + const released = await boundedCleanup(runtime.port.release({ + receipt, + idempotencyKey: releaseKey(claim.idempotencyKey), + })); + if (released.action !== 'release' || released.status !== 'completed' + || released.released !== true || !nonempty(released.id)) { + throw new NodeSpawnError('The node did not confirm exact agent release', + released.errorCode ?? 'release_unconfirmed'); + } + const { cleanupPending: _cleanupPending, ...withoutPending } = claim; + const updated: NodeSpawnClaim = { ...withoutPending, + release: { invocationId: released.id, status: released.status } }; + await recordNodeSpawn(path, updated); + return { claim: updated }; + } catch (error) { + const updated: NodeSpawnClaim = { ...claim, cleanupPending: { + attempts: (claim.cleanupPending?.attempts ?? 0) + 1, + lastError: safeError(error), + updatedAt: Date.now(), + } }; + try { await recordNodeSpawn(path, updated); } + catch (stateError) { + return { claim: updated, + error: new NodeSpawnError( + `${safeError(error)}; cleanup state was not durable: ${safeError(stateError)}`, + 'release_unconfirmed', error) }; + } + return { claim: updated, error }; + } +} + +async function boundedCleanup(pending: Promise): Promise { + let timer: NodeJS.Timeout | undefined; + const deadline = new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new NodeSpawnError( + 'Exact node cleanup did not confirm within its independent deadline', 'release_unconfirmed')), + CLEANUP_ATTEMPT_TIMEOUT_MS); + }); + // A transport that ignores cancellation may settle after this attempt has + // durably recorded cleanupPending. Fold that late outcome into this promise + // so it cannot become an unhandled rejection; reconciliation reuses the + // same idempotency key for the exact release. + const outcome = pending.then(value => ({ ok: true as const, value }), + error => ({ ok: false as const, error })); + try { + const settled = await Promise.race([outcome, deadline]); + if (!settled.ok) throw settled.error; + return settled.value; + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} + +function releaseKey(key: string): string { + return `release_${createHash('sha256').update(key).digest('hex').slice(0, 32)}`; +} + +function safeError(error: unknown): string { + return error instanceof Error && error.message ? error.message : 'Node-spawn task failed'; +} diff --git a/packages/sdk/src/node-spawn-deadline.ts b/packages/sdk/src/node-spawn-deadline.ts new file mode 100644 index 000000000..77d74f139 --- /dev/null +++ b/packages/sdk/src/node-spawn-deadline.ts @@ -0,0 +1,43 @@ +import type { NodeSpawnReceipt } from './node-spawn.js'; + +/** Races a side-effect-free node read/runtime acquisition against its caller deadline. */ +export async function waitForNodeOperation(pending: Promise, signal: AbortSignal): Promise { + signal.throwIfAborted(); + let onAbort: (() => void) | undefined; + const aborted = new Promise((_resolve, reject) => { + onAbort = () => reject(signal.reason ?? new DOMException('The operation was aborted', 'AbortError')); + signal.addEventListener('abort', onAbort, { once: true }); + }); + // Fold a late rejection from a transport that ignored cancellation. + const outcome = pending.then(value => ({ ok: true as const, value }), error => ({ ok: false as const, error })); + try { + const settled = await Promise.race([outcome, aborted]); + if (!settled.ok) throw settled.error; + return settled.value; + } finally { + if (onAbort !== undefined) signal.removeEventListener('abort', onAbort); + } +} + +/** Preserves the exact receipt when spawn confirmation settles after cancellation. */ +export async function waitForNodeSpawn( + pending: Promise, + signal: AbortSignal, + onLate: (outcome: { spawn?: NodeSpawnReceipt; error?: unknown }) => void, +): Promise { + signal.throwIfAborted(); + let onAbort: (() => void) | undefined; + const aborted = new Promise((_resolve, reject) => { + onAbort = () => reject(signal.reason ?? new DOMException('The operation was aborted', 'AbortError')); + signal.addEventListener('abort', onAbort, { once: true }); + }); + try { return await Promise.race([pending, aborted]); } + catch (error) { + if (signal.aborted) { + void pending.then(spawn => onLate({ spawn }), lateError => onLate({ error: lateError })); + } + throw error; + } finally { + if (onAbort !== undefined) signal.removeEventListener('abort', onAbort); + } +} diff --git a/packages/sdk/src/node-spawn-dispatch.ts b/packages/sdk/src/node-spawn-dispatch.ts new file mode 100644 index 000000000..eff7f977c --- /dev/null +++ b/packages/sdk/src/node-spawn-dispatch.ts @@ -0,0 +1,197 @@ +import { waitForNodeSpawn } from './node-spawn-deadline.js'; +import { claimNodeSpawnDispatch, hasNodeSpawnDispatchFence, + type NodeSpawnClaim } from './node-spawn-state.js'; +import { NodeSpawnError, NodeSpawnPreDispatchError, + validNodeSpawnIdentity, validNodeSpawnReceipt, + type NodeSpawnPort, type NodeSpawnReceipt, type NodeSpawnRequest } from './node-spawn.js'; + +interface FencedNodeSpawnOptions { + port: NodeSpawnPort; + request: NodeSpawnRequest & { timeoutMs: number }; + claimPath: string; + claim: NodeSpawnClaim; + signal?: AbortSignal; + settleLate(outcome: { spawn?: NodeSpawnReceipt; error?: unknown }): Promise; + replayExistingFence?: boolean; +} + +interface ExactPortIdentity { + prototype: object; + spawn: NodeSpawnPort['spawn']; + targets: NodeSpawnPort['targets']; + status: NodeSpawnPort['status']; + release: NodeSpawnPort['release']; +} + +interface ReplayableFencedRegistration { + identity: ExactPortIdentity; + validateState: () => boolean; + replay: NodeSpawnPort['spawn']; +} + +interface ExactPortRegistration { + identity: ExactPortIdentity; + validateState: () => boolean; +} + +const retryablePreflightPorts = new WeakMap(); +const replayableFencedPorts = new WeakMap(); + +/** + * Registers the exact built-in method set that may perform proven side-effect- + * free preflight before publishing the durable dispatch fence. Replaced or + * overridden methods remain untrusted and are fenced before entry. + */ +export function registerRetryablePreflightPort( + port: NodeSpawnPort, + identity: ExactPortIdentity, + validateState: () => boolean, +): void { + if (hasExactPortMethods(port, identity) && validateState()) { + retryablePreflightPorts.set(port, { identity, validateState }); + } +} + +/** Registers an internal handler whose server contract replays a fenced dispatch by idempotency key. */ +export function registerReplayableFencedPort( + port: NodeSpawnPort, + identity: ExactPortIdentity, + validateState: () => boolean, + replay: NodeSpawnPort['spawn'], +): void { + if (hasExactPortMethods(port, identity) && validateState()) { + replayableFencedPorts.set(port, { identity, validateState, replay }); + } +} + +/** True only while the exact registered built-in method set remains installed. */ +export function hasReplayableFencedDispatch(port: NodeSpawnPort): boolean { + return replayableFencedRegistration(port) !== undefined; +} + +function hasRetryablePreflight(port: NodeSpawnPort): boolean { + const registration = retryablePreflightPorts.get(port); + return registration !== undefined + && hasExactPortMethods(port, registration.identity) + && registration.validateState(); +} + +function replayableFencedRegistration( + port: NodeSpawnPort, +): ReplayableFencedRegistration | undefined { + const registration = replayableFencedPorts.get(port); + return registration !== undefined + && hasExactPortMethods(port, registration.identity) + && registration.validateState() + ? registration : undefined; +} + +function hasExactPortMethods( + port: NodeSpawnPort, + identity: ExactPortIdentity, +): boolean { + if (Object.getPrototypeOf(port) !== identity.prototype + || Object.getOwnPropertyDescriptor(port, 'spawn') !== undefined + || Object.getOwnPropertyDescriptor(port, 'targets') !== undefined + || Object.getOwnPropertyDescriptor(port, 'status') !== undefined + || Object.getOwnPropertyDescriptor(port, 'release') !== undefined) return false; + const spawn = Object.getOwnPropertyDescriptor(identity.prototype, 'spawn'); + const targets = Object.getOwnPropertyDescriptor(identity.prototype, 'targets'); + const status = Object.getOwnPropertyDescriptor(identity.prototype, 'status'); + const release = Object.getOwnPropertyDescriptor(identity.prototype, 'release'); + return spawn?.value === identity.spawn && targets?.value === identity.targets + && status?.value === identity.status && release?.value === identity.release; +} + +/** Enters exactly one irreversible spawn behind a durable, one-way fence. */ +export async function dispatchFencedNodeSpawn( + options: FencedNodeSpawnOptions, +): Promise { + const retryablePreflight = hasRetryablePreflight(options.port); + let replay: NodeSpawnPort['spawn'] | undefined; + let dispatchPublished = false; + if (options.replayExistingFence === true) { + if (!await hasNodeSpawnDispatchFence(options.claimPath, options.claim)) { + throw new NodeSpawnError( + 'A fenced node-spawn replay requires the exact built-in replay contract and durable fence', + 'spawn_dispatch_fence_missing', + ); + } + // Recheck exact port identity after the asynchronous durable-fence read and + // immediately before selecting the internal replay handler. + replay = replayableFencedRegistration(options.port)?.replay; + if (replay === undefined) { + throw new NodeSpawnError( + 'A fenced node-spawn replay requires the exact built-in replay contract and durable fence', + 'spawn_dispatch_fence_missing', + ); + } + dispatchPublished = true; + } + let dispatchHookEntered = false; + let pendingSpawn: Promise; + if (!retryablePreflight) { + // Arbitrary public ports cannot prove that their own spawn preflight is + // free of side effects. Publish before resolving or entering spawn, so a + // process crash cannot leave a provider-contacted claim looking retryable. + options.signal?.throwIfAborted(); + await claimNodeSpawnDispatch(options.claimPath, options.claim); + dispatchPublished = true; + } + try { + const spawn = replay ?? options.port.spawn.bind(options.port); + pendingSpawn = spawn(options.request, async () => { + dispatchHookEntered = true; + options.signal?.throwIfAborted(); + if (dispatchPublished) return; + await claimNodeSpawnDispatch(options.claimPath, options.claim); + dispatchPublished = true; + }); + } catch (error) { + pendingSpawn = Promise.reject(error); + } + const spawnTimeout = AbortSignal.timeout(options.request.timeoutMs); + const spawnSignal = options.signal === undefined + ? spawnTimeout : AbortSignal.any([options.signal, spawnTimeout]); + let spawn: NodeSpawnReceipt; + try { + spawn = await waitForNodeSpawn(pendingSpawn, spawnSignal, outcome => { + void options.settleLate(outcome).catch(() => undefined); + }); + } catch (error) { + if (!dispatchPublished && dispatchHookEntered) { + // Publication may fail before the atomic link (safe to retry) or after + // it (permanently ambiguous). Observe once without creating a fence. + dispatchPublished = await hasNodeSpawnDispatchFence(options.claimPath, options.claim); + } else if (!dispatchPublished && !(error instanceof NodeSpawnPreDispatchError)) { + // An untrusted port did not prove that it stopped before dispatch. + await claimNodeSpawnDispatch(options.claimPath, options.claim); + dispatchPublished = true; + } + throw error; + } + if (!dispatchPublished) { + await claimNodeSpawnDispatch(options.claimPath, options.claim); + dispatchPublished = true; + throw new NodeSpawnError( + 'The node-spawn port returned without publishing its required dispatch fence', + 'spawn_dispatch_fence_missing', + undefined, + validNodeSpawnIdentity(spawn) ? spawn : undefined, + ); + } + if (!await hasNodeSpawnDispatchFence(options.claimPath, options.claim)) { + throw new NodeSpawnError( + 'The node-spawn port did not leave its required dispatch fence durable', + 'spawn_dispatch_fence_missing', + undefined, + validNodeSpawnIdentity(spawn) ? spawn : undefined, + ); + } + if (!validNodeSpawnReceipt(spawn) || spawn.name !== options.request.name) { + throw new NodeSpawnError('The node spawn did not return a valid ready identity', + 'spawn_identity_mismatch', undefined, + validNodeSpawnIdentity(spawn) ? spawn : undefined); + } + return spawn; +} diff --git a/packages/sdk/src/node-spawn-local.ts b/packages/sdk/src/node-spawn-local.ts new file mode 100644 index 000000000..b55768ea7 --- /dev/null +++ b/packages/sdk/src/node-spawn-local.ts @@ -0,0 +1,550 @@ +import { isAbsolute, relative, sep } from 'node:path'; + +import { + NodeSpawnError, + NodeSpawnPreDispatchError, + normalizeNodeSpawnStatus, + terminalStatus, + type NodeReleaseRequest, + type NodeSpawnAction, + type NodeSpawnInvocation, + type NodeSpawnIdentity, + type NodeSpawnPort, + type NodeSpawnReceipt, + type NodeSpawnRequest, + type NodeSpawnTarget, + validIdempotencyKey, + type BeforeNodeDispatch, +} from './node-spawn.js'; +import { registerReplayableFencedPort, + registerRetryablePreflightPort } from './node-spawn-dispatch.js'; + +/** The structural core-bridge surface Flows consumes. Kept here so the package stays optional. */ +export interface CoreBridgeClient { + readonly hello: { + bridge_version: number; + daemon_version: string; + pid: number; + methods: string[]; + topics: Array<'status' | 'activity' | 'shutdown' | 'invalidated' | 'notify' | 'sign_in'>; + }; + close(): void; +} + +export interface CallerDescriptor { + readonly schema: 'agent-relay-service-caller/2'; + readonly schemaVersion: 2; + readonly auth: unknown; + readonly callerId: string; + readonly callerName: string; + readonly publisher: { readonly id: string; readonly name: string }; + readonly workspace: { + readonly site_url: string; + readonly base_url: string; + readonly workspace_id: string; + readonly relaycast_workspace_id: string; + }; + readonly account: { readonly user_id: string }; + readonly credential: unknown; + readonly createdAt: string; + readonly rotatedAt?: string; + readonly expiresAt: string; + readonly harnesses: readonly string[]; + readonly directoryScope: readonly string[]; + readonly ownsOnly: boolean; + readonly socket: string; + readonly inbox: { + readonly transport: string; + readonly open: string; + readonly next: string; + readonly ack: string; + readonly fail: string; + readonly close: string; + readonly disconnect: string; + }; + readonly desktopVersion: string; +} + +interface CoreTarget { + id: string; + name: string; + live: boolean; + handlers_live: boolean; + stale: boolean; + capabilities: string[]; + active_agents: number; + reserved_agents: number; + max_agents: number; +} + +interface CoreInvocation { + id: string; + agent_id: string | null; + status: string; + node_id: string | null; + name: string | null; + spawned: boolean; + ready: boolean; + released: boolean; + task_unconfirmed: boolean; + error_code: string | null; +} + +interface CoreSpawnInvokeParams { + caller_name?: string; + node_id: string; + node_name: string; + name: string; + harness: string; + task: string; + directory: string; + idempotency_key: string; +} + +interface CoreSpawnStatusParams { + caller_name?: string; + action: NodeSpawnAction; + invocation_id: string; +} + +interface CoreSpawnReleaseParams { + caller_name?: string; + node_id: string; + node_name: string; + name: string; + expected_agent_id: string; + idempotency_key: string; +} + +export interface CoreInboxDelivery { + item_id: string; + message_id: string; + sender: { id: string; name: string }; + text?: string; + failure?: { code: 'result_too_large' | 'result_write_failed'; bytes: number; limit: number }; + invocation_id: string; + sent_at: string; +} + +/** Typed API required from @agent-relay/core-bridge's schema-v2 release. */ +export interface CoreBridgeApi { + connectCore(options?: { socket?: string; signal?: AbortSignal }): Promise; + readCallerDescriptor(path: string): Promise; + authenticate(client: CoreBridgeClient, credential: unknown, options?: { signal?: AbortSignal }): Promise; + requireServiceCallerMethods(client: CoreBridgeClient): void; + assertServiceCallerIdentity( + descriptor: CallerDescriptor, + identity: Awaited>, + ): void; + serviceCallerIdentity(client: CoreBridgeClient, options?: { signal?: AbortSignal }): Promise<{ + publisher: { id: string; name: string }; + node: { id: string; name: string }; + base_url: string; + workspace_id: string; + relaycast_workspace_id: string; + }>; + spawnTargets(client: CoreBridgeClient, options?: { signal?: AbortSignal }): Promise<{ targets: CoreTarget[] }>; + spawnInvoke(client: CoreBridgeClient, params: CoreSpawnInvokeParams, options?: { signal?: AbortSignal; actionTimeoutMs?: number }): Promise; + spawnStatus(client: CoreBridgeClient, params: CoreSpawnStatusParams, options?: { signal?: AbortSignal }): Promise; + spawnRelease(client: CoreBridgeClient, params: CoreSpawnReleaseParams, options?: { signal?: AbortSignal; actionTimeoutMs?: number }): Promise; + serviceCallerInboxOpen(client: CoreBridgeClient, invocationId: string, options?: { signal?: AbortSignal }): Promise<{ iterator_id: string }>; + serviceCallerInboxNext(client: CoreBridgeClient, iteratorId: string, timeoutMs: number, options?: { signal?: AbortSignal }): Promise<{ delivery: CoreInboxDelivery | null }>; + serviceCallerInboxAck(client: CoreBridgeClient, itemId: string, options?: { signal?: AbortSignal }): Promise; + serviceCallerInboxFail(client: CoreBridgeClient, itemId: string, options?: { signal?: AbortSignal }): Promise; + serviceCallerInboxClose(client: CoreBridgeClient, iteratorId: string, options?: { signal?: AbortSignal }): Promise; + serviceCallerDisconnect(client: CoreBridgeClient, options?: { signal?: AbortSignal }): Promise; +} + +const REQUIRED_CORE_BRIDGE_METHODS = { + connectCore: true, + readCallerDescriptor: true, + authenticate: true, + requireServiceCallerMethods: true, + assertServiceCallerIdentity: true, + serviceCallerIdentity: true, + spawnTargets: true, + spawnInvoke: true, + spawnStatus: true, + spawnRelease: true, + serviceCallerInboxOpen: true, + serviceCallerInboxNext: true, + serviceCallerInboxAck: true, + serviceCallerInboxFail: true, + serviceCallerInboxClose: true, + serviceCallerDisconnect: true, +} as const satisfies Record; + +const loadedCoreBridges = new WeakSet(); +const LOCAL_FENCED_REPLAY = Symbol('LocalNodeSpawnPort.fencedReplay'); +const LOCAL_STATE_IS_EXACT = Symbol('LocalNodeSpawnPort.stateIsExact'); + +interface LocalNodeSpawnState { + core: CoreBridgeClient; + bridge: Pick; + scope: Pick; + exactNode: { readonly id: string; readonly name: string }; + bridgeMethods: readonly unknown[]; + exactNodeId: string; + exactNodeName: string; + internalMethods: readonly unknown[]; +} + +const localNodeSpawnStates = new WeakMap(); + +/** Fails before local admission when an installed optional bridge lacks the required contract. */ +export function assertCoreBridgeApi(value: unknown): asserts value is CoreBridgeApi { + const module = (typeof value === 'object' && value !== null) + ? value as Record + : {}; + const missingMethods = Object.keys(REQUIRED_CORE_BRIDGE_METHODS) + .filter(method => typeof module[method] !== 'function'); + if (missingMethods.length === 0) return; + throw new NodeSpawnError( + `Installed @agent-relay/core-bridge is missing required schema-v2 methods: ${missingMethods.join(', ')}. ` + + 'Install >=18.1.0 <19.', + 'core_bridge_unavailable', + { missingMethods, requiredVersion: '>=18.1.0 <19' }, + ); +} + +/** Loads the optional bridge without making remote-only installs depend on Desktop support. */ +export async function loadCoreBridge(): Promise { + const specifier = '@agent-relay/core-bridge'; + let bridge: unknown; + try { + bridge = await import(specifier); + } catch (error) { + throw new NodeSpawnError( + 'Local Desktop node spawn requires optional @agent-relay/core-bridge >=18.1.0 <19 ' + + 'with schema-v2 service-caller support', + 'core_bridge_unavailable', + error, + ); + } + assertCoreBridgeApi(bridge); + loadedCoreBridges.add(bridge); + return bridge; +} + +/** Enables replay only for the exact bridge module admitted by loadCoreBridge(). */ +export function enableLoadedCoreBridgeCapabilities( + port: LocalNodeSpawnPort, + bridge: CoreBridgeApi, +): void { + if (!loadedCoreBridges.has(bridge)) return; + const validateState = port[LOCAL_STATE_IS_EXACT].bind(port); + const replay = port[LOCAL_FENCED_REPLAY].bind(port); + registerRetryablePreflightPort(port, LOCAL_NODE_SPAWN_IDENTITY, validateState); + registerReplayableFencedPort( + port, + LOCAL_NODE_SPAWN_IDENTITY, + validateState, + replay, + ); +} + +/** Native local spawn adapter over one already authenticated schema-v2 bridge connection. */ +export class LocalNodeSpawnPort implements NodeSpawnPort { + readonly dispatchFenceVersion = 1 as const; + + constructor( + private readonly core: CoreBridgeClient, + private readonly bridge: Pick, + private readonly scope: Pick, + private readonly exactNode: { readonly id: string; readonly name: string }, + ) { + localNodeSpawnStates.set(this, { + core, + bridge, + scope, + exactNode, + bridgeMethods: [ + bridge.spawnTargets, + bridge.spawnInvoke, + bridge.spawnStatus, + bridge.spawnRelease, + ], + exactNodeId: exactNode.id, + exactNodeName: exactNode.name, + internalMethods: [ + this.invokeSpawn, + this.wait, + this.cleanupIdentity, + ], + }); + } + + private [LOCAL_STATE_IS_EXACT](): boolean { + const state = localNodeSpawnStates.get(this); + return state !== undefined + && this.core === state.core + && this.bridge === state.bridge + && this.scope === state.scope + && this.exactNode === state.exactNode + && this.exactNode.id === state.exactNodeId + && this.exactNode.name === state.exactNodeName + && this.bridge.spawnTargets === state.bridgeMethods[0] + && this.bridge.spawnInvoke === state.bridgeMethods[1] + && this.bridge.spawnStatus === state.bridgeMethods[2] + && this.bridge.spawnRelease === state.bridgeMethods[3] + && this.invokeSpawn === state.internalMethods[0] + && this.wait === state.internalMethods[1] + && this.cleanupIdentity === state.internalMethods[2]; + } + + async targets(): Promise { + const result = await this.bridge.spawnTargets(this.core); + if (!result || !Array.isArray(result.targets)) { + throw new NodeSpawnError('The core returned invalid spawn targets', 'invalid_core_response'); + } + return result.targets.map(value => { + if (!value || typeof value.id !== 'string' || value.id === '' + || typeof value.name !== 'string' || value.name === '') { + throw new NodeSpawnError('The core returned a spawn target without an exact identity', 'invalid_core_response'); + } + return { + id: value.id, + name: value.name, + live: value.live === true, + handlersLive: value.handlers_live === true, + stale: value.stale === true, + capabilities: Array.isArray(value.capabilities) + ? value.capabilities.filter((entry): entry is string => typeof entry === 'string' && entry.length > 0) + : [], + activeAgents: Number(value.active_agents), + reservedAgents: Number(value.reserved_agents), + maxAgents: Number(value.max_agents), + }; + }); + } + + async spawn(request: NodeSpawnRequest, beforeDispatch: BeforeNodeDispatch): Promise { + let allTargets: NodeSpawnTarget[]; + let target: NodeSpawnTarget | undefined; + try { + if (!request.directory || !isAbsolute(request.directory)) { + throw new NodeSpawnError('Local node spawn requires an absolute directory', 'spawn_invalid_input'); + } + if (!this.scope.harnesses.includes(request.harness) + || !this.scope.directoryScope.some(root => withinDirectory(root, request.directory!))) { + throw new NodeSpawnError( + 'The Desktop service caller does not authorize this harness and directory', + 'outside_scope', + ); + } + if (!validIdempotencyKey(request.idempotencyKey)) { + throw new NodeSpawnError('Invalid node-spawn idempotency key', 'spawn_invalid_input'); + } + if (request.model !== undefined || request.repo !== undefined || request.sandboxOnly !== undefined) { + throw new NodeSpawnError( + 'The Desktop core bridge does not accept model, repository, or sandbox placement overrides', + 'spawn_invalid_input', + ); + } + allTargets = await this.targets(); + if (request.node !== undefined && request.node !== this.exactNode.name) { + throw new NodeSpawnError('The requested node differs from the authenticated Desktop node', + 'spawn_identity_mismatch'); + } + if (request.nodeId !== undefined && request.nodeId !== this.exactNode.id) { + throw new NodeSpawnError('The requested node id differs from the authenticated Desktop node', + 'spawn_identity_mismatch'); + } + target = allTargets.find(candidate => candidate.id === this.exactNode.id + && candidate.name === this.exactNode.name + && candidate.live && candidate.handlersLive && !candidate.stale + && candidate.capabilities.includes(`spawn:${request.harness}`)); + if (!target?.id) { + throw new NodeSpawnError('No eligible local Agent Relay spawn target', 'no_eligible_node'); + } + } catch (error) { + throw new NodeSpawnPreDispatchError(error); + } + return this.invokeSpawn( + { ...request, directory: request.directory }, + beforeDispatch, + { id: target.id, name: target.name }, + allTargets, + ); + } + + private async [LOCAL_FENCED_REPLAY]( + request: NodeSpawnRequest, + beforeDispatch: BeforeNodeDispatch, + ): Promise { + if (!request.directory || !isAbsolute(request.directory)) { + throw new NodeSpawnError('Local node spawn requires an absolute directory', 'spawn_invalid_input'); + } + if (!validIdempotencyKey(request.idempotencyKey)) { + throw new NodeSpawnError('Invalid node-spawn idempotency key', 'spawn_invalid_input'); + } + if (request.model !== undefined || request.repo !== undefined || request.sandboxOnly !== undefined) { + throw new NodeSpawnError( + 'The Desktop core bridge does not accept model, repository, or sandbox placement overrides', + 'spawn_invalid_input', + ); + } + if (!request.node || !request.nodeId) { + throw new NodeSpawnError('Fenced local replay requires the durable exact node identity', + 'spawn_identity_mismatch'); + } + // The original v3 claim was already admitted and fenced. Repeating the + // exact logical request lets Relay replay that durable claim even if the + // caller's current directory scope or live-target view has since changed. + return this.invokeSpawn( + { ...request, directory: request.directory }, + beforeDispatch, + { id: request.nodeId, name: request.node }, + ); + } + + private async invokeSpawn( + request: NodeSpawnRequest & { directory: string }, + beforeDispatch: BeforeNodeDispatch, + target: { readonly id: string; readonly name: string }, + knownTargets: readonly { readonly id?: string; readonly name: string }[] = [target], + ): Promise { + await beforeDispatch(); + const raw = await this.bridge.spawnInvoke(this.core, { + node_id: target.id, + node_name: target.name, + name: request.name, + harness: request.harness, + task: request.task, + directory: request.directory, + idempotency_key: request.idempotencyKey, + }, { actionTimeoutMs: request.timeoutMs }); + // spawn.invoke is the action receipt; spawn.status is also the boundary + // that enriches it from the core's exact same-workspace registration. + const confirmation = await this.wait('spawn', raw.id, request.timeoutMs, + mapCoreInvocation('spawn', raw)); + const settled = confirmation.invocation; + const settledTarget = knownTargets.find(candidate => candidate.id === settled.nodeId); + const cleanup = confirmation.cleanup ?? (settled.agentId && settled.name && settledTarget?.id + ? { id: settled.id, nodeId: settledTarget.id, nodeName: settledTarget.name, + name: settled.name, agentId: settled.agentId } + : undefined); + if (settled.status !== 'completed' || !settled.spawned || !settled.ready + || settled.taskUnconfirmed !== false || !settled.agentId || !settled.name) { + throw new NodeSpawnError('The local node did not prove a ready spawned agent', + settled.taskUnconfirmed !== false ? 'spawn_task_unconfirmed' : settled.errorCode, undefined, cleanup); + } + if (settled.name !== request.name || settled.nodeId !== target.id) { + throw new NodeSpawnError('The local spawn result did not preserve its requested identity', + 'spawn_identity_mismatch', undefined, cleanup); + } + return { ...settled, action: 'spawn', nodeName: target.name, name: settled.name, + agentId: settled.agentId, spawned: true, ready: true }; + } + + async status(action: NodeSpawnAction, invocationId: string): Promise { + return mapCoreInvocation(action, await this.bridge.spawnStatus(this.core, { + action, + invocation_id: invocationId, + })); + } + + async release(request: NodeReleaseRequest): Promise { + if (!validIdempotencyKey(request.idempotencyKey)) { + throw new NodeSpawnError('Invalid node-release idempotency key', 'spawn_invalid_input'); + } + const receipt = request.receipt; + if (!receipt.nodeId || receipt.nodeId !== this.exactNode.id + || receipt.nodeName !== this.exactNode.name) { + throw new NodeSpawnError('Local release requires the authenticated Desktop node identity', + 'release_identity_mismatch'); + } + const raw = await this.bridge.spawnRelease(this.core, { + node_id: receipt.nodeId, + node_name: receipt.nodeName, + name: receipt.name, + expected_agent_id: receipt.agentId, + idempotency_key: request.idempotencyKey, + }); + const settled = raw.status === 'completed' + ? mapCoreInvocation('release', raw) + : (await this.wait('release', raw.id)).invocation; + if (settled.status !== 'completed' || settled.released !== true) { + throw new NodeSpawnError('The local node did not confirm exact agent release', + settled.errorCode ?? 'release_unconfirmed'); + } + return settled; + } + + private async wait( + action: NodeSpawnAction, + id: string, + timeoutMs = 360_000, + initial?: NodeSpawnInvocation, + ): Promise<{ invocation: NodeSpawnInvocation; cleanup?: NodeSpawnIdentity }> { + const deadline = Date.now() + timeoutMs; + let cleanup = this.cleanupIdentity(action, initial); + while (Date.now() < deadline) { + let value: NodeSpawnInvocation; + try { + value = await this.status(action, id); + } catch (error) { + if (cleanup === undefined) throw error; + if (error instanceof NodeSpawnError) { + throw new NodeSpawnError(error.message, error.code, error.cause ?? error, cleanup); + } + throw new NodeSpawnError(`Could not confirm local ${action}`, `${action}_unconfirmed`, error, cleanup); + } + const observed = this.cleanupIdentity(action, value); + if (cleanup !== undefined && value.spawned === true + && ((value.id !== '' && cleanup.id !== value.id) + || (value.nodeId !== undefined && cleanup.nodeId !== value.nodeId) + || (value.name !== undefined && cleanup.name !== value.name) + || (value.agentId !== undefined && cleanup.agentId !== value.agentId))) { + throw new NodeSpawnError('The local spawn confirmation changed its exact identity', + 'spawn_identity_mismatch', undefined, cleanup); + } + cleanup ??= observed; + if (terminalStatus(value.status)) return { invocation: value, cleanup }; + await new Promise(resolve => setTimeout(resolve, 500)); + } + throw new NodeSpawnError(`Timed out waiting for local ${action} confirmation`, + `${action}_unconfirmed`, undefined, cleanup); + } + + private cleanupIdentity( + action: NodeSpawnAction, + value?: NodeSpawnInvocation, + ): NodeSpawnIdentity | undefined { + if (action !== 'spawn' || value?.spawned !== true || value.id.trim() === '' + || value.nodeId !== this.exactNode.id || !value.name || value.name.trim() === '' + || !value.agentId || value.agentId.trim() === '') return undefined; + return { id: value.id, nodeId: value.nodeId, nodeName: this.exactNode.name, + name: value.name, agentId: value.agentId }; + } +} + +const LOCAL_NODE_SPAWN_IDENTITY = Object.freeze({ + prototype: LocalNodeSpawnPort.prototype, + spawn: LocalNodeSpawnPort.prototype.spawn, + targets: LocalNodeSpawnPort.prototype.targets, + status: LocalNodeSpawnPort.prototype.status, + release: LocalNodeSpawnPort.prototype.release, +}); + +function withinDirectory(root: string, candidate: string): boolean { + const path = relative(root, candidate); + return path === '' || (path !== '..' && !path.startsWith(`..${sep}`) + && !isAbsolute(path)); +} + +function mapCoreInvocation(action: NodeSpawnAction, value: CoreInvocation): NodeSpawnInvocation { + return { + id: value.id, + action, + status: normalizeNodeSpawnStatus(value.status), + nodeId: value.node_id ?? undefined, + name: value.name ?? undefined, + agentId: value.agent_id ?? undefined, + spawned: value.spawned, + ready: value.ready, + taskUnconfirmed: value.task_unconfirmed, + released: value.released, + errorCode: value.error_code ?? undefined, + }; +} diff --git a/packages/sdk/src/node-spawn-remote.ts b/packages/sdk/src/node-spawn-remote.ts new file mode 100644 index 000000000..bc8ec39e9 --- /dev/null +++ b/packages/sdk/src/node-spawn-remote.ts @@ -0,0 +1,280 @@ +import { setTimeout as delay } from 'node:timers/promises'; +import { + NodeSpawnError, + NodeSpawnPreDispatchError, + isRecord, + normalizeNodeSpawnStatus, + nonempty, + terminalStatus, + validIdempotencyKey, + type BeforeNodeDispatch, + type NodeReleaseRequest, + type NodeSpawnAction, + type NodeSpawnIdentity, + type NodeSpawnInvocation, + type NodeSpawnPort, + type NodeSpawnReceipt, + type NodeSpawnRequest, + type NodeSpawnTarget, +} from './node-spawn.js'; + +interface RelayAgent { id: string; name: string } +interface RelayNode { + id?: string; nodeId?: string; name: string; status: string; live?: boolean; + handlersLive?: boolean; capabilities: Array<{ name: string }>; + activeAgents?: number; maxAgents?: number; +} +interface RelayInvocation { + invocationId: string; actionName: string; status: string; handlerNodeId?: string | null; + dispatchedNodeId?: string | null; output?: Record | null; error?: string | null; +} +interface PlacementAck extends RelayInvocation { + node?: RelayNode; + placement: { node?: string; confirmed: boolean; state: string }; + confirmation?: RelayInvocation; +} + +interface PlacementFailure extends Error { + code?: unknown; + node?: unknown; + invocationId?: unknown; + receipt?: unknown; +} + +export interface RemoteNodeSpawnClient { + agents: { get(name: string): Promise }; + nodes: { list(options?: Record): Promise }; + placement: { spawn(input: Record): Promise }; + commands: { getInvocation(name: string, invocationId: string): Promise }; +} + +/** Fleet node-spawn adapter backed only by the supported Agent Relay SDK. */ +export class RemoteNodeSpawnPort implements NodeSpawnPort { + readonly dispatchFenceVersion = 1 as const; + + constructor(private readonly client: RemoteNodeSpawnClient) {} + + async targets(): Promise { + return (await this.client.nodes.list()).map(node => ({ + id: node.nodeId ?? node.id, + name: node.name, + live: node.live ?? node.status === 'online', + handlersLive: node.handlersLive ?? false, + capabilities: node.capabilities.map(capability => capability.name), + activeAgents: node.activeAgents, + maxAgents: node.maxAgents, + })); + } + + async spawn(request: NodeSpawnRequest, beforeDispatch: BeforeNodeDispatch): Promise { + if (!validIdempotencyKey(request.idempotencyKey)) { + throw new NodeSpawnPreDispatchError( + new NodeSpawnError('Invalid node-spawn idempotency key', 'spawn_invalid_input')); + } + const capability = `spawn:${request.harness}`; + let ack: PlacementAck; + try { + await beforeDispatch(); + ack = await this.client.placement.spawn({ + capability, + ...(request.node === undefined ? {} : { node: request.node }), + ...(request.repo === undefined ? {} : { repo: request.repo }), + ...(request.sandboxOnly === undefined ? {} : { sandboxOnly: request.sandboxOnly }), + failFast: true, + confirm: true, + verifyReady: true, + confirmTimeoutMs: request.timeoutMs ?? 360_000, + input: { + name: request.name, + task: request.task, + idempotency_key: request.idempotencyKey, + ...(request.model === undefined ? {} : { model: request.model }), + ...(request.directory === undefined ? {} : { worker_cwd: request.directory }), + }, + }); + } catch (error) { + const cleanup = placementFailureCleanup(error, request); + if (cleanup !== undefined) { + const failure = error as PlacementFailure; + const output = isRecord(failure.receipt) && isRecord(failure.receipt.output) + ? failure.receipt.output : undefined; + const taskUnconfirmed = output?.warning === 'spawn_task_unconfirmed' + || (isRecord(output?.warning) && output.warning.code === 'spawn_task_unconfirmed'); + throw new NodeSpawnError( + error instanceof Error ? error.message : 'Fleet placement failed after dispatch', + taskUnconfirmed ? 'spawn_task_unconfirmed' + : nonempty(failure.code) ? failure.code : 'spawn_unconfirmed', + error, + cleanup, + ); + } + throw error; + } + const confirmation = ack.confirmation; + const output = confirmation?.output; + const reportedName = nonempty(output?.name) ? output.name : request.name; + const nodeId = ack.node?.nodeId ?? ack.node?.id + ?? confirmation?.dispatchedNodeId ?? confirmation?.handlerNodeId ?? undefined; + let nodeName = ack.node?.name ?? ack.placement.node ?? request.node; + if (!nodeName && nodeId) { + nodeName = await this.resolveNodeName(nodeId); + } + const taskUnconfirmed = output?.warning === 'spawn_task_unconfirmed' + || (isRecord(output?.warning) && output.warning.code === 'spawn_task_unconfirmed'); + let cleanup: NodeSpawnIdentity | undefined; + if (confirmation && output?.spawned === true && nodeName) { + const outputAgentId = nonempty(output.agent_id) ? output.agent_id : undefined; + const outputCleanup: NodeSpawnIdentity | undefined = outputAgentId === undefined ? undefined : { + id: confirmation.invocationId, + nodeId, + nodeName, + name: reportedName, + agentId: outputAgentId, + }; + let agent: RelayAgent; + try { + agent = await this.resolveAgent(reportedName); + } catch (error) { + if (error instanceof NodeSpawnError) { + throw new NodeSpawnError(error.message, error.code, error.cause ?? error, outputCleanup); + } + throw error; + } + const exactAgentName = nonempty(agent.name) ? agent.name : reportedName; + if (!nonempty(agent.id)) { + throw new NodeSpawnError('The spawned agent identity could not be resolved exactly', + 'spawn_identity_mismatch', undefined, outputCleanup); + } + if (exactAgentName !== reportedName || (outputAgentId !== undefined && outputAgentId !== agent.id)) { + throw new NodeSpawnError('The spawned agent identity could not be resolved exactly', + 'spawn_identity_mismatch', undefined, outputCleanup); + } + cleanup = { + id: confirmation.invocationId, + nodeId, + nodeName, + name: exactAgentName, + agentId: agent.id, + }; + } + if (!confirmation || confirmation.status.toLowerCase() !== 'completed' + || output?.spawned !== true || output.ready !== true) { + throw new NodeSpawnError('The fleet node did not prove a ready spawned agent', + 'spawn_unconfirmed', undefined, cleanup); + } + if (!cleanup) { + throw new NodeSpawnError('The spawn result did not identify its exact agent and node', 'spawn_identity_unavailable'); + } + if (ack.placement.confirmed !== true) { + throw new NodeSpawnError('The fleet placement was not confirmed', + 'spawn_placement_unconfirmed', undefined, cleanup); + } + if (taskUnconfirmed) { + throw new NodeSpawnError('The fleet node could not confirm initial task delivery', + 'spawn_task_unconfirmed', undefined, cleanup); + } + if (reportedName !== request.name) { + throw new NodeSpawnError('The spawn result named a different agent', + 'spawn_identity_mismatch', undefined, cleanup); + } + return { + ...cleanup, + action: 'spawn', + status: 'completed', + spawned: true, + ready: true, + taskUnconfirmed: false, + }; + } + + async status(action: NodeSpawnAction, invocationId: string): Promise { + return mapInvocation(action, await this.client.commands.getInvocation(action, invocationId)); + } + + async release(request: NodeReleaseRequest): Promise { + if (!validIdempotencyKey(request.idempotencyKey)) { + throw new NodeSpawnError('Invalid node-release idempotency key', 'spawn_invalid_input'); + } + const receipt = request.receipt; + const ack = await this.client.placement.spawn({ + capability: 'release', + actionName: 'release', + node: receipt.nodeName, + failFast: true, + confirm: false, + input: { + name: receipt.name, + expected_agent_id: receipt.agentId, + idempotency_key: request.idempotencyKey, + reason: request.reason ?? 'Relayflows attempt ended', + }, + }); + const deadline = Date.now() + 60_000; + let invocation = mapInvocation('release', ack); + while (!terminalStatus(invocation.status) && Date.now() < deadline) { + await delay(500); + invocation = await this.status('release', ack.invocationId); + } + if (invocation.status !== 'completed' || invocation.released !== true) { + throw new NodeSpawnError('The fleet node did not confirm exact agent release', invocation.errorCode ?? 'release_unconfirmed'); + } + return invocation; + } + + private async resolveAgent(name: string): Promise { + let last: unknown; + for (let attempt = 0; attempt < 10; attempt += 1) { + try { return await this.client.agents.get(name); } + catch (error) { last = error; await delay(250); } + } + throw new NodeSpawnError('The ready agent was not visible in the Relay roster', 'spawn_identity_unavailable', last); + } + + private async resolveNodeName(id: string): Promise { + for (let attempt = 0; attempt < 10; attempt += 1) { + try { + const nodes = await this.client.nodes.list(); + const name = nodes.find(node => (node.nodeId ?? node.id) === id)?.name; + if (name) return name; + } catch { /* Roster convergence is retried within the confirmation budget. */ } + if (attempt < 9) await delay(250); + } + return undefined; + } +} + +function placementFailureCleanup(error: unknown, request: NodeSpawnRequest): NodeSpawnIdentity | undefined { + if (!(error instanceof Error)) return undefined; + const failure = error as PlacementFailure; + if (!isRecord(failure.receipt)) return undefined; + const receipt = failure.receipt; + const output = isRecord(receipt.output) ? receipt.output : undefined; + const invocationId = nonempty(failure.invocationId) ? failure.invocationId + : nonempty(receipt.invocationId) ? receipt.invocationId : undefined; + const nodeName = nonempty(failure.node) ? failure.node : request.node; + const nodeId = nonempty(receipt.dispatchedNodeId) ? receipt.dispatchedNodeId + : nonempty(receipt.handlerNodeId) ? receipt.handlerNodeId : undefined; + const agentId = nonempty(output?.agent_id) ? output.agent_id : undefined; + const name = nonempty(output?.name) ? output.name : request.name; + if (invocationId === undefined || nodeName === undefined || agentId === undefined + || output?.spawned !== true || name !== request.name) return undefined; + return { id: invocationId, nodeId, nodeName, name, agentId }; +} + +function mapInvocation(action: NodeSpawnAction, value: RelayInvocation): NodeSpawnInvocation { + const output = value.output; + const status = value.status.toLowerCase(); + return { + id: value.invocationId, + action, + status: normalizeNodeSpawnStatus(status), + nodeId: value.dispatchedNodeId ?? value.handlerNodeId ?? undefined, + name: nonempty(output?.name) ? output.name : undefined, + spawned: output?.spawned === true, + ready: output?.ready === true, + released: output?.released === true, + taskUnconfirmed: output?.warning === 'spawn_task_unconfirmed' + || (isRecord(output?.warning) && output.warning.code === 'spawn_task_unconfirmed'), + errorCode: value.error?.split(':', 1)[0]?.trim() || undefined, + }; +} diff --git a/packages/sdk/src/node-spawn-result.ts b/packages/sdk/src/node-spawn-result.ts new file mode 100644 index 000000000..9ef63949b --- /dev/null +++ b/packages/sdk/src/node-spawn-result.ts @@ -0,0 +1,203 @@ +import { canonicalize } from './canonical.js'; +import { isRecord, nonempty } from './node-spawn.js'; + +export const NODE_RESULT_PREFIX = 'RELAYFLOWS_RESULT_V1 '; + +export interface NodeResultEnvelope { + version: 1; + run_id: string; + step_id: string; + dispatch_id: string; + status: 'completed' | 'failed'; + output?: unknown; + error?: string; + accounting?: { tokens_input?: number; tokens_output?: number }; +} + +export interface NodeResultCorrelation { + runId: string; + stepId: string; + dispatchId: string; +} + +export interface NodeResultTerminalFailure { + code: 'result_too_large' | 'result_write_failed'; + bytes: number; + limit: number; +} + +interface InboxItem { + id: string; + message: { id: string; text: string; from: { id?: string; name?: string } }; + terminalFailure?: NodeResultTerminalFailure; + /** Local Desktop delivery teardown; settlement uses a separate authenticated connection after this resolves. */ + settlementReady?: Promise<{ failure?: unknown }>; +} + +export interface NodeResultInboxClient { + inbox: { + subscribe(input?: { signal?: AbortSignal; invocationId?: string }): AsyncIterable; + ack(input: { inboxItemId: string; state?: 'delivered' | 'read'; metadata?: Record }): Promise; + fail(input: { inboxItemId: string; error: string; retry?: boolean; metadata?: Record }): Promise; + }; +} + +export interface NodeResultDelivery { + inboxItemId: string; + messageId: string; + envelope: NodeResultEnvelope; + settlementAction: 'ack' | 'fail'; + /** Non-rejecting current-process teardown outcome. Durable replay intentionally omits it. */ + settlementReady?: Promise<{ failure?: unknown }>; + /** Durably records the journal settlement that must precede inbox settlement. */ + prepareSettlement?(input: { attempt: number; completionReason: string }): Promise; + /** Called only after the result has been durably appended to the run journal. */ + ack(): Promise; +} + +export function nodeResultInstruction(correlation: NodeResultCorrelation): string { + const template = { + version: 1, + run_id: correlation.runId, + step_id: correlation.stepId, + dispatch_id: correlation.dispatchId, + status: 'completed', + output: { summary: 'replace with the final task result' }, + }; + return `When the task is finished, use the Agent Relay report command supplied in your spawn instructions exactly once. ` + + `The entire report body must be this prefix followed by one compact JSON object: ${NODE_RESULT_PREFIX}${canonicalize(template)}. ` + + 'Keep version, run_id, step_id, and dispatch_id unchanged. Set status to completed and output to the final JSON value, ' + + 'or set status to failed and error to a concise reason. Wait for the report command to succeed before ending the turn.'; +} + +export async function waitForNodeResult( + client: NodeResultInboxClient, + expected: { agentId: string; agentName: string; invocationId: string } & NodeResultCorrelation, + signal?: AbortSignal, +): Promise { + const iterator = client.inbox.subscribe({ signal, invocationId: expected.invocationId })[Symbol.asyncIterator](); + let closing: Promise | undefined; + const close = (): Promise => { + closing ??= Promise.resolve().then(() => iterator.return?.()).then(() => undefined); + return closing; + }; + try { + for (;;) { + const next = await abortableNext(iterator, signal, close); + if (next.done) break; + const item = next.value; + signal?.throwIfAborted(); + if (item.message.from.id !== expected.agentId || item.message.from.name !== expected.agentName) continue; + if (item.terminalFailure !== undefined) { + const failure = item.terminalFailure; + const reason = `${failure.code} (bytes=${failure.bytes}, limit=${failure.limit})`; + return rejectedDelivery(client, item, expected, reason); + } + if (!item.message.text.startsWith(NODE_RESULT_PREFIX)) continue; + let raw: unknown; + try { raw = JSON.parse(item.message.text.slice(NODE_RESULT_PREFIX.length)); } + catch { raw = undefined; } + let envelope: NodeResultEnvelope; + try { envelope = readNodeResult(raw, expected); } + catch (error) { + const reason = error instanceof Error ? error.message : 'Malformed Relayflows result'; + return rejectedDelivery(client, item, expected, reason); + } + return { + inboxItemId: item.id, + messageId: item.message.id, + envelope, + settlementAction: 'ack', + settlementReady: settlementOutcome(item), + ack: async () => { await awaitSettlementReady(item); await client.inbox.ack({ inboxItemId: item.id, state: 'read', + metadata: { relayflows_result: 'journaled' } }); }, + }; + } + signal?.throwIfAborted(); + throw new Error('Agent Relay result inbox ended before the spawned agent reported completion'); + } finally { + // A client that ignores cancellation may leave next() pending forever. + // Preserve prompt lease loss while still closing only this subscription. + // The runtime owner closes the shared agent connection after every + // concurrent task has drained. + if (signal?.aborted) { + void close().catch(() => undefined); + } else { + // The outcome is already decided. Iterator teardown must not replace it + // or hold the worker lease forever when return() rejects or never settles. + void close().catch(() => undefined); + } + } +} + +function rejectedDelivery( + client: NodeResultInboxClient, + item: InboxItem, + expected: NodeResultCorrelation, + reason: string, +): NodeResultDelivery { + return { + inboxItemId: item.id, + messageId: item.message.id, + envelope: { version: 1, run_id: expected.runId, step_id: expected.stepId, + dispatch_id: expected.dispatchId, status: 'failed', error: reason }, + settlementAction: 'fail', + settlementReady: settlementOutcome(item), + ack: async () => { await awaitSettlementReady(item); await client.inbox.fail({ inboxItemId: item.id, error: reason, retry: false, + metadata: { relayflows_result: 'rejected' } }); }, + }; +} + +function settlementOutcome(item: InboxItem): Promise<{ failure?: unknown }> | undefined { + return item.settlementReady?.then(outcome => outcome, failure => ({ failure })); +} + +async function awaitSettlementReady(item: InboxItem): Promise { + const teardown = await item.settlementReady; + if (teardown?.failure !== undefined) throw teardown.failure; +} + +async function abortableNext( + iterator: AsyncIterator, + signal: AbortSignal | undefined, + close: () => Promise, +): Promise> { + signal?.throwIfAborted(); + const pending = iterator.next(); + if (signal === undefined) return pending; + let onAbort: (() => void) | undefined; + const aborted = new Promise((_resolve, reject) => { + onAbort = () => reject(signal.reason ?? new DOMException('The operation was aborted', 'AbortError')); + signal.addEventListener('abort', onAbort, { once: true }); + }); + try { return await Promise.race([pending, aborted]); } + catch (error) { + if (signal.aborted) { + void pending.catch(() => undefined); + void close().catch(() => undefined); + } + throw error; + } finally { + if (onAbort !== undefined) signal.removeEventListener('abort', onAbort); + } +} + +export function readNodeResult(value: unknown, expected: NodeResultCorrelation): NodeResultEnvelope { + if (!isRecord(value) || value.version !== 1 || value.run_id !== expected.runId + || value.step_id !== expected.stepId || value.dispatch_id !== expected.dispatchId + || (value.status !== 'completed' && value.status !== 'failed')) { + throw new Error('Spawned agent result has invalid attempt correlation'); + } + if (value.status === 'completed' && !Object.hasOwn(value, 'output')) { + throw new Error('Spawned agent completion omitted its output'); + } + if (value.status === 'failed' && !nonempty(value.error)) { + throw new Error('Spawned agent failure omitted its reason'); + } + if (value.accounting !== undefined && (!isRecord(value.accounting) + || Object.entries(value.accounting).some(([key, amount]) => !['tokens_input', 'tokens_output'].includes(key) + || typeof amount !== 'number' || !Number.isSafeInteger(amount) || amount < 0))) { + throw new Error('Spawned agent result has invalid accounting'); + } + return value as unknown as NodeResultEnvelope; +} diff --git a/packages/sdk/src/node-spawn-runtime.ts b/packages/sdk/src/node-spawn-runtime.ts new file mode 100644 index 000000000..dbe672ac9 --- /dev/null +++ b/packages/sdk/src/node-spawn-runtime.ts @@ -0,0 +1,419 @@ +import { isIP } from 'node:net'; +import { isAbsolute } from 'node:path'; +import type { NodeResultInboxClient, NodeResultTerminalFailure } from './node-spawn-result.js'; +import { + enableLoadedCoreBridgeCapabilities, + LocalNodeSpawnPort, + loadCoreBridge, + type CallerDescriptor, + type CoreBridgeApi, + type CoreBridgeClient, + type CoreInboxDelivery, +} from './node-spawn-local.js'; +import { NodeSpawnError } from './node-spawn.js'; +import { RemoteNodeSpawnPort, type RemoteNodeSpawnClient } from './node-spawn-remote.js'; +import type { NodeSpawnPort } from './node-spawn.js'; + +export interface NodeSpawnPublisher { id: string; name: string } +export interface NodeSpawnRuntime { + port: NodeSpawnPort; + inbox: NodeResultInboxClient; + publisher: NodeSpawnPublisher; + baseUrl: string; + node?: string; + /** Exact local node id paired with node; remote placement leaves this unset. */ + nodeId?: string; + repo?: string; + sandboxOnly?: boolean; + /** Explicit authority family; Desktop alone may receive the worker's local run root. */ + authority?: 'desktop' | 'remote'; + /** Closes the publisher connection after every task using it has drained. */ + disconnect?(): Promise; +} + +export interface NodeSpawnRuntimeOwner { + get(signal?: AbortSignal): Promise; + /** Returns only authority already established by configuration or a successful get(). */ + peek(): NodeSpawnRuntime | undefined; + close(): Promise; +} + +/** Applies the same authority precedence for runtime creation and worker directory policy. */ +export function mergeNodeSpawnEnvironment( + environment: NodeJS.ProcessEnv | undefined, + serviceCallerFile: string | undefined, + inherited: NodeJS.ProcessEnv = process.env, +): NodeJS.ProcessEnv { + return { + ...inherited, + ...environment, + ...(serviceCallerFile === undefined + ? {} + : { RELAYFLOW_SERVICE_CALLER_FILE: serviceCallerFile }), + }; +} + +/** Returns an exact nonempty Desktop capability path; paths are never normalized. */ +export function explicitServiceCallerFile(env: NodeJS.ProcessEnv): string | undefined { + const serviceCallerFile = env.RELAYFLOW_SERVICE_CALLER_FILE; + return serviceCallerFile === undefined || serviceCallerFile === '' ? undefined : serviceCallerFile; +} + +interface RelayRuntimeClient extends RemoteNodeSpawnClient, NodeResultInboxClient { + agents: RemoteNodeSpawnClient['agents'] & { me(): Promise }; + events: { disconnect(): Promise }; +} + +const PUBLISHER_DISCOVERY_TIMEOUT_MS = 15_000; + +export async function createRemoteNodeSpawnRuntime( + env: NodeJS.ProcessEnv = process.env, + options: { signal?: AbortSignal } = {}, +): Promise { + const agentToken = env.RELAY_AGENT_TOKEN?.trim(); + if (!agentToken) throw new NodeSpawnError( + 'Node-spawn transport requires a dedicated publisher RELAY_AGENT_TOKEN.', 'publisher_token_required'); + const baseUrl = validateNodeSpawnBaseUrl(env.RELAY_BASE_URL?.trim() || 'https://cast.agentrelay.com'); + let RelaycastMessagingClient: new (options: Record) => RelayRuntimeClient; + try { + ({ RelaycastMessagingClient } = await waitForRuntime(import('@agent-relay/sdk'), options.signal) as unknown as { + RelaycastMessagingClient: new (options: Record) => RelayRuntimeClient; + }); + } catch (error) { + if (options.signal?.aborted === true) throw options.signal.reason ?? error; + throw new NodeSpawnError('Node-spawn transport requires optional @agent-relay/sdk >=17.2.1 <19.', + 'relay_sdk_unavailable', error); + } + const sandbox = env.RELAYFLOW_NODE_SANDBOX_ONLY?.trim(); + if (sandbox !== undefined && sandbox !== '' && sandbox !== '0' && sandbox !== '1') { + throw new NodeSpawnError('RELAYFLOW_NODE_SANDBOX_ONLY must be 0 or 1', 'spawn_invalid_input'); + } + const client = new RelaycastMessagingClient({ baseUrl, agentToken }); + const timeout = AbortSignal.timeout(PUBLISHER_DISCOVERY_TIMEOUT_MS); + const signal = options.signal === undefined ? timeout : AbortSignal.any([options.signal, timeout]); + let publisher: NodeSpawnPublisher; + try { + publisher = await discoverPublisher(client, signal); + if (!publisher.id?.trim() || !publisher.name?.trim()) { + throw new NodeSpawnError('The node-spawn publisher token has no exact Relay identity', 'publisher_identity_missing'); + } + } catch (error) { + await client.events.disconnect().catch(() => undefined); + throw error; + } + return { + port: new RemoteNodeSpawnPort(client), + inbox: { inbox: client.inbox }, + publisher, + baseUrl, + authority: 'remote', + disconnect: () => client.events.disconnect(), + ...(env.RELAYFLOW_NODE_NAME?.trim() ? { node: env.RELAYFLOW_NODE_NAME.trim() } : {}), + ...(env.RELAYFLOW_NODE_REPO?.trim() ? { repo: env.RELAYFLOW_NODE_REPO.trim() } : {}), + ...(sandbox === '1' ? { sandboxOnly: true } : {}), + }; +} + +/** + * Opens one explicitly configured Desktop service caller. The caller file is + * the authority: there is no directory scan and no fallback to a remote token. + */ +export async function createLocalNodeSpawnRuntime( + serviceCallerFile: string, + options: { signal?: AbortSignal; bridge?: CoreBridgeApi } = {}, +): Promise { + if (!serviceCallerFile || !isAbsolute(serviceCallerFile) || serviceCallerFile.includes('\0')) { + throw new NodeSpawnError( + 'RELAYFLOW_SERVICE_CALLER_FILE must be an absolute path to caller.json', + 'caller_file_invalid', + ); + } + const internallyLoadedBridge = options.bridge === undefined; + const bridge = options.bridge ?? await loadCoreBridge(); + const descriptor = await bridge.readCallerDescriptor(serviceCallerFile); + const core = await bindCore(bridge, descriptor, options.signal); + try { + const identity = await bridge.serviceCallerIdentity(core, { signal: options.signal }); + bridge.assertServiceCallerIdentity(descriptor, identity); + if (!identity.node.id.trim() || !identity.node.name.trim()) { + throw new NodeSpawnError( + 'Desktop service-caller identity omitted its exact node identity', + 'caller_identity_mismatch', + ); + } + const port = new LocalNodeSpawnPort(core, bridge, descriptor, identity.node); + if (internallyLoadedBridge) enableLoadedCoreBridgeCapabilities(port, bridge); + return { + port, + inbox: localResultInbox(bridge, descriptor), + publisher: identity.publisher, + baseUrl: identity.base_url, + authority: 'desktop', + node: identity.node.name, + nodeId: identity.node.id, + disconnect: async () => { + await disconnectBoundCore(bridge, core); + }, + }; + } catch (error) { + await disconnectBoundCore(bridge, core, error); + throw error; // unreachable; keeps the return type explicit + } +} + +/** Selects exactly one authority family and refuses ambiguous credentials. */ +export async function createNodeSpawnRuntime( + env: NodeJS.ProcessEnv = process.env, + options: { signal?: AbortSignal; bridge?: CoreBridgeApi } = {}, +): Promise { + const serviceCallerFile = explicitServiceCallerFile(env); + const agentToken = env.RELAY_AGENT_TOKEN?.trim(); + if (serviceCallerFile !== undefined && agentToken) { + throw new NodeSpawnError( + 'RELAYFLOW_SERVICE_CALLER_FILE and RELAY_AGENT_TOKEN name different node-spawn authorities; choose one', + 'ambiguous_publisher_authority', + ); + } + if (serviceCallerFile !== undefined) return createLocalNodeSpawnRuntime(serviceCallerFile, options); + return createRemoteNodeSpawnRuntime(env, options); +} + +/** One lazy publisher connection shared by every relay task on a worker. */ +export function createNodeSpawnRuntimeOwner( + provided?: NodeSpawnRuntime, + factory: (signal: AbortSignal) => Promise = signal => + createNodeSpawnRuntime(process.env, { signal }), +): NodeSpawnRuntimeOwner { + const controller = new AbortController(); + let pending: Promise | undefined; + let established = provided; + let closed = false; + let closePromise: Promise | undefined; + const get = async (signal?: AbortSignal): Promise => { + signal?.throwIfAborted(); + if (closed) throw new NodeSpawnError('Node-spawn runtime owner is closed', 'task_cancelled'); + if (established !== undefined) return established; + if (pending === undefined) { + const created = factory(controller.signal).then(runtime => { + established = runtime; + return runtime; + }); + pending = created; + void created.catch(() => { if (pending === created) pending = undefined; }); + } + return waitForRuntime(pending, signal); + }; + const closeOnce = async (): Promise => { + closed = true; + controller.abort(new Error('Node-spawn runtime owner closed')); + if (provided !== undefined) return; + const runtime = await pending?.catch(() => undefined); + await runtime?.disconnect?.().catch(() => undefined); + }; + return { get, peek: () => established, close: () => closePromise ??= closeOnce() }; +} + +async function bindCore( + bridge: CoreBridgeApi, + descriptor: CallerDescriptor, + signal?: AbortSignal, +): Promise { + const core = await bridge.connectCore({ socket: descriptor.socket, signal }); + try { + bridge.requireServiceCallerMethods(core); + await bridge.authenticate(core, descriptor.credential, { signal }); + return core; + } catch (error) { + core.close(); + throw error; + } +} + +function localResultInbox(bridge: CoreBridgeApi, descriptor: CallerDescriptor): NodeResultInboxClient { + const settle = async (action: 'ack' | 'fail', itemId: string): Promise => { + const core = await bindCore(bridge, descriptor); + let failure: unknown; + try { + if (action === 'ack') await bridge.serviceCallerInboxAck(core, itemId); + else await bridge.serviceCallerInboxFail(core, itemId); + } catch (error) { failure = error; } + await disconnectBoundCore(bridge, core, failure); + }; + return { inbox: { + subscribe(input = {}) { + const invocationId = input.invocationId?.trim(); + if (!invocationId) { + throw new NodeSpawnError( + 'Local result inbox requires the exact spawn invocation id', + 'result_identity_missing', + ); + } + return localInboxSubscription( + bridge, + descriptor, + invocationId, + input.signal, + ); + }, + async ack(input) { await settle('ack', input.inboxItemId); }, + async fail(input) { await settle('fail', input.inboxItemId); }, + } }; +} + +function localInboxSubscription( + bridge: CoreBridgeApi, + descriptor: CallerDescriptor, + invocationId: string, + signal?: AbortSignal, +): AsyncIterable<{ id: string; message: { id: string; text: string; from: { id?: string; name?: string } } }> { + return { [Symbol.asyncIterator]() { + let core: CoreBridgeClient | undefined; + let binding: Promise | undefined; + let opening: Promise | undefined; + let iteratorId: string | undefined; + let closed = false; + let closePromise: Promise | undefined; + let resolveClosed!: (result: { failure?: unknown }) => void; + const closedPromise = new Promise<{ failure?: unknown }>(resolve => { + resolveClosed = resolve; + }); + const closeOnce = async (): Promise => { + closed = true; + let failure: unknown; + if (opening !== undefined) { + try { await opening; } + catch (error) { failure = error; } + } + const bound = core; + if (bound === undefined) { + resolveClosed(failure === undefined ? {} : { failure }); + if (failure !== undefined) throw failure; + return; + } + try { + if (iteratorId !== undefined) await bridge.serviceCallerInboxClose(bound, iteratorId); + } catch (error) { failure ??= error; } + try { + await disconnectBoundCore(bridge, bound, failure); + resolveClosed({}); + } catch (error) { + resolveClosed({ failure: error }); + throw error; + } + }; + const close = (): Promise => closePromise ??= closeOnce(); + const ensureOpen = (): Promise => opening ??= (async () => { + binding ??= bindCore(bridge, descriptor, signal); + core = await binding; + if (closed) return; + const opened = await bridge.serviceCallerInboxOpen(core, invocationId, { signal }); + iteratorId = opened.iterator_id; + })(); + return { + async next() { + for (;;) { + signal?.throwIfAborted(); + if (closed) return { done: true, value: undefined }; + if (core === undefined) { + await ensureOpen(); + if (closed) { + await close(); + return { done: true, value: undefined }; + } + } + const bound = core; + if (bound === undefined || iteratorId === undefined) { + throw new NodeSpawnError('Desktop inbox binding did not return an iterator', + 'invalid_core_response'); + } + const result = await bridge.serviceCallerInboxNext(bound, iteratorId, 30_000, { signal }); + if (result.delivery !== null) { + if (result.delivery.invocation_id !== invocationId) { + throw new NodeSpawnError( + 'Desktop returned a result outside the subscribed spawn invocation', + 'result_identity_mismatch', + ); + } + return { done: false as const, value: mapInboxDelivery(result.delivery, closedPromise) }; + } + } + }, + async return() { + await close(); + return { done: true as const, value: undefined }; + }, + }; + } }; +} + +async function disconnectBoundCore( + bridge: Pick, + core: CoreBridgeClient, + priorFailure?: unknown, +): Promise { + let disconnectFailure: unknown; + try { await bridge.serviceCallerDisconnect(core); } + catch (error) { disconnectFailure = error; } + finally { core.close(); } + if (priorFailure !== undefined) throw priorFailure; + if (disconnectFailure !== undefined) throw disconnectFailure; +} + +function mapInboxDelivery(delivery: CoreInboxDelivery, settlementReady: Promise<{ failure?: unknown }>) { + return { + id: delivery.item_id, + settlementReady, + message: { + id: delivery.message_id, + text: delivery.text ?? '', + from: { id: delivery.sender.id, name: delivery.sender.name }, + }, + ...(delivery.failure === undefined ? {} : { + terminalFailure: delivery.failure satisfies NodeResultTerminalFailure, + }), + }; +} + +async function waitForRuntime(pending: Promise, signal?: AbortSignal): Promise { + if (signal === undefined) return pending; + let onAbort: (() => void) | undefined; + const aborted = new Promise((_resolve, reject) => { + onAbort = () => reject(signal.reason ?? new DOMException('The operation was aborted', 'AbortError')); + signal.addEventListener('abort', onAbort, { once: true }); + }); + try { return await Promise.race([pending, aborted]); } + finally { if (onAbort !== undefined) signal.removeEventListener('abort', onAbort); } +} + +async function discoverPublisher(client: RelayRuntimeClient, signal: AbortSignal): Promise { + let onAbort: (() => void) | undefined; + try { + signal.throwIfAborted(); + const aborted = new Promise((_resolve, reject) => { + onAbort = () => reject(signal.reason ?? new DOMException('The operation was aborted', 'AbortError')); + signal.addEventListener('abort', onAbort, { once: true }); + }); + return await Promise.race([client.agents.me(), aborted]); + } catch (error) { + throw error; + } finally { + if (onAbort !== undefined) signal.removeEventListener('abort', onAbort); + } +} + +function validateNodeSpawnBaseUrl(raw: string): string { + let url: URL; + try { url = new URL(raw); } + catch (error) { + throw new NodeSpawnError('Invalid Agent Relay base URL', 'spawn_invalid_input', error); + } + if (url.username || url.password || url.search || url.hash || url.pathname !== '/' + || !['https:', 'http:'].includes(url.protocol)) { + throw new NodeSpawnError('Invalid Agent Relay base URL', 'spawn_invalid_input'); + } + const loopback = url.hostname === '[::1]' || (isIP(url.hostname) === 4 && url.hostname.startsWith('127.')); + if (url.protocol !== 'https:' && !loopback) { + throw new NodeSpawnError('Agent Relay credentials require HTTPS outside literal loopback addresses', 'spawn_invalid_input'); + } + return url.origin; +} diff --git a/packages/sdk/src/node-spawn-settlement.ts b/packages/sdk/src/node-spawn-settlement.ts new file mode 100644 index 000000000..1443e553f --- /dev/null +++ b/packages/sdk/src/node-spawn-settlement.ts @@ -0,0 +1,406 @@ +import type { JournalClient } from './journal-client.js'; +import type { JournalEvent } from './journal-reader.js'; +import type { NodeResultDelivery } from './node-spawn-result.js'; +import { nodeSpawnCleanupRequired, releaseNodeSpawnClaim } from './node-spawn-cleanup.js'; +import { readNodeSpawnClaim, readNodeSpawnClaims, recordNodeSpawn, + type NodeSpawnClaim } from './node-spawn-state.js'; +import type { NodeSpawnRuntime } from './node-spawn-runtime.js'; +import { NodeSpawnError, isRecord } from './node-spawn.js'; + +const SETTLEMENT_ATTEMPT_TIMEOUT_MS = 10_000; + +interface LiveTeardown { + settled: boolean; + failure?: unknown; + outcome: Promise<{ failure?: unknown }>; +} + +interface LiveTeardownGroup { + generation: number; + barriers: Set; + settlementToken?: object; +} + +type Reconciliation = 'complete' | 'pending' | 'foreign'; + +/** Process-generation state keyed by durable claim path; a real restart intentionally resets it. */ +const liveTeardowns = new Map(); +const settlementMutations = new Map>(); + +/** True only when a durable claim belongs to this already-established publisher authority. */ +export function nodeSpawnClaimBelongsToRuntime( + claim: Pick, + runtime: Pick, +): boolean { + return claim.baseUrl === runtime.baseUrl + && claim.publisherId === runtime.publisher.id + && claim.publisherName === runtime.publisher.name; +} + +export function durableNodeResultDelivery( + runtime: NodeSpawnRuntime, + path: string, + initial: NodeSpawnClaim, + observed?: NodeResultDelivery, +): NodeResultDelivery { + const result = initial.result; + if (result === undefined) throw new NodeSpawnError('Node result settlement requires a durable result'); + if (observed !== undefined && (observed.inboxItemId !== result.inboxItemId + || observed.messageId !== result.messageId + || observed.settlementAction !== result.settlementAction)) { + throw new NodeSpawnError('Observed node result differs from its durable settlement identity', + 'result_identity_mismatch'); + } + if (observed?.settlementReady !== undefined) { + rememberLiveTeardown(path, observed.settlementReady); + } + let claim = initial; + let settling: Promise | undefined; + const settleOnce = (): Promise => serializeSettlementMutation(path, async () => { + const durable = await loadDurableClaim(path, result); + const current = durable.result; + if (current === undefined) throw new NodeSpawnError('Durable node result disappeared before settlement'); + const expected = claim.result?.settlement; + if (expected === undefined || current.settlement === undefined) { + throw new NodeSpawnError('Node result cannot settle before its journal intent is durable', + 'result_settlement_unprepared'); + } + if (current.settlement.attempt !== expected.attempt + || current.settlement.completionReason !== expected.completionReason) { + throw new NodeSpawnError('Node result settlement conflicts with its journal identity', + 'result_identity_mismatch'); + } + if (current.settlement.settledAt !== undefined) { + if (current.settlement.settledAs !== current.settlementAction) { + throw new NodeSpawnError('Node result settled with a different inbox action', + 'result_identity_mismatch'); + } + claim = durable; + return; + } + const liveSettlement = await settleAfterLiveTeardowns(runtime, path, current.settlementAction, + current.inboxItemId, current.envelope.error ?? 'Relayflows rejected an invalid result'); + const updated: NodeSpawnClaim = { ...durable, result: { ...current, settlement: { + ...current.settlement, + settledAt: Date.now(), settledAs: current.settlementAction, + } } }; + try { + await recordNodeSpawn(path, updated); + } catch (error) { + releaseLiveSettlement(path, liveSettlement); + throw error; + } + claim = updated; + completeLiveSettlement(path, liveSettlement); + }); + const settle = (): Promise => { + if (settling !== undefined) return settling; + const attempt = settleOnce(); + settling = attempt; + void attempt.catch(() => { if (settling === attempt) settling = undefined; }); + return attempt; + }; + return { + inboxItemId: result.inboxItemId, + messageId: result.messageId, + envelope: result.envelope, + settlementAction: result.settlementAction, + ...(observed?.settlementReady === undefined ? {} : { settlementReady: observed.settlementReady }), + prepareSettlement: input => serializeSettlementMutation(path, async () => { + const durable = await loadDurableClaim(path, result); + const current = durable.result; + if (current === undefined) throw new NodeSpawnError('Durable node result disappeared before settlement'); + const prior = current.settlement; + if (prior !== undefined && prior.completionReason !== input.completionReason) { + throw new NodeSpawnError('Node result settlement conflicts with its journal identity', + 'result_identity_mismatch'); + } + if (prior?.settledAt !== undefined && prior.attempt !== input.attempt) { + throw new NodeSpawnError('A settled node result cannot move to a different journal attempt', + 'result_identity_mismatch'); + } + if (prior?.settledAt !== undefined && prior.settledAs !== current.settlementAction) { + throw new NodeSpawnError('Node result settled with a different inbox action', + 'result_identity_mismatch'); + } + if (prior !== undefined && input.attempt < prior.attempt) { + throw new NodeSpawnError('Node result settlement cannot move back to an older journal attempt', + 'result_identity_mismatch'); + } + if (prior !== undefined && prior.attempt === input.attempt) { + claim = durable; + return; + } + const updated: NodeSpawnClaim = { ...durable, result: { ...current, settlement: + prior !== undefined && prior.attempt === input.attempt ? prior : { + attempt: input.attempt, + completionReason: input.completionReason, + preparedAt: Date.now(), + } } }; + await recordNodeSpawn(path, updated); + claim = updated; + }), + ack: settle, + }; +} + +async function loadDurableClaim( + path: string, + expected: NonNullable, +): Promise { + const durable = await readNodeSpawnClaim(path); + const current = durable.result; + if (current === undefined + || current.inboxItemId !== expected.inboxItemId + || current.messageId !== expected.messageId + || current.status !== expected.status + || current.settlementAction !== expected.settlementAction) { + throw new NodeSpawnError('Durable node result differs from its delivery identity', + 'result_identity_mismatch'); + } + return durable; +} + +async function serializeSettlementMutation(path: string, mutate: () => Promise): Promise { + const prior = settlementMutations.get(path) ?? Promise.resolve(); + let release!: () => void; + const occupied = new Promise(resolve => { release = resolve; }); + const queued = prior.then(() => occupied, () => occupied); + settlementMutations.set(path, queued); + await prior.catch(() => undefined); + try { + return await mutate(); + } finally { + release(); + if (settlementMutations.get(path) === queued) settlementMutations.delete(path); + } +} + +/** Reconciles exact cleanup and journaled result settlement left by a crash or failed call. */ +export async function reconcileNodeResultSettlements( + dataDir: string, + runtime: NodeSpawnRuntime, + journal: Pick, +): Promise { + let pending = 0; + for (const entry of await readNodeSpawnClaims(dataDir)) { + const status = await serializeSettlementMutation(entry.path, + () => reconcileNodeResultSettlement(entry.path, runtime, journal)); + if (status === 'pending') pending += 1; + } + return pending; +} + +async function reconcileNodeResultSettlement( + path: string, + runtime: NodeSpawnRuntime, + journal: Pick, +): Promise { + let claim = await readNodeSpawnClaim(path); + if (!nodeSpawnClaimBelongsToRuntime(claim, runtime)) { + return 'foreign'; + } + if (nodeSpawnCleanupRequired(claim)) { + const cleanup = await releaseNodeSpawnClaim(runtime, path, claim); + claim = cleanup.claim; + if (cleanup.error !== undefined) throw cleanup.error; + } + const result = claim.result; + const settlement = result?.settlement; + if (result === undefined || settlement === undefined || settlement.settledAt !== undefined) return 'complete'; + const live = liveTeardowns.get(path); + if (live !== undefined) { + const failed = [...live.barriers].find(item => item.settled && item.failure !== undefined); + if (failed !== undefined) throw failed.failure; + if (live.settlementToken !== undefined || [...live.barriers].some(item => !item.settled)) return 'pending'; + } + const terminal = await terminalEntry(journal, claim.runId, claim.stepId, settlement.attempt); + if (terminal === undefined) return 'pending'; + const exact = terminalMatchesResult(terminal, claim, settlement, result); + const action = exact ? result.settlementAction : 'fail'; + const error = exact ? (result.envelope.error ?? 'Relayflows rejected an invalid result') + : 'The journal completed this attempt with a different terminal outcome'; + const liveSettlement = await settleAfterLiveTeardowns( + runtime, path, action, result.inboxItemId, error); + const updated: NodeSpawnClaim = { ...claim, result: { ...result, settlement: { + ...settlement, settledAt: Date.now(), settledAs: action, + } } }; + try { + await recordNodeSpawn(path, updated); + } catch (writeError) { + releaseLiveSettlement(path, liveSettlement); + throw writeError; + } + completeLiveSettlement(path, liveSettlement); + return 'complete'; +} + +function rememberLiveTeardown( + path: string, + outcome: Promise<{ failure?: unknown }>, +): void { + const normalized = outcome.then(result => result, failure => ({ failure })); + const live: LiveTeardown = { settled: false, outcome: normalized }; + let group = liveTeardowns.get(path); + if (group === undefined) { + group = { generation: 0, barriers: new Set() }; + liveTeardowns.set(path, group); + } + if (group.settlementToken !== undefined) { + throw new NodeSpawnError('Node result gained a live teardown after settlement started', + 'result_settlement_unconfirmed'); + } + group.generation += 1; + group.barriers.add(live); + void normalized.then(result => { + live.settled = true; + live.failure = result.failure; + if (result.failure === undefined) { + liveTeardowns.get(path)?.barriers.delete(live); + } + }); +} + +interface LiveSettlement { + group: LiveTeardownGroup; + generation: number; + token: object; +} + +async function settleAfterLiveTeardowns( + runtime: NodeSpawnRuntime, + path: string, + action: 'ack' | 'fail', + inboxItemId: string, + error: string, +): Promise { + let active: LiveSettlement | undefined; + try { + return await boundedOperation(async signal => { + for (;;) { + let group = liveTeardowns.get(path); + if (group === undefined) { + group = { generation: 0, barriers: new Set() }; + liveTeardowns.set(path, group); + } + if (group.settlementToken !== undefined) { + throw new NodeSpawnError('Node result settlement is already in progress', + 'result_settlement_unconfirmed'); + } + const generation = group.generation; + const outcomes = group.barriers.size === 0 + ? [] + : await Promise.all([...group.barriers].map(item => item.outcome)); + const failed = outcomes.find(item => item.failure !== undefined); + if (failed?.failure !== undefined) throw failed.failure; + signal.throwIfAborted(); + const current = liveTeardowns.get(path); + if (current !== group || current.generation !== generation) continue; + const token = {}; + current.settlementToken = token; + active = { group: current, generation, token }; + try { + await issueInboxSettlement(runtime, action, inboxItemId, error); + signal.throwIfAborted(); + return active; + } catch (settlementError) { + releaseLiveSettlement(path, active); + throw settlementError; + } + } + }); + } catch (error) { + if (active !== undefined) releaseLiveSettlement(path, active); + throw error; + } +} + +function releaseLiveSettlement(path: string, settlement: LiveSettlement): void { + const current = liveTeardowns.get(path); + if (current === settlement.group && current.generation === settlement.generation + && current.settlementToken === settlement.token) { + current.settlementToken = undefined; + } +} + +function completeLiveSettlement(path: string, settlement: LiveSettlement): void { + const current = liveTeardowns.get(path); + if (current === settlement.group && current.generation === settlement.generation + && current.settlementToken === settlement.token && current.barriers.size === 0) { + liveTeardowns.delete(path); + } +} + +function terminalMatchesResult( + terminal: JournalEvent, + claim: NodeSpawnClaim, + settlement: NonNullable['settlement']>, + result: NonNullable, +): boolean { + const payload = isRecord(terminal.payload) ? terminal.payload : {}; + const tail = isRecord(payload.trajectory_tail) ? payload.trajectory_tail : {}; + const relayTask = isRecord(tail.relay_task) ? tail.relay_task : {}; + const execution = isRecord(relayTask.task_execution) ? relayTask.task_execution : {}; + return payload.completionReason === settlement.completionReason + && execution.run_id === claim.runId + && execution.step_id === claim.stepId + && execution.dispatch_id === claim.idempotencyKey + && execution.delivery_id === result.inboxItemId + && execution.message_id === result.messageId; +} + +async function terminalEntry( + journal: Pick, + runId: string, + stepId: string, + attempt: number, +): Promise { + let fromSeq = 1; + while (true) { + const entries = (await journal.journalRead(runId, fromSeq, 256)).entries as JournalEvent[]; + const terminal = entries.find(entry => entry.entry_type === 'step.completed' + && entry.step_id === stepId && entry.attempt === attempt); + if (terminal !== undefined || entries.length < 256) return terminal; + const last = entries.at(-1); + if (last === undefined || !Number.isSafeInteger(last.seq) || last.seq < fromSeq) { + throw new NodeSpawnError('Journal settlement scan did not advance', 'result_settlement_unconfirmed'); + } + fromSeq = last.seq + 1; + } +} + +function issueInboxSettlement( + runtime: NodeSpawnRuntime, + action: 'ack' | 'fail', + inboxItemId: string, + error: string, +): Promise { + const pending = action === 'ack' + ? runtime.inbox.inbox.ack({ inboxItemId, state: 'read', metadata: { relayflows_result: 'journaled' } }) + : runtime.inbox.inbox.fail({ inboxItemId, error, retry: false, + metadata: { relayflows_result: 'rejected' } }); + return pending; +} + +async function boundedOperation(operation: (signal: AbortSignal) => Promise): Promise { + const controller = new AbortController(); + let timer: NodeJS.Timeout | undefined; + const deadline = new Promise((_resolve, reject) => { + timer = setTimeout(() => { + const error = new NodeSpawnError( + 'Node-result inbox settlement did not confirm within its independent deadline', + 'result_settlement_unconfirmed'); + controller.abort(error); + reject(error); + }, SETTLEMENT_ATTEMPT_TIMEOUT_MS); + }); + const pending = operation(controller.signal) + .then(value => ({ ok: true as const, value }), error => ({ ok: false as const, error })); + try { + const settled = await Promise.race([pending, deadline]); + if (!settled.ok) throw settled.error; + return settled.value; + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} diff --git a/packages/sdk/src/node-spawn-state.ts b/packages/sdk/src/node-spawn-state.ts new file mode 100644 index 000000000..a2fc9cbb1 --- /dev/null +++ b/packages/sdk/src/node-spawn-state.ts @@ -0,0 +1,297 @@ +import { createHash, randomUUID } from 'node:crypto'; +import { link, mkdir, open, readFile, readdir, realpath, rename, unlink } from 'node:fs/promises'; +import { basename, dirname, join } from 'node:path'; +import { canonicalize } from './canonical.js'; +import { readNodeResult, type NodeResultEnvelope } from './node-spawn-result.js'; +import { isRecord, nonempty, validIdempotencyKey, validNodeSpawnIdentity, validNodeSpawnReceipt, + type NodeSpawnIdentity, type NodeSpawnReceipt } from './node-spawn.js'; + +export interface NodeSpawnClaim { + version: 2 | 3; + baseUrl: string; + publisherId: string; + publisherName: string; + runId: string; + stepId: string; + idempotencyKey: string; + agentName: string; + timeoutMs: number; + request: Record; + startedAt: number; + spawn?: NodeSpawnReceipt; + cleanup?: NodeSpawnIdentity; + failure?: { code: string; message: string }; + result?: { inboxItemId: string; messageId: string; status: 'completed' | 'failed'; + envelope: NodeResultEnvelope; settlementAction: 'ack' | 'fail'; receivedAt: number; + settlement?: { attempt: number; completionReason: string; preparedAt: number; + settledAt?: number; settledAs?: 'ack' | 'fail' } }; + cleanupPending?: { attempts: number; lastError: string; updatedAt: number }; + release?: { invocationId: string; status: string }; +} + +interface NodeSpawnDispatchFence { + version: 1; + claimDigest: string; +} + +/** A durable write-ahead fence already owns the only permitted dispatch. */ +export class NodeSpawnDispatchFencedError extends Error { + constructor() { + super('A prior node-spawn dispatch entered the ambiguous pre-ack window; refusing a second spawn'); + this.name = 'NodeSpawnDispatchFencedError'; + } +} + +export async function claimNodeSpawn(dataDir: string, proposed: NodeSpawnClaim): Promise<{ + claim: NodeSpawnClaim; created: boolean; path: string; +}> { + const path = await nodeSpawnClaimPath( + dataDir, proposed.runId, proposed.stepId, proposed.idempotencyKey); + return claimNodeSpawnAtPath(path, proposed); +} + +export async function claimNodeSpawnAtPath(path: string, proposed: NodeSpawnClaim): Promise<{ + claim: NodeSpawnClaim; created: boolean; path: string; +}> { + const directory = dirname(path); + await mkdir(directory, { recursive: true }); + await syncDirectory(dirname(directory)); + const digest = basename(path, '.json'); + const temporary = join(directory, `.${digest}.${randomUUID()}.tmp`); + const file = await open(temporary, 'wx', 0o600); + try { await file.writeFile(canonicalize(proposed)); await file.sync(); } + finally { await file.close(); } + try { + // A hard link publishes the already-complete inode atomically and refuses + // to replace a claim won by another worker. + await link(temporary, path); + await syncDirectory(directory); + return { claim: proposed, created: true, path }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + const prior = decodeNodeSpawnClaim(JSON.parse(await readFile(path, 'utf8')) as unknown, path); + if (!sameClaim(prior, proposed)) throw new Error('Node-spawn dispatch conflicts with its durable identity or input'); + return { claim: prior, created: false, path }; + } finally { + await unlink(temporary).catch(() => undefined); + } +} + +export async function nodeSpawnClaimPath( + dataDir: string, + runId: string, + stepId: string, + idempotencyKey: string, +): Promise { + await mkdir(dataDir, { recursive: true }); + const canonicalDataDir = await realpath(dataDir); + const digest = createHash('sha256') + .update(canonicalize([runId, stepId, idempotencyKey])) + .digest('hex'); + return join(canonicalDataDir, 'node-spawns', `${digest}.json`); +} + +export function nodeSpawnDispatchFencePath(claimPath: string): string { + return `${claimPath}.dispatch`; +} + +/** Reads and validates the one-way dispatch fence for a v3 claim. */ +export async function hasNodeSpawnDispatchFence( + claimPath: string, + claim: NodeSpawnClaim, +): Promise { + const fencePath = nodeSpawnDispatchFencePath(claimPath); + let text: string; + try { text = await readFile(fencePath, 'utf8'); } + catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; + throw invalidDispatchFence(fencePath, error); + } + let value: unknown; + try { value = JSON.parse(text) as unknown; } + catch (error) { throw invalidDispatchFence(fencePath, error); } + decodeDispatchFence(value, fencePath, claim); + return true; +} + +/** + * Atomically publishes the write-ahead fact immediately before dispatch. + * The marker is permanent: a crash after it is indistinguishable from a sent + * request, so no later worker may publish or dispatch again. + */ +export async function claimNodeSpawnDispatch( + claimPath: string, + claim: NodeSpawnClaim, +): Promise { + if (claim.version !== 3) throw new Error('Legacy node-spawn claims cannot enter the fenced dispatch protocol'); + const fencePath = nodeSpawnDispatchFencePath(claimPath); + const directory = dirname(fencePath); + const temporary = `${fencePath}.${randomUUID()}.tmp`; + const fence: NodeSpawnDispatchFence = { + version: 1, + claimDigest: dispatchClaimDigest(claim), + }; + const file = await open(temporary, 'wx', 0o600); + try { + try { await file.writeFile(canonicalize(fence)); await file.sync(); } + finally { await file.close(); } + try { + await link(temporary, fencePath); + await syncDirectory(directory); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + await hasNodeSpawnDispatchFence(claimPath, claim); + throw new NodeSpawnDispatchFencedError(); + } + } finally { + await unlink(temporary).catch(() => undefined); + } +} + +export async function recordNodeSpawn(path: string, claim: NodeSpawnClaim): Promise { + const temporary = `${path}.${randomUUID()}.tmp`; + const file = await open(temporary, 'wx', 0o600); + try { await file.writeFile(canonicalize(claim)); await file.sync(); } + finally { await file.close(); } + await rename(temporary, path); + await syncDirectory(dirname(path)); +} + +export async function readNodeSpawnClaims(dataDir: string): Promise> { + let canonicalDataDir: string; + try { canonicalDataDir = await realpath(dataDir); } + catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; + throw error; + } + const directory = join(canonicalDataDir, 'node-spawns'); + let names: string[]; + try { names = await readdir(directory); } + catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; + throw error; + } + const claims: Array<{ path: string; claim: NodeSpawnClaim }> = []; + for (const name of names.sort()) { + if (!/^[a-f0-9]{64}\.json$/.test(name)) continue; + const path = join(directory, name); + claims.push({ path, claim: await readNodeSpawnClaim(path) }); + } + return claims; +} + +export async function readNodeSpawnClaim(path: string): Promise { + const value = JSON.parse(await readFile(path, 'utf8')) as unknown; + return decodeNodeSpawnClaim(value, path); +} + +function sameClaim(prior: NodeSpawnClaim, proposed: NodeSpawnClaim): boolean { + const { version: _priorVersion, + spawn: _priorSpawn, cleanup: _priorCleanup, failure: _priorFailure, + result: _priorResult, cleanupPending: _priorCleanupPending, + release: _priorRelease, ...priorIdentity } = prior; + const { version: _version, + spawn: _spawn, cleanup: _cleanup, failure: _failure, result: _result, + cleanupPending: _cleanupPending, release: _release, ...proposedIdentity } = proposed; + const { task: _priorRenderedTask, ...priorRequestIdentity } = priorIdentity.request; + const { task: _renderedTask, ...proposedRequestIdentity } = proposedIdentity.request; + return canonicalize({ ...priorIdentity, version: proposed.version, request: priorRequestIdentity, + startedAt: proposed.startedAt }) === canonicalize({ + ...proposedIdentity, version: proposed.version, request: proposedRequestIdentity, + }); +} + +function decodeNodeSpawnClaim(value: unknown, label: string): NodeSpawnClaim { + const invalid = (): never => { throw new Error(`Invalid durable node-spawn claim: ${label}`); }; + const record = isRecord(value) ? value : invalid(); + const requestRecord = isRecord(record.request) ? record.request : invalid(); + if ((record.version !== 2 && record.version !== 3) + || ![record.baseUrl, record.publisherId, record.publisherName, record.runId, record.stepId, record.agentName] + .every(nonempty) + || !validIdempotencyKey(record.idempotencyKey) + || !positiveInteger(record.timeoutMs) || record.timeoutMs > 86_400_000 + || ![requestRecord.cli, requestRecord.task, requestRecord.task_identity].every(nonempty) + || !safeTime(record.startedAt)) invalid(); + for (const field of ['model', 'worker_cwd', 'node', 'node_id', 'repo'] as const) { + if (requestRecord[field] !== undefined && !nonempty(requestRecord[field])) invalid(); + } + if (requestRecord.node_id !== undefined && requestRecord.node === undefined) invalid(); + if (requestRecord.sandbox_only !== undefined + && typeof requestRecord.sandbox_only !== 'boolean') invalid(); + const claim = record as unknown as NodeSpawnClaim; + if (record.spawn !== undefined && !validNodeSpawnReceipt(record.spawn)) invalid(); + if (record.cleanup !== undefined && !validNodeSpawnIdentity(record.cleanup)) invalid(); + if (record.spawn !== undefined && record.cleanup !== undefined) invalid(); + if (record.failure !== undefined) { + const failure = isRecord(record.failure) ? record.failure : invalid(); + if (!nonempty(failure.code) || !nonempty(failure.message) + || (record.spawn === undefined && record.cleanup === undefined)) invalid(); + } + if (record.cleanupPending !== undefined) { + const pending = isRecord(record.cleanupPending) ? record.cleanupPending : invalid(); + if (!positiveInteger(pending.attempts) || !nonempty(pending.lastError) || !safeTime(pending.updatedAt) + || (record.spawn === undefined && record.cleanup === undefined) || record.release !== undefined) invalid(); + } + if (record.release !== undefined) { + const release = isRecord(record.release) ? record.release : invalid(); + if (!nonempty(release.invocationId) || release.status !== 'completed' + || (record.spawn === undefined && record.cleanup === undefined)) invalid(); + } + if (record.result !== undefined) { + const result = isRecord(record.result) ? record.result : invalid(); + if (!nonempty(result.inboxItemId) + || !nonempty(result.messageId) + || !['completed', 'failed'].includes(String(result.status)) + || !['ack', 'fail'].includes(String(result.settlementAction)) + || !safeTime(result.receivedAt)) invalid(); + let envelope: NodeResultEnvelope; + try { + envelope = readNodeResult(result.envelope, { + runId: claim.runId, stepId: claim.stepId, dispatchId: claim.idempotencyKey, + }); + } catch { invalid(); } + if (envelope!.status !== result.status || record.spawn === undefined) invalid(); + const settlement = result.settlement; + if (settlement !== undefined) { + if (!isRecord(settlement) || !positiveInteger(settlement.attempt) + || !nonempty(settlement.completionReason) || !safeTime(settlement.preparedAt) + || (settlement.settledAt !== undefined && !safeTime(settlement.settledAt)) + || (settlement.settledAs !== undefined && !['ack', 'fail'].includes(String(settlement.settledAs))) + || (settlement.settledAt === undefined) !== (settlement.settledAs === undefined)) invalid(); + } + } + return claim; +} + +function dispatchClaimDigest(claim: NodeSpawnClaim): string { + const { spawn: _spawn, cleanup: _cleanup, failure: _failure, + result: _result, cleanupPending: _cleanupPending, release: _release, + ...identity } = claim; + return createHash('sha256').update(canonicalize(identity)).digest('hex'); +} + +function decodeDispatchFence(value: unknown, label: string, claim: NodeSpawnClaim): NodeSpawnDispatchFence { + const record = isRecord(value) ? value : undefined; + if (record?.version !== 1 + || typeof record.claimDigest !== 'string' + || !/^[a-f0-9]{64}$/.test(record.claimDigest) + || record.claimDigest !== dispatchClaimDigest(claim)) { + throw invalidDispatchFence(label); + } + return record as unknown as NodeSpawnDispatchFence; +} + +function invalidDispatchFence(label: string, cause?: unknown): Error { + return new Error(`Invalid durable node-spawn dispatch fence: ${label}`, { cause }); +} + +const safeTime = (value: unknown): value is number => + typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; + +const positiveInteger = (value: unknown): value is number => + typeof value === 'number' && Number.isSafeInteger(value) && value > 0; + +async function syncDirectory(path: string): Promise { + const directory = await open(path, 'r'); + try { await directory.sync(); } finally { await directory.close(); } +} diff --git a/packages/sdk/src/node-spawn.ts b/packages/sdk/src/node-spawn.ts new file mode 100644 index 000000000..566414164 --- /dev/null +++ b/packages/sdk/src/node-spawn.ts @@ -0,0 +1,153 @@ +/** Native Agent Relay node-spawn contract used by Flows. */ +export interface NodeSpawnTarget { + id?: string; + name: string; + live: boolean; + handlersLive: boolean; + stale?: boolean; + capabilities: string[]; + activeAgents?: number; + reservedAgents?: number; + maxAgents?: number; +} + +export interface NodeSpawnRequest { + name: string; + harness: string; + task: string; + model?: string; + directory?: string; + idempotencyKey: string; + node?: string; + /** Exact local node identity retained only for durable fenced replay. */ + nodeId?: string; + repo?: string; + timeoutMs?: number; + sandboxOnly?: boolean; +} + +export type NodeSpawnAction = 'spawn' | 'release'; +export type NodeSpawnStatus = 'pending' | 'dispatched' | 'running' | 'completed' | 'failed' + | 'denied' | 'cancelled' | 'timed_out'; + +export interface NodeSpawnInvocation { + id: string; + action: NodeSpawnAction; + status: NodeSpawnStatus; + nodeId?: string; + nodeName?: string; + name?: string; + agentId?: string; + spawned: boolean; + ready: boolean; + released?: boolean; + taskUnconfirmed?: boolean; + errorCode?: string; +} + +export interface NodeSpawnReceipt extends NodeSpawnInvocation { + action: 'spawn'; + name: string; + agentId: string; + nodeName: string; + spawned: true; + ready: true; +} + +/** Exact identity sufficient for a safe release, even when readiness failed. */ +export interface NodeSpawnIdentity { + id: string; + nodeId?: string; + nodeName: string; + name: string; + agentId: string; +} + +export interface NodeReleaseRequest { + receipt: NodeSpawnIdentity; + idempotencyKey: string; + reason?: string; +} + +/** Requests the durable write-ahead fact before an irreversible spawn call. */ +export type BeforeNodeDispatch = () => Promise; + +/** Credentials and transport stay inside an implementation of this port. */ +export interface NodeSpawnPort { + /** Declares participation in the durable dispatch-fence protocol. */ + readonly dispatchFenceVersion: 1; + targets(): Promise; + spawn(request: NodeSpawnRequest, beforeDispatch: BeforeNodeDispatch): Promise; + status(action: NodeSpawnAction, invocationId: string): Promise; + release(request: NodeReleaseRequest): Promise; +} + +export class NodeSpawnError extends Error { + constructor( + message: string, + public readonly code = 'node_spawn_failed', + public readonly cause?: unknown, + /** Exact identity to release when spawn happened but later verification failed. */ + public readonly cleanup?: NodeSpawnIdentity, + ) { + super(message); + this.name = 'NodeSpawnError'; + } +} + +/** An official port proved that it failed before entering its dispatch hook. */ +export class NodeSpawnPreDispatchError extends NodeSpawnError { + constructor(error: unknown) { + super( + error instanceof Error ? error.message : 'Node-spawn preflight failed', + error instanceof NodeSpawnError ? error.code : 'node_spawn_preflight_failed', + error, + ); + this.name = 'NodeSpawnPreDispatchError'; + } +} + +export const isRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value); + +export const nonempty = (value: unknown): value is string => + typeof value === 'string' && value.trim().length > 0; + +export function validNodeSpawnIdentity(value: unknown): value is NodeSpawnIdentity { + if (!isRecord(value)) return false; + return [value.id, value.nodeName, value.name, value.agentId].every(nonempty) + && (value.nodeId === undefined || nonempty(value.nodeId)); +} + +export function validNodeSpawnReceipt(value: unknown): value is NodeSpawnReceipt { + return validNodeSpawnIdentity(value) + && (value as unknown as Record).action === 'spawn' + && (value as unknown as Record).status === 'completed' + && (value as unknown as Record).spawned === true + && (value as unknown as Record).ready === true; +} + +export function validSegment(value: unknown): value is string { + return typeof value === 'string' && value.length > 0 && value.length <= 200 + && /^[A-Za-z0-9_-]+$/.test(value); +} + +export function validIdempotencyKey(value: unknown): value is string { + return validSegment(value) && value.length >= 16; +} + +export function terminalStatus(status: string): boolean { + return ['completed', 'failed', 'denied', 'cancelled', 'canceled', 'timed_out', 'timeout'].includes(status); +} + +export function normalizeNodeSpawnStatus(status: string): NodeSpawnStatus { + const normalized = status.toLowerCase(); + if (normalized === 'completed') return 'completed'; + if (normalized === 'denied') return 'denied'; + if (normalized === 'cancelled' || normalized === 'canceled') return 'cancelled'; + if (normalized === 'timed_out' || normalized === 'timeout') return 'timed_out'; + if (normalized === 'failed') return 'failed'; + if (normalized === 'running') return 'running'; + if (normalized === 'dispatched') return 'dispatched'; + return 'pending'; +} diff --git a/packages/sdk/src/relayflowd-path.ts b/packages/sdk/src/relayflowd-path.ts index f5d4ec5f4..696ca3885 100644 --- a/packages/sdk/src/relayflowd-path.ts +++ b/packages/sdk/src/relayflowd-path.ts @@ -4,7 +4,7 @@ // with a minimal PATH, and that is exactly the case where a `which` lookup // fails to see the binary the installer placed next to its own launcher. The // multi-anchor order below is ported from ../relay's -// packages/harness-driver/src/broker-path.ts for that reason. +// Agent Relay's launcher resolution for that reason. import { createRequire } from 'node:module'; import { accessSync, constants, existsSync, realpathSync, statSync } from 'node:fs'; diff --git a/packages/sdk/src/worker-cli.ts b/packages/sdk/src/worker-cli.ts index 05e0ac044..258e9f922 100644 --- a/packages/sdk/src/worker-cli.ts +++ b/packages/sdk/src/worker-cli.ts @@ -47,6 +47,8 @@ import { import { pinCliAlias } from './cli/pinned-cli-alias.js'; import { prepareDevinRuntime } from './devin-runtime.js'; import { promptOperand } from './adapters/base.js'; +import type { NodeSpawnRuntime } from './node-spawn-runtime.js'; +import { NodeSpawnError } from './node-spawn.js'; export type { CliTransportCause, CliTransportEvidence, CliTransportPhase } from './cli-transport-evidence.js'; export { agentCompletionReason } from './cli-transport-evidence.js'; @@ -68,6 +70,11 @@ export function cliInvocationArgv0(cli: string, cliIdentity?: string): string { export interface WorkerCliResult { relay_task?: import('./agent-relay-receipt.js').RelayTaskReceipt; + /** Settled only after the journal accepts stepComplete; never journaled. */ + relay_delivery?: { + prepareSettlement?(input: { attempt: number; completionReason: string }): Promise; + ack(): Promise; + }; tokens_input?: number; tokens_output?: number; exit_code: number | null; @@ -119,8 +126,14 @@ export interface AgentRelayContext { runId: string; stepId: string; idempotencyKey: string; + attempt: number; + /** Authored journal input without per-attempt recovery evidence. */ + taskIdentity?: string; dataDir?: string; resultSchema?: unknown; + nodeSpawnRuntime?: NodeSpawnRuntime; + nodeSpawnRuntimeProvider?: (signal?: AbortSignal) => Promise; + nodeSpawnCleanupPending?: () => void; } export async function runAgentCli( @@ -168,7 +181,14 @@ export async function runAgentCli( return { exit_code: null, stdout_tail: '', stderr_tail: error instanceof Error ? error.message : 'step secret grant is invalid' }; } - const effectiveModel = resolveCliModel(cliIdentity ?? cli, model); + // A relay node owns its harness default. `model === undefined` therefore + // has to survive this boundary unchanged: resolving the local adapter's + // default here would turn an omitted model into a Desktop placement + // override. Direct CLI execution still needs the adapter default for its + // invocation, usage and pricing paths. + const effectiveModel = transport === 'relay' + ? model + : resolveCliModel(cliIdentity ?? cli, model); const argv0 = cliInvocationArgv0(cli, cliIdentity); if (resume !== undefined && (mode !== 'agent' || transport === 'relay' || !agentResumable(kind))) { @@ -177,7 +197,10 @@ export async function runAgentCli( stderr_tail: `resume is refused for CLI ${JSON.stringify(cli)} (${mode}, ${transport}): only a directly spawned Claude or Codex agent can continue a recorded session` }; } if (mode === 'agent' && transport === 'relay') { - return runViaAgentRelay(kind, instruction, wakeContext, effectiveModel, relayContext, cwd, signal); + return runViaAgentRelay( + kind, instruction, wakeContext, effectiveModel, relayContext, cwd, signal, stepTimeoutMs, + childEnvironment, + ); } // Artifact detection brackets the spawn: the directory the CLI runs in is @@ -386,21 +409,29 @@ async function runViaAgentRelay( kind: CliAdapterKind, instruction: string, wakeContext: unknown, model: string | undefined, context: AgentRelayContext | undefined, worker_cwd: string | undefined, signal: AbortSignal | undefined, + timeoutMs: number | undefined, + environment: NodeJS.ProcessEnv, ): Promise { try { if (kind === 'relayflows-wrapper-v1') throw new Error('Relay task transport does not support same-process wrappers.'); if (!context?.dataDir) throw new Error('Relay task transport requires a durable data directory and journal dispatch identity.'); - const task = instruction + (wakeContext === undefined ? '' : `\n\nWake context (journaled):\n${JSON.stringify(wakeContext)}`) - + '\n\nReport the final task output with the injected agent_result tool and final=true. Wait for its successful durable acknowledgment before exiting.'; + const wake = wakeContext === undefined ? '' + : `\n\nWake context (journaled):\n${JSON.stringify(wakeContext)}`; + const task = instruction + wake; + const taskIdentity = (context.taskIdentity ?? instruction) + wake; const received = await runAgentRelayTask({ - cli: kind, task, model, worker_cwd, result_schema: context.resultSchema, + cli: kind, task, taskIdentity, model, worker_cwd, result_schema: context.resultSchema, runId: context.runId, stepId: context.stepId, idempotencyKey: context.idempotencyKey, - dataDir: context.dataDir, - }, { signal }); - const receipt = { ...received, error: received.error === null ? null : redactRelayError(received.error) }; + attempt: context.attempt, dataDir: context.dataDir, + ...(timeoutMs === undefined ? {} : { timeoutMs }), + }, { signal, runtime: context.nodeSpawnRuntime, runtimeProvider: context.nodeSpawnRuntimeProvider, + onCleanupPending: context.nodeSpawnCleanupPending }); + const receipt = { ...received.receipt, + error: received.receipt.error === null ? null : redactRelayError(received.receipt.error) }; const accounting = receipt.task_execution.accounting; const result: WorkerCliResult = { - relay_task: receipt, exit_code: receipt.status === 'completed' ? 0 : 1, + relay_task: receipt, relay_delivery: received.delivery, + exit_code: receipt.status === 'completed' ? 0 : 1, stdout_tail: receipt.status === 'completed' ? JSON.stringify(receipt.output) : '', stderr_tail: receipt.status === 'failed' ? `Relay task failed: ${receipt.error}` : '', ...(accounting?.tokens_input === undefined ? {} : { tokens_input: accounting.tokens_input }), @@ -409,10 +440,32 @@ async function runViaAgentRelay( return requirePricedUsage(result, model); } catch (error) { signal?.throwIfAborted(); + // A durable exact release remains pending. Do not journal worker_error: + // leaving this attempt incomplete lets the kernel redispatch it, and the + // saved node-spawn claim resumes cleanup without spawning another agent. + if (error instanceof AgentRelayTransportError && error.cleanupPending) throw error; const detail = error instanceof AgentRelayTransportError ? error.message : error instanceof Error ? error.message : 'Relay task transport failed'; - return { exit_code: null, stdout_tail: '', stderr_tail: redactRelayError(detail) }; + const timedOut = error instanceof AgentRelayTransportError && relayTaskTimedOut(error); + return { exit_code: null, stdout_tail: '', stderr_tail: redactRelayError(detail), + ...(timedOut ? { transport: transportEvidence({ + phase: 'timeout', cause: 'timeout', exitCode: null, signal: null, + stderr: detail, retryable: false, + }, environment) } : {}) }; + } +} + +function relayTaskTimedOut(error: AgentRelayTransportError): boolean { + let current: unknown = error; + const seen = new Set(); + while (current !== undefined && current !== null && !seen.has(current)) { + seen.add(current); + if (current instanceof NodeSpawnError && current.code === 'task_timeout') return true; + if (current instanceof DOMException && current.name === 'TimeoutError') return true; + current = current instanceof AgentRelayTransportError || current instanceof NodeSpawnError + ? current.cause : undefined; } + return false; } /** diff --git a/packages/sdk/src/worker-input.ts b/packages/sdk/src/worker-input.ts index 6cc68f5ff..1a5c7408e 100644 --- a/packages/sdk/src/worker-input.ts +++ b/packages/sdk/src/worker-input.ts @@ -2,9 +2,22 @@ import type { StepDispatchEvent } from './protocol.js'; /** Memory precedes the final, visibly delimited JSON input segment. */ export function workerInstruction(instruction: string, dispatch: StepDispatchEvent): string { + return renderWorkerInstruction(instruction, dispatch, true); +} + +/** Stable dispatch input, excluding recovery evidence that changes between attempts. */ +export function workerInstructionIdentity(instruction: string, dispatch: StepDispatchEvent): string { + return renderWorkerInstruction(instruction, dispatch, false); +} + +function renderWorkerInstruction( + instruction: string, + dispatch: StepDispatchEvent, + includeRecovery: boolean, +): string { const withMemory = dispatch.memory === undefined ? instruction : `${instruction}\n\nMemory context (journaled):\n${JSON.stringify(dispatch.memory.pack)}`; - const noted = `${withMemory}${recoveryNote(dispatch)}`; + const noted = includeRecovery ? `${withMemory}${recoveryNote(dispatch)}` : withMemory; if (dispatch.input === undefined) { if (typeof dispatch.spec === 'object' && dispatch.spec !== null && 'input' in dispatch.spec) { diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index 3d82c1b3d..6efabe542 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -1,5 +1,4 @@ import { brokerInstruction } from './communication/broker-spec.js'; -import { BrokerSessions } from './communication/broker-worker.js'; import { communicationInstruction } from './communication/spec.js'; import { workerSpend } from './worker-spend.js'; import { reportedCost } from './reported-cost.js'; @@ -13,7 +12,7 @@ import { agentCompletionReason } from './cli-transport-evidence.js'; import { agentStepCwd } from './agent-cwd.js'; import { resolveCliModel } from './cli-adapter.js'; import { withWorkerLease } from './worker-lease.js'; -import { workerInstruction } from './worker-input.js'; +import { workerInstruction, workerInstructionIdentity } from './worker-input.js'; import { helperCall } from './yaml-helpers.js'; import { completeHelperDispatch } from './yaml-helper-effect.js'; import { @@ -23,6 +22,11 @@ import { redactText, type TranscriptDigest, } from './agent-transcript.js'; +import { createNodeSpawnRuntime, createNodeSpawnRuntimeOwner, explicitServiceCallerFile, + mergeNodeSpawnEnvironment, type NodeSpawnRuntime, type NodeSpawnRuntimeOwner } from './node-spawn-runtime.js'; +import { nodeSpawnCleanupRequired } from './node-spawn-cleanup.js'; +import { nodeSpawnClaimBelongsToRuntime, reconcileNodeResultSettlements } from './node-spawn-settlement.js'; +import { readNodeSpawnClaims } from './node-spawn-state.js'; export { MODEL_ENV, WAKE_CONTEXT_ENV } from './worker-cli.js'; @@ -43,6 +47,10 @@ export interface AgentWorkerOptions { runRoot?: string; /** Source environment; each provider CLI receives only its allowlist and step grants. */ environment?: NodeJS.ProcessEnv; + /** Optional approved node-spawn runtime; otherwise relay steps use the agent-token fleet client. */ + nodeSpawnRuntime?: NodeSpawnRuntime; + /** Explicit Desktop schema-v2 handoff; never discovered from an app-data directory. */ + serviceCallerFile?: string; } /** @@ -62,17 +70,29 @@ export interface AgentWorkerOptions { * shutdown. */ export class AgentWorker extends EventEmitter { - private readonly brokerSessions: BrokerSessions; private attached = false; private closing = false; + private closePromise?: Promise; + private settlementTimer?: NodeJS.Timeout; + private settlementDrain?: Promise; + private settlementReconcile?: Promise; + private readonly localDesktopAuthority: boolean; private readonly inFlight: Set> = new Set(); + private readonly nodeSpawnRuntimeOwner: NodeSpawnRuntimeOwner; constructor( private readonly client: JournalClient, private readonly options: AgentWorkerOptions, ) { super(); - this.brokerSessions = new BrokerSessions(options.runRoot, options.environment); + const runtimeEnvironment = mergeNodeSpawnEnvironment(options.environment, options.serviceCallerFile); + this.localDesktopAuthority = options.nodeSpawnRuntime === undefined + ? explicitServiceCallerFile(runtimeEnvironment) !== undefined + : options.nodeSpawnRuntime.authority === 'desktop'; + this.nodeSpawnRuntimeOwner = createNodeSpawnRuntimeOwner( + options.nodeSpawnRuntime, + signal => createNodeSpawnRuntime(runtimeEnvironment, { signal }), + ); } async attach(): Promise { @@ -80,6 +100,14 @@ export class AgentWorker extends EventEmitter { if (this.closing) throw new Error('agent worker: cannot attach a closed worker (construct a new one)'); this.client.on('step.dispatch', this.onDispatch); try { + // An explicit Desktop handoff is fully checkable before this worker is + // admitted. Bind and validate it now so a missing, expired or mismatched + // caller cannot let earlier deterministic steps run and fail only when + // the first Relay dispatch eventually asks for the lazy runtime. + // Remote fleet authority stays lazy: a direct-only local worker does not + // need to contact Relay merely because RELAY_AGENT_TOKEN is present. + if (this.localDesktopAuthority) await this.nodeSpawnRuntimeOwner.get(); + await this.reconcileRelaySettlementsAfterDrain(); await this.client.workerAttach( this.options.workerId, ['agent'], @@ -90,41 +118,102 @@ export class AgentWorker extends EventEmitter { this.attached = true; } catch (error) { this.client.off('step.dispatch', this.onDispatch); + if (this.settlementTimer !== undefined) { + clearTimeout(this.settlementTimer); + this.settlementTimer = undefined; + } + // An attach attempt can create the lazy Relay runtime while reconciling + // a prior inbox settlement. Once attachment fails, this worker is not + // reusable: close that owned connection before the caller loses the + // worker instance. + this.closing = true; + try { + const close = this.nodeSpawnRuntimeOwner.close(); + this.closePromise = close; + await close; + } catch (closeError) { + throw new AggregateError([error, closeError], + 'Agent worker could not attach or close its node runtime'); + } throw error; } } /** * Async, drain-aware shutdown. Awaits every dispatch in-flight; ignores - * dispatches that arrive after close() begins. Idempotent. + * dispatches that arrive after close() begins. Concurrent and later callers + * receive the same promise, so none can observe shutdown before the drain. * * A prior synchronous close() did NOT drain, so a caller shutting * mid-dispatch could silently lose an in-flight step's stepComplete * journal write when the socket was immediately shut. This closes * that hole. */ - async close(): Promise { - if (this.closing) return; + close(): Promise { + this.closePromise ??= this.closeOnce(); + return this.closePromise; + } + + private async closeOnce(): Promise { this.closing = true; this.client.off('step.dispatch', this.onDispatch); const pending = Array.from(this.inFlight); if (pending.length > 0) { await Promise.allSettled(pending); } - await this.brokerSessions.close(); - this.attached = false; + if (this.settlementTimer !== undefined) { + clearTimeout(this.settlementTimer); + this.settlementTimer = undefined; + } + let settlementError: unknown; + try { + await this.reconcileRelaySettlementsAfterDrain(); + } catch (error) { + settlementError = error; + } + try { + await this.nodeSpawnRuntimeOwner.close(); + } catch (closeError) { + if (settlementError !== undefined) { + throw new AggregateError([settlementError, closeError], + 'Agent worker could not reconcile relay settlement or close its node runtime'); + } + throw closeError; + } finally { + this.attached = false; + } + if (settlementError !== undefined) throw settlementError; } private readonly onDispatch = (dispatch: StepDispatchEvent): void => { if (this.closing) return; if (dispatch.step_type !== 'agent') return; - const running: Promise = this.execute(dispatch).catch((error: unknown) => { + const running: Promise = this.executeAfterSettlementDrain(dispatch).catch((error: unknown) => { this.emit('error', error, dispatch); }); this.inFlight.add(running); - void running.finally(() => { this.inFlight.delete(running); }); + void running.finally(() => { + this.inFlight.delete(running); + // A task's own ack settles its durable outbox entry after stepComplete. + // Reconcile only after the dispatch drain: scanning while another relay + // task is between its journal append and ack can race that task's own + // settlement and issue the same inbox operation twice. + if (this.inFlight.size === 0) { + void this.reconcileRelaySettlementsAfterDrain().catch(error => this.emit('error', error)); + } + }); }; + private async executeAfterSettlementDrain(dispatch: StepDispatchEvent): Promise { + // A dispatch that arrives after a drain can race the drain's asynchronous + // claim scan. Keep it in the drain count, but do not let it prepare or + // settle a new delivery until the existing scan has finished. + while (this.settlementDrain !== undefined) { + await this.settlementDrain; + } + await this.execute(dispatch); + } + private async execute(dispatch: StepDispatchEvent): Promise { const spec = dispatch.spec as Partial; const helper = helperCall(spec); @@ -135,16 +224,14 @@ export class AgentWorker extends EventEmitter { } const broker = brokerInstruction(spec.instruction); if (broker) { - if (!this.options.dataDir) throw new Error('Broker agents require a worker data directory'); - await this.brokerSessions.complete(this.client, dispatch, broker, this.options.dataDir); + await this.retiredNodeSession(dispatch, + 'transport: broker was removed with the Relay broker. Use transport: relay for one-shot fleet execution; retained local sessions require the pending Desktop service-caller contract.'); return; } const communication = communicationInstruction(spec.instruction); if (communication) { - if (!this.options.dataDir) throw new Error('Agent communication requires a worker data directory'); - const { completeCommunicationDispatch } = await import('./communication/worker.js'); - await completeCommunicationDispatch(this.client, dispatch, communication, this.options.dataDir, - this.options.runRoot, this.options.environment); + await this.retiredNodeSession(dispatch, + 'Agent communication is unavailable without the retired broker/PTy path. Native node-spawn communication remains fail-closed until an authorized local service caller exists.'); return; } let humanIntervention = false; @@ -155,23 +242,35 @@ export class AgentWorker extends EventEmitter { // the agent's filesystem. A refusal completes the step the way a missing // CLI does — journaled as `worker_error` with the reason — rather than // spawning into a directory nobody established is inside the run root. - const cwd = agentStepCwd(spec, dispatch.step_id, this.options.runRoot); + const localDesktopRelay = spec.transport === 'relay' && this.localDesktopAuthority; + // An adapter default is useful for readiness, pricing and direct CLI + // execution, but is not an authored placement override. Relay harnesses + // select their own default when the journaled step omitted `model`. + const executionModel = spec.transport === 'relay' ? spec.model : effectiveModel; + const cwd = agentStepCwd(localDesktopRelay ? { ...spec, transport: 'direct' } : spec, + dispatch.step_id, this.options.runRoot); const completed: WorkerCliResult = await withWorkerLease(this.client, dispatch, signal => 'refusal' in cwd ? Promise.resolve({ exit_code: null, stdout_tail: '', stderr_tail: cwd.refusal }) : typeof spec.cli === 'string' && typeof spec.instruction === 'string' - ? runAgentCli(spec.cli, workerInstruction(spec.instruction, dispatch), dispatch.wake_context, effectiveModel, undefined, signal, 'agent', this.options.dataDir === undefined ? undefined : { + ? runAgentCli(spec.cli, workerInstruction(spec.instruction, dispatch), dispatch.wake_context, executionModel, undefined, signal, 'agent', this.options.dataDir === undefined ? undefined : { dataDir: this.options.dataDir, runId: dispatch.run_id, stepId: dispatch.step_id, attempt: dispatch.attempt, onReady: this.options.onPtyReady, onDrive: () => { humanIntervention = true; }, - }, cwd.directory ?? (spec.transport === 'relay' ? undefined : this.options.runRoot), + }, cwd.directory ?? (localDesktopRelay + ? (this.options.runRoot ?? process.cwd()) + : (spec.transport === 'relay' ? undefined : this.options.runRoot)), spec.transport === 'relay' ? 'relay' : 'direct', { runId: dispatch.run_id, stepId: dispatch.step_id, idempotencyKey: dispatch.idempotency_key, - dataDir: this.options.dataDir, resultSchema: spec.verification?.json_schema }, + attempt: dispatch.attempt, taskIdentity: workerInstructionIdentity(spec.instruction, dispatch), + dataDir: this.options.dataDir, resultSchema: spec.verification?.json_schema, + nodeSpawnRuntime: this.options.nodeSpawnRuntime, + nodeSpawnRuntimeProvider: signal => this.nodeSpawnRuntimeOwner.get(signal), + nodeSpawnCleanupPending: () => this.scheduleSettlementReconcile() }, this.options.environment, spec.cli_identity, spec.timeout_ms, spec.resume, spec.secrets, this.client.supportsFeature?.('step_secret_scoping') ?? true) : Promise.resolve({ exit_code: null, stdout_tail: '', stderr_tail: 'agent step has no declared CLI' })); - const { result, usage } = workerSpend(completed, effectiveModel); - const cost = reportedCost(completed, effectiveModel); + const { result, usage } = workerSpend(completed, executionModel); + const cost = reportedCost(completed, executionModel); const completionReason = agentCompletionReason(result); // Output shape: if the CLI's stdout parses as JSON, promote THAT @@ -200,7 +299,7 @@ export class AgentWorker extends EventEmitter { // `transcript` is not part of the wrapper either: it is evidence about the // attempt, journaled in `trajectory_tail` below on success and failure // alike, where the kernel already accepts and bounds it (16 KiB). - const { transcript, transport, ...rawWrapper } = result; + const { transcript, transport, relay_delivery: relayDelivery, ...rawWrapper } = result; // `stderr_tail` is evidence, not authored output. Redact and bound it // before it reaches either the worker-failure render or the journal -- // against the environment the CLI ran with as well as this host's, since a @@ -221,6 +320,7 @@ export class AgentWorker extends EventEmitter { ...(transport === undefined ? {} : { transport }), }; + await relayDelivery?.prepareSettlement?.({ attempt: dispatch.attempt, completionReason }); await this.client.stepComplete( dispatch.run_id, dispatch.step_id, @@ -239,6 +339,83 @@ export class AgentWorker extends EventEmitter { : dispatch.pins, }, ); + // Relay delivery is transport state, not execution truth. Only settle it + // after the journal has accepted the attempt's terminal fact. + await relayDelivery?.ack(); + } + + private reconcileRelaySettlementsAfterDrain(): Promise { + if (this.settlementDrain !== undefined) return this.settlementDrain; + const generation = Array.from(this.inFlight); + const running = (async () => { + if (generation.length > 0) await Promise.allSettled(generation); + await this.reconcileRelaySettlements(); + })(); + this.settlementDrain = running; + void running.finally(() => { + if (this.settlementDrain === running) this.settlementDrain = undefined; + }).catch(() => undefined); + return running; + } + + private async reconcileRelaySettlements(): Promise { + if (this.settlementReconcile !== undefined) return this.settlementReconcile; + const running = this.reconcileRelaySettlementsOnce(); + this.settlementReconcile = running; + void running.finally(() => { + if (this.settlementReconcile === running) this.settlementReconcile = undefined; + }).catch(() => undefined); + return running; + } + + private async reconcileRelaySettlementsOnce(): Promise { + if (this.options.dataDir === undefined || (this.closing && !this.attached)) return; + // Durable claims are not authority. A direct-only worker must not read + // ambient credentials or contact Relay merely because another publisher + // left state in the shared data directory. Explicit Desktop admission or + // a real Relay dispatch establishes the only runtime this worker may use. + const runtime = this.nodeSpawnRuntimeOwner.peek(); + if (runtime === undefined) return; + const claims = await readNodeSpawnClaims(this.options.dataDir); + const hasPending = claims.some(entry => nodeSpawnClaimBelongsToRuntime(entry.claim, runtime) + && (nodeSpawnCleanupRequired(entry.claim) + || (entry.claim.result?.settlement !== undefined + && entry.claim.result.settlement.settledAt === undefined))); + if (!hasPending) return; + let pending: number; + try { + pending = await reconcileNodeResultSettlements(this.options.dataDir, runtime, this.client); + } catch (error) { + this.scheduleSettlementReconcile(); + throw error; + } + if (this.settlementTimer !== undefined) { + clearTimeout(this.settlementTimer); + this.settlementTimer = undefined; + } + if (pending > 0) this.scheduleSettlementReconcile(); + } + + private scheduleSettlementReconcile(): void { + if (this.closing || this.settlementTimer !== undefined) return; + this.settlementTimer = setTimeout(() => { + this.settlementTimer = undefined; + void this.reconcileRelaySettlementsAfterDrain().catch(error => this.emit('error', error)); + }, 5_000); + this.settlementTimer.unref(); + } + + private async retiredNodeSession(dispatch: StepDispatchEvent, error: string): Promise { + await withWorkerLease(this.client, dispatch, async signal => { signal.throwIfAborted(); }); + await this.client.stepComplete( + dispatch.run_id, dispatch.step_id, dispatch.attempt, dispatch.idempotency_key, + 'worker_error', { + output: { error }, + usage: { tokens_in: 0, tokens_out: 0, dollars_unmetered: true }, + started_pins: dispatch.pins, + end_pins: dispatch.pins, + }, + ); } } diff --git a/packages/sdk/tests/agent-relay-hardening.test.ts b/packages/sdk/tests/agent-relay-hardening.test.ts index 58c3e1869..0fedd0a7f 100644 --- a/packages/sdk/tests/agent-relay-hardening.test.ts +++ b/packages/sdk/tests/agent-relay-hardening.test.ts @@ -1,91 +1,176 @@ -import { readFile } from "node:fs/promises"; -import { describe, expect, it, vi } from "vitest"; -import { runAgentRelayTask } from "../src/agent-relay-transport.js"; -import { readTaskReceipt } from "../src/agent-relay-receipt.js"; +import { describe, expect, it, vi } from 'vitest'; +import { readNodeResult, waitForNodeResult, NODE_RESULT_PREFIX } from '../src/node-spawn-result.js'; -const wire = JSON.parse( - await readFile( - new URL("./fixtures/relay-task/relaycast-436.json", import.meta.url), - "utf8", - ), -); -const id = wire.ack.data.invocation_id; -const request = { - cli: "claude", - task: "fixture", - runId: "run", - stepId: "step", - idempotencyKey: "key", - dataDir: "/unused", -}; +const correlation = { runId: 'run_1', stepId: 'step_1', dispatchId: 'dispatch_1234567890' }; +const expected = { agentId: 'agent_1', agentName: 'worker', invocationId: 'inv_1', ...correlation }; -describe("Relay credential origin", () => { - for (const baseUrl of [ - "http://cast.agentrelay.com", - "http://192.168.1.1", - "http://localhost.example", - "http://127.0.0.1.example", - ]) { - it(`refuses bearer credentials at ${baseUrl} before any request`, async () => { - const fetchMock = vi.fn(); - await expect( - runAgentRelayTask(request, { - baseUrl, - agentToken: "fixture-token", - fetch: fetchMock, - }), - ).rejects.toThrow(/require HTTPS/); - expect(fetchMock).not.toHaveBeenCalled(); - }); - } - for (const baseUrl of [ - "https://cast.agentrelay.com", - "http://127.0.0.1:1234", - "http://127.9.8.7:1234", - "http://[::1]:1234", - ]) { - it(`allows ${baseUrl} to reach the authenticated identity check`, async () => { - const fetchMock = vi.fn(async () => new Response("", { status: 401 })); - await expect( - runAgentRelayTask(request, { - baseUrl, - agentToken: "fixture-token", - fetch: fetchMock, - }), - ).rejects.toThrow("HTTP 401"); - expect(fetchMock).toHaveBeenCalledOnce(); +describe('node-spawn result hardening', () => { + it('requires immutable attempt correlation and terminal evidence', () => { + expect(() => readNodeResult({ version: 1, run_id: 'other', step_id: 'step_1', + dispatch_id: correlation.dispatchId, status: 'completed', output: {} }, correlation)).toThrow('correlation'); + expect(() => readNodeResult({ version: 1, run_id: 'run_1', step_id: 'step_1', + dispatch_id: correlation.dispatchId, status: 'failed' }, correlation)).toThrow('reason'); + for (const amount of [0.5, Number.MAX_SAFE_INTEGER + 1]) { + expect(() => readNodeResult({ version: 1, run_id: 'run_1', step_id: 'step_1', + dispatch_id: correlation.dispatchId, status: 'completed', output: {}, + accounting: { tokens_input: amount } }, correlation)).toThrow('accounting'); + } + }); + + it('ignores a forged name match without the exact immutable sender id', async () => { + const ack = vi.fn(); + const subscribe = vi.fn(async function* () { + for (const [id, sender] of [['forged', 'attacker'], ['real', 'agent_1']] as const) yield { + id, message: { id: `message_${id}`, from: { id: sender, name: 'worker' }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_1', step_id: 'step_1', + dispatch_id: correlation.dispatchId, status: 'completed', output: { id } }) }, + }; }); - } -}); + const delivery = await waitForNodeResult({ + inbox: { + subscribe, + ack: async input => { ack(input); }, fail: async () => {}, + }, + }, expected); + expect(delivery.envelope.output).toEqual({ id: 'real' }); + expect(subscribe).toHaveBeenCalledWith(expect.objectContaining({ invocationId: 'inv_1' })); + expect(ack).not.toHaveBeenCalled(); + }); -describe("Relay acceptance identity", () => { - it("allows generation and acceptance time to appear once on the same execution", () => { - const pending = readTaskReceipt(wire.dispatched.data, id, wire.input); - expect( - readTaskReceipt(wire.running.data, id, wire.input, pending).status, - ).toBe("running"); + it('dead-letters a malformed exact-sender completion only after journal settlement', async () => { + const fail = vi.fn(async () => ({})); + const delivery = await waitForNodeResult({ inbox: { + subscribe: async function* () { yield { id: 'delivery_bad', message: { id: 'message_bad', + from: { id: 'agent_1', name: 'worker' }, text: NODE_RESULT_PREFIX + '{bad' } }; }, + ack: async () => ({}), fail, + } }, expected); + expect(delivery.envelope.status).toBe('failed'); + expect(fail).not.toHaveBeenCalled(); + await delivery.ack(); + expect(fail).toHaveBeenCalledWith(expect.objectContaining({ inboxItemId: 'delivery_bad', retry: false })); }); - it("refuses an execution change before the generation exists", () => { - const pending = readTaskReceipt(wire.dispatched.data, id, wire.input); - const changed = structuredClone(wire.running.data); - changed.task_execution.execution_id += "/replacement"; - expect(() => readTaskReceipt(changed, id, wire.input, pending)).toThrow( - /changed its execution/, - ); + + it('dead-letters an array-valued result status instead of accepting a malformed claim value', async () => { + const fail = vi.fn(async () => ({})); + const delivery = await waitForNodeResult({ inbox: { + subscribe: async function* () { yield { id: 'delivery_array_status', message: { + id: 'message_array_status', from: { id: 'agent_1', name: 'worker' }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_1', step_id: 'step_1', + dispatch_id: correlation.dispatchId, status: ['completed'], output: { ok: true } }), + } }; }, + ack: async () => ({}), fail, + } }, expected); + expect(delivery.envelope).toMatchObject({ status: 'failed', error: expect.stringContaining('correlation') }); + expect(delivery.settlementAction).toBe('fail'); + expect(fail).not.toHaveBeenCalled(); + await delivery.ack(); + expect(fail).toHaveBeenCalledWith(expect.objectContaining({ + inboxItemId: 'delivery_array_status', retry: false, + })); + }); + + it('closes only the normal inbox subscription without acknowledging the delivery', async () => { + const returned = vi.fn(async () => { throw new Error('iterator close failed'); }); + let delivered = false; + const ack = vi.fn(async () => ({})); + const delivery = await waitForNodeResult({ inbox: { + subscribe: () => ({ [Symbol.asyncIterator]: () => ({ + next: async () => { + if (delivered) return new Promise>(() => {}); + delivered = true; + return { done: false as const, value: { id: 'delivery_1', message: { id: 'message_1', + from: { id: 'agent_1', name: 'worker' }, text: NODE_RESULT_PREFIX + JSON.stringify({ + version: 1, run_id: 'run_1', step_id: 'step_1', dispatch_id: correlation.dispatchId, + status: 'completed', output: { ok: true }, + }) } } }; + }, + return: returned, + }) }), + ack, fail: async () => ({}), + } }, expected); + expect(returned).toHaveBeenCalledOnce(); + expect(ack).not.toHaveBeenCalled(); + await delivery.ack(); + expect(ack).toHaveBeenCalledOnce(); + }); + + it('preserves a received result when iterator close throws synchronously', async () => { + const returned = vi.fn(() => { throw new Error('synchronous iterator close failed'); }); + const delivery = await waitForNodeResult({ inbox: { + subscribe: () => ({ [Symbol.asyncIterator]: () => ({ + next: async () => ({ done: false as const, value: { id: 'delivery_1', message: { + id: 'message_1', from: { id: 'agent_1', name: 'worker' }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_1', step_id: 'step_1', + dispatch_id: correlation.dispatchId, status: 'completed', output: { ok: true } }), + } } }), + return: returned, + }) }), + ack: async () => ({}), fail: async () => ({}), + } }, expected); + expect(delivery.envelope.output).toEqual({ ok: true }); + expect(returned).toHaveBeenCalledOnce(); + }); + + it('does not wait for an uncooperative normal iterator close after receiving a result', async () => { + const returned = vi.fn(() => new Promise>(() => {})); + const delivery = await waitForNodeResult({ inbox: { + subscribe: () => ({ [Symbol.asyncIterator]: () => ({ + next: async () => ({ done: false as const, value: { id: 'delivery_1', message: { + id: 'message_1', from: { id: 'agent_1', name: 'worker' }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_1', step_id: 'step_1', + dispatch_id: correlation.dispatchId, status: 'completed', output: { ok: true } }), + } } }), + return: returned, + }) }), + ack: async () => ({}), fail: async () => ({}), + } }, expected); + expect(delivery.envelope.output).toEqual({ ok: true }); + expect(returned).toHaveBeenCalledOnce(); + }); + + it('cancels an initial inbox read even when the client ignores its signal option', async () => { + const returned = vi.fn(); + const controller = new AbortController(); + const waiting = waitForNodeResult({ inbox: { + subscribe: () => ({ [Symbol.asyncIterator]: () => ({ + next: () => new Promise>(() => {}), + return: async () => { returned(); return { done: true, value: undefined }; }, + }) }), + ack: async () => ({}), fail: async () => ({}), + } }, expected, controller.signal); + controller.abort(new Error('lease lost')); + await expect(waiting).rejects.toThrow('lease lost'); + expect(returned).toHaveBeenCalledOnce(); + }); + + it('does not wait for an uncooperative iterator close after cancellation', async () => { + const returned = vi.fn(() => new Promise>(() => {})); + const controller = new AbortController(); + const waiting = waitForNodeResult({ inbox: { + subscribe: () => ({ [Symbol.asyncIterator]: () => ({ + next: () => new Promise>(() => {}), + return: returned, + }) }), + ack: async () => ({}), fail: async () => ({}), + } }, expected, controller.signal); + controller.abort(new Error('lease lost')); + await expect(waiting).rejects.toThrow('lease lost'); + expect(returned).toHaveBeenCalledOnce(); + }); + + it('preserves cancellation when a detached iterator close rejects', async () => { + const returned = vi.fn(async () => { throw new Error('close failed'); }); + const controller = new AbortController(); + const waiting = waitForNodeResult({ inbox: { + subscribe: () => ({ [Symbol.asyncIterator]: () => ({ + next: () => new Promise>(() => {}), + return: returned, + }) }), + ack: async () => ({}), fail: async () => ({}), + } }, expected, controller.signal); + controller.abort(new Error('lease lost')); + await expect(waiting).rejects.toThrow('lease lost'); + expect(returned).toHaveBeenCalledOnce(); + await new Promise(resolve => setImmediate(resolve)); }); - for (const field of ["worker_generation", "accepted_at"]) { - it(`refuses changed or removed ${field} after acceptance`, () => { - const running = readTaskReceipt(wire.running.data, id, wire.input); - const changed = structuredClone(wire.completed.data); - changed.task_execution[field] = - field === "accepted_at" ? "2026-01-01T00:00:00.000Z" : "replacement"; - expect(() => readTaskReceipt(changed, id, wire.input, running)).toThrow( - /changed its execution/, - ); - delete changed.task_execution[field]; - expect(() => readTaskReceipt(changed, id, wire.input, running)).toThrow( - /acceptance/, - ); - }); - } }); diff --git a/packages/sdk/tests/agent-relay-transport.test.ts b/packages/sdk/tests/agent-relay-transport.test.ts index 52c7a3469..b53f9ed6d 100644 --- a/packages/sdk/tests/agent-relay-transport.test.ts +++ b/packages/sdk/tests/agent-relay-transport.test.ts @@ -1,479 +1,596 @@ -import { createHash } from "node:crypto"; -import { mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { afterEach, describe, expect, it, vi } from "vitest"; -import { - runAgentRelayTask, - readAgentRelayEnv, - type AgentRelayTaskRequest, -} from "../src/agent-relay-transport.js"; +import { mkdtemp, rm } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { AgentRelayTransportError, runAgentRelayTask, + type AgentRelayTaskRequest } from '../src/agent-relay-transport.js'; +import { NODE_RESULT_PREFIX } from '../src/node-spawn-result.js'; +import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; +import { NodeSpawnError, type NodeSpawnPort, type NodeSpawnReceipt } from '../src/node-spawn.js'; +import { EventEmitter } from 'node:events'; +import { AgentWorker } from '../src/worker.js'; +import type { JournalClient } from '../src/journal-client.js'; +import type { StepDispatchEvent } from '../src/protocol.js'; +import { readNodeSpawnClaims, recordNodeSpawn } from '../src/node-spawn-state.js'; -const directories: string[] = []; -afterEach(async () => { - vi.restoreAllMocks(); - vi.unstubAllEnvs(); - vi.unstubAllGlobals(); - await Promise.all( - directories - .splice(0) - .map((path) => rm(path, { recursive: true, force: true })), - ); +const request = (dataDir: string): AgentRelayTaskRequest => ({ + cli: 'codex', task: 'Implement the change', runId: 'run_1', stepId: 'edit', + idempotencyKey: 'dispatch_1234567890', attempt: 1, dataDir, timeoutMs: 10_000, }); -const ok = (data: unknown) => - new Response(JSON.stringify({ ok: true, data }), { - status: 200, - headers: { "content-type": "application/json" }, - }); -async function fixture() { - const dataDir = await mkdtemp(join(tmpdir(), "flows-relay-task-")); - directories.push(dataDir); - const request: AgentRelayTaskRequest = { - cli: "claude", - task: "return output", - runId: "run", - stepId: "step", - idempotencyKey: "journal-key", - dataDir, +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +async function dataDirectory(label: string): Promise { + const directory = await mkdtemp(join(tmpdir(), label)); + roots.push(directory); + return directory; +} + +function fixture(result: 'completed' | 'failed' = 'completed'): { runtime: NodeSpawnRuntime; spawn: ReturnType; + release: ReturnType; status: ReturnType; + ack: ReturnType; fail: ReturnType } { + const receipt: NodeSpawnReceipt = { + id: 'inv_spawn_1', action: 'spawn', status: 'completed', nodeId: 'node_1', nodeName: 'desktop-1', + name: 'placeholder', agentId: 'agent_1', spawned: true, ready: true, }; - const invocationId = - "inv_idem_" + - createHash("sha256") - .update( - [ - "action-invoke-v1", - "workspace", - "caller", - "task.run", - "journal-key", - ].join("\0"), - ) - .digest("hex"); - let input: Record; - let status = "completed"; - let caller = "caller"; - let result: unknown = { answer: 42 }; - let generation = "generation-1"; - let losePost = false; - const deadline = new Date(Date.now() + 86400000).toISOString(); - let reads = 0; - const frames: { url: string; init?: RequestInit }[] = []; - const fetchMock = vi.fn( - async (url: string | URL | Request, init?: RequestInit) => { - const address = String(url); - frames.push({ url: address, init }); - expect((init?.headers as Record).authorization).toBe( - "Bearer fixture-agent-token", - ); - expect(init?.redirect).toBe("error"); - if (address.endsWith("/v1/agent")) - return ok({ - id: caller, - workspace_id: "workspace", - name: "caller-name", - }); - if (address.endsWith("/v1/workspace")) return ok({ id: "workspace" }); - if (address.endsWith("/invoke")) { - expect(init?.method).toBe("POST"); - expect( - (init?.headers as Record)["Idempotency-Key"], - ).toBe("journal-key"); - input = JSON.parse(String(init?.body)).input; - if (losePost) throw new Error("response lost after engine commit"); - return ok({ - invocation_id: invocationId, - action_name: "task.run", - input, - status: "completed", - }); - } - expect(address).toBe( - `https://cast.agentrelay.com/v1/actions/task.run/invocations/${invocationId}`, - ); - reads++; - return ok({ - invocation_id: invocationId, - action_name: "task.run", - caller_id: "caller", - input, - status, - task_execution: { - execution_id: `${invocationId}/1`, - run_id: "run", - step_id: "step", - dispatch_id: "journal-key", - deadline, - ...(status === "pending" - ? {} - : { - worker_generation: generation, - accepted_at: "2026-01-01T00:00:00.000Z", - }), - accounting: { tokens_input: 7, tokens_output: 3 }, - }, - output: status === "completed" ? result : null, - error: status === "failed" ? "task_deadline_exceeded" : null, - completed_at: ["completed", "failed"].includes(status) - ? "2026-01-01T00:00:01.000Z" - : null, - }); - }, - ); - return { - request, - fetchMock, - frames, - invocationId, - run: (signal?: AbortSignal) => - runAgentRelayTask(request, { - agentToken: "fixture-agent-token", - fetch: fetchMock as typeof fetch, - pollMs: 1, - signal, - }), - setStatus: (value: string) => { - status = value; - }, - setCaller: (value: string) => { - caller = value; - }, - setOutput: (value: unknown) => { - result = value; - }, - setGeneration: (value: string) => { - generation = value; - }, - losePost: () => { - losePost = true; - }, - get reads() { - return reads; - }, - posts: () => frames.filter((frame) => frame.init?.method === "POST"), + const spawn = vi.fn(async (input, beforeDispatch) => { + await beforeDispatch(); + return { ...receipt, name: input.name }; + }); + const release = vi.fn(async () => ({ id: 'inv_release_1', action: 'release' as const, + status: 'completed' as const, spawned: false, ready: false, released: true })); + const status = vi.fn(async () => ({ ...receipt, name: receipt.name })); + const ack = vi.fn(async () => ({})); + const fail = vi.fn(async () => ({})); + const port: NodeSpawnPort = { + dispatchFenceVersion: 1, targets: async () => [], spawn, status, release, }; + const runtime: NodeSpawnRuntime = { + port, + publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + inbox: { inbox: { + subscribe: async function* () { + const name = spawn.mock.results[0]?.value === undefined + ? receipt.name : (await spawn.mock.results[0]!.value).name; + yield { + id: 'delivery_1', + message: { + id: 'message_1', + from: { id: 'agent_1', name }, + text: NODE_RESULT_PREFIX + JSON.stringify({ + version: 1, run_id: 'run_1', step_id: 'edit', dispatch_id: 'dispatch_1234567890', + status: result, + ...(result === 'completed' ? { output: { summary: 'done' } } : { error: 'agent reported failure' }), + accounting: { tokens_input: 10, tokens_output: 5 }, + }), + }, + }; + }, + ack, + fail, + } }, + }; + return { runtime, spawn, release, status, ack, fail }; } -describe("durable Relay task transport", () => { - it("requires an agent token; a workspace key is not an invocation credential", () => { - expect(() => - readAgentRelayEnv({ RELAY_API_KEY: "fixture-workspace-key" }), - ).toThrow(/RELAY_AGENT_TOKEN/); - expect(readAgentRelayEnv({ RELAY_AGENT_TOKEN: " token " })).toEqual({ - agentToken: "token", - baseUrl: "https://cast.agentrelay.com", +describe('native node-spawn task transport', () => { + it('spawns once, captures exact-sender inbox evidence, and leaves ack to the journal caller', async () => { + const dataDir = await dataDirectory('flows-node-spawn-'); + const f = fixture(); + const result = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.spawn.mock.calls[0]![0]).toMatchObject({ harness: 'codex', idempotencyKey: 'dispatch_1234567890' }); + expect(f.spawn.mock.calls[0]![0]).not.toHaveProperty('directory'); + expect(f.release).toHaveBeenCalledWith(expect.objectContaining({ + receipt: expect.objectContaining({ agentId: 'agent_1', nodeName: 'desktop-1' }), + })); + expect(result.receipt).toMatchObject({ + action_name: 'spawn', status: 'completed', output: { summary: 'done' }, + task_execution: { agent_id: 'agent_1', delivery_id: 'delivery_1', release_invocation_id: 'inv_release_1' }, }); + expect(f.ack).not.toHaveBeenCalled(); + await result.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + await result.delivery.ack(); + expect(f.ack).toHaveBeenCalledWith(expect.objectContaining({ inboxItemId: 'delivery_1', state: 'read' })); + }); + + it('replays a durable result without a second spawn, release, or remote status lookup', async () => { + const dataDir = await dataDirectory('flows-node-recover-'); + const f = fixture(); + await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + const recovered = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + expect(f.spawn).toHaveBeenCalledTimes(1); + expect(f.release).toHaveBeenCalledTimes(1); + expect(f.status).not.toHaveBeenCalled(); + expect(recovered.receipt.status).toBe('completed'); }); - it("uses supported routes and snake_case data envelopes, returning only final GET output", async () => { - const f = await fixture(); - const receipt = await f.run(); - expect(receipt.output).toEqual({ answer: 42 }); - expect(receipt.status).toBe("completed"); - expect(f.posts()).toHaveLength(1); - expect(f.reads).toBe(1); - expect( - JSON.parse(String(f.posts()[0]!.init!.body)).input.task_context, - ).toEqual({ - run_id: "run", - step_id: "step", - dispatch_id: "journal-key", - timeout_ms: 86400000, + + it('retains the original inbox teardown barrier across a failed release and cached retry', async () => { + const dataDir = await dataDirectory('flows-node-release-retry-teardown-'); + const f = fixture(); + let finishTeardown!: () => void; + const settlementReady = new Promise<{ failure?: unknown }>(resolve => { + finishTeardown = () => resolve({}); }); + let delivered = false; + const subscribe = vi.fn(() => ({ [Symbol.asyncIterator]() { return { + async next() { + if (delivered) return new Promise>(() => undefined); + delivered = true; + const name = (await f.spawn.mock.results[0]!.value).name; + return { done: false as const, value: { id: 'delivery_1', settlementReady, + message: { id: 'message_1', from: { id: 'agent_1', name }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_1', + step_id: 'edit', dispatch_id: 'dispatch_1234567890', status: 'completed', + output: { summary: 'done' }, accounting: { tokens_input: 10, tokens_output: 5 } }) } } }; + }, + async return() { return { done: true as const, value: undefined }; }, + }; } })); + f.runtime.inbox.inbox.subscribe = subscribe; + f.release.mockRejectedValueOnce(new Error('release unavailable')); + + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('release unavailable'); + const recovered = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + expect(subscribe).toHaveBeenCalledOnce(); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).toHaveBeenCalledTimes(2); + await recovered.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + const settling = recovered.delivery.ack(); + const settledBeforeTeardown = await Promise.race([ + settling.then(() => true), + new Promise(resolve => setTimeout(() => resolve(false), 100)), + ]); + expect(settledBeforeTeardown).toBe(false); + expect(f.ack).not.toHaveBeenCalled(); + expect(f.fail).not.toHaveBeenCalled(); + finishTeardown(); + await settling; + expect(f.ack).toHaveBeenCalledOnce(); + expect(f.fail).not.toHaveBeenCalled(); + }); + + it('replays across changed recovery text while stable task identity remains exact', async () => { + const dataDir = await dataDirectory('flows-node-recover-task-identity-'); + const f = fixture(); + const initial = { ...request(dataDir), taskIdentity: 'Implement the change', + result_schema: { type: 'object', required: ['summary'] } }; + await runAgentRelayTask(initial, { runtime: f.runtime }); + const recovered = await runAgentRelayTask({ + ...initial, + task: `${initial.task}\n\nPrevious attempt (journaled):\n${JSON.stringify({ + completion_reason: 'crashed', trajectory_tail: { note: 'retry' }, + })}`, + attempt: 2, + }, { runtime: f.runtime }); + expect(recovered.receipt.status).toBe('completed'); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).toHaveBeenCalledOnce(); + + await expect(runAgentRelayTask({ + ...initial, taskIdentity: 'A different authored task', attempt: 2, + }, { runtime: f.runtime })).rejects.toThrow('conflicts with its durable identity or input'); + await expect(runAgentRelayTask({ + ...initial, result_schema: { type: 'object', required: ['different'] }, attempt: 2, + }, { runtime: f.runtime })).rejects.toThrow('conflicts with its durable identity or input'); }); - it("keeps readiness/interim states pending even when the POST acknowledgment says completed", async () => { - const f = await fixture(); - f.setStatus("running"); - let settled = false; - const running = f.run().then((value) => { - settled = true; - return value; + + it('fails closed across an ambiguous pre-ack crash instead of spawning twice', async () => { + const dataDir = await dataDirectory('flows-node-ambiguous-'); + const f = fixture(); + f.spawn.mockImplementationOnce(async (_input, beforeDispatch) => { + await beforeDispatch(); + throw new Error('connection dropped'); }); - await vi.waitFor(() => expect(f.reads).toBeGreaterThan(1)); - expect(settled).toBe(false); - f.setStatus("completed"); - expect((await running).output).toEqual({ answer: 42 }); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })).rejects.toThrow('connection dropped'); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })).rejects.toThrow('ambiguous pre-ack'); + expect(f.spawn).toHaveBeenCalledTimes(1); }); - it("reconciles an ambiguous POST exclusively by GET without issuing another POST", async () => { - const f = await fixture(); - f.losePost(); - expect((await f.run()).status).toBe("completed"); - expect(f.posts()).toHaveLength(1); - expect(f.reads).toBe(1); + + it('durably releases exact identity attached to a post-spawn verification failure', async () => { + const dataDir = await dataDirectory('flows-node-postspawn-failure-'); + const f = fixture(); + f.spawn.mockImplementationOnce(async (_input, beforeDispatch) => { + await beforeDispatch(); + throw new NodeSpawnError('task delivery unconfirmed', + 'spawn_task_unconfirmed', undefined, { + id: 'inv_spawn_1', nodeId: 'node_1', nodeName: 'desktop-1', + name: 'worker_1', agentId: 'agent_1', + }); + }); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('task delivery unconfirmed'); + expect(f.release).toHaveBeenCalledWith(expect.objectContaining({ + receipt: expect.objectContaining({ agentId: 'agent_1', nodeName: 'desktop-1' }), + })); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('task delivery unconfirmed'); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).toHaveBeenCalledOnce(); }); - it("reopens the durable dispatch after interruption and reads the original task", async () => { - const f = await fixture(); - f.setStatus("running"); - const abort = new AbortController(); - const running = f.run(abort.signal); - const rejection = expect(running).rejects.toThrow(); - await vi.waitFor(() => expect(f.reads).toBeGreaterThan(0)); - abort.abort(new Error("lease lost")); - await rejection; - f.setStatus("completed"); - expect((await f.run()).status).toBe("completed"); - expect(f.posts()).toHaveLength(1); + + it('releases the exact confirmed agent before propagating cancellation', async () => { + const dataDir = await dataDirectory('flows-node-cancel-'); + const f = fixture(); + const controller = new AbortController(); + f.spawn.mockImplementationOnce(async (input, beforeDispatch) => { + await beforeDispatch(); + queueMicrotask(() => controller.abort(new Error('lease lost'))); + return { id: 'inv_spawn_1', action: 'spawn' as const, status: 'completed' as const, + nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, agentId: 'agent_1', + spawned: true as const, ready: true as const }; + }); + const order: string[] = []; + f.release.mockImplementationOnce(async () => { + order.push('release'); + return { id: 'inv_release_1', action: 'release' as const, + status: 'completed' as const, spawned: false, ready: false, released: true }; + }); + try { + await runAgentRelayTask(request(dataDir), { runtime: f.runtime, signal: controller.signal }); + throw new Error('expected cancellation'); + } catch (error) { + expect(error).toMatchObject({ message: 'lease lost' }); + order.push('rejected'); + } + expect(order).toEqual(['release', 'rejected']); + expect(f.release).toHaveBeenCalledWith(expect.objectContaining({ + receipt: expect.objectContaining({ agentId: 'agent_1', nodeName: 'desktop-1' }), + })); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })).rejects.toMatchObject({ + cause: expect.objectContaining({ code: 'task_cancelled' }), + }); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).toHaveBeenCalledOnce(); + expect(f.status).not.toHaveBeenCalled(); }); - it("refuses changed caller, canonical endpoint, or input before another invocation", async () => { - const f = await fixture(); - await f.run(); - f.setCaller("replacement"); - await expect(f.run()).rejects.toThrow(/durable caller/); - f.setCaller("caller"); - f.request.task = "changed"; - await expect(f.run()).rejects.toThrow(/durable caller/); - f.request.task = "return output"; - await expect( - runAgentRelayTask(f.request, { - baseUrl: "https://other.example", - agentToken: "fixture-agent-token", - fetch: f.fetchMock as typeof fetch, - }), - ).rejects.toThrow(/durable caller/); - expect(f.posts()).toHaveLength(1); + + it('refuses to extend the original durable execution deadline on redispatch', async () => { + const dataDir = await dataDirectory('flows-node-deadline-identity-'); + const f = fixture(); + const controller = new AbortController(); + f.spawn.mockImplementationOnce(async (input, beforeDispatch) => { + await beforeDispatch(); + queueMicrotask(() => controller.abort(new Error('worker stopped'))); + return { id: 'inv_spawn_1', action: 'spawn' as const, status: 'completed' as const, + nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, agentId: 'agent_1', + spawned: true as const, ready: true as const }; + }); + await expect(runAgentRelayTask({ ...request(dataDir), timeoutMs: 1_000 }, { + runtime: f.runtime, signal: controller.signal, + })).rejects.toThrow('worker stopped'); + await expect(runAgentRelayTask({ ...request(dataDir), timeoutMs: 60_000 }, { runtime: f.runtime })) + .rejects.toThrow('conflicts with its durable identity or input'); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).toHaveBeenCalledOnce(); }); - it("recovers an old terminal receipt without redispatch after its reconciliation deadline", async () => { - const f = await fixture(); - await f.run(); - const directory = join(f.request.dataDir, "relay-tasks"); - const [name] = await readdir(directory); - const path = join(directory, name!); - const claim = JSON.parse(await readFile(path, "utf8")); - claim.startedAt = Date.now() - 2 * 86400000; - await writeFile(path, JSON.stringify(claim)); - expect((await f.run()).status).toBe("completed"); - expect(f.posts()).toHaveLength(1); + + it('bounds a recovered spawn status lookup by the original deadline and releases exact identity', async () => { + vi.useFakeTimers(); + try { + const dataDir = await dataDirectory('flows-node-status-stall-'); + const f = fixture(); + const bounded = { ...request(dataDir), timeoutMs: 1_000 }; + await runAgentRelayTask(bounded, { runtime: f.runtime }); + const [entry] = await readNodeSpawnClaims(dataDir); + const { result: _result, release: _release, ...spawnOnly } = entry!.claim; + await recordNodeSpawn(entry!.path, spawnOnly); + f.status.mockImplementationOnce(() => new Promise(() => undefined)); + const pending = expect(runAgentRelayTask(bounded, { runtime: f.runtime })) + .rejects.toThrow(); + await vi.waitFor(() => expect(f.status).toHaveBeenCalledOnce()); + await vi.advanceTimersByTimeAsync(1_001); + await pending; + expect(f.release).toHaveBeenCalledTimes(2); + const [recovered] = await readNodeSpawnClaims(dataDir); + expect(recovered!.claim.release).toBeDefined(); + } finally { + vi.useRealTimers(); + } }); - it("fails on corrupt durable state rather than discarding it and replaying", async () => { - const f = await fixture(); - await f.run(); - const directory = join(f.request.dataDir, "relay-tasks"); - const [name] = await readdir(directory); - await writeFile(join(directory, name!), "{"); - await expect(f.run()).rejects.toThrow(); - expect(f.posts()).toHaveLength(1); + + it('returns promptly from a cancelled confirmation and cleans up its late exact receipt', async () => { + const dataDir = await dataDirectory('flows-node-confirm-cancel-'); + const f = fixture(); + const controller = new AbortController(); + let confirm!: (receipt: NodeSpawnReceipt) => void; + f.spawn.mockImplementationOnce(async (input, beforeDispatch) => { + await beforeDispatch(); + return new Promise(resolve => { + confirm = receipt => resolve({ ...receipt, name: input.name }); + }); + }); + const pending = runAgentRelayTask(request(dataDir), { runtime: f.runtime, signal: controller.signal }); + await vi.waitFor(() => expect(f.spawn).toHaveBeenCalledOnce()); + controller.abort(new Error('lease lost during confirmation')); + await expect(pending).rejects.toThrow('lease lost during confirmation'); + expect(f.release).not.toHaveBeenCalled(); + confirm({ id: 'inv_spawn_1', action: 'spawn', status: 'completed', nodeId: 'node_1', + nodeName: 'desktop-1', name: 'placeholder', agentId: 'agent_1', spawned: true, ready: true }); + await vi.waitFor(() => expect(f.release).toHaveBeenCalledOnce()); }); - it("refuses before POST when the durable dispatch cannot be written", async () => { - const f = await fixture(); - const file = join(f.request.dataDir, "not-a-directory"); - await writeFile(file, "fixture"); - f.request.dataDir = file; - await expect(f.run()).rejects.toThrow(); - expect(f.posts()).toHaveLength(0); + + it('rechecks cancellation after durable claiming immediately before spawn dispatch', async () => { + const dataDir = await dataDirectory('flows-node-prespawn-cancel-'); + const f = fixture(); + const controller = new AbortController(); + const pending = runAgentRelayTask(request(dataDir), { runtime: f.runtime, signal: controller.signal }); + queueMicrotask(() => controller.abort(new Error('lease lost before dispatch'))); + await expect(pending).rejects.toThrow('lease lost before dispatch'); + expect(f.spawn).not.toHaveBeenCalled(); }); - it("preserves explicit terminal failure and scalar/array/null output", async () => { - const f = await fixture(); - f.setStatus("failed"); - expect((await f.run()).error).toBe("task_deadline_exceeded"); - for (const output of [null, 42, "text", [1, 2]]) { - const next = await fixture(); - next.setOutput(output); - expect((await next.run()).output).toEqual(output); - } + + it('acks the inbox delivery only after stepComplete has entered the journal', async () => { + const dataDir = await dataDirectory('flows-node-journal-'); + const f = fixture(); + const order: string[] = []; + f.ack.mockImplementation(async () => { order.push('ack'); return {}; }); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), + stepComplete: vi.fn(async () => { expect(f.ack).not.toHaveBeenCalled(); order.push('journal'); return {}; }), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_1', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + const errors: unknown[] = []; + worker.on('error', error => errors.push(error)); + await worker.attach(); + const dispatch: StepDispatchEvent = { + run_id: 'run_1', step_id: 'edit', attempt: 1, step_type: 'agent', + spec: { cli: 'codex', instruction: 'Implement the change', transport: 'relay', model: 'codex-medium' }, + idempotency_key: 'dispatch_1234567890', lease_id: 'lease_1', + lease_deadline_ms: Date.now() + 30_000, pins: {}, + }; + client.emit('step.dispatch', dispatch); + await worker.close(); + expect(errors).toEqual([]); + expect(order).toEqual(['journal', 'ack']); + expect(client.stepComplete).toHaveBeenCalledWith('run_1', 'edit', 1, 'dispatch_1234567890', 'success', + expect.objectContaining({ output: { summary: 'done' }, trajectory_tail: { + relay_task: expect.objectContaining({ invocation_id: 'inv_spawn_1', status: 'completed' }), + } })); }); - it("rejects a changed accepted generation instead of accepting a stale terminal receipt", async () => { - const f = await fixture(); - f.setStatus("running"); - const running = f.run(); - const rejection = expect(running).rejects.toThrow(/generation/); - await vi.waitFor(() => expect(f.reads).toBeGreaterThan(0)); - f.setGeneration("replacement"); - f.setStatus("completed"); - await rejection; + + it('keeps a result unacked and durably retries exact cleanup when release is unconfirmed', async () => { + const dataDir = await dataDirectory('flows-node-cleanup-'); + const f = fixture(); + f.release.mockRejectedValueOnce(new Error('release response lost')); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toMatchObject({ cleanupPending: true }); + expect(f.ack).not.toHaveBeenCalled(); + const recovered = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).toHaveBeenCalledTimes(2); + expect(recovered.receipt.status).toBe('completed'); }); - it("rejects wrong correlation and masks arbitrary HTTP error bodies", async () => { - const f = await fixture(); - await expect( - runAgentRelayTask(f.request, { - agentToken: "fixture-token", - fetch: vi.fn( - async () => new Response("secret-response-body", { status: 401 }), - ) as typeof fetch, - }), - ).rejects.toThrow("HTTP 401"); - const changed = vi.fn( - async (url: string | URL | Request, init?: RequestInit) => { - const response = await f.fetchMock(url, init); - const body = (await response.json()) as { - data: { task_execution: { dispatch_id: string } }; - }; - if (String(url).includes("/invocations/")) - body.data.task_execution.dispatch_id = "wrong"; - return new Response(JSON.stringify(body), { status: 200 }); - }, - ); - await expect( - runAgentRelayTask(f.request, { - agentToken: "fixture-agent-token", - fetch: changed as typeof fetch, - }), - ).rejects.toThrow(/correlation/); + + it('bounds a stalled exact release and retries it from durable cleanup state', async () => { + vi.useFakeTimers(); + try { + const dataDir = await dataDirectory('flows-node-cleanup-stall-'); + const f = fixture(); + f.release.mockImplementationOnce(() => new Promise(() => undefined)); + const task = runAgentRelayTask({ ...request(dataDir), timeoutMs: 60_000 }, { runtime: f.runtime }); + const rejection = expect(task).rejects.toMatchObject({ cleanupPending: true }); + await vi.waitFor(() => expect(f.release).toHaveBeenCalledOnce()); + await vi.advanceTimersByTimeAsync(15_001); + await rejection; + expect(f.ack).not.toHaveBeenCalled(); + const claims = await readNodeSpawnClaims(dataDir); + expect(claims[0]!.claim.cleanupPending).toMatchObject({ + attempts: 1, + lastError: expect.stringContaining('independent deadline'), + }); + const recovered = await runAgentRelayTask({ ...request(dataDir), timeoutMs: 60_000 }, { runtime: f.runtime }); + expect(f.release).toHaveBeenCalledTimes(2); + expect(recovered.receipt.status).toBe('completed'); + } finally { + vi.useRealTimers(); + } }); -}); -import { EventEmitter } from "node:events"; -import type { JournalClient } from "../src/journal-client.js"; -import type { StepDispatchEvent } from "../src/protocol.js"; -import { AgentWorker } from "../src/worker.js"; + it('leaves failed cleanup nonterminal so a worker redispatch can finish exact release', async () => { + const dataDir = await dataDirectory('flows-node-worker-cleanup-'); + const f = fixture(); + f.release.mockRejectedValueOnce(new Error('release response lost')); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), + stepComplete: vi.fn(async () => ({})), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_1', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + await worker.attach(); + const dispatch = (attempt: number): StepDispatchEvent => ({ + run_id: 'run_1', step_id: 'edit', attempt, step_type: 'agent', + spec: { cli: 'codex', instruction: 'Implement the change', transport: 'relay' }, + idempotency_key: 'dispatch_1234567890', lease_id: `lease_${attempt}`, + lease_deadline_ms: Date.now() + 30_000, pins: {}, + }); + const failed = new Promise(resolve => worker.once('error', resolve)); + client.emit('step.dispatch', dispatch(1)); + const failure = await failed; + expect(failure).toBeInstanceOf(AgentRelayTransportError); + expect(failure).toMatchObject({ cleanupPending: true }); + expect(client.stepComplete).not.toHaveBeenCalled(); + expect(f.ack).not.toHaveBeenCalled(); + client.emit('step.dispatch', dispatch(2)); + await vi.waitFor(() => expect(client.stepComplete).toHaveBeenCalledOnce()); + await worker.close(); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).toHaveBeenCalledTimes(2); + expect(client.stepComplete).toHaveBeenCalledWith('run_1', 'edit', 2, + 'dispatch_1234567890', 'success', expect.any(Object)); + expect(f.ack).toHaveBeenCalledOnce(); + }); -async function workerFixture(f: Awaited>, runRoot?: string) { - vi.stubEnv("RELAY_AGENT_TOKEN", "fixture-agent-token"); - vi.stubEnv("RELAY_BASE_URL", "https://cast.agentrelay.com"); - vi.stubGlobal("fetch", f.fetchMock); - const client = Object.assign(new EventEmitter(), { - workerAttach: vi.fn(async () => ({})), - stepHeartbeat: vi.fn(async () => ({ - lease_deadline_ms: Date.now() + 30000, - })), - stepComplete: vi.fn(async (..._args: unknown[]) => ({})), + it('fails immediately when a prior error already released an agent without a result', async () => { + const dataDir = await dataDirectory('flows-node-released-without-result-'); + const f = fixture(); + f.runtime.inbox.inbox.subscribe = async function* () {}; + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('inbox ended'); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('inbox ended'); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).toHaveBeenCalledOnce(); + expect(f.status).not.toHaveBeenCalled(); }); - const worker = new AgentWorker(client as unknown as JournalClient, { - workerId: "fixture-worker", - pins: {}, - dataDir: f.request.dataDir, - ...(runRoot === undefined ? {} : { runRoot }), + + it('retries pending cleanup before failing a result-less spawn recovery', async () => { + const dataDir = await dataDirectory('flows-node-resultless-cleanup-'); + const f = fixture(); + f.runtime.inbox.inbox.subscribe = async function* () {}; + f.release.mockRejectedValueOnce(new Error('release response lost')); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toMatchObject({ cleanupPending: true }); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('inbox ended'); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).toHaveBeenCalledTimes(2); + expect(f.status).not.toHaveBeenCalled(); }); - const errors: unknown[] = []; - worker.on("error", (error) => errors.push(error)); - await worker.attach(); - const dispatch: StepDispatchEvent = { - run_id: "run", - step_id: "step", - attempt: 1, - step_type: "agent", - spec: { - cli: "claude", - instruction: "return output", - transport: "relay", - model: "codex-medium", - verification: { json_schema: { type: "array" } }, - }, - idempotency_key: "journal-key", - lease_id: "lease", - lease_deadline_ms: Date.now() + 30000, - pins: {}, - wake_context: { triggering_event: { type: "fixture" } }, - }; - return { client, worker, errors, dispatch }; -} -describe("Relay completion at the journal boundary", () => { - it("does not complete at readiness and journals exact output, receipt, and priced accounting", async () => { - const f = await fixture(); - f.setStatus("running"); - f.setOutput([1, 2]); - const w = await workerFixture(f); - w.client.emit("step.dispatch", w.dispatch); - await vi.waitFor(() => expect(f.reads).toBe(1)); - expect(w.client.stepComplete).not.toHaveBeenCalled(); - f.setStatus("completed"); - await w.worker.close(); - expect(w.errors).toEqual([]); - expect(w.client.stepComplete).toHaveBeenCalledTimes(1); - expect(w.client.stepComplete.mock.calls[0]).toEqual([ - "run", - "step", - 1, - "journal-key", - "success", - expect.objectContaining({ - output: [1, 2], - usage: { tokens_in: 7, tokens_out: 3, dollars: "0.000038" }, - trajectory_tail: { - relay_task: expect.objectContaining({ - invocation_id: f.invocationId, - status: "completed", - }), - }, - }), - ]); - const input = JSON.parse(String(f.posts()[0]!.init!.body)).input; - expect(input.task).toContain("Wake context (journaled)"); - expect(input.task).toContain("final=true"); - expect(input.result_schema).toEqual({ type: "array" }); + it('releases a durably failed spawn before any status or inbox recovery', async () => { + const dataDir = await dataDirectory('flows-node-failure-before-cleanup-'); + const f = fixture(); + await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + const [entry] = await readNodeSpawnClaims(dataDir); + const { result: _result, release: _release, cleanupPending: _pending, ...spawned } = entry!.claim; + await recordNodeSpawn(entry!.path, { ...spawned, + failure: { code: 'spawn_cancelled', message: 'worker stopped after spawn' } }); + f.release.mockClear(); + f.status.mockClear(); + f.runtime.inbox.inbox.subscribe = async function* () { throw new Error('inbox must not be read'); }; + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('worker stopped after spawn'); + expect(f.release).toHaveBeenCalledOnce(); + expect(f.status).not.toHaveBeenCalled(); }); - it("does not forward the local run root as worker_cwd to a remote relay worker", async () => { - const f = await fixture(); - const runRoot = await mkdtemp(join(tmpdir(), "flows-relay-runroot-")); - directories.push(runRoot); - const w = await workerFixture(f, runRoot); - w.client.emit("step.dispatch", w.dispatch); - await vi.waitFor(() => expect(f.posts()).toHaveLength(1)); - await w.worker.close(); - expect(w.errors).toEqual([]); - const input = JSON.parse(String(f.posts()[0]!.init!.body)).input; - expect(input.worker_cwd).toBeUndefined(); + + it('keeps the original execution deadline but never expires a durably observed result', async () => { + vi.useFakeTimers({ toFake: ['Date'] }); + try { + const started = new Date('2026-11-07T00:00:00.000Z'); + vi.setSystemTime(started); + const dataDir = await dataDirectory('flows-node-deadline-'); + const f = fixture(); + const bounded = { ...request(dataDir), timeoutMs: 1_000 }; + await runAgentRelayTask(bounded, { runtime: f.runtime }); + vi.setSystemTime(started.getTime() + 1_001); + await expect(runAgentRelayTask(bounded, { runtime: f.runtime })).resolves.toBeDefined(); + expect(f.status).not.toHaveBeenCalled(); + vi.setSystemTime(started.getTime() + 31_001); + f.runtime.inbox.inbox.subscribe = async function* () { throw new Error('must use durable envelope'); }; + await expect(runAgentRelayTask(bounded, { runtime: f.runtime })).resolves.toBeDefined(); + expect(f.status).not.toHaveBeenCalled(); + expect(f.spawn).toHaveBeenCalledOnce(); + } finally { + vi.useRealTimers(); + } }); - it("journals terminal task failure as worker_error with its explicit reason", async () => { - const f = await fixture(); - f.setStatus("failed"); - const w = await workerFixture(f); - w.client.emit("step.dispatch", w.dispatch); - await w.worker.close(); - expect(w.errors).toEqual([]); - expect(w.client.stepComplete.mock.calls[0]![4]).toBe("worker_error"); - expect(JSON.stringify(w.client.stepComplete.mock.calls[0])).toContain( - "task_deadline_exceeded", - ); + + it('preserves a timely durable result when exact release crosses the task deadline', async () => { + const taskDeadline = new AbortController(); + const nativeTimeout = AbortSignal.timeout.bind(AbortSignal); + const timeout = vi.spyOn(AbortSignal, 'timeout').mockImplementation(milliseconds => + milliseconds < 2_000 ? taskDeadline.signal : nativeTimeout(milliseconds)); + try { + const dataDir = await dataDirectory('flows-node-result-before-deadline-'); + const f = fixture(); + let confirmRelease!: (receipt: Awaited>) => void; + f.release.mockImplementationOnce(() => new Promise(resolve => { confirmRelease = resolve; })); + const task = runAgentRelayTask({ ...request(dataDir), timeoutMs: 1_000 }, { runtime: f.runtime }); + await vi.waitFor(() => expect(f.release).toHaveBeenCalledOnce()); + + taskDeadline.abort(new DOMException('The operation was aborted due to timeout', 'TimeoutError')); + confirmRelease({ id: 'inv_release_after_deadline', action: 'release', status: 'completed', + spawned: false, ready: false, released: true }); + + const completed = await task; + expect(completed.receipt).toMatchObject({ + status: 'completed', + task_execution: { delivery_id: 'delivery_1', release_invocation_id: 'inv_release_after_deadline' }, + }); + const [persisted] = await readNodeSpawnClaims(dataDir); + expect(persisted!.claim.result).toMatchObject({ inboxItemId: 'delivery_1', messageId: 'message_1' }); + expect(persisted!.claim.release).toMatchObject({ invocationId: 'inv_release_after_deadline' }); + expect(f.ack).not.toHaveBeenCalled(); + await completed.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + await completed.delivery.ack(); + expect(f.ack).toHaveBeenCalledOnce(); + } finally { + timeout.mockRestore(); + } }); - it("aborts polling on rejected renewal and never writes a stale completion", async () => { - const f = await fixture(); - f.setStatus("running"); - const w = await workerFixture(f); - w.client.stepHeartbeat - // 3s lease, so worker-lease.ts renews at ~1s: after the durable claim's - // three fsyncs (agent-relay-state.ts `claimRelayTask`) and the POST, - // while `runAgentRelayTask` sits in its 1s poll delay. The old 150ms - // lease renewed at ~50ms and raced those fsyncs, which a loaded CI - // runner loses. `mockImplementationOnce` also reads the clock at the - // heartbeat rather than at mock setup, so the window is the stated one. - .mockImplementationOnce(async () => ({ lease_deadline_ms: Date.now() + 3_000 })) - .mockRejectedValueOnce(new Error("lease replaced")); - w.client.emit("step.dispatch", w.dispatch); - // The POST is the precondition for the abort being mid-poll, not a - // side effect of the timing: wait for it rather than assuming it landed. - await vi.waitFor(() => { expect(f.posts()).toHaveLength(1); }); - await w.worker.close(); - expect(w.client.stepComplete).not.toHaveBeenCalled(); - expect(String(w.errors[0])).toContain("lease replaced"); - expect(f.posts()).toHaveLength(1); + + it('journals a failed agent envelope before acknowledging its delivery', async () => { + const dataDir = await dataDirectory('flows-node-failed-result-'); + const f = fixture('failed'); + const order: string[] = []; + f.ack.mockImplementation(async () => { order.push('ack'); return {}; }); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), + stepComplete: vi.fn(async () => { expect(f.ack).not.toHaveBeenCalled(); order.push('journal'); return {}; }), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_1', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + const errors: unknown[] = []; + worker.on('error', error => errors.push(error)); + await worker.attach(); + client.emit('step.dispatch', { + run_id: 'run_1', step_id: 'edit', attempt: 1, step_type: 'agent', + spec: { cli: 'codex', instruction: 'Implement the change', transport: 'relay' }, + idempotency_key: 'dispatch_1234567890', lease_id: 'lease_1', + lease_deadline_ms: Date.now() + 30_000, pins: {}, + } satisfies StepDispatchEvent); + await worker.close(); + expect(errors).toEqual([]); + expect(order).toEqual(['journal', 'ack']); + expect(client.stepComplete).toHaveBeenCalledWith('run_1', 'edit', 1, 'dispatch_1234567890', + expect.not.stringMatching(/^success$/), expect.objectContaining({ + trajectory_tail: { relay_task: expect.objectContaining({ status: 'failed', error: 'agent reported failure' }) }, + })); }); -}); -import { readFile } from "node:fs/promises"; -it("matches exact HTTP fixtures generated by Relaycast #436, including its deterministic invocation ID", async () => { - const wire = JSON.parse( - await readFile( - new URL("./fixtures/relay-task/relaycast-436.json", import.meta.url), - "utf8", - ), - ); - const f = await fixture(); - let reads = 0; - const fetchMock = vi.fn( - async (url: string | URL | Request, init?: RequestInit) => { - if (String(url).endsWith("/v1/agent")) - return new Response(JSON.stringify(wire.identity), { status: 200 }); - if (String(url).endsWith("/invoke")) { - expect(JSON.parse(String(init?.body))).toEqual({ input: wire.input }); - return new Response(JSON.stringify(wire.ack), { status: 201 }); - } - expect(String(url)).toBe( - `https://cast.agentrelay.com/v1/actions/task.run/invocations/${wire.ack.data.invocation_id}`, - ); - return new Response( - JSON.stringify( - [wire.dispatched, wire.running, wire.completed][reads++], - ), - { status: 200 }, - ); - }, - ); - const result = await runAgentRelayTask(f.request, { - agentToken: "fixture-token", - fetch: fetchMock as typeof fetch, - pollMs: 1, + it('journals a malformed exact-sender envelope before dead-lettering its delivery', async () => { + const dataDir = await dataDirectory('flows-node-malformed-result-'); + const f = fixture(); + f.runtime.inbox.inbox.subscribe = async function* () { + const name = (await f.spawn.mock.results[0]!.value).name; + yield { id: 'delivery_1', message: { id: 'message_1', + from: { id: 'agent_1', name }, + text: NODE_RESULT_PREFIX + '{bad', + } }; + }; + const order: string[] = []; + f.fail.mockImplementation(async () => { order.push('fail'); return {}; }); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), + stepComplete: vi.fn(async () => { expect(f.fail).not.toHaveBeenCalled(); order.push('journal'); return {}; }), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_1', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + const errors: unknown[] = []; + worker.on('error', error => errors.push(error)); + await worker.attach(); + client.emit('step.dispatch', { + run_id: 'run_1', step_id: 'edit', attempt: 1, step_type: 'agent', + spec: { cli: 'codex', instruction: 'Implement the change', transport: 'relay' }, + idempotency_key: 'dispatch_1234567890', lease_id: 'lease_1', + lease_deadline_ms: Date.now() + 30_000, pins: {}, + } satisfies StepDispatchEvent); + await worker.close(); + expect(errors).toEqual([]); + expect(order).toEqual(['journal', 'fail']); + expect(f.ack).not.toHaveBeenCalled(); + expect(f.fail).toHaveBeenCalledWith(expect.objectContaining({ inboxItemId: 'delivery_1', retry: false })); }); - expect(result.output).toEqual(wire.completed.data.output); - expect(result.task_execution).toEqual(wire.completed.data.task_execution); - expect(reads).toBe(3); }); diff --git a/packages/sdk/tests/authored-broker.test.ts b/packages/sdk/tests/authored-broker.test.ts deleted file mode 100644 index 3a41aa623..000000000 --- a/packages/sdk/tests/authored-broker.test.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { expect, it, vi } from 'vitest'; -import { authoredBroker } from '../src/authored-broker.js'; -import { brokerAgentName, brokerInstruction } from '../src/communication/broker-spec.js'; -import { toKernelSpec } from '../src/compile.js'; -import type { StepSpec } from '../src/spec.js'; -function setup(owner = 'root') { - const run = vi.fn(async (step: StepSpec) => { - const kernel = toKernelSpec({ version: '0.1.0', name: 'demo', steps: [step] }); - expect(kernel.steps[0]?.type).toBe('agent'); - expect(kernel.steps[0]).not.toHaveProperty('transport'); - const broker = step.type === 'agent' ? brokerInstruction(step.instruction)! : undefined!; - return { completionReason: 'success', summary: '{"ok":true}', artifacts: [], agentName: broker.agentName }; - }); - return { run, broker: authoredBroker({ name: 'My flow!' }, owner, 'local-worker', run) }; -} -it('lowers broker options into an agent instruction and preserves the JSON summary', async () => { - const { broker, run } = setup(); - const result = await broker.agent('agent-0', 'Implement #42', { transport: 'broker', task: 'Implement', cli: 'codex', - cwd: '/external/worktree', keepAlive: true, skills: ['review-fix-signoff-loop'] }); - expect(result.summary).toBe('{"ok":true}'); - const step = run.mock.calls[0]![0]; - expect(step).toMatchObject({ type: 'agent', timeoutMs: 3_600_000, transportRetries: 0, - surfaces: { streams: [{ stream: 'local-worker' }] } }); - expect(brokerInstruction((step as { instruction: string }).instruction)).toMatchObject({ - cwd: '/external/worktree', channel: 'flow-root', owner: 'root', flow: 'My flow!', step: 'Implement #42', - }); - await broker.message('agent-1', result.agentName!, 'Fix', { timeout: '5s' }); - expect(run.mock.calls[1]![0]).toMatchObject({ timeoutMs: 5000 }); - await broker.release('agent-2', result.agentName!); - await broker.release('agent-3', result.agentName!); - expect(run).toHaveBeenCalledTimes(4); -}); -it('names are sanitized, stable on replay, distinct across runs and duplicate display names', () => { - const name = brokerAgentName('Flow / 🚀', 'Step 42', 'root', 'agent-1'); - expect(name).toMatch(/^[a-z0-9][a-z0-9_-]{0,63}$/); - expect(brokerAgentName('Flow / 🚀', 'Step 42', 'root', 'agent-1')).toBe(name); - expect(brokerAgentName('Flow / 🚀', 'Step 42', 'other', 'agent-1')).not.toBe(name); - expect(brokerAgentName('Flow / 🚀', 'Step 42', 'root', 'agent-2')).not.toBe(name); -}); -it('only accepts retained agents created by this flow', async () => { - const { broker, run } = setup(); - await expect(broker.message('agent-1', 'someone-else', 'Fix')).rejects.toThrow('No retained'); - await expect(broker.release('agent-1', 'someone-else')).rejects.toThrow('No retained'); - expect(run).not.toHaveBeenCalled(); -}); -it('forwards a named gate on a retained message', async () => { - const { broker, run } = setup(); - const result = await broker.agent('agent-0', 'implement', { transport: 'broker', task: 'Implement', cli: 'codex', keepAlive: true }); - const gate = { type: 'artifact_exists' as const, path: 'result.json' }; - await broker.message('agent-1', result.agentName!, 'Fix', undefined, gate); - expect(run.mock.calls[1]![0]).toMatchObject({ verification: gate }); -}); -it('carries declared secrets onto spawn, message, and release', async () => { - const { broker, run } = setup(); - const result = await broker.agent('agent-0', 'implement', { transport: 'broker', task: 'Implement', cli: 'codex', - keepAlive: true, secrets: ['GH_TOKEN'] }, undefined, ['GH_TOKEN']); - expect(run.mock.calls[0]![0]).toMatchObject({ secrets: ['GH_TOKEN'] }); - await broker.message('agent-1', result.agentName!, 'Fix'); - expect(run.mock.calls[1]![0]).toMatchObject({ secrets: ['GH_TOKEN'] }); - await broker.release('agent-2', result.agentName!); - expect(run.mock.calls[2]![0]).toMatchObject({ secrets: ['GH_TOKEN'] }); -}); -it('refuses retained sessions without a local worker pin', async () => { - const run = vi.fn(); - const broker = authoredBroker({ name: 'demo' }, 'root', undefined, run); - await expect(broker.agent('agent-0', 'worker', { transport: 'broker', task: 'Work', keepAlive: true })) - .rejects.toThrow('local agent worker stream'); - expect(run).not.toHaveBeenCalled(); -}); -it.each([ - { skills: ['../unsafe'] }, { keepAlive: 'yes' }, { channel: 'bad channel' }, - { resume: 'previous' }, { maxIterations: 2 }, { transportRetries: 1 }, { cwd: '' }, -])('refuses unsupported or malformed options before admission: %j', async options => { - const { broker, run } = setup(); - await expect(broker.agent('agent-0', 'worker', { transport: 'broker', task: 'Work', ...options } as any)).rejects.toThrow(); - expect(run).not.toHaveBeenCalled(); -}); diff --git a/packages/sdk/tests/authored-parallel-agents.test.ts b/packages/sdk/tests/authored-parallel-agents.test.ts index c3503368e..11fe97cd0 100644 --- a/packages/sdk/tests/authored-parallel-agents.test.ts +++ b/packages/sdk/tests/authored-parallel-agents.test.ts @@ -1,6 +1,6 @@ import { existsSync, mkdirSync, readFileSync, symlinkSync, writeFileSync } from 'node:fs'; import { join, resolve } from 'node:path'; -import { afterEach, describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { flow } from '@relayflows/surface'; import { AuthoredFlowExecutionError, executeAuthoredFlow } from '../src/authored-flow-executor.js'; import { attachLocalAgent } from '../src/local-agent.js'; @@ -15,7 +15,7 @@ afterEach(async () => { }); /** The fixture's wrapper, but each session holds for a while and journals when it ran. */ -async function slowAgents(capacity: number, failFirstSession = false) { +async function slowAgents(capacity: number, failFirstSession = false, holdMs = 400) { const fixture = chainFixture(); closes.push(() => fixture.close()); const spans = join(fixture.root, 'spans.jsonl'); @@ -28,7 +28,7 @@ const request = await receiveWrapperRequest(); if (request) { writeFileSync(${JSON.stringify(started)}, ''); const start = Date.now(); - await new Promise(done => setTimeout(done, 400)); + await new Promise(done => setTimeout(done, ${holdMs})); appendFileSync(${JSON.stringify(spans)}, JSON.stringify({ start, end: Date.now() }) + '\\n'); ${failFirstSession ? `if (!existsSync(${JSON.stringify(join(fixture.root, 'failed-once'))})) { writeFileSync(${JSON.stringify(join(fixture.root, 'failed-once'))}, ''); @@ -143,6 +143,38 @@ describe('authored steps under local workers with capacity', () => { expect(peakOverlap(readSpans())).toBe(1); }); + it('does not attribute an intentionally aborted lease snapshot to the authored agent callback', async () => { + const { fixture, client, agent } = await slowAgents(1, false, 2_500); + const originalRunGet = client.runGet.bind(client); + let snapshotHeld = false; + let snapshotsAbortedByCompletion = 0; + vi.spyOn(client, 'runGet').mockImplementation(async (runId, options) => { + const snapshot = await originalRunGet(runId, options); + if (snapshotHeld || options?.signal === undefined || snapshot.steps['agent-1']?.state !== 'running') { + return snapshot; + } + snapshotHeld = true; + await new Promise((resolve, reject) => { + const timer = setTimeout(resolve, 5_000); + options?.signal?.addEventListener('abort', () => { + snapshotsAbortedByCompletion += 1; + clearTimeout(timer); + reject(options.signal!.reason); + }, { once: true }); + }); + return snapshot; + }); + const completesDuringSnapshot = flow('completes-during-snapshot', async f => { + await f.agent('agent', { task: 'finish while the lease snapshot is pending' }); + f.done('success'); + }); + const result = await executeAuthoredFlow(completesDuringSnapshot, client, undefined, { + flowPath: fixture.flowPath, localAgentStream: agent.stream, workerCapacity: 1, + }); + expect(result.completionReason).toBe('success'); + expect(snapshotsAbortedByCompletion).toBe(1); + }, 10_000); + it('never starts queued agents once the body has failed', async () => { const { fixture, client, agent, readSpans, firstStarted } = await slowAgents(1); const failing = flow('fails-while-queued', async f => { diff --git a/packages/sdk/tests/broker-environment.test.ts b/packages/sdk/tests/broker-environment.test.ts deleted file mode 100644 index c6b88cef3..000000000 --- a/packages/sdk/tests/broker-environment.test.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { beforeEach, expect, it, vi } from 'vitest'; -const fake = vi.hoisted(() => ({ resolve: vi.fn(), spawn: vi.fn() })); -vi.mock('node:module', () => ({ createRequire: () => ({ resolve: fake.resolve }) })); -vi.mock('node:child_process', () => ({ spawnSync: fake.spawn })); -import { checkBrokerEnvironment } from '../src/communication/broker-preflight.js'; -beforeEach(() => vi.resetAllMocks()); -it('refuses missing packages before attempting a connection', () => { - fake.resolve.mockImplementation(() => { throw new Error('MODULE_NOT_FOUND'); }); - expect(() => checkBrokerEnvironment('/run')).toThrow('optional @agent-relay'); - expect(fake.spawn).not.toHaveBeenCalled(); -}); -it.each([1, null])('refuses unavailable or timed-out local broker probes (%s)', status => { - fake.resolve.mockReturnValue('/installed/driver.js'); fake.spawn.mockReturnValue({ status }); - expect(() => checkBrokerEnvironment('/run')).toThrow('reachable local Relay broker'); -}); -it('uses SDK connect and healthCheck without spawning a broker or agent', () => { - fake.resolve.mockReturnValue('/installed/driver.js'); fake.spawn.mockReturnValue({ status: 0 }); - expect(() => checkBrokerEnvironment('/run')).not.toThrow(); - const args = fake.spawn.mock.calls[0]![1] as string[]; - expect(args[2]).toContain('HarnessDriverClient.connect'); - expect(args[2]).toContain('client.healthCheck'); - expect(args[2]).not.toContain('.spawn'); - expect(args.slice(-2)).toEqual(['file:///installed/driver.js', '/run']); - expect(fake.spawn.mock.calls[0]![2]).toMatchObject({ timeout: 10_000 }); -}); diff --git a/packages/sdk/tests/broker-flow.test.ts b/packages/sdk/tests/broker-flow.test.ts deleted file mode 100644 index a33cebfa0..000000000 --- a/packages/sdk/tests/broker-flow.test.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { afterEach, expect, it, vi } from 'vitest'; -import { flow } from '@relayflows/surface'; -import { executeAuthoredFlow } from '../src/authored-flow-executor.js'; -import { attachLocalAgent } from '../src/local-agent.js'; -import { chainFixture } from './flow-chain-fixture.js'; -const fake = vi.hoisted(() => ({ - invoke: undefined as undefined | ((request: { operation: string; values: string[] }) => Promise), - release: vi.fn(async () => {}), shutdown: vi.fn(async () => {}), spawn: vi.fn(), send: vi.fn(), -})); -vi.mock('../src/authored-preflight.js', () => ({ authoredPreflight: () => async (flow: unknown) => ({ - flow, report: { ok: true, diagnostics: [], resolutions: [], gates: [] }, -}) })); -vi.mock('../src/communication/relay.js', () => ({ connectLocalBroker: async () => ({ - spawnPty: fake.spawn, sendMessage: fake.send, shutdown: fake.shutdown, - onEvent: () => () => {}, connectEvents: () => {}, -}) })); -vi.mock('../src/communication/skills.js', () => ({ installBrokerSkills: async () => {} })); -vi.mock('../src/communication/tools.js', () => ({ openCommunicationTools: async (invoke: typeof fake.invoke) => { - fake.invoke = invoke; - return { path: '/tmp/fake.sock', token: 'fake-token', helperPath: '/tmp/fake.mjs', close: async () => {} }; -} })); -const cleanups: Array<() => Promise> = []; -afterEach(async () => { for (const close of cleanups.splice(0).reverse()) await close(); vi.clearAllMocks(); }); -it('journals spawn, message, and release through the authored surface and worker', async () => { - const fixture = chainFixture(); cleanups.push(() => fixture.close()); - const client = await fixture.connect(); - const worker = await attachLocalAgent(client, fixture.data, undefined, undefined, 2, fixture.root); - cleanups.push(() => worker.close()); - const complete = (text: string, json: string) => { - const turn = /complete '([^']+)'/.exec(text)![1]!; - setTimeout(() => void fake.invoke!({ operation: 'complete', values: [turn, json] }), 1); - }; - fake.spawn.mockImplementation(async input => { - complete(input.task, '{"pr":42}'); - return { name: input.name, release: fake.release, waitForReady: async () => ({ reason: 'ready' }) }; - }); - fake.send.mockImplementation(async input => { - complete(input.text, '{"fixed":true}'); return { event_id: 'event', targets: [input.to] }; - }); - const definition = flow('broker-flow', async f => { - const agent = await f.agent('implement', { task: 'Open PR', transport: 'broker', cli: 'codex', keepAlive: true }); - expect(JSON.parse(agent.summary)).toEqual({ pr: 42 }); - const reply = await f.message(agent.agentName!, 'Fix', { timeout: '5s' }); - expect(JSON.parse(reply.summary)).toEqual({ fixed: true }); - await f.release(agent.agentName!); - expect(fake.release).toHaveBeenCalledTimes(1); - f.done('success'); - }); - const report = await executeAuthoredFlow(definition, client, undefined, { - flowPath: fixture.flowPath, localAgentStream: worker.stream, dataDir: fixture.data, workerCapacity: 2, - }); - expect(report.completionReason).toBe('success'); - const agents = report.journalSteps.filter(step => step.id.startsWith('agent-')); - expect(agents).toHaveLength(3); - for (const step of agents) { - const entries = (await client.journalRead(step.runId, 1)).entries; - expect(entries.some(entry => entry.entry_type === 'step.completed' && entry.payload.completionReason === 'success')).toBe(true); - } - await worker.close(); expect(fake.release).toHaveBeenCalledTimes(1); -}); - -it('runs a named gate on f.message against the broker result', async () => { - const fixture = chainFixture(); cleanups.push(() => fixture.close()); - const client = await fixture.connect(); - const worker = await attachLocalAgent(client, fixture.data, undefined, undefined, 2, fixture.root); - cleanups.push(() => worker.close()); - const complete = (text: string, json: string) => { - const turn = /complete '([^']+)'/.exec(text)![1]!; - setTimeout(() => void fake.invoke!({ operation: 'complete', values: [turn, json] }), 1); - }; - fake.spawn.mockImplementation(async input => { - complete(input.task, '{"pr":42}'); - return { name: input.name, release: fake.release, waitForReady: async () => ({ reason: 'ready' }) }; - }); - fake.send.mockImplementation(async input => { - complete(input.text, '{"fixed":true}'); return { event_id: 'event', targets: [input.to] }; - }); - const definition = flow('broker-message-gate', async f => { - const agent = await f.agent('implement', { task: 'Open PR', transport: 'broker', cli: 'codex', keepAlive: true }); - await f.message(agent.agentName!, 'Fix').gate({ type: 'artifact_exists', path: 'result.json' }); - f.done('success'); - }); - await expect(executeAuthoredFlow(definition, client, undefined, { - flowPath: fixture.flowPath, localAgentStream: worker.stream, dataDir: fixture.data, workerCapacity: 2, - })).rejects.toThrow(/agent-\d+\.gate/); - expect(fake.spawn).toHaveBeenCalledTimes(1); - expect(fake.send).toHaveBeenCalledTimes(1); -}); - -it('releases a retained session after a budgeted named-gate failure', async () => { - const fixture = chainFixture(); cleanups.push(() => fixture.close()); - const client = await fixture.connect(); - const worker = await attachLocalAgent(client, fixture.data, undefined, undefined, 2, fixture.root); - cleanups.push(() => worker.close()); - const complete = (text: string, json: string) => { - const turn = /complete '([^']+)'/.exec(text)![1]!; - setTimeout(() => void fake.invoke!({ operation: 'complete', values: [turn, json] }), 1); - }; - fake.spawn.mockImplementation(async input => { - complete(input.task, '{"pr":42}'); - return { name: input.name, release: fake.release, waitForReady: async () => ({ reason: 'ready' }) }; - }); - fake.send.mockImplementation(async input => { - complete(input.text, '{"fixed":true}'); return { event_id: 'event', targets: [input.to] }; - }); - const definition = flow('broker-release-after-failure', { budget: '$1/run' }, async f => { - const agent = await f.agent('implement', { task: 'Open PR', transport: 'broker', cli: 'codex', keepAlive: true }); - try { - await f.message(agent.agentName!, 'Fix').gate({ type: 'artifact_exists', path: 'result.json' }); - } finally { - await f.release(agent.agentName!); - } - f.done('success'); - }); - await expect(executeAuthoredFlow(definition, client, undefined, { - flowPath: fixture.flowPath, localAgentStream: worker.stream, dataDir: fixture.data, workerCapacity: 2, - })).rejects.toThrow(/agent-\d+\.gate/); - expect(fake.release).toHaveBeenCalled(); -}); - -it.each(['named', 'predicate'] as const)('refuses a retained spawn with a %s gate before allocating a PTY', async gate => { - const fixture = chainFixture(); cleanups.push(() => fixture.close()); - const client = await fixture.connect(); - const definition = flow('broker-gated', async f => { - const agent = f.agent('implement', { task: 'Open PR', transport: 'broker', cli: 'codex', keepAlive: true }); - if (gate === 'predicate') await agent.gate(() => false); - else await agent.gate({ type: 'artifact_exists', path: 'result.json' }); - f.done('success'); - }); - await expect(executeAuthoredFlow(definition, client, undefined, { - flowPath: fixture.flowPath, localAgentStream: 'fake-local-worker', dataDir: fixture.data, - })).rejects.toThrow('Retained broker agents do not support gates'); - expect(fake.spawn).not.toHaveBeenCalled(); -}); diff --git a/packages/sdk/tests/broker-preflight.test.ts b/packages/sdk/tests/broker-preflight.test.ts index c197d81a4..afbfe8cac 100644 --- a/packages/sdk/tests/broker-preflight.test.ts +++ b/packages/sdk/tests/broker-preflight.test.ts @@ -1,51 +1,19 @@ -import { afterEach, expect, it, vi } from 'vitest'; -import { checkAuthoredFlow } from '../src/cli/check.js'; +import { expect, it } from 'vitest'; import { budgetDiagnostics } from '../src/budget-preflight.js'; import { compileSpec } from '../src/compile.js'; -import { cliProbeKey, type CliProbeOutcome } from '../src/preflight.js'; import { BROKER_TYPE } from '../src/communication/broker-spec.js'; -const environment = vi.hoisted(() => vi.fn()); -vi.mock('../src/communication/broker-preflight.js', () => ({ checkBrokerEnvironment: environment })); -afterEach(() => vi.resetAllMocks()); -function spec(operation: 'spawn' | 'message' | 'release' = 'spawn') { +import { checkBrokerEnvironment } from '../src/communication/broker-preflight.js'; + +function spec() { return { version: '0.1.0', budget: '$1/run', steps: [{ id: 'agent', type: 'agent', cli: 'claude', - model: 'claude-sonnet-4-6', instruction: JSON.stringify({ type: BROKER_TYPE, operation, + model: 'claude-sonnet-4-6', instruction: JSON.stringify({ type: BROKER_TYPE, operation: 'spawn', owner: 'root', flow: 'test', step: 'implement', agentName: 'test-agent', task: 'Work', skills: [], channel: 'flow-root', keepAlive: false, timeoutMs: 1000 }) }] }; } -it('warns that even a priced broker model cannot be bounded by token or dollar ceilings', () => { - const warnings = budgetDiagnostics(compileSpec(spec())); - expect(warnings).toEqual([expect.objectContaining({ kind: 'budget_unmetered', stepId: 'agent', - message: expect.stringContaining('Token and dollar ceilings cannot bound its spend') })]); -}); -it.each([undefined, { tokens: 100 }])('warns about unmetered sessions even without a dollar budget', budget => { - expect(budgetDiagnostics(compileSpec({ ...spec(), budget }))).toEqual([ + +it('keeps legacy broker instructions unmetered while refusing their runtime', () => { + expect(budgetDiagnostics(compileSpec(spec() as any))).toEqual([ expect.objectContaining({ kind: 'budget_unmetered', stepId: 'agent' }), ]); -}); -it.each([false, true])('checks broker prerequisites before returning an admissible flow (available=%s)', available => { - if (!available) environment.mockImplementation(() => { throw new Error('missing local broker'); }); - const cache = new Map([[cliProbeKey({ stepId: 'agent', cli: 'claude', source: 'step', - model: 'claude-sonnet-4-6' }, true), { result: { exists: true, authenticated: true, modelAvailable: true, executable: '/usr/bin/true' } }]]); - const checked = checkAuthoredFlow(spec() as any, '/tmp/test.flow.ts', { - projectConfig: { directory: '/tmp', models: [], executors: [] }, probeCache: cache, - }); - expect(environment).toHaveBeenCalledTimes(1); - expect(checked.report.ok).toBe(available); - if (!available) { - expect(checked.flow).toBeUndefined(); - expect(checked.report.diagnostics).toContainEqual(expect.objectContaining({ severity: 'refusal', kind: 'probe_failed', message: 'missing local broker' })); - } - expect(checked.report.diagnostics).toContainEqual(expect.objectContaining({ kind: 'budget_unmetered' })); -}); -it.each(['message', 'release'] as const)('leaves broker %s steps to the worker instead of probing again', operation => { - environment.mockImplementation(() => { throw new Error('missing local broker'); }); - const cache = new Map([[cliProbeKey({ stepId: 'agent', cli: 'claude', source: 'step', - model: 'claude-sonnet-4-6' }, true), { result: { exists: true, authenticated: true, modelAvailable: true, executable: '/usr/bin/true' } }]]); - const checked = checkAuthoredFlow(spec(operation) as any, '/tmp/test.flow.ts', { - projectConfig: { directory: '/tmp', models: [], executors: [] }, probeCache: cache, - }); - expect(environment).not.toHaveBeenCalled(); - expect(checked.report.ok).toBe(true); - expect(checked.flow).toBeDefined(); + expect(() => checkBrokerEnvironment()).toThrow('transport: broker was removed'); }); diff --git a/packages/sdk/tests/broker-retirement.test.ts b/packages/sdk/tests/broker-retirement.test.ts new file mode 100644 index 000000000..bd015f238 --- /dev/null +++ b/packages/sdk/tests/broker-retirement.test.ts @@ -0,0 +1,64 @@ +import { chmodSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, expect, it, vi } from 'vitest'; +import { authoredBroker } from '../src/authored-broker.js'; +import { checkAuthoredFlow } from '../src/cli/check.js'; +import { checkBrokerEnvironment } from '../src/communication/broker-preflight.js'; +import { checkCommunicationEnvironment } from '../src/communication/preflight.js'; +import type { FlowSpec } from '../src/spec.js'; + +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); + +it('removes harness-driver from the published dependency contract', async () => { + const manifest = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8')); + expect(JSON.stringify({ dependencies: manifest.dependencies, peerDependencies: manifest.peerDependencies, + optionalDependencies: manifest.optionalDependencies })).not.toContain('@agent-relay/harness-driver'); + expect(manifest.peerDependencies['@agent-relay/sdk']).toBe('>=17.2.1 <19'); + expect(manifest.peerDependencies['@agent-relay/core-bridge']).toBe('>=18.1.0 <19'); + expect(manifest.peerDependenciesMeta['@agent-relay/core-bridge']).toEqual({ optional: true }); +}); + +it('fails legacy retained sessions before creating a journal child run', async () => { + const run = vi.fn(); + const broker = authoredBroker({ name: 'demo' }, 'root', 'local-worker', run); + await expect(broker.agent('step', 'worker', { transport: 'broker', task: 'work' })) + .rejects.toThrow('transport: broker was removed'); + await expect(broker.message('step', 'worker', 'continue')).rejects.toThrow('transport: broker was removed'); + expect(run).not.toHaveBeenCalled(); +}); + +it('has no health-check fallback that can restart or reconnect to a broker', () => { + expect(() => checkBrokerEnvironment('/workspace')).toThrow('transport: broker was removed'); +}); + +it('refuses a compiled broker spawn through the public check path', () => { + const root = mkdtempSync(join(tmpdir(), 'broker-retirement-')); + roots.push(root); + const cli = join(root, 'codex'); + writeFileSync(cli, '#!/bin/sh\nexit 0\n'); + chmodSync(cli, 0o755); + const instruction = JSON.stringify({ type: 'relayflows.broker.v1', operation: 'spawn', + owner: 'owner', flow: 'flow', step: 'step', agentName: 'agent', task: 'work', + channel: 'default', keepAlive: false, timeoutMs: 1000, skills: [] }); + const flow: FlowSpec = { version: '0.1.0', cli, steps: [ + { id: 'step', type: 'agent', instruction }, + ] }; + const checked = checkAuthoredFlow(flow, join(root, 'flow.yaml'), { + projectConfig: { directory: root, models: [], executors: [] }, + }); + expect(checked.report.ok).toBe(false); + expect(checked.report.diagnostics).toEqual(expect.arrayContaining([ + expect.objectContaining({ severity: 'refusal', kind: 'probe_failed', + message: expect.stringContaining('transport: broker was removed') }), + ])); + expect(checked.flow).toBeUndefined(); +}); + +it('fails communication admission until the native local caller contract exists', () => { + expect(() => checkCommunicationEnvironment({ version: '0.1.0', steps: [{ id: 'a', type: 'agent', cli: 'codex', + instruction: JSON.stringify({ type: 'relayflows.communication.v1', instruction: 'talk', incoming: [], + outgoing: ['b'], timeoutMs: 1000 }) }] })).toThrow('Native node-spawn communication'); +}); diff --git a/packages/sdk/tests/broker-worker.test.ts b/packages/sdk/tests/broker-worker.test.ts deleted file mode 100644 index 6b20dda97..000000000 --- a/packages/sdk/tests/broker-worker.test.ts +++ /dev/null @@ -1,226 +0,0 @@ -import { afterEach, beforeEach, expect, it, vi } from 'vitest'; -import type { JournalClient } from '../src/journal-client.js'; -import type { StepDispatchEvent } from '../src/protocol.js'; -import { BrokerSessions } from '../src/communication/broker-worker.js'; -import { BROKER_TYPE, type BrokerInstruction } from '../src/communication/broker-spec.js'; -const fake = vi.hoisted(() => ({ - event: undefined as undefined | ((event: { kind: string; name: string }) => void), - invoke: undefined as undefined | ((request: { operation: string; values: string[] }) => Promise), - release: vi.fn(async () => {}), shutdown: vi.fn(async () => {}), close: vi.fn(async () => {}), - ready: vi.fn(async () => ({ reason: 'ready' })), spawn: vi.fn(), send: vi.fn(), install: vi.fn(), -})); -vi.mock('../src/communication/relay.js', () => ({ connectLocalBroker: async () => ({ - spawnPty: fake.spawn, sendMessage: fake.send, shutdown: fake.shutdown, connectEvents: () => {}, onEvent: (callback: typeof fake.event) => { fake.event = callback; return () => {}; }, -}) })); -vi.mock('../src/communication/skills.js', () => ({ installBrokerSkills: fake.install })); -vi.mock('../src/communication/tools.js', () => ({ openCommunicationTools: async (invoke: typeof fake.invoke) => { - fake.invoke = invoke; - return { path: '/tmp/fake.sock', token: 'fake-token', helperPath: '/tmp/fake.mjs', close: fake.close }; -} })); -let sessions: BrokerSessions; -beforeEach(() => { - vi.clearAllMocks(); - sessions = new BrokerSessions(process.cwd()); - fake.ready.mockResolvedValue({ reason: 'ready' }); - const complete = (text: string) => { - const turn = /complete '([^']+)'/.exec(text)![1]!; - setTimeout(() => void fake.invoke!({ operation: 'complete', values: [turn, '{"ok":true}'] }), 1); - }; - fake.spawn.mockImplementation(async input => { - complete(input.task); - return { name: input.name, release: fake.release, waitForReady: fake.ready }; - }); - fake.send.mockImplementation(async input => { complete(input.text); return { event_id: 'event', targets: [input.to] }; }); -}); -afterEach(async () => sessions.close()); -const instruction: BrokerInstruction = { type: BROKER_TYPE, operation: 'spawn', owner: 'root', flow: 'demo', step: 'implement', - agentName: 'demo-implement-unique', task: 'Implement', skills: ['review-fix-signoff-loop'], channel: 'flow-root', keepAlive: true, timeoutMs: 2000 }; -function fixture() { - const client = { stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), stepComplete: vi.fn(async (..._args: unknown[]) => ({})) }; - let number = 0; - return { client, execute: (change: Partial = {}) => sessions.complete(client as unknown as JournalClient, { - run_id: `child-${++number}`, step_id: 'agent', attempt: 1, idempotency_key: `key-${number}`, pins: {}, - lease_id: 'lease', lease_deadline_ms: Date.now() + 30_000, - spec: { type: 'agent', cli: 'codex', model: 'gpt-5.4', instruction: '' }, - } as StepDispatchEvent, { ...instruction, ...change }, '/tmp/flows-broker-tests') }; -} -it('spawns with channel/team/metadata/skills, then reuses the retained session for a message', async () => { - const f = fixture(); await f.execute(); - expect(fake.install).toHaveBeenCalledWith(instruction.skills, 'codex', process.cwd(), expect.any(AbortSignal)); - expect(fake.spawn).toHaveBeenCalledWith(expect.objectContaining({ name: instruction.agentName, - skipRelayPrompt: false, channels: ['flow-root'], team: 'flow-root', cwd: process.cwd(), model: 'gpt-5.4', - harnessConfig: expect.objectContaining({ env: expect.objectContaining({ RELAYFLOW_RUN_ID: 'root', RELAYFLOW_FLOW_NAME: 'demo', RELAYFLOW_STEP_NAME: 'implement' }) }), - })); - expect(fake.release).not.toHaveBeenCalled(); - await f.execute({ operation: 'message', task: 'Fix review comments' }); - expect(fake.spawn).toHaveBeenCalledTimes(1); - expect(fake.send).toHaveBeenCalledWith(expect.objectContaining({ to: instruction.agentName, text: expect.stringContaining('Fix review comments') })); - expect(f.client.stepComplete).toHaveBeenLastCalledWith('child-2', 'agent', 1, 'key-2', 'success', expect.objectContaining({ - output: { completionReason: 'success', summary: '{"ok":true}', artifacts: [], agentName: instruction.agentName }, - })); - await sessions.close(); expect(fake.release).toHaveBeenCalledTimes(1); expect(fake.shutdown).toHaveBeenCalledTimes(1); -}); -it('passes a declared secret into the broker PTY and withholds other delivered secrets', async () => { - await sessions.close(); - sessions = new BrokerSessions(process.cwd(), { - OPENAI_API_KEY: 'codex-key', GH_TOKEN: 'repository-secret', OTHER_SECRET: 'granted', - RELAY_SECRET_ENV_NAMES: '["OPENAI_API_KEY","GH_TOKEN","OTHER_SECRET"]', RELAY_API_KEY: 'broker-key', - }); - const client = { supportsFeature: () => true, stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), - stepComplete: vi.fn(async () => ({})) }; - await sessions.complete(client as unknown as JournalClient, { - run_id: 'child-secret', step_id: 'agent', attempt: 1, idempotency_key: 'key-secret', pins: {}, - lease_id: 'lease', lease_deadline_ms: Date.now() + 30_000, - spec: { type: 'agent', cli: 'codex', instruction: '', secrets: ['OTHER_SECRET'] }, - } as StepDispatchEvent, instruction, '/tmp/flows-broker-tests'); - const env = fake.spawn.mock.calls[0]![0].harnessConfig.env as Record; - expect(env.OTHER_SECRET).toBe('granted'); - expect(env.OPENAI_API_KEY).toBe('codex-key'); - expect(env).not.toHaveProperty('GH_TOKEN'); - expect(env.RELAY_API_KEY).toBe(''); -}); -it('explicit release is idempotent and later messages fail closed', async () => { - const f = fixture(); await f.execute(); await f.execute({ operation: 'release' }); await f.execute({ operation: 'release' }); - expect(fake.release).toHaveBeenCalledTimes(1); - await f.execute({ operation: 'message' }); - expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('worker_error'); expect(fake.send).not.toHaveBeenCalled(); -}); -it('releases a non-retained agent before reporting success', async () => { - const f = fixture(); await f.execute({ keepAlive: false }); - expect(fake.release).toHaveBeenCalledTimes(1); - expect(fake.release.mock.invocationCallOrder[0]).toBeLessThan(f.client.stepComplete.mock.invocationCallOrder[0]!); -}); -it('rejects stale, duplicate, and non-JSON completions', async () => { - const f = fixture(); await f.execute(); - await expect(fake.invoke!({ operation: 'complete', values: ['child-1:agent:1', '{}'] })).rejects.toThrow('stale'); - fake.send.mockImplementationOnce(async input => { - await expect(fake.invoke!({ operation: 'complete', values: ['child-1:agent:1', '{}'] })).rejects.toThrow('stale'); - await expect(fake.invoke!({ operation: 'complete', values: ['child-2:agent:1', 'not json'] })).rejects.toThrow(); - await fake.invoke!({ operation: 'complete', values: ['child-2:agent:1', '[]'] }); - return { event_id: 'event', targets: [input.to] }; - }); - await f.execute({ operation: 'message' }); expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('success'); -}); -it('times out bounded messages and releases the session', async () => { - const f = fixture(); await f.execute(); fake.send.mockResolvedValue({ event_id: 'event', targets: [instruction.agentName] }); - await f.execute({ operation: 'message', timeoutMs: 10 }); - expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('timeout'); expect(fake.release).toHaveBeenCalledTimes(1); -}); -it('refuses missing delivery targets', async () => { - const f = fixture(); await f.execute(); fake.send.mockResolvedValue({ event_id: 'unsupported_operation', targets: [] }); - await f.execute({ operation: 'message' }); - expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('worker_error'); expect(fake.release).toHaveBeenCalledTimes(1); -}); -it('releases retained resources when the journal write fails', async () => { - const f = fixture(); f.client.stepComplete.mockRejectedValueOnce(new Error('journal unavailable')); - await expect(f.execute()).rejects.toThrow('journal unavailable'); - expect(fake.release).toHaveBeenCalledTimes(1); expect(fake.close).toHaveBeenCalledTimes(1); -}); -it('refuses invalid cwd and failed skills before spawn', async () => { - const f = fixture(); await f.execute({ cwd: '/this-directory-does-not-exist' }); expect(fake.spawn).not.toHaveBeenCalled(); - fake.install.mockRejectedValueOnce(new Error('unknown canonical skill')); - await f.execute(); expect(fake.spawn).not.toHaveBeenCalled(); expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('worker_error'); -}); -it('releases on readiness failure', async () => { - fake.spawn.mockImplementationOnce(async input => ({ name: input.name, release: fake.release, waitForReady: fake.ready })); - fake.ready.mockResolvedValue({ reason: 'timeout' }); - const f = fixture(); await f.execute(); expect(fake.release).toHaveBeenCalledTimes(1); - expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('worker_error'); -}); -it('times out a spawn that never returns and releases a PTY that arrives later', async () => { - let resolveSpawn: (value: unknown) => void = () => {}; - fake.spawn.mockImplementationOnce(() => new Promise(resolve => { resolveSpawn = resolve; })); - const f = fixture(); - await f.execute({ timeoutMs: 100 }); - expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('timeout'); - const lateRelease = vi.fn(async () => {}); - resolveSpawn({ name: 'late', release: lateRelease, waitForReady: fake.ready }); - await new Promise(resolve => setTimeout(resolve, 20)); - expect(lateRelease).toHaveBeenCalledWith('Flow broker session ended', { deleteIdentity: true }); - expect(fake.release).not.toHaveBeenCalled(); -}); -it('a late release failure stays inside the timed-out spawn', async () => { - let resolveSpawn: (value: unknown) => void = () => {}; - fake.spawn.mockImplementationOnce(() => new Promise(resolve => { resolveSpawn = resolve; })); - const rejections: unknown[] = []; - const onRejection = (error: unknown) => { rejections.push(error); }; - process.on('unhandledRejection', onRejection); - try { - const f = fixture(); - await f.execute({ timeoutMs: 100 }); - const lateRelease = vi.fn(async () => { throw new Error('late release failed'); }); - resolveSpawn({ name: 'late', release: lateRelease, waitForReady: fake.ready }); - await new Promise(resolve => setTimeout(resolve, 30)); - expect(lateRelease).toHaveBeenCalledTimes(1); - expect(rejections).toEqual([]); - expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('timeout'); - } finally { - process.off('unhandledRejection', onRejection); - } -}); -it('times out even when message submission never responds', async () => { - const f = fixture(); await f.execute(); fake.send.mockImplementationOnce(() => new Promise(() => {})); - await f.execute({ operation: 'message', timeoutMs: 10 }); - expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('timeout'); expect(fake.release).toHaveBeenCalledTimes(1); -}); -it('retains a failed release for retry instead of acknowledging it as removed', async () => { - const f = fixture(); await f.execute(); fake.release.mockRejectedValueOnce(new Error('broker unavailable')); - await f.execute({ operation: 'release' }); - expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('worker_error'); - expect(fake.close).not.toHaveBeenCalled(); - await f.execute({ operation: 'release' }); - expect(fake.release).toHaveBeenCalledTimes(2); expect(fake.close).toHaveBeenCalledTimes(1); -}); -it('a replacement worker refuses messages to lost sessions without spawning a replacement', async () => { - const f = fixture(); await f.execute({ operation: 'message' }); - expect(fake.spawn).not.toHaveBeenCalled(); expect(fake.send).not.toHaveBeenCalled(); - expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('worker_error'); -}); -it('releases on lease loss without writing a worker-error completion', async () => { - const { JournalRequestTimeoutError } = await import('../src/journal-client.js'); - const { isLeaseLost } = await import('../src/worker-lease.js'); - fake.spawn.mockImplementationOnce(async input => ({ name: input.name, release: fake.release, waitForReady: fake.ready })); - const client = { stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 100, lease_ttl_ms: 100 })), stepComplete: vi.fn() }; - client.stepHeartbeat.mockImplementationOnce(async () => ({ lease_deadline_ms: Date.now() + 100, lease_ttl_ms: 100 })); - client.stepHeartbeat.mockImplementation(async () => { throw new JournalRequestTimeoutError('step.heartbeat', 10); }); - const error = await sessions.complete(client as unknown as JournalClient, { - run_id: 'child', step_id: 'agent', attempt: 1, idempotency_key: 'key', pins: {}, - lease_id: 'lease', lease_deadline_ms: Date.now() + 100, lease_ttl_ms: 100, - spec: { type: 'agent', cli: 'codex', instruction: '' }, - } as StepDispatchEvent, instruction, '/tmp/flows-broker-tests').catch(error => error); - expect(isLeaseLost(error)).toBe(true); expect(client.stepComplete).not.toHaveBeenCalled(); - expect(fake.release).toHaveBeenCalledTimes(1); -}); - -it('accepts completion even when the PTY exits before the helper response flushes', async () => { - fake.spawn.mockImplementationOnce(async input => { - const turn = /complete '([^']+)'/.exec(input.task)![1]!; - await fake.invoke!({ operation: 'complete', values: [turn, '{}'] }); - fake.event!({ kind: 'agent_exit', name: input.name }); - return { name: input.name, release: fake.release, waitForReady: fake.ready }; - }); - const f = fixture(); await f.execute({ keepAlive: false }); - expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('success'); -}); - -it('retires an idle exited session before a later message can send', async () => { - const f = fixture(); await f.execute(); - fake.event!({ kind: 'agent_exit', name: instruction.agentName }); - await f.execute({ operation: 'message' }); - expect(fake.send).not.toHaveBeenCalled(); - expect(fake.release).toHaveBeenCalledTimes(1); - expect(fake.close).toHaveBeenCalledTimes(1); - expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('worker_error'); -}); - -it('retries failed idle-exit cleanup without sending to the dead PTY', async () => { - const f = fixture(); await f.execute(); - fake.release.mockRejectedValueOnce(new Error('broker unavailable')); - fake.event!({ kind: 'agent_exit', name: instruction.agentName }); - await new Promise(resolve => setTimeout(resolve, 0)); - await f.execute({ operation: 'message' }); - expect(fake.send).not.toHaveBeenCalled(); - expect(fake.release).toHaveBeenCalledTimes(2); - expect(fake.close).toHaveBeenCalledTimes(1); - expect(f.client.stepComplete.mock.calls.at(-1)?.[4]).toBe('worker_error'); -}); diff --git a/packages/sdk/tests/check-helper-communication.test.ts b/packages/sdk/tests/check-helper-communication.test.ts index 36f0ad069..bc33713dd 100644 --- a/packages/sdk/tests/check-helper-communication.test.ts +++ b/packages/sdk/tests/check-helper-communication.test.ts @@ -53,7 +53,7 @@ it.each([false, true])('keeps the communication refusal and budget warning wheth const report = communicatingYaml('slack', mounted)(true); expect(report.ok).toBe(false); expect(report.diagnostics).toEqual(expect.arrayContaining([ - expect.objectContaining({ severity: 'refusal', kind: 'probe_failed', message: expect.stringContaining('RELAY_API_KEY') }), + expect.objectContaining({ severity: 'refusal', kind: 'probe_failed', message: expect.stringContaining('Native node-spawn communication') }), expect.objectContaining({ severity: 'warning', kind: 'budget_unmetered' }), ])); expect(report.diagnostics.some(d => d.kind === 'helper_credential_unresolved')).toBe(!mounted); diff --git a/packages/sdk/tests/cli.test.ts b/packages/sdk/tests/cli.test.ts index cd1ad962c..e3ddfa1c3 100644 --- a/packages/sdk/tests/cli.test.ts +++ b/packages/sdk/tests/cli.test.ts @@ -9,6 +9,7 @@ import { parse as parseYaml, stringify as stringifyYaml } from 'yaml'; import { afterEach, describe, expect, it } from 'vitest'; import { runCli, type CheckReport, type CliIo } from '../src/cli.js'; import { checkFlow } from '../src/cli/check.js'; +import { toKernelSpec } from '../src/compile.js'; import { socketPathFor } from '../src/daemon-connection.js'; import { runFlow } from '../src/cli/run.js'; import { runAgentCli } from '../src/worker-cli.js'; @@ -460,6 +461,58 @@ steps: ]); }); + it('does not turn an adapter default into a relay placement override', () => { + const directory = temporaryProject('flows-relay-default-model-'); + const cli = executableFixture(directory, 'claude', `if [ "$1 $2" = "auth status" ]; then exit 0; fi +if [ "$1" = "-p" ]; then exit 0; fi +exit 7`); + writeFileSync(join(directory, 'flows.json'), JSON.stringify({ + models: ['claude-opus-5', 'declared-relay-model', 'named-relay-model'], + })); + const path = join(directory, 'relay.flow.yaml'); + writeFileSync(path, `version: '0.1.0' +agents: + reviewer: + cli: ${JSON.stringify(cli)} + model: named-relay-model +steps: + - id: defaulted + type: agent + cli: ${JSON.stringify(cli)} + transport: relay + instruction: Review. + - id: declared + type: agent + cli: ${JSON.stringify(cli)} + model: declared-relay-model + transport: relay + instruction: Review. + - id: named + type: agent + agent: reviewer + transport: relay + instruction: Review. +`); + + const checked = checkFlow(path); + expect(checked.report.ok, checked.report.diagnostics).toBe(true); + expect(checked.report.resolutions).toEqual([ + { stepId: 'defaulted', cli, source: 'step', model: 'claude-opus-5', modelSource: 'adapter' }, + { stepId: 'declared', cli, source: 'step', model: 'declared-relay-model', modelSource: 'step' }, + { stepId: 'named', cli, source: 'named', model: 'named-relay-model', modelSource: 'named' }, + ]); + expect(checked.flow?.steps).toEqual([ + expect.objectContaining({ id: 'defaulted', transport: 'relay', cli }), + expect.objectContaining({ id: 'declared', transport: 'relay', cli, model: 'declared-relay-model' }), + expect.objectContaining({ id: 'named', transport: 'relay', cli, agent: 'reviewer' }), + ]); + expect(checked.flow?.steps[0]).not.toHaveProperty('model'); + expect(checked.flow?.steps[2]).not.toHaveProperty('model'); + expect(toKernelSpec(checked.flow!).steps[2]).toMatchObject({ + id: 'named', transport: 'relay', cli, model: 'named-relay-model', + }); + }); + it('keeps the project config path with the CLI clause when the step declares the model', async () => { const directory = temporaryProject('flows-project-model-'); const flowDirectory = join(directory, 'nested'); diff --git a/packages/sdk/tests/communication-environment-preflight.test.ts b/packages/sdk/tests/communication-environment-preflight.test.ts index d59a956e8..54fbaea7b 100644 --- a/packages/sdk/tests/communication-environment-preflight.test.ts +++ b/packages/sdk/tests/communication-environment-preflight.test.ts @@ -14,19 +14,10 @@ it('ordinary flows never inspect Relay credentials, packages, or broker', () => checkCommunicationEnvironment({ version: '0.1.0', steps: [{ id: 'plain', type: 'agent', instruction: 'hello' }] }); expect(mocks.resolve).not.toHaveBeenCalled(); expect(mocks.spawn).not.toHaveBeenCalled(); }); -it('refuses missing credentials before package or daemon access', () => { - vi.stubEnv('RELAY_API_KEY', undefined); - expect(() => checkCommunicationEnvironment(spec)).toThrow('requires RELAY_API_KEY'); +it.each([undefined, '', 'rk_live_test'])('always refuses retired communication without probing the broker (credential=%s)', credential => { + vi.stubEnv('RELAY_API_KEY', credential); + expect(() => checkCommunicationEnvironment(spec)).toThrow('broker/PTy was removed'); + expect(mocks.resolve).not.toHaveBeenCalled(); expect(mocks.spawn).not.toHaveBeenCalled(); -}); -it('refuses missing optional packages', () => { - vi.stubEnv('RELAY_API_KEY', 'rk_live_test'); mocks.resolve.mockImplementation(() => { throw Error('MODULE_NOT_FOUND'); }); - expect(() => checkCommunicationEnvironment(spec)).toThrow('optional @agent-relay'); -}); -it.each(['missing', 'not-executable', 'ready'])('checks the broker executable: %s', state => { - vi.stubEnv('RELAY_API_KEY', 'rk_live_test'); mocks.resolve.mockReturnValue('/installed/broker-path.js'); - mocks.spawn.mockReturnValue({ status: state === 'missing' ? 1 : 0, stdout: '/installed/broker' }); - if (state === 'not-executable') mocks.access.mockImplementation(() => { throw Error('EACCES'); }); - if (state === 'ready') expect(() => checkCommunicationEnvironment(spec)).not.toThrow(); - else expect(() => checkCommunicationEnvironment(spec)).toThrow('executable Relay broker'); + expect(mocks.access).not.toHaveBeenCalled(); }); diff --git a/packages/sdk/tests/communication-feature-secret-scoping.test.ts b/packages/sdk/tests/communication-feature-secret-scoping.test.ts deleted file mode 100644 index a0ac18688..000000000 --- a/packages/sdk/tests/communication-feature-secret-scoping.test.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { EventEmitter } from 'node:events'; -import { mkdtempSync, rmSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { expect, it, vi } from 'vitest'; -import { completeCommunicationDispatch } from '../src/communication/worker.js'; -import type { JournalClient } from '../src/journal-client.js'; -import type { StepDispatchEvent } from '../src/protocol.js'; - -const state = vi.hoisted(() => ({ environments: [] as Array> })); -vi.mock('../src/communication/relay.js', () => ({ acquireRelayRuntime: async () => ({ - prefix: 'flow-test', - broker: { - onEvent: () => () => {}, - spawnPty: async (input: { harnessConfig: { env: Record } }) => { - state.environments.push(input.harnessConfig.env); - return { waitForReady: async () => ({ reason: 'failed' }), release: async () => {} }; - }, - }, - close: async () => {}, -}) })); - -it('communication worker uses the negotiated capability to scope its broker child', async () => { - const root = mkdtempSync(join(tmpdir(), 'flows-communication-capability-')); - state.environments.length = 0; - try { - for (const scoped of [true, false]) { - const client = new EventEmitter() as EventEmitter & Record; - client.supportsFeature = (name: string) => name === 'step_secret_scoping' && scoped; - client.stepHeartbeat = async () => ({ lease_deadline_ms: Date.now() + 30_000 }); - client.stepComplete = async () => ({}); - const dispatch = { - run_id: 'run', step_id: 'agent', step_type: 'agent', attempt: 1, - idempotency_key: `key-${scoped}`, lease_id: 'lease', lease_deadline_ms: Date.now() + 30_000, - pins: { workspace: [], streams: [] }, - spec: { cli: 'claude', instruction: 'task' }, - } satisfies StepDispatchEvent; - await completeCommunicationDispatch(client as unknown as JournalClient, dispatch, - { type: 'relayflows.communication.v1', instruction: 'task', incoming: [], outgoing: [], timeoutMs: 1000 }, - root, root, { - ANTHROPIC_API_KEY: 'delivered-provider-key', GH_TOKEN: 'user-repository-secret', - RELAY_SECRET_ENV_NAMES: '["ANTHROPIC_API_KEY","GH_TOKEN"]', - RELAY_API_KEY: 'broker-key', RELAYFLOW_V2_SIGNAL_READ_TOKEN: 'private-signal-token', - }); - const env = state.environments.at(-1)!; - expect(env.ANTHROPIC_API_KEY).toBe('delivered-provider-key'); - expect(env).not.toHaveProperty('GH_TOKEN'); - expect(env.RELAY_API_KEY).toBe(''); - expect(env).not.toHaveProperty('RELAYFLOW_V2_SIGNAL_READ_TOKEN'); - } - expect(state.environments).toHaveLength(2); - const client = new EventEmitter() as EventEmitter & Record; - client.supportsFeature = (name: string) => name === 'step_secret_scoping'; - client.stepHeartbeat = async () => ({ lease_deadline_ms: Date.now() + 30_000 }); - client.stepComplete = async () => ({}); - const dispatch = { - run_id: 'run', step_id: 'agent', step_type: 'agent', attempt: 1, - idempotency_key: 'key-granted', lease_id: 'lease', lease_deadline_ms: Date.now() + 30_000, - pins: { workspace: [], streams: [] }, - spec: { cli: 'claude', instruction: 'task', secrets: ['GH_TOKEN'] }, - } satisfies StepDispatchEvent; - await completeCommunicationDispatch(client as unknown as JournalClient, dispatch, - { type: 'relayflows.communication.v1', instruction: 'task', incoming: [], outgoing: [], timeoutMs: 1000 }, - root, root, { - ANTHROPIC_API_KEY: 'delivered-provider-key', GH_TOKEN: 'user-repository-secret', - RELAY_SECRET_ENV_NAMES: '["ANTHROPIC_API_KEY","GH_TOKEN"]', - RELAY_API_KEY: 'broker-key', - }); - expect(state.environments.at(-1)!.GH_TOKEN).toBe('user-repository-secret'); - expect(state.environments.at(-1)!.ANTHROPIC_API_KEY).toBe('delivered-provider-key'); - } finally { rmSync(root, { recursive: true, force: true }); } -}, 10_000); diff --git a/packages/sdk/tests/communication-mixed-resume.test.ts b/packages/sdk/tests/communication-mixed-resume.test.ts deleted file mode 100644 index 8133bd5c8..000000000 --- a/packages/sdk/tests/communication-mixed-resume.test.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; -import { join } from 'node:path'; -import { tmpdir } from 'node:os'; -import { expect, it, vi } from 'vitest'; -import { compileSpec, toKernelSpec } from '../src/compile.js'; -import { JournalClient } from '../src/journal-client.js'; -import { ensureDaemon } from '../src/daemon-lifecycle.js'; -import { resumeFlow, socketFor } from '../src/cli/run.js'; -import type { StepDispatchEvent } from '../src/protocol.js'; - -const state = vi.hoisted(() => ({ started: new Set(), ready: Promise.resolve(), release: () => {} })); -vi.mock('../src/communication/preflight.js', () => ({ checkCommunicationEnvironment: () => {}, CommunicationEnvironmentError: class extends Error {} })); -vi.mock('../src/communication/relay.js', () => ({ loadRelayModules: async () => ({}) })); -vi.mock('../src/worker-cli.js', () => ({ runAgentCli: async () => { - // Ordinary work cannot finish until BOTH conversation peers have capacity. - await state.ready; - return { exit_code: 0, stdout_tail: 'ordinary completed', stderr_tail: '' }; -} })); -vi.mock('../src/communication/worker.js', () => ({ requireCommunicationCli: () => {}, - completeCommunicationDispatch: async (client: JournalClient, dispatch: StepDispatchEvent) => { - state.started.add(dispatch.step_id); - if (state.started.size === 2) state.release(); - await state.ready; - await client.stepComplete(dispatch.run_id, dispatch.step_id, dispatch.attempt, dispatch.idempotency_key, - 'success', { output: { summary: 'peer started concurrently' }, started_pins: dispatch.pins, end_pins: dispatch.pins }); - }, -})); - -it('resumes mixed ordinary and linked agents through the real daemon without stealing peer capacity', async () => { - const root = mkdtempSync(join(tmpdir(), 'communication-resume-')); - const dataDir = join(root, 'data'); - state.started.clear(); - let reject!: (error: Error) => void; - state.ready = new Promise((yes, no) => { state.release = yes; reject = no; }); - void state.ready.catch(() => {}); - const timeout = setTimeout(() => reject(new Error('ordinary work occupied a conversation worker')), 5000); - const client = new JournalClient(socketFor(dataDir)); - try { - expect((await ensureDaemon(dataDir)).kind).toBe('attached'); - await client.connect(); await client.hello('mixed-resume-test'); - const spec = toKernelSpec(compileSpec({ version: '0.1.0', cli: 'test-cli', - communication: { links: [{ from: 'a', to: 'b' }, { from: 'b', to: 'a' }] }, - steps: ['ordinary1', 'ordinary2', 'a', 'b'].map(id => ({ id, type: 'agent', cli: 'test-cli', instruction: id })), - })); - const parked = await client.runStart(spec); - expect(parked.status).toBe('parked'); - const resumed = await resumeFlow(parked.run_id, dataDir, { localAgent: true, signal: AbortSignal.timeout(8000) }); - expect(resumed.report, JSON.stringify(resumed.report)).toMatchObject({ status: 'completed', completionReason: 'success', completedSteps: 4 }); - expect(state.started).toEqual(new Set(['a', 'b'])); - const entries = (await client.journalRead(parked.run_id, 1)).entries; - expect(entries.filter(entry => entry.entry_type === 'step.completed')).toHaveLength(4); - } finally { - clearTimeout(timeout); state.release(); client.close(); - try { process.kill(JSON.parse(readFileSync(join(dataDir, 'connection.json'), 'utf8')).pid, 'SIGTERM'); } - finally { rmSync(root, { recursive: true, force: true }); } - } -}, 15_000); diff --git a/packages/sdk/tests/communication-refusal.test.ts b/packages/sdk/tests/communication-refusal.test.ts index 9f328b259..f4a4ffde7 100644 --- a/packages/sdk/tests/communication-refusal.test.ts +++ b/packages/sdk/tests/communication-refusal.test.ts @@ -5,7 +5,7 @@ import { expect, it, vi } from 'vitest'; import { checkFlow } from '../src/cli/check.js'; import { runFlow } from '../src/cli/run.js'; -it('check and run refuse missing Relay prerequisites before creating a daemon or journal', async () => { +it('check and run refuse retired broker communication before creating a daemon or journal', async () => { const root = mkdtempSync(join(tmpdir(), 'communication-refusal-')); vi.stubEnv('RELAY_API_KEY', undefined); try { @@ -19,11 +19,11 @@ it('check and run refuse missing Relay prerequisites before creating a daemon or const checked = checkFlow(path); expect(checked.report.ok).toBe(false); expect(checked.flow).toBeUndefined(); - expect(checked.report.diagnostics).toContainEqual(expect.objectContaining({ severity: 'refusal', kind: 'probe_failed', message: expect.stringContaining('RELAY_API_KEY') })); + expect(checked.report.diagnostics).toContainEqual(expect.objectContaining({ severity: 'refusal', kind: 'probe_failed', message: expect.stringContaining('broker/PTy was removed') })); const dataDir = join(root, 'data'); const run = await runFlow(path, dataDir, { localAgent: true, daemon: { spawn: false } }); expect(run.exitCode).toBe(2); - expect(run.report.diagnostics).toContainEqual(expect.objectContaining({ kind: 'probe_failed', message: expect.stringContaining('RELAY_API_KEY') })); + expect(run.report.diagnostics).toContainEqual(expect.objectContaining({ kind: 'probe_failed', message: expect.stringContaining('broker/PTy was removed') })); expect(run.report.diagnostics.some(d => d.kind === 'protocol_error' || d.kind === 'daemon_unreachable')).toBe(false); expect(existsSync(dataDir)).toBe(false); } finally { vi.unstubAllEnvs(); rmSync(root, { recursive: true, force: true }); } diff --git a/packages/sdk/tests/communication-review.test.ts b/packages/sdk/tests/communication-review.test.ts index 798babe30..411c88240 100644 --- a/packages/sdk/tests/communication-review.test.ts +++ b/packages/sdk/tests/communication-review.test.ts @@ -70,7 +70,7 @@ it('communication agents drop delivered secrets by default on a scoped runtime', expect(agentEnvironment('claude', source, undefined, false).ANTHROPIC_API_KEY).toBe('delivered-provider-key'); expect(() => agentEnvironment('claude', { ...source, RELAY_SECRET_ENV_NAMES: JSON.stringify(['RELAY_API_KEY']), - }, ['RELAY_API_KEY'])).toThrow('Relay broker credentials'); + }, ['RELAY_API_KEY'])).toThrow('Agent Relay control credentials'); }); it('legacy communication agents still strip delivered repository names', () => { diff --git a/packages/sdk/tests/communication-worker.test.ts b/packages/sdk/tests/communication-worker.test.ts deleted file mode 100644 index 8d69f5556..000000000 --- a/packages/sdk/tests/communication-worker.test.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { it, expect, vi, beforeEach } from 'vitest'; -import { readlinkSync } from 'node:fs'; -import type { JournalClient } from '../src/journal-client.js'; -import type { StepDispatchEvent } from '../src/protocol.js'; -import { completeCommunicationDispatch } from '../src/communication/worker.js'; -const mocks = vi.hoisted(() => ({ - invoke: undefined as undefined | ((request: { operation: string; values: string[] }) => Promise), - release: vi.fn(async () => {}), close: vi.fn(async () => {}), toolClose: vi.fn(async () => {}), - ready: vi.fn(async () => ({ reason: 'ready' })), spawn: vi.fn(), -})); -vi.mock('../src/communication/relay.js', () => ({ acquireRelayRuntime: async () => ({ - prefix: 'managed', channel: 'wf-run', close: mocks.close, - broker: { onEvent: () => () => {}, spawnPty: mocks.spawn }, messaging: {}, -}) })); -vi.mock('../src/communication/tools.js', () => ({ openCommunicationTools: async (invoke: typeof mocks.invoke) => { - mocks.invoke = invoke; - return { path: '/tmp/test.sock', token: 'test-session-token', helperPath: '/tmp/tool.mjs', close: mocks.toolClose }; -} })); -beforeEach(() => { - vi.clearAllMocks(); - mocks.ready.mockResolvedValue({ reason: 'ready' }); - mocks.spawn.mockImplementation(async () => { - setTimeout(() => void mocks.invoke!({ operation: 'complete', values: ['done'] }), 5); - return { name: 'managed-agent', generation: 'generation', release: mocks.release, waitForReady: mocks.ready }; - }); -}); -function fixture(cli = 'claude', environment?: NodeJS.ProcessEnv, cliIdentity?: string) { - const client = { stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30000 })), - stepComplete: vi.fn(async () => ({})), channelReceive: vi.fn(async () => null) }; - const dispatch = { run_id: 'run', step_id: 'agent', attempt: 1, idempotency_key: 'key', pins: {}, - lease_id: 'lease', lease_deadline_ms: Date.now() + 30000, - spec: { type: 'agent', cli, ...(cliIdentity === undefined ? {} : { cli_identity: cliIdentity }), instruction: '' } } as StepDispatchEvent; - return { client, execute: () => completeCommunicationDispatch(client as unknown as JournalClient, dispatch, - { type: 'relayflows.communication.v1', instruction: 'test', incoming: ['peer'], outgoing: [], timeoutMs: 1000 }, - '/tmp/data', undefined, environment) }; -} -it('releases the exact owned identity and runtime before successful completion; never fabricates measured usage', async () => { - const f = fixture(); await f.execute(); - expect(mocks.release).toHaveBeenCalledWith('Flow communication attempt ended', { deleteIdentity: true }); - expect(mocks.close).toHaveBeenCalledTimes(1); - expect(mocks.toolClose).toHaveBeenCalledTimes(1); - expect(f.client.stepComplete).toHaveBeenCalledWith('run', 'agent', 1, 'key', 'success', expect.objectContaining({ - usage: { tokens_in: 0, tokens_out: 0, dollars_unmetered: true }, output: { summary: 'done' }, - })); -}); -it('journals readiness failure and still releases all acquired resources', async () => { - mocks.ready.mockResolvedValue({ reason: 'timeout' }); - const f = fixture(); await f.execute(); - expect(f.client.stepComplete).toHaveBeenCalledWith('run', 'agent', 1, 'key', 'worker_error', expect.objectContaining({ output: { error: 'Communication agent did not become ready: timeout' } })); - expect(mocks.release).toHaveBeenCalledTimes(1); - expect(mocks.close).toHaveBeenCalledTimes(1); -}); - -it.each(['claude', 'codex', 'gemini', 'cursor-agent', 'droid', 'opencode', 'aider', 'goose', 'grok', 'devin', 'pi', 'deepagents', '/opt/custom/tool'])('delegates %s launch and injection to Relay without Claude-only flags', async cli => { - const f = fixture(cli); await f.execute(); - expect(mocks.spawn).toHaveBeenCalledWith(expect.objectContaining({ - cli: cli.split('/').at(-1), - harnessConfig: expect.objectContaining({ command: `'${cli}'`, args: [], runtime: 'pty', - env: expect.objectContaining({ RELAYFLOW_COMMUNICATION_SOCKET: '/tmp/test.sock', RELAYFLOW_COMMUNICATION_TOKEN: 'test-session-token' }), - delivery: { mode: 'pty-injection', format: 'relay-block' } }), - })); - expect(f.client.stepComplete).toHaveBeenCalledWith('run', 'agent', 1, 'key', 'success', expect.anything()); -}); - -it('quotes an executable path as one command without losing spaces or apostrophes', async () => { - const f = fixture("/opt/agent tools/tool's cli"); await f.execute(); - expect(mocks.spawn.mock.calls[0]![0].harnessConfig.command).toBe("'/opt/agent tools/tool'\"'\"'s cli'"); -}); - -it('launches communication agents with the isolated provider credential', async () => { - const f = fixture('codex', { ...process.env, OPENAI_API_KEY: 'house-key' }); - await f.execute(); - expect(mocks.spawn.mock.calls[0]![0].harnessConfig.env).toMatchObject({ OPENAI_API_KEY: 'house-key' }); -}); - -it('uses the proved identity for a canonical generic communication executable', async () => { - let linkedTarget: string | undefined; - mocks.spawn.mockImplementationOnce(async input => { - const quotedCommand = (input as { harnessConfig: { command: string } }).harnessConfig.command; - linkedTarget = readlinkSync(quotedCommand.slice(1, -1)); - setTimeout(() => void mocks.invoke!({ operation: 'complete', values: ['done'] }), 5); - return { name: 'managed-agent', generation: 'generation', release: mocks.release, waitForReady: mocks.ready }; - }); - const f = fixture('/store/provider-cli.js', { ...process.env, OPENAI_API_KEY: 'house-key' }, 'codex'); - await f.execute(); - expect(linkedTarget).toBe('/store/provider-cli.js'); - expect(mocks.spawn).toHaveBeenCalledWith(expect.objectContaining({ - cli: 'codex', - harnessConfig: expect.objectContaining({ - command: expect.stringMatching(/^'\/tmp\/data\/communication\/cli-[^/]+\/codex'$/u), - env: expect.objectContaining({ OPENAI_API_KEY: 'house-key' }), - }), - })); -}); -it('leaves a heartbeat-lost attempt to the kernel instead of completing it as worker_error', async () => { - const { JournalRequestTimeoutError } = await import('../src/journal-client.js'); - const { isLeaseLost } = await import('../src/worker-lease.js'); - mocks.spawn.mockImplementation(async () => ({ name: 'managed-agent', generation: 'generation', release: mocks.release, waitForReady: mocks.ready })); - const f = fixture(); - f.client.stepHeartbeat.mockImplementation(async () => { throw new JournalRequestTimeoutError('step.heartbeat', 10); }); - (f as unknown as { client: { stepHeartbeat: unknown } }).client.stepHeartbeat = f.client.stepHeartbeat; - const dispatchDeadline = Date.now() + 200; - const error = await completeCommunicationDispatch(f.client as unknown as JournalClient, { - run_id: 'run', step_id: 'agent', attempt: 1, idempotency_key: 'key', pins: {}, lease_id: 'lease', - lease_deadline_ms: dispatchDeadline, spec: { type: 'agent', cli: 'claude', instruction: '' }, - } as StepDispatchEvent, { type: 'relayflows.communication.v1', instruction: 'test', incoming: ['peer'], outgoing: [], timeoutMs: 60_000 }, - '/tmp/data').catch(caught => caught); - expect(isLeaseLost(error)).toBe(true); - expect(f.client.stepComplete).not.toHaveBeenCalled(); -}); -it('leaves a retried attempt whose history read was interrupted to the kernel', async () => { - const { JournalRequestTimeoutError } = await import('../src/journal-client.js'); - const { isReadInterruptionError } = await import('../src/journal-read-policy.js'); - const f = fixture(); - (f.client as unknown as { journalRead: () => Promise }).journalRead = async () => { - throw new JournalRequestTimeoutError('journal.read', 10, 3, 300_000, 300_000); - }; - const error = await completeCommunicationDispatch(f.client as unknown as JournalClient, { - run_id: 'run', step_id: 'agent', attempt: 2, idempotency_key: 'key', pins: {}, lease_id: 'lease', - lease_deadline_ms: Date.now() + 30_000, spec: { type: 'agent', cli: 'claude', instruction: '' }, - } as StepDispatchEvent, { type: 'relayflows.communication.v1', instruction: 'test', incoming: ['peer'], outgoing: [], timeoutMs: 1000 }, - '/tmp/data').catch(caught => caught); - expect(isReadInterruptionError(error)).toBe(true); - expect(f.client.stepComplete).not.toHaveBeenCalled(); -}); -it('scopes the attempt history reads to its lease and releases them after', async () => { - const f = fixture(); - const release = vi.fn(); - const scopeReads = vi.fn((_signal: AbortSignal) => release); - (f.client as unknown as { scopeReads: typeof scopeReads }).scopeReads = scopeReads; - await f.execute(); - expect(scopeReads).toHaveBeenCalledTimes(1); - expect(scopeReads.mock.calls[0]![0]).toBeInstanceOf(AbortSignal); - expect(release).toHaveBeenCalledTimes(1); -}); diff --git a/packages/sdk/tests/input-binding.test.ts b/packages/sdk/tests/input-binding.test.ts index e0a8f20d5..819699c04 100644 --- a/packages/sdk/tests/input-binding.test.ts +++ b/packages/sdk/tests/input-binding.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest'; import { compileSpec, compileYaml, kernelToAuthoring, toKernelSpec } from '../src/compile.js'; import type { FlowSpec } from '../src/spec.js'; import { preflight } from '../src/preflight.js'; -import { workerInstruction } from '../src/worker-input.js'; +import { workerInstruction, workerInstructionIdentity } from '../src/worker-input.js'; import type { StepDispatchEvent } from '../src/protocol.js'; const schema = { @@ -85,4 +85,17 @@ steps: expect(prompt).toBe(`Task\n\nMemory context (journaled):\n"memory"\n\ninput:\n${JSON.stringify(input)}`); expect(() => workerInstruction('Task', { spec: { input: consumer.input } } as StepDispatchEvent)).toThrow('did not resolve'); }); + + it('keeps memory and input in task identity while excluding per-attempt recovery evidence', () => { + const dispatch = { + spec: { input: consumer.input }, + input: { message: 'stable input' }, + memory: { pack: 'stable memory' }, + recovery: { previous_completion_reason: 'crashed', trajectory_tail: { attempt: 1 } }, + } as unknown as StepDispatchEvent; + expect(workerInstruction('Task', dispatch)).toContain('Previous attempt (journaled)'); + expect(workerInstructionIdentity('Task', dispatch)).toBe( + `Task\n\nMemory context (journaled):\n"stable memory"\n\ninput:\n${JSON.stringify(dispatch.input)}`, + ); + }); }); diff --git a/packages/sdk/tests/live-kernel.test.ts b/packages/sdk/tests/live-kernel.test.ts index 75f2b5768..98a644f61 100644 --- a/packages/sdk/tests/live-kernel.test.ts +++ b/packages/sdk/tests/live-kernel.test.ts @@ -1159,7 +1159,6 @@ steps: // exported. That is exactly the ambient-state failure this field was // added to remove, so it has to be enforced by the spawn, not by luck. // Without this test nothing else would catch it. - process.env.RELAYFLOW_MODEL = 'leaked-parent-model'; const dataDir = temporaryDirectory('flows-live-model-unset-'); await startDaemon(dataDir); const cli = join(TESTDATA, 'preflight', 'echo-model-cli'); @@ -1171,7 +1170,10 @@ steps: }); await worker.attach(); - const started = await client.runStart(toKernelSpec(compileYaml(` + const inheritedModel = process.env.RELAYFLOW_MODEL; + process.env.RELAYFLOW_MODEL = 'leaked-parent-model'; + try { + const started = await client.runStart(toKernelSpec(compileYaml(` version: '0.1.0' steps: - id: probe @@ -1180,22 +1182,23 @@ steps: instruction: Report the model env var. `))); - expect(await waitForStep(client, started.run_id, 'probe', 'done')).toMatchObject({ - type: 'agent', - state: 'done', - }); - const entries = (await client.journalRead(started.run_id)).entries; - const completed = entries.find( - (entry) => (entry as { entry_type: string; step_id?: string }).entry_type === 'step.completed' - && (entry as { step_id?: string }).step_id === 'probe', - ) as { payload: { output: { story_title: string } } } | undefined; - expect(completed).toBeDefined(); - // UNSET, not EMPTY and not the leaked parent value. - expect(completed!.payload.output.story_title).toBe('model:UNSET'); - - delete process.env.RELAYFLOW_MODEL; - - await worker.close(); + expect(await waitForStep(client, started.run_id, 'probe', 'done')).toMatchObject({ + type: 'agent', + state: 'done', + }); + const entries = (await client.journalRead(started.run_id)).entries; + const completed = entries.find( + (entry) => (entry as { entry_type: string; step_id?: string }).entry_type === 'step.completed' + && (entry as { step_id?: string }).step_id === 'probe', + ) as { payload: { output: { story_title: string } } } | undefined; + expect(completed).toBeDefined(); + // UNSET, not EMPTY and not the leaked parent value. + expect(completed!.payload.output.story_title).toBe('model:UNSET'); + } finally { + if (inheritedModel === undefined) delete process.env.RELAYFLOW_MODEL; + else process.env.RELAYFLOW_MODEL = inheritedModel; + await worker.close(); + } }, 30_000); it('hn-monitor analyze-story reaches done through the real Claude analyzer CLI', async (context) => { @@ -1364,7 +1367,7 @@ steps: const unreachable = invokeCli([ 'run', '--no-spawn', '--data-dir', absentDir, join(TESTDATA, 'hello-deterministic.flow.yaml'), ]); - expect(unreachable.status).toBe(2); + expect(unreachable.status, unreachable.stderr).toBe(2); expect(unreachable.stderr).toContain('REFUSED [daemon_unreachable]'); expect(unreachable.stderr).toContain(absentSocket); expect(unreachable.stderr).toContain('relayflowd --data-dir'); diff --git a/packages/sdk/tests/node-spawn-core-bridge-public.test.ts b/packages/sdk/tests/node-spawn-core-bridge-public.test.ts new file mode 100644 index 000000000..a5f729b08 --- /dev/null +++ b/packages/sdk/tests/node-spawn-core-bridge-public.test.ts @@ -0,0 +1,10 @@ +import * as publicCoreBridge181 from '@agent-relay/core-bridge'; +import { describe, expect, it } from 'vitest'; + +import { assertCoreBridgeApi } from '../src/node-spawn-local.js'; + +describe('public core-bridge release contract', () => { + it('accepts the exact installed 18.1.0 schema-v2 lifecycle surface', () => { + expect(() => assertCoreBridgeApi(publicCoreBridge181)).not.toThrow(); + }); +}); diff --git a/packages/sdk/tests/node-spawn-core-bridge.test.ts b/packages/sdk/tests/node-spawn-core-bridge.test.ts new file mode 100644 index 000000000..8912ead2b --- /dev/null +++ b/packages/sdk/tests/node-spawn-core-bridge.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { assertCoreBridgeApi, loadCoreBridge } from '../src/node-spawn-local.js'; + +// 18.0.0 implements only these six CoreBridgeApi methods. Undefined exports let +// Vitest expose the same missing lifecycle surface through a virtual module. +const publicCoreBridge180Shape = vi.hoisted(() => ({ + connectCore: vi.fn(), + authenticate: vi.fn(), + spawnTargets: vi.fn(), + spawnInvoke: vi.fn(), + spawnStatus: vi.fn(), + spawnRelease: vi.fn(), + readCallerDescriptor: undefined, + requireServiceCallerMethods: undefined, + assertServiceCallerIdentity: undefined, + serviceCallerIdentity: undefined, + serviceCallerInboxOpen: undefined, + serviceCallerInboxNext: undefined, + serviceCallerInboxAck: undefined, + serviceCallerInboxFail: undefined, + serviceCallerInboxClose: undefined, + serviceCallerDisconnect: undefined, +})); + +vi.mock('@agent-relay/core-bridge', () => publicCoreBridge180Shape); + +const schemaV2CoreBridge181Shape = { + ...publicCoreBridge180Shape, + readCallerDescriptor: vi.fn(), + requireServiceCallerMethods: vi.fn(), + assertServiceCallerIdentity: vi.fn(), + serviceCallerIdentity: vi.fn(), + serviceCallerInboxOpen: vi.fn(), + serviceCallerInboxNext: vi.fn(), + serviceCallerInboxAck: vi.fn(), + serviceCallerInboxFail: vi.fn(), + serviceCallerInboxClose: vi.fn(), + serviceCallerDisconnect: vi.fn(), +}; + +describe('optional core-bridge contract', () => { + it('fails closed while loading the incomplete public 18.0 module shape', async () => { + let failure: unknown; + try { + await loadCoreBridge(); + } catch (error) { + failure = error; + } + expect(failure).toMatchObject({ + code: 'core_bridge_unavailable', + message: expect.stringContaining('Install >=18.1.0 <19'), + cause: { + missingMethods: [ + 'readCallerDescriptor', + 'requireServiceCallerMethods', + 'assertServiceCallerIdentity', + 'serviceCallerIdentity', + 'serviceCallerInboxOpen', + 'serviceCallerInboxNext', + 'serviceCallerInboxAck', + 'serviceCallerInboxFail', + 'serviceCallerInboxClose', + 'serviceCallerDisconnect', + ], + requiredVersion: '>=18.1.0 <19', + }, + }); + }); + + it('accepts the complete 18.1 schema-v2 lifecycle module shape', () => { + expect(() => assertCoreBridgeApi(schemaV2CoreBridge181Shape)).not.toThrow(); + }); +}); diff --git a/packages/sdk/tests/node-spawn-dispatch-fence.test.ts b/packages/sdk/tests/node-spawn-dispatch-fence.test.ts new file mode 100644 index 000000000..505edb0a4 --- /dev/null +++ b/packages/sdk/tests/node-spawn-dispatch-fence.test.ts @@ -0,0 +1,535 @@ +import { existsSync } from 'node:fs'; +import { chmod, mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { runAgentRelayTask, type AgentRelayTaskRequest } from '../src/agent-relay-transport.js'; +import { registerRetryablePreflightPort } from '../src/node-spawn-dispatch.js'; +import { LocalNodeSpawnPort, type CoreBridgeClient } from '../src/node-spawn-local.js'; +import { NODE_RESULT_PREFIX } from '../src/node-spawn-result.js'; +import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; +import { claimNodeSpawnDispatch, nodeSpawnClaimPath, nodeSpawnDispatchFencePath, + readNodeSpawnClaims, recordNodeSpawn } from '../src/node-spawn-state.js'; +import { NodeSpawnError, NodeSpawnPreDispatchError, + type NodeSpawnPort, type NodeSpawnReceipt } from '../src/index.js'; + +const roots: string[] = []; +const TEST_LOCAL_IDENTITY = { + prototype: LocalNodeSpawnPort.prototype, + spawn: LocalNodeSpawnPort.prototype.spawn, + targets: LocalNodeSpawnPort.prototype.targets, + status: LocalNodeSpawnPort.prototype.status, + release: LocalNodeSpawnPort.prototype.release, +}; + +afterEach(async () => { + vi.useRealTimers(); + await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); +}); + +async function dataDirectory(label: string): Promise { + const directory = await mkdtemp(join(tmpdir(), label)); + roots.push(directory); + return directory; +} + +function request(dataDir: string, timeoutMs = 10_000): AgentRelayTaskRequest { + return { + cli: 'codex', task: 'Implement the change', runId: 'run_fence', stepId: 'edit', + idempotencyKey: 'dispatch_fence123', attempt: 1, dataDir, timeoutMs, + worker_cwd: '/tmp', + }; +} + +function officialLocalRuntime(options: { + eligible(): boolean; + invoked(name: string): void; +}, registerInternalPreflight = true): NodeSpawnRuntime { + const core: CoreBridgeClient = { + hello: { + bridge_version: 1, + daemon_version: 'test', + pid: process.pid, + methods: [], + topics: [], + }, + close() {}, + }; + const invocation = (name: string) => ({ + id: 'inv_spawn_1', + agent_id: 'agent_1', + status: 'completed', + node_id: 'node_1', + name, + spawned: true, + ready: true, + released: false, + task_unconfirmed: false, + error_code: null, + }); + let spawnedName = ''; + const bridge: ConstructorParameters[1] = { + spawnTargets: vi.fn(async () => ({ + targets: options.eligible() ? [{ + id: 'node_1', + name: 'desktop-1', + live: true, + handlers_live: true, + stale: false, + capabilities: ['spawn:codex'], + active_agents: 0, + reserved_agents: 0, + max_agents: 4, + }] : [], + })), + spawnInvoke: vi.fn(async (_core, input) => { + spawnedName = input.name; + options.invoked(input.name); + return invocation(input.name); + }), + spawnStatus: vi.fn(async () => invocation(spawnedName)), + spawnRelease: vi.fn(async () => ({ + ...invocation(spawnedName), + spawned: false, + ready: false, + released: true, + })), + }; + const port = new LocalNodeSpawnPort(core, bridge, { + harnesses: ['codex'], + directoryScope: ['/tmp'], + }, { id: 'node_1', name: 'desktop-1' }); + if (registerInternalPreflight) { + registerRetryablePreflightPort(port, TEST_LOCAL_IDENTITY, () => true); + } + return { + authority: 'desktop', + publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + port, + inbox: { inbox: { + subscribe: async function* () { + yield { id: 'delivery_1', message: { id: 'message_1', + from: { id: 'agent_1', name: spawnedName }, + text: NODE_RESULT_PREFIX + JSON.stringify({ + version: 1, run_id: 'run_fence', step_id: 'edit', + dispatch_id: 'dispatch_fence123', status: 'completed', output: { ok: true }, + }) } }; + }, + ack: vi.fn(async () => ({})), + fail: vi.fn(async () => ({})), + } }, + }; +} + +function readyReceipt(name: string): NodeSpawnReceipt { + return { + id: 'inv_spawn_1', action: 'spawn', status: 'completed', nodeId: 'node_1', + nodeName: 'desktop-1', name, agentId: 'agent_1', spawned: true, ready: true, + }; +} + +function runtime(spawn: NodeSpawnPort['spawn']): NodeSpawnRuntime { + let spawnedName = ''; + const wrapped = vi.fn(async (input, beforeDispatch) => { + const receipt = await spawn(input, beforeDispatch); + spawnedName = receipt.name; + return receipt; + }); + return { + authority: 'desktop', + publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + port: { + dispatchFenceVersion: 1, + targets: async () => [], + spawn: wrapped, + status: vi.fn(), + release: vi.fn(async () => ({ + id: 'inv_release_1', action: 'release' as const, status: 'completed' as const, + spawned: false, ready: false, released: true, + })), + }, + inbox: { inbox: { + subscribe: async function* () { + yield { id: 'delivery_1', message: { id: 'message_1', + from: { id: 'agent_1', name: spawnedName }, + text: NODE_RESULT_PREFIX + JSON.stringify({ + version: 1, run_id: 'run_fence', step_id: 'edit', + dispatch_id: 'dispatch_fence123', status: 'completed', output: { ok: true }, + }) } }; + }, + ack: vi.fn(async () => ({})), + fail: vi.fn(async () => ({})), + } }, + }; +} + +const preDispatchFailure = (message: string, code = 'no_eligible_node'): NodeSpawnPreDispatchError => + new NodeSpawnPreDispatchError(new NodeSpawnError(message, code)); + +describe('durable node-spawn dispatch fence', () => { + it('lets an official port retry proven preflight once when eligible', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-retry-'); + let eligible = false; + let invokes = 0; + const worker = officialLocalRuntime({ + eligible: () => eligible, + invoked() { invokes += 1; }, + }); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('No eligible local Agent Relay spawn target'); + const [claim] = await readNodeSpawnClaims(dataDir); + expect(claim!.claim.version).toBe(3); + await expect(readFile(nodeSpawnDispatchFencePath(claim!.path), 'utf8')) + .rejects.toMatchObject({ code: 'ENOENT' }); + + eligible = true; + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .resolves.toMatchObject({ receipt: { status: 'completed' } }); + expect(invokes).toBe(1); + }); + + it('fences an arbitrary public port before entering any of its code', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-public-port-'); + let calls = 0; + let fenceAtEntry = false; + const worker = runtime(async () => { + calls += 1; + const [entry] = await readNodeSpawnClaims(dataDir); + await readFile(nodeSpawnDispatchFencePath(entry!.path), 'utf8'); + fenceAtEntry = true; + throw preDispatchFailure('custom preflight cannot be trusted'); + }); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('custom preflight cannot be trusted'); + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('ambiguous pre-ack'); + expect(fenceAtEntry).toBe(true); + expect(calls).toBe(1); + }); + + it('never replays an injected local bridge after a lost non-idempotent reply', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-injected-local-'); + let workersCreated = 0; + const worker = officialLocalRuntime({ + eligible: () => true, + invoked() { + workersCreated += 1; + throw new Error('lost reply after non-idempotent create'); + }, + }, false); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('lost reply after non-idempotent create'); + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('ambiguous pre-ack'); + expect(workersCreated).toBe(1); + }); + + it('never grants retryable target preflight to an injected local bridge', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-injected-preflight-'); + let eligible = false; + let workersCreated = 0; + const worker = officialLocalRuntime({ + eligible: () => eligible, + invoked() { workersCreated += 1; }, + }, false); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('No eligible local Agent Relay spawn target'); + const [claim] = await readNodeSpawnClaims(dataDir); + await expect(readFile(nodeSpawnDispatchFencePath(claim!.path), 'utf8')) + .resolves.toContain('claimDigest'); + eligible = true; + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('ambiguous pre-ack'); + expect(workersCreated).toBe(0); + }); + + it('rejects an old public custom port before it creates a claim or invokes', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-old-port-'); + const spawn = vi.fn(async (input: Parameters[0]) => readyReceipt(input.name)); + const legacyPort = { + targets: async () => [], + spawn, + status: vi.fn(), + release: vi.fn(), + }; + // The public contract makes the migration compile-time visible too. + // @ts-expect-error dispatchFenceVersion is required for the fenced protocol. + const incompatible: NodeSpawnPort = legacyPort; + const worker = runtime(async () => readyReceipt('unused')); + for (const incompatiblePort of [ + incompatible, + { ...legacyPort, dispatchFenceVersion: 2 } as unknown as NodeSpawnPort, + ]) { + worker.port = incompatiblePort; + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('does not implement durable dispatch-fence protocol version 1'); + } + expect(spawn).not.toHaveBeenCalled(); + await expect(readNodeSpawnClaims(dataDir)).resolves.toEqual([]); + }); + + it('permanently refuses retry after the port crossed the fence and then failed', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-post-'); + let invokes = 0; + const worker = runtime(async (_input, beforeDispatch) => { + await beforeDispatch(); + invokes += 1; + throw new Error('connection dropped after invoke'); + }); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('connection dropped after invoke'); + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('ambiguous pre-ack'); + expect(invokes).toBe(1); + }); + + it('lets exactly one independent fence contender enter its irreversible call', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-race-'); + const worker = officialLocalRuntime({ eligible: () => false, invoked() {} }); + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('No eligible local Agent Relay spawn target'); + const [entry] = await readNodeSpawnClaims(dataDir); + let invokes = 0; + const contender = async (): Promise => { + await claimNodeSpawnDispatch(entry!.path, entry!.claim); + invokes += 1; + }; + + const results = await Promise.allSettled([contender(), contender()]); + expect(results.filter(result => result.status === 'fulfilled')).toHaveLength(1); + expect(results.filter(result => result.status === 'rejected')).toHaveLength(1); + expect(invokes).toBe(1); + }); + + it('keeps a receipt-less legacy v2 claim ambiguous', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-legacy-'); + let calls = 0; + const worker = runtime(async () => { + calls += 1; + throw preDispatchFailure('prepare legacy claim'); + }); + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('prepare legacy claim'); + const [entry] = await readNodeSpawnClaims(dataDir); + await recordNodeSpawn(entry!.path, { ...entry!.claim, version: 2 }); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('ambiguous pre-ack'); + expect(calls).toBe(1); + }); + + it('rejects a string claim protocol version instead of coercing it', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-string-version-'); + let calls = 0; + const worker = runtime(async () => { + calls += 1; + throw preDispatchFailure('prepare malformed claim'); + }); + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('prepare malformed claim'); + const [entry] = await readNodeSpawnClaims(dataDir); + await writeFile(entry!.path, JSON.stringify({ ...entry!.claim, version: '3' })); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('Invalid durable node-spawn claim'); + expect(calls).toBe(1); + }); + + it.each([ + { label: 'malformed JSON', content: '{' }, + { label: 'wrong claim digest', content: JSON.stringify({ version: 1, claimDigest: '0'.repeat(64) }) }, + ])('fails closed on a $label dispatch fence', async ({ content }) => { + const dataDir = await dataDirectory('flows-dispatch-fence-invalid-'); + let calls = 0; + const worker = runtime(async () => { + calls += 1; + throw preDispatchFailure('prepare invalid fence'); + }); + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('prepare invalid fence'); + const [entry] = await readNodeSpawnClaims(dataDir); + await writeFile(nodeSpawnDispatchFencePath(entry!.path), content); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('Invalid durable node-spawn dispatch fence'); + expect(calls).toBe(1); + }); + + it('treats a non-ENOENT fence read failure as invalid durable state', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-read-failure-'); + let invokes = 0; + const worker = officialLocalRuntime({ + eligible: () => false, + invoked() { invokes += 1; }, + }); + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('No eligible local Agent Relay spawn target'); + const [entry] = await readNodeSpawnClaims(dataDir); + await mkdir(nodeSpawnDispatchFencePath(entry!.path)); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('Invalid durable node-spawn dispatch fence'); + expect(invokes).toBe(0); + }); + + it('does not call the remote effect when dispatch-fence publication fails', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-write-failure-'); + let eligible = false; + let remoteCalls = 0; + const worker = officialLocalRuntime({ + eligible: () => eligible, + invoked() { remoteCalls += 1; }, + }); + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toThrow('No eligible local Agent Relay spawn target'); + const [entry] = await readNodeSpawnClaims(dataDir); + eligible = true; + await chmod(dirname(entry!.path), 0o500); + try { + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .rejects.toMatchObject({ code: 'EACCES' }); + } finally { + await chmod(dirname(entry!.path), 0o700); + } + expect(remoteCalls).toBe(0); + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .resolves.toMatchObject({ receipt: { status: 'completed' } }); + expect(remoteCalls).toBe(1); + }); + + it('retains the original claim deadline across a safe pre-dispatch retry', async () => { + vi.useFakeTimers(); + const dataDir = await dataDirectory('flows-dispatch-fence-deadline-'); + let invokes = 0; + const worker = officialLocalRuntime({ + eligible: () => false, + invoked() { invokes += 1; }, + }); + await expect(runAgentRelayTask(request(dataDir, 1_000), { runtime: worker })) + .rejects.toThrow('No eligible local Agent Relay spawn target'); + await vi.advanceTimersByTimeAsync(1_001); + + await expect(runAgentRelayTask(request(dataDir, 1_000), { runtime: worker })) + .rejects.toThrow('deadline elapsed'); + expect(invokes).toBe(0); + }); + + it('treats an overridden built-in spawn method as an arbitrary port', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-overridden-port-'); + const worker = officialLocalRuntime({ eligible: () => true, invoked() {} }); + let fenceAtEntry = false; + const originalSpawn = worker.port.spawn.bind(worker.port); + worker.port.spawn = vi.fn(async (input, beforeDispatch) => { + const [entry] = await readNodeSpawnClaims(dataDir); + await readFile(nodeSpawnDispatchFencePath(entry!.path), 'utf8'); + fenceAtEntry = true; + return originalSpawn(input, beforeDispatch); + }); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .resolves.toMatchObject({ receipt: { status: 'completed' } }); + expect(fenceAtEntry).toBe(true); + }); + + it('does not read a custom spawn accessor before publishing the fence', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-custom-accessor-'); + const worker = runtime(async input => readyReceipt(input.name)); + const spawn = worker.port.spawn; + const observations: boolean[] = []; + const claimPath = await nodeSpawnClaimPath( + dataDir, 'run_fence', 'edit', 'dispatch_fence123'); + Object.defineProperty(worker.port, 'spawn', { + configurable: true, + get() { + observations.push(existsSync(nodeSpawnDispatchFencePath(claimPath))); + return spawn; + }, + }); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .resolves.toMatchObject({ receipt: { status: 'completed' } }); + expect(observations).toEqual([true]); + }); + + it('does not read a registered local spawn accessor before publishing the fence', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-local-accessor-'); + const worker = officialLocalRuntime({ eligible: () => true, invoked() {} }); + const spawn = worker.port.spawn; + const observations: boolean[] = []; + const claimPath = await nodeSpawnClaimPath( + dataDir, 'run_fence', 'edit', 'dispatch_fence123'); + Object.defineProperty(worker.port, 'spawn', { + configurable: true, + get() { + observations.push(existsSync(nodeSpawnDispatchFencePath(claimPath))); + return spawn; + }, + }); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .resolves.toMatchObject({ receipt: { status: 'completed' } }); + expect(observations).toEqual([true]); + }); + + it('treats an overridden built-in target preflight as arbitrary code', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-overridden-targets-'); + const worker = officialLocalRuntime({ eligible: () => true, invoked() {} }); + let fenceAtEntry = false; + const originalTargets = worker.port.targets.bind(worker.port); + worker.port.targets = vi.fn(async () => { + const [entry] = await readNodeSpawnClaims(dataDir); + await readFile(nodeSpawnDispatchFencePath(entry!.path), 'utf8'); + fenceAtEntry = true; + return originalTargets(); + }); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .resolves.toMatchObject({ receipt: { status: 'completed' } }); + expect(fenceAtEntry).toBe(true); + }); + + it('does not trust a local spawn prototype replaced before construction', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-replaced-prototype-'); + const originalSpawn = LocalNodeSpawnPort.prototype.spawn; + let fenceAtEntry = false; + LocalNodeSpawnPort.prototype.spawn = async function (input, beforeDispatch) { + const [entry] = await readNodeSpawnClaims(dataDir); + await readFile(nodeSpawnDispatchFencePath(entry!.path), 'utf8'); + fenceAtEntry = true; + return originalSpawn.call(this, input, beforeDispatch); + }; + try { + const worker = officialLocalRuntime({ eligible: () => true, invoked() {} }); + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .resolves.toMatchObject({ receipt: { status: 'completed' } }); + expect(fenceAtEntry).toBe(true); + } finally { + LocalNodeSpawnPort.prototype.spawn = originalSpawn; + } + }); + + it('keeps an arbitrary port fenced when it returns without calling the hook', async () => { + const dataDir = await dataDirectory('flows-dispatch-fence-bypass-'); + let calls = 0; + let fenceAtEntry = false; + const worker = runtime(async input => { + calls += 1; + const [entry] = await readNodeSpawnClaims(dataDir); + await readFile(nodeSpawnDispatchFencePath(entry!.path), 'utf8'); + fenceAtEntry = true; + return readyReceipt(input.name); + }); + + await expect(runAgentRelayTask(request(dataDir), { runtime: worker })) + .resolves.toMatchObject({ receipt: { status: 'completed' } }); + expect(fenceAtEntry).toBe(true); + expect(calls).toBe(1); + }); +}); diff --git a/packages/sdk/tests/node-spawn-live.test.ts b/packages/sdk/tests/node-spawn-live.test.ts new file mode 100644 index 000000000..3f2458a76 --- /dev/null +++ b/packages/sdk/tests/node-spawn-live.test.ts @@ -0,0 +1,306 @@ +import { + accessSync, + constants, + existsSync, + lstatSync, + mkdtempSync, + readdirSync, + rmSync, + statSync, +} from 'node:fs'; +import { homedir, tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { spawn, type ChildProcess } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { afterEach, beforeAll, describe, expect, it } from 'vitest'; +import { compileYaml, toKernelSpec } from '../src/compile.js'; +import { socketPathFor } from '../src/daemon-connection.js'; +import { JournalClient } from '../src/journal-client.js'; +import { NODE_RESULT_PREFIX } from '../src/node-spawn-result.js'; +import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; +import type { NodeSpawnIdentity, NodeSpawnReceipt } from '../src/node-spawn.js'; +import { AgentWorker } from '../src/worker.js'; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..', '..'); +const TOOLCHAIN_TARGET = + process.env['CARGO_TARGET_DIR'] ?? + join( + process.env['RELAYFLOWS_TOOLCHAIN_HOME'] ?? + join(homedir(), '.relayflows-toolchain'), + 'target', + ); +const RELAYFLOWD = resolve(process.env['RELAYFLOWD_BIN'] ?? locateRelayflowd()); +const roots: string[] = []; +const daemons: ChildProcess[] = []; +const clients: JournalClient[] = []; + +beforeAll(() => accessSync(RELAYFLOWD, constants.X_OK)); + +afterEach(async () => { + for (const client of clients.splice(0)) client.close(); + for (const daemon of daemons.splice(0)) await stopDaemon(daemon); + for (const root of roots.splice(0)) + rmSync(root, { recursive: true, force: true }); +}); + +describe('node-spawn transport through the real journal', () => { + it('runs a relay agent step, releases exact identity, journals, then acknowledges', async () => { + const dataDir = mkdtempSync(join(tmpdir(), 'flows-node-spawn-live-')); + roots.push(dataDir); + await startDaemon(dataDir); + const client = await connectClient(dataDir); + await client.hello('node-spawn-live-test'); + + let runId = ''; + let allowSpawn!: () => void; + const runReady = new Promise((resolveReady) => { + allowSpawn = resolveReady; + }); + let resolveSpawn!: (receipt: NodeSpawnReceipt) => void; + const spawned = new Promise((resolveReceipt) => { + resolveSpawn = resolveReceipt; + }); + let dispatchId = ''; + const releases: Array<{ + receipt: NodeSpawnIdentity; + idempotencyKey: string; + }> = []; + const ackStates: string[] = []; + let acknowledgeEntered!: () => void; + const ackEntered = new Promise((resolveEntered) => { + acknowledgeEntered = resolveEntered; + }); + let allowAck!: () => void; + const ackReady = new Promise((resolveReady) => { + allowAck = resolveReady; + }); + const runtime: NodeSpawnRuntime = { + baseUrl: 'https://cast.agentrelay.com', + publisher: { id: 'publisher_1', name: 'flows-publisher' }, + port: { + dispatchFenceVersion: 1, + targets: async () => [], + spawn: async (input, beforeDispatch) => { + await runReady; + await beforeDispatch(); + dispatchId = input.idempotencyKey; + const receipt: NodeSpawnReceipt = { + id: 'spawn_1', + action: 'spawn', + status: 'completed', + nodeId: 'node_1', + nodeName: 'desktop-1', + name: input.name, + agentId: 'agent_1', + spawned: true, + ready: true, + }; + resolveSpawn(receipt); + return receipt; + }, + status: async () => { + throw new Error('first-attempt run must not poll a prior spawn'); + }, + release: async (input) => { + releases.push({ + receipt: input.receipt, + idempotencyKey: input.idempotencyKey, + }); + return { + id: 'release_1', + action: 'release', + status: 'completed', + spawned: false, + ready: false, + released: true, + }; + }, + }, + inbox: { + inbox: { + subscribe: async function* () { + const receipt = await spawned; + yield { + id: 'delivery_1', + message: { + id: 'message_1', + from: { id: receipt.agentId, name: receipt.name }, + text: + NODE_RESULT_PREFIX + + JSON.stringify({ + version: 1, + run_id: runId, + step_id: 'edit', + dispatch_id: dispatchId, + status: 'completed', + output: { summary: 'node-spawn journal round trip' }, + accounting: { tokens_input: 3, tokens_output: 5 }, + }), + }, + }; + }, + ack: async () => { + const snapshot = await client.runGet(runId); + ackStates.push(snapshot.steps['edit']?.state ?? 'missing'); + acknowledgeEntered(); + await ackReady; + return {}; + }, + fail: async () => { + throw new Error('valid result must not fail its delivery'); + }, + }, + }, + }; + const worker = new AgentWorker(client, { + workerId: 'node-spawn-live-worker', + pins: { + workspace: [{ surface: 'repo', revision_id: 'rev-a' }], + streams: [], + }, + dataDir, + nodeSpawnRuntime: runtime, + }); + const workerErrors: unknown[] = []; + worker.on('error', (error) => workerErrors.push(error)); + await worker.attach(); + + try { + const started = await client.runStart( + toKernelSpec( + compileYaml(` +version: '0.1.0' +steps: + - id: edit + type: agent + cli: codex + model: codex-medium + transport: relay + instruction: Perform the declared work. +`), + ), + ); + runId = started.run_id; + allowSpawn(); + const step = await waitForStep(client, runId, 'edit', 'done'); + const completed = (await client.journalRead(runId, 1, 100)).entries.find( + (entry) => + (entry as { entry_type?: string; step_id?: string }).entry_type === + 'step.completed' && + (entry as { step_id?: string }).step_id === 'edit', + ); + + expect(workerErrors).toEqual([]); + expect(step).toMatchObject({ state: 'done', type: 'agent' }); + expect(completed).toMatchObject({ + payload: { + completionReason: 'success', + output: { summary: 'node-spawn journal round trip' }, + }, + }); + expect(releases).toEqual([ + { + receipt: expect.objectContaining({ + id: 'spawn_1', + nodeId: 'node_1', + nodeName: 'desktop-1', + agentId: 'agent_1', + }), + idempotencyKey: expect.stringMatching(/^release_[a-f0-9]{32}$/), + }, + ]); + await ackEntered; + expect(ackStates).toEqual(['done']); + allowAck(); + await worker.close(); + } finally { + allowSpawn(); + acknowledgeEntered(); + allowAck(); + await worker.close(); + } + }, 20_000); +}); + +function locateRelayflowd(): string { + const direct = join(TOOLCHAIN_TARGET, 'debug', 'relayflowd'); + if (existsSync(direct)) return direct; + if (!existsSync(TOOLCHAIN_TARGET)) + return join(ROOT, 'kernel', 'target', 'debug', 'relayflowd'); + return ( + readdirSync(TOOLCHAIN_TARGET) + .map((entry) => join(TOOLCHAIN_TARGET, entry, 'debug', 'relayflowd')) + .filter((candidate) => existsSync(candidate)) + .sort((a, b) => statSync(b).mtimeMs - statSync(a).mtimeMs)[0] ?? + join(ROOT, 'kernel', 'target', 'debug', 'relayflowd') + ); +} + +async function startDaemon(dataDir: string): Promise { + const daemon = spawn(RELAYFLOWD, ['--data-dir', dataDir, 'serve'], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + daemons.push(daemon); + const stderr: Buffer[] = []; + daemon.stderr?.on('data', (chunk: Buffer) => stderr.push(chunk)); + const socket = socketPathFor(dataDir); + const deadline = Date.now() + 5_000; + while (Date.now() < deadline) { + if (existsSync(socket) && lstatSync(socket).isSocket()) { + const ready = new JournalClient(socket, { requestTimeoutMs: 1_000 }); + try { + await ready.connect(); + await ready.hello('node-spawn-live-readiness'); + return; + } catch { + /* Startup is still in progress. */ + } finally { + ready.close(); + } + } + if (daemon.exitCode !== null || daemon.signalCode !== null) { + throw new Error( + `relayflowd exited before readiness: ${Buffer.concat(stderr).toString('utf8')}`, + ); + } + await delay(20); + } + throw new Error(`relayflowd did not bind ${socket} within 5000ms`); +} + +async function connectClient(dataDir: string): Promise { + const client = new JournalClient(socketPathFor(dataDir), { + requestTimeoutMs: 5_000, + }); + clients.push(client); + await client.connect(); + return client; +} + +async function stopDaemon(daemon: ChildProcess): Promise { + if (daemon.exitCode !== null || daemon.signalCode !== null) return; + const exited = new Promise((resolveExit) => + daemon.once('exit', () => resolveExit()), + ); + daemon.kill('SIGTERM'); + await exited; +} + +async function waitForStep( + client: JournalClient, + runId: string, + stepId: string, + state: string, +) { + const deadline = Date.now() + 10_000; + while (Date.now() < deadline) { + const snapshot = await client.runGet(runId); + if (snapshot.steps[stepId]?.state === state) return snapshot.steps[stepId]; + await delay(20); + } + throw new Error(`step ${stepId} did not reach ${state} within 10000ms`); +} + +function delay(ms: number): Promise { + return new Promise((resolveDelay) => setTimeout(resolveDelay, ms)); +} diff --git a/packages/sdk/tests/node-spawn-local-replay-gap.test.ts b/packages/sdk/tests/node-spawn-local-replay-gap.test.ts new file mode 100644 index 000000000..fc6cdf02c --- /dev/null +++ b/packages/sdk/tests/node-spawn-local-replay-gap.test.ts @@ -0,0 +1,336 @@ +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { runAgentRelayTask, type AgentRelayTaskRequest } from '../src/agent-relay-transport.js'; +import { hasReplayableFencedDispatch } from '../src/node-spawn-dispatch.js'; +import { loadCoreBridge } from '../src/node-spawn-local.js'; +import { createLocalNodeSpawnRuntime } from '../src/node-spawn-runtime.js'; +import { nodeSpawnDispatchFencePath, readNodeSpawnClaims } from '../src/node-spawn-state.js'; + +const coreBridge = vi.hoisted(() => { + const state = { + directoryScope: ['/tmp'] as string[], + invokeCalls: [] as Array<{ + key: string; task: string; directory: string; nodeId: string; nodeName: string; + }>, + commits: new Set(), + dropFirstReply: true, + releases: 0, + subscriptions: 0, + targetCalls: 0, + preflightFailures: 0, + spawnedName: '', + nodeId: 'node_local', + committedNodeId: '', + }; + const core = { + hello: { bridge_version: 1, daemon_version: 'test', pid: 1, methods: [], topics: [] }, + close: vi.fn(), + }; + const invocation = () => ({ + id: 'inv_spawn_replayed', + agent_id: 'agent_replayed', + status: 'completed', + node_id: state.committedNodeId, + name: state.spawnedName, + spawned: true, + ready: true, + released: false, + task_unconfirmed: false, + error_code: null, + }); + const bridge = { + connectCore: vi.fn(async () => core), + readCallerDescriptor: vi.fn(async () => ({ + schema: 'agent-relay-service-caller/2', + schemaVersion: 2, + auth: {}, + callerId: 'caller_local', + callerName: 'flows-caller', + publisher: { id: 'publisher_local', name: 'flows-publisher' }, + workspace: { + site_url: 'https://agentrelay.com', + base_url: 'https://cast.agentrelay.com', + workspace_id: 'workspace_local', + relaycast_workspace_id: 'relaycast_local', + }, + account: { user_id: 'user_local' }, + credential: {}, + createdAt: '2026-01-01T00:00:00.000Z', + expiresAt: '2027-01-01T00:00:00.000Z', + harnesses: ['codex'], + directoryScope: [...state.directoryScope], + ownsOnly: true, + socket: '/tmp/core.sock', + inbox: { + transport: 'core', open: 'open', next: 'next', ack: 'ack', fail: 'fail', + close: 'close', disconnect: 'disconnect', + }, + desktopVersion: 'test', + })), + authenticate: vi.fn(async () => ({})), + requireServiceCallerMethods: vi.fn(), + assertServiceCallerIdentity: vi.fn(), + serviceCallerIdentity: vi.fn(async () => ({ + publisher: { id: 'publisher_local', name: 'flows-publisher' }, + node: { id: state.nodeId, name: 'desktop-local' }, + base_url: 'https://cast.agentrelay.com', + workspace_id: 'workspace_local', + relaycast_workspace_id: 'relaycast_local', + })), + spawnTargets: vi.fn(async () => { + state.targetCalls += 1; + if (state.preflightFailures > 0) { + state.preflightFailures -= 1; + throw new Error('target lookup failed before dispatch'); + } + return { targets: [{ + id: state.nodeId, name: 'desktop-local', live: true, handlers_live: true, + stale: false, capabilities: ['spawn:codex'], active_agents: 0, + reserved_agents: 0, max_agents: 4, + }] }; + }), + spawnInvoke: vi.fn(async (_core: unknown, input: { + name: string; task: string; directory: string; idempotency_key: string; + node_id: string; node_name: string; + }) => { + state.spawnedName = input.name; + state.committedNodeId ||= input.node_id; + state.invokeCalls.push({ + key: input.idempotency_key, + task: input.task, + directory: input.directory, + nodeId: input.node_id, + nodeName: input.node_name, + }); + state.commits.add(input.idempotency_key); + if (state.dropFirstReply) { + state.dropFirstReply = false; + throw new Error('bridge reply lost after strict server commit'); + } + return invocation(); + }), + spawnStatus: vi.fn(async () => invocation()), + spawnRelease: vi.fn(async () => { + state.releases += 1; + return { ...invocation(), spawned: false, ready: false, released: true }; + }), + serviceCallerInboxOpen: vi.fn(async () => { + state.subscriptions += 1; + return { iterator_id: 'iterator_replayed' }; + }), + serviceCallerInboxNext: vi.fn(async () => ({ delivery: { + item_id: 'delivery_replayed', + message_id: 'message_replayed', + sender: { id: 'agent_replayed', name: state.spawnedName }, + text: 'RELAYFLOWS_RESULT_V1 ' + JSON.stringify({ + version: 1, + run_id: 'run_local_replay', + step_id: 'edit', + dispatch_id: 'local_replay_key_123', + status: 'completed', + output: { ok: true }, + }), + invocation_id: 'inv_spawn_replayed', + sent_at: '2026-01-01T00:00:00.000Z', + } })), + serviceCallerInboxAck: vi.fn(async () => ({})), + serviceCallerInboxFail: vi.fn(async () => ({})), + serviceCallerInboxClose: vi.fn(async () => ({})), + serviceCallerDisconnect: vi.fn(async () => ({})), + }; + return { bridge, state }; +}); + +vi.mock('@agent-relay/core-bridge', () => coreBridge.bridge); + +const roots: string[] = []; + +beforeEach(() => { + coreBridge.state.directoryScope = ['/tmp']; + coreBridge.state.invokeCalls.length = 0; + coreBridge.state.commits.clear(); + coreBridge.state.dropFirstReply = true; + coreBridge.state.releases = 0; + coreBridge.state.subscriptions = 0; + coreBridge.state.targetCalls = 0; + coreBridge.state.preflightFailures = 0; + coreBridge.state.spawnedName = ''; + coreBridge.state.nodeId = 'node_local'; + coreBridge.state.committedNodeId = ''; + vi.clearAllMocks(); +}); + +afterEach(async () => { + await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); +}); + +async function dataDirectory(): Promise { + const directory = await mkdtemp(join(tmpdir(), 'flows-local-replay-gap-')); + roots.push(directory); + return directory; +} + +function request( + dataDir: string, + workerCwd = '/tmp', + task = 'Implement the exact change', +): AgentRelayTaskRequest { + return { + cli: 'codex', + task, + taskIdentity: 'task_sha256_exact', + runId: 'run_local_replay', + stepId: 'edit', + idempotencyKey: 'local_replay_key_123', + attempt: 1, + dataDir, + timeoutMs: 10_000, + worker_cwd: workerCwd, + }; +} + +describe('official local strict replay', () => { + it('replays an accepted same-key claim after current directory scope is revoked', async () => { + const dataDir = await dataDirectory(); + const crashedRuntime = await createLocalNodeSpawnRuntime('/tmp/caller.json'); + + await expect(runAgentRelayTask(request(dataDir), { runtime: crashedRuntime })) + .rejects.toThrow('bridge reply lost after strict server commit'); + coreBridge.state.directoryScope = ['/revoked-root']; + const restartedRuntime = await createLocalNodeSpawnRuntime('/tmp/caller.json'); + const recovered = await Promise.all([ + runAgentRelayTask(request(dataDir, '/tmp', 'Changed rendering after SDK upgrade'), + { runtime: restartedRuntime }), + runAgentRelayTask(request(dataDir, '/tmp', 'Changed rendering after SDK upgrade'), + { runtime: restartedRuntime }), + ]); + expect(recovered).toEqual([ + expect.objectContaining({ receipt: expect.objectContaining({ status: 'completed' }) }), + expect.objectContaining({ receipt: expect.objectContaining({ status: 'completed' }) }), + ]); + + expect(coreBridge.state.invokeCalls).toHaveLength(2); + expect(coreBridge.state.invokeCalls[0]).toEqual(coreBridge.state.invokeCalls[1]); + expect(coreBridge.state.invokeCalls[0]).toMatchObject({ + key: 'local_replay_key_123', + directory: '/tmp', + nodeId: 'node_local', + nodeName: 'desktop-local', + }); + expect(coreBridge.state.invokeCalls[0]!.task).toContain('Implement the exact change'); + expect([...coreBridge.state.commits]).toEqual(['local_replay_key_123']); + expect(coreBridge.state.targetCalls).toBe(1); + expect(coreBridge.state.subscriptions).toBe(1); + expect(coreBridge.state.releases).toBe(1); + }); + + it('still applies current directory scope before a new unfenced dispatch', async () => { + const dataDir = await dataDirectory(); + coreBridge.state.directoryScope = ['/revoked-root']; + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json'); + + await expect(runAgentRelayTask(request(dataDir), { runtime })) + .rejects.toThrow('does not authorize this harness and directory'); + expect(coreBridge.state.invokeCalls).toHaveLength(0); + expect(coreBridge.state.targetCalls).toBe(0); + const [claim] = await readNodeSpawnClaims(dataDir); + await expect(readFile(nodeSpawnDispatchFencePath(claim!.path), 'utf8')) + .rejects.toMatchObject({ code: 'ENOENT' }); + }); + + it('keeps the original request across a safe preflight retry and fenced replay', async () => { + const dataDir = await dataDirectory(); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json'); + coreBridge.state.preflightFailures = 1; + + await expect(runAgentRelayTask(request(dataDir), { runtime })) + .rejects.toThrow('target lookup failed before dispatch'); + expect(coreBridge.state.invokeCalls).toHaveLength(0); + const [claim] = await readNodeSpawnClaims(dataDir); + await expect(readFile(nodeSpawnDispatchFencePath(claim!.path), 'utf8')) + .rejects.toMatchObject({ code: 'ENOENT' }); + + await expect(runAgentRelayTask( + request(dataDir, '/tmp', 'Changed rendering before the safe retry'), { runtime }, + )).rejects.toThrow('bridge reply lost after strict server commit'); + const recovered = await runAgentRelayTask( + request(dataDir, '/tmp', 'Changed rendering again after response loss'), { runtime }, + ); + + expect(recovered.receipt.status).toBe('completed'); + expect(coreBridge.state.invokeCalls).toHaveLength(2); + expect(coreBridge.state.invokeCalls[0]).toEqual(coreBridge.state.invokeCalls[1]); + expect(coreBridge.state.invokeCalls[0]!.task).toContain('Implement the exact change'); + expect(coreBridge.state.targetCalls).toBe(2); + }); + + it('refuses fenced recovery after the exact Desktop node identity changes', async () => { + const dataDir = await dataDirectory(); + const crashedRuntime = await createLocalNodeSpawnRuntime('/tmp/caller.json'); + + await expect(runAgentRelayTask(request(dataDir), { runtime: crashedRuntime })) + .rejects.toThrow('bridge reply lost after strict server commit'); + coreBridge.state.nodeId = 'node_after_reenrolment'; + const restartedRuntime = await createLocalNodeSpawnRuntime('/tmp/caller.json'); + + await expect(runAgentRelayTask(request(dataDir), { runtime: restartedRuntime })) + .rejects.toThrow('conflicts with its durable identity or input'); + expect(coreBridge.state.invokeCalls).toHaveLength(1); + }); + + it('rejects a durable local node id without its paired node name', async () => { + const dataDir = await dataDirectory(); + coreBridge.state.directoryScope = ['/revoked-root']; + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json'); + + await expect(runAgentRelayTask(request(dataDir), { runtime })) + .rejects.toThrow('does not authorize this harness and directory'); + const [entry] = await readNodeSpawnClaims(dataDir); + const claim = JSON.parse(await readFile(entry!.path, 'utf8')) as { + request: Record; + }; + delete claim.request.node; + await writeFile(entry!.path, JSON.stringify(claim)); + + await expect(runAgentRelayTask(request(dataDir), { runtime })) + .rejects.toThrow('Invalid durable node-spawn claim'); + expect(coreBridge.state.invokeCalls).toHaveLength(0); + }); + + it.each(['spawn', 'targets', 'status', 'release'] as const)( + 'revokes replay trust when authentic port method %s is overridden', async method => { + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json'); + expect(hasReplayableFencedDispatch(runtime.port)).toBe(true); + const original = runtime.port[method].bind(runtime.port); + Object.defineProperty(runtime.port, method, { value: vi.fn(original) }); + expect(hasReplayableFencedDispatch(runtime.port)).toBe(false); + }, + ); + + it.each(['core', 'bridge', 'scope', 'exactNode'] as const)( + 'revokes replay trust when authentic port state %s is replaced', async field => { + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json'); + expect(hasReplayableFencedDispatch(runtime.port)).toBe(true); + Object.defineProperty(runtime.port, field, { value: {}, configurable: true }); + expect(hasReplayableFencedDispatch(runtime.port)).toBe(false); + }, + ); + + it.each(['invokeSpawn', 'wait', 'cleanupIdentity'] as const)( + 'revokes replay trust when internal replay method %s is overridden', async method => { + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json'); + expect(hasReplayableFencedDispatch(runtime.port)).toBe(true); + Object.defineProperty(runtime.port, method, { value: vi.fn(), configurable: true }); + expect(hasReplayableFencedDispatch(runtime.port)).toBe(false); + }, + ); + + it('does not trust even the official module when supplied through options.bridge', async () => { + const bridge = await loadCoreBridge(); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge }); + expect(hasReplayableFencedDispatch(runtime.port)).toBe(false); + }); +}); diff --git a/packages/sdk/tests/node-spawn-ports.test.ts b/packages/sdk/tests/node-spawn-ports.test.ts new file mode 100644 index 000000000..2c8404a90 --- /dev/null +++ b/packages/sdk/tests/node-spawn-ports.test.ts @@ -0,0 +1,451 @@ +import { describe, expect, it, vi } from 'vitest'; +import { RelayPlacementError } from '@agent-relay/sdk/messaging'; +import { + LocalNodeSpawnPort, + type CoreBridgeApi, + type CoreBridgeClient, +} from '../src/node-spawn-local.js'; +import { RemoteNodeSpawnPort } from '../src/node-spawn-remote.js'; + +interface TestCore { + call(method: string, params?: Record): Promise; +} + +const beforeDispatch = async (): Promise => undefined; + +function localPort(core: TestCore): LocalNodeSpawnPort { + const client = { + hello: { bridge_version: 2, daemon_version: 'test', pid: 1, methods: [], topics: [] }, + close() {}, + } satisfies CoreBridgeClient; + const bridge = { + spawnTargets: () => core.call('spawn.targets'), + spawnInvoke: (_client, params) => core.call('spawn.invoke', { ...params }), + spawnStatus: (_client, params) => core.call('spawn.status', { ...params }), + spawnRelease: (_client, params) => core.call('spawn.release', { ...params }), + } satisfies Pick; + return new LocalNodeSpawnPort(client, bridge, { + harnesses: ['codex'], + directoryScope: ['/workspace'], + }, { id: 'node_1', name: 'desktop-1' }); +} + +describe('local node-spawn port', () => { + it('uses bound bridge v2 spawn methods with no raw caller credential and exact release identity', async () => { + const calls: Array<[string, Record | undefined]> = []; + const dispatchOrder: string[] = []; + const core: TestCore = { + async call(method: string, params?: Record): Promise { + calls.push([method, params]); + if (method === 'spawn.invoke') dispatchOrder.push('spawn.invoke'); + if (method === 'spawn.targets') return { targets: [{ id: 'node_1', name: 'desktop-1', live: true, + handlers_live: true, stale: false, capabilities: ['spawn:codex'], active_agents: 0, + reserved_agents: 0, max_agents: 2 }] } as T; + if (method === 'spawn.invoke' || (method === 'spawn.status' && params?.action === 'spawn')) return { id: 'inv_1', status: 'completed', node_id: 'node_1', + agent_id: 'agent_1', name: 'worker_1', spawned: true, ready: true, task_unconfirmed: false } as T; + return { id: 'rel_1', status: 'completed', name: 'worker_1', spawned: false, + ready: false, released: true, task_unconfirmed: false } as T; + }, + }; + const port = localPort(core); + const receipt = await port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', + directory: '/workspace', idempotencyKey: 'dispatch_1234567890' }, async () => { + dispatchOrder.push('dispatch.fence'); + }); + await port.release({ receipt, idempotencyKey: 'release_1234567890' }); + expect(calls[1]).toEqual(['spawn.invoke', expect.objectContaining({ node_id: 'node_1', name: 'worker_1' })]); + expect(calls[2]).toEqual(['spawn.status', expect.objectContaining({ invocation_id: 'inv_1' })]); + expect(calls[3]).toEqual(['spawn.release', expect.objectContaining({ + expected_agent_id: 'agent_1', node_id: 'node_1', node_name: 'desktop-1', + name: 'worker_1', idempotency_key: 'release_1234567890' })]); + expect(dispatchOrder).toEqual(['dispatch.fence', 'spawn.invoke']); + for (const [, params] of calls) { + if (params === undefined) continue; + expect(params).not.toHaveProperty('caller_name'); + expect(params).not.toHaveProperty('caller_secret'); + } + }); + + it('refuses local-only fields the Desktop core bridge cannot preserve', async () => { + const calls = vi.fn(); + const core: TestCore = { async call(): Promise { calls(); throw new Error('unused'); } }; + const port = localPort(core); + await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', directory: '/workspace', + model: 'codex-medium', idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)).rejects.toMatchObject({ + code: 'spawn_invalid_input', + }); + expect(calls).not.toHaveBeenCalled(); + }); + + it.each([ + { harness: 'claude', directory: '/workspace', label: 'harness' }, + { harness: 'codex', directory: '/other', label: 'directory' }, + { harness: 'codex', directory: '/workspace-sibling', label: 'directory prefix' }, + ])('refuses a $label outside the caller-file scope before bridge dispatch', async request => { + const calls = vi.fn(); + const port = localPort({ async call(): Promise { calls(); throw new Error('unused'); } }); + await expect(port.spawn({ name: 'worker_1', harness: request.harness, task: 'work', + directory: request.directory, idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)) + .rejects.toMatchObject({ code: 'outside_scope' }); + expect(calls).not.toHaveBeenCalled(); + }); + + it.each([ + { label: 'invalid input', request: { directory: undefined }, targetReadFails: false }, + { label: 'invalid scope', request: { directory: '/workspace', harness: 'claude' }, targetReadFails: false }, + { label: 'target read failure', request: { directory: '/workspace' }, targetReadFails: true }, + { label: 'target mismatch', request: { directory: '/workspace', node: 'desktop-other' }, targetReadFails: false }, + ])('does not enter the dispatch hook for $label', async testCase => { + const entered = vi.fn(async () => undefined); + const core: TestCore = { async call(method: string): Promise { + if (method !== 'spawn.targets') throw new Error('spawn must not be invoked'); + if (testCase.targetReadFails) throw new Error('target read failed'); + return { targets: [{ id: 'node_1', name: 'desktop-1', live: true, + handlers_live: true, stale: false, capabilities: ['spawn:codex'] }] } as T; + } }; + await expect(localPort(core).spawn({ + name: 'worker_1', harness: testCase.request.harness ?? 'codex', task: 'work', + idempotencyKey: 'dispatch_1234567890', + ...(testCase.request.directory === undefined ? {} : { directory: testCase.request.directory }), + ...(testCase.request.node === undefined ? {} : { node: testCase.request.node }), + }, entered)).rejects.toBeInstanceOf(Error); + expect(entered).not.toHaveBeenCalled(); + }); + + it('refuses local targets without an exact string id instead of fabricating one', async () => { + const core: TestCore = { async call(): Promise { + return { targets: [{ id: undefined, name: 'desktop-1', live: true, + handlers_live: true, capabilities: ['spawn:codex'] }] } as T; + } }; + const port = localPort(core); + await expect(port.targets()).rejects.toMatchObject({ code: 'invalid_core_response' }); + }); + + it.each([ + { live: false, handlers_live: true }, + { live: true, handlers_live: false }, + ])('does not dispatch to an explicitly named node unless both liveness checks pass (%j)', async readiness => { + const calls = vi.fn(); + const core: TestCore = { async call(method: string): Promise { + calls(method); + if (method === 'spawn.targets') return { targets: [{ id: 'node_1', name: 'desktop-1', + ...readiness, stale: false, capabilities: ['spawn:codex'] }] } as T; + throw new Error('spawn must not be invoked'); + } }; + const port = localPort(core); + await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', node: 'desktop-1', + directory: '/workspace', idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)).rejects.toMatchObject({ + code: 'no_eligible_node', + }); + expect(calls).toHaveBeenCalledTimes(1); + }); + + it('does not dispatch to a duplicate node name with a different authenticated id', async () => { + const calls = vi.fn(); + const core: TestCore = { async call(method: string): Promise { + calls(method); + if (method === 'spawn.targets') return { targets: [{ id: 'node_other', name: 'desktop-1', + live: true, handlers_live: true, stale: false, capabilities: ['spawn:codex'] }] } as T; + throw new Error('spawn must not be invoked'); + } }; + await expect(localPort(core).spawn({ name: 'worker_1', harness: 'codex', task: 'work', + node: 'desktop-1', directory: '/workspace', idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)) + .rejects.toMatchObject({ code: 'no_eligible_node' }); + expect(calls).toHaveBeenCalledTimes(1); + }); + + it('stops polling on a denied terminal status', async () => { + const status = vi.fn(); + const core: TestCore = { async call(method: string): Promise { + if (method === 'spawn.targets') return { targets: [{ id: 'node_1', name: 'desktop-1', live: true, + handlers_live: true, stale: false, capabilities: ['spawn:codex'] }] } as T; + if (method === 'spawn.invoke') return { id: 'inv_1', status: 'pending', spawned: false, + ready: false, task_unconfirmed: false } as T; + if (method === 'spawn.status') { + status(); + return { id: 'inv_1', status: 'denied', spawned: false, ready: false, + task_unconfirmed: false, error_code: 'outside_scope' } as T; + } + throw new Error(`unexpected core method: ${method}`); + } }; + const port = localPort(core); + await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', directory: '/workspace', + idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)).rejects.toMatchObject({ code: 'outside_scope' }); + expect(status).toHaveBeenCalledOnce(); + }); + + it('refuses a completed local spawn whose returned node identity changed', async () => { + const core: TestCore = { async call(method: string): Promise { + if (method === 'spawn.targets') return { targets: [ + { id: 'node_1', name: 'desktop-1', live: true, + handlers_live: true, stale: false, capabilities: ['spawn:codex'] }, + { id: 'node_other', name: 'desktop-2', live: true, + handlers_live: true, stale: false, capabilities: ['spawn:codex'] }, + ] } as T; + if (method === 'spawn.invoke') return { id: 'inv_1', status: 'pending', spawned: false, + ready: false, task_unconfirmed: false } as T; + return { id: 'inv_1', status: 'completed', node_id: 'node_other', agent_id: 'agent_1', + name: 'worker_1', spawned: true, ready: true, task_unconfirmed: false } as T; + } }; + const port = localPort(core); + await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', directory: '/workspace', + idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)).rejects.toMatchObject({ + code: 'spawn_identity_mismatch', + cleanup: { nodeId: 'node_other', nodeName: 'desktop-2', name: 'worker_1', agentId: 'agent_1' }, + }); + }); + + it('refuses a completed local spawn without explicit task delivery confirmation', async () => { + const core: TestCore = { async call(method: string): Promise { + if (method === 'spawn.targets') return { targets: [{ id: 'node_1', name: 'desktop-1', live: true, + handlers_live: true, stale: false, capabilities: ['spawn:codex'] }] } as T; + if (method === 'spawn.invoke') return { id: 'inv_1', status: 'pending', spawned: false, + ready: false, task_unconfirmed: false } as T; + if (method === 'spawn.status') return { id: 'inv_1', status: 'completed', node_id: 'node_1', + agent_id: 'agent_1', name: 'worker_1', spawned: true, ready: true } as T; + throw new Error(`unexpected core method: ${method}`); + } }; + const port = localPort(core); + await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', directory: '/workspace', + idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)).rejects.toMatchObject({ + code: 'spawn_task_unconfirmed', + cleanup: { nodeId: 'node_1', nodeName: 'desktop-1', name: 'worker_1', agentId: 'agent_1' }, + }); + }); + + it('retains exact cleanup identity when local spawn confirmation times out', async () => { + vi.useFakeTimers(); + try { + const status = vi.fn(); + const core: TestCore = { async call(method: string): Promise { + if (method === 'spawn.targets') return { targets: [{ id: 'node_1', name: 'desktop-1', live: true, + handlers_live: true, stale: false, capabilities: ['spawn:codex'] }] } as T; + if (method === 'spawn.invoke') return { id: 'inv_1', status: 'pending', spawned: false, + ready: false, task_unconfirmed: false } as T; + if (method === 'spawn.status') { + status(); + return { id: 'inv_1', status: 'running', node_id: 'node_1', agent_id: 'agent_1', + name: 'worker_1', spawned: true, ready: false, task_unconfirmed: false } as T; + } + throw new Error(`unexpected core method: ${method}`); + } }; + const failure = expect(localPort(core).spawn({ name: 'worker_1', harness: 'codex', task: 'work', + directory: '/workspace', idempotencyKey: 'dispatch_1234567890', timeoutMs: 750 }, beforeDispatch)) + .rejects.toMatchObject({ code: 'spawn_unconfirmed', cleanup: { + id: 'inv_1', nodeId: 'node_1', nodeName: 'desktop-1', name: 'worker_1', agentId: 'agent_1', + } }); + await vi.waitFor(() => expect(status).toHaveBeenCalledOnce()); + await vi.advanceTimersByTimeAsync(1_000); + await failure; + } finally { + vi.useRealTimers(); + } + }); + + it('retains exact cleanup identity when later local spawn polling fails', async () => { + vi.useFakeTimers(); + try { + const pollFailure = new Error('core disconnected'); + let statusCalls = 0; + const core: TestCore = { async call(method: string): Promise { + if (method === 'spawn.targets') return { targets: [{ id: 'node_1', name: 'desktop-1', live: true, + handlers_live: true, stale: false, capabilities: ['spawn:codex'] }] } as T; + if (method === 'spawn.invoke') return { id: 'inv_1', status: 'pending', spawned: false, + ready: false, task_unconfirmed: false } as T; + if (method === 'spawn.status' && statusCalls++ === 0) return { id: 'inv_1', status: 'running', + node_id: 'node_1', agent_id: 'agent_1', name: 'worker_1', spawned: true, + ready: false, task_unconfirmed: false } as T; + if (method === 'spawn.status') throw pollFailure; + throw new Error(`unexpected core method: ${method}`); + } }; + const failure = expect(localPort(core).spawn({ name: 'worker_1', harness: 'codex', task: 'work', + directory: '/workspace', idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)) + .rejects.toMatchObject({ code: 'spawn_unconfirmed', cause: pollFailure, cleanup: { + id: 'inv_1', nodeId: 'node_1', nodeName: 'desktop-1', name: 'worker_1', agentId: 'agent_1', + } }); + await vi.waitFor(() => expect(statusCalls).toBe(1)); + await vi.advanceTimersByTimeAsync(500); + await failure; + } finally { + vi.useRealTimers(); + } + }); + + it('fails closed on a conflicting later spawn identity and retains the first exact cleanup', async () => { + let statusCalls = 0; + const core: TestCore = { async call(method: string): Promise { + if (method === 'spawn.targets') return { targets: [{ id: 'node_1', name: 'desktop-1', live: true, + handlers_live: true, stale: false, capabilities: ['spawn:codex'] }] } as T; + if (method === 'spawn.invoke') return { id: 'inv_1', status: 'pending', spawned: false, + ready: false, task_unconfirmed: false } as T; + if (method === 'spawn.status' && statusCalls++ === 0) return { id: 'inv_1', status: 'running', + node_id: 'node_1', agent_id: 'agent_1', name: 'worker_1', spawned: true, + ready: false, task_unconfirmed: false } as T; + if (method === 'spawn.status') return { id: 'inv_1', status: 'completed', node_id: 'node_1', + agent_id: 'agent_other', name: 'worker_1', spawned: true, + ready: true, task_unconfirmed: false } as T; + throw new Error(`unexpected core method: ${method}`); + } }; + const pending = localPort(core).spawn({ name: 'worker_1', harness: 'codex', task: 'work', + directory: '/workspace', idempotencyKey: 'dispatch_1234567890' }, beforeDispatch); + await expect(pending).rejects.toMatchObject({ code: 'spawn_identity_mismatch', cleanup: { + id: 'inv_1', nodeId: 'node_1', nodeName: 'desktop-1', name: 'worker_1', agentId: 'agent_1', + } }); + }); +}); + +describe('remote node-spawn port', () => { + it('confirms readiness, resolves exact identity, and targets release to the same node', async () => { + const dispatchOrder: string[] = []; + const placement = vi.fn() + .mockImplementationOnce(async () => { + dispatchOrder.push('placement.spawn'); + return { invocationId: 'ack_1', actionName: 'spawn', status: 'pending', + node: { id: 'node_1', name: 'desktop-1', status: 'online', capabilities: [] }, + placement: { node: 'desktop-1', confirmed: true, state: 'ready' }, + confirmation: { invocationId: 'inv_1', actionName: 'spawn', status: 'completed', + output: { spawned: true, ready: true, name: 'worker_1' } } }; + }) + .mockResolvedValueOnce({ invocationId: 'rel_1', actionName: 'release', status: 'pending', + placement: { node: 'desktop-1', confirmed: false, state: 'accepted' } }); + const getInvocation = vi.fn().mockResolvedValue({ invocationId: 'rel_1', actionName: 'release', + status: 'completed', output: { released: true, name: 'worker_1' } }); + const port = new RemoteNodeSpawnPort({ + agents: { get: async name => ({ id: 'agent_1', name }) }, + nodes: { list: async () => [] }, placement: { spawn: placement }, commands: { getInvocation }, + }); + const receipt = await port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', + idempotencyKey: 'dispatch_1234567890' }, async () => { + dispatchOrder.push('dispatch.fence'); + }); + await port.release({ receipt, idempotencyKey: 'release_1234567890' }); + expect(receipt).toMatchObject({ agentId: 'agent_1', nodeName: 'desktop-1', ready: true }); + expect(placement.mock.calls[0]![0]).toMatchObject({ confirm: true, verifyReady: true, + input: { idempotency_key: 'dispatch_1234567890' } }); + expect(placement.mock.calls[1]![0]).toMatchObject({ capability: 'release', actionName: 'release', + node: 'desktop-1', input: { name: 'worker_1', expected_agent_id: 'agent_1', + idempotency_key: 'release_1234567890' } }); + expect(getInvocation).toHaveBeenCalledWith('release', 'rel_1'); + expect(dispatchOrder).toEqual(['dispatch.fence', 'placement.spawn']); + }); + + it('returns exact cleanup identity when task delivery verification fails after spawn', async () => { + const port = new RemoteNodeSpawnPort({ + agents: { get: async name => ({ id: 'agent_1', name }) }, + nodes: { list: async () => [] }, + placement: { spawn: async () => ({ invocationId: 'ack_1', actionName: 'spawn', status: 'pending', + node: { id: 'node_1', name: 'desktop-1', status: 'online', capabilities: [] }, + placement: { node: 'desktop-1', confirmed: true, state: 'ready' }, + confirmation: { invocationId: 'inv_1', actionName: 'spawn', status: 'completed', + output: { spawned: true, ready: true, name: 'worker_1', warning: 'spawn_task_unconfirmed' } } }) }, + commands: { getInvocation: vi.fn() }, + }); + await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', + idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)).rejects.toMatchObject({ + code: 'spawn_task_unconfirmed', + cleanup: { id: 'inv_1', nodeId: 'node_1', nodeName: 'desktop-1', + name: 'worker_1', agentId: 'agent_1' }, + }); + }); + + it('retains cleanup identity from the real SDK placement error receipt', async () => { + const receipt = { invocationId: 'inv_1', actionName: 'spawn', status: 'completed', + dispatchedNodeId: 'node_1', output: { spawned: true, ready: true, + name: 'worker_1', agent_id: 'agent_1', warning: 'spawn_task_unconfirmed' } }; + const port = new RemoteNodeSpawnPort({ + agents: { get: async name => ({ id: 'agent_1', name }) }, + nodes: { list: async () => [] }, + placement: { spawn: async () => { throw new RelayPlacementError('spawn_failed', 'task unconfirmed', { + capability: 'spawn:codex', node: 'desktop-1', attempts: 1, + invocationId: 'inv_1', state: 'failed', dispatchState: 'dispatched', receipt, + }); } }, + commands: { getInvocation: vi.fn() }, + }); + await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', + idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)).rejects.toMatchObject({ + code: 'spawn_task_unconfirmed', + cleanup: { id: 'inv_1', nodeId: 'node_1', nodeName: 'desktop-1', + name: 'worker_1', agentId: 'agent_1' }, + }); + }); + + it('never substitutes a conflicting roster id for the handler cleanup identity', async () => { + const port = new RemoteNodeSpawnPort({ + agents: { get: async name => ({ id: 'agent_roster_replacement', name }) }, + nodes: { list: async () => [] }, + placement: { spawn: async () => ({ invocationId: 'ack_1', actionName: 'spawn', status: 'pending', + node: { id: 'node_1', name: 'desktop-1', status: 'online', capabilities: [] }, + placement: { node: 'desktop-1', confirmed: true, state: 'ready' }, + confirmation: { invocationId: 'inv_1', actionName: 'spawn', status: 'completed', + output: { spawned: true, ready: true, name: 'worker_1', agent_id: 'agent_spawned' } } }) }, + commands: { getInvocation: vi.fn() }, + }); + await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', + idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)).rejects.toMatchObject({ + code: 'spawn_identity_mismatch', + cleanup: { agentId: 'agent_spawned', name: 'worker_1', nodeName: 'desktop-1' }, + }); + }); + + it('retains the handler cleanup identity when the ready-agent roster is unavailable', async () => { + vi.useFakeTimers(); + try { + const port = new RemoteNodeSpawnPort({ + agents: { get: async () => { throw new Error('roster unavailable'); } }, + nodes: { list: async () => [] }, + placement: { spawn: async () => ({ invocationId: 'ack_1', actionName: 'spawn', status: 'pending', + node: { id: 'node_1', name: 'desktop-1', status: 'online', capabilities: [] }, + placement: { node: 'desktop-1', confirmed: true, state: 'ready' }, + confirmation: { invocationId: 'inv_1', actionName: 'spawn', status: 'completed', + output: { spawned: true, ready: true, name: 'worker_1', agent_id: 'agent_spawned' } } }) }, + commands: { getInvocation: vi.fn() }, + }); + const pending = port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', + idempotencyKey: 'dispatch_1234567890' }, beforeDispatch); + await vi.runAllTimersAsync(); + await expect(pending).rejects.toMatchObject({ + code: 'spawn_identity_unavailable', + cleanup: { id: 'inv_1', nodeId: 'node_1', nodeName: 'desktop-1', + name: 'worker_1', agentId: 'agent_spawned' }, + }); + } finally { + vi.useRealTimers(); + } + }); + + it('recovers the selected node name from its dispatch id when placement metadata lags', async () => { + const list = vi.fn(async () => [{ id: 'node_1', name: 'desktop-1', status: 'online', capabilities: [] }]); + const port = new RemoteNodeSpawnPort({ + agents: { get: async name => ({ id: 'agent_1', name }) }, + nodes: { list }, + placement: { spawn: async () => ({ invocationId: 'ack_1', actionName: 'spawn', status: 'pending', + placement: { confirmed: true, state: 'ready' }, + confirmation: { invocationId: 'inv_1', actionName: 'spawn', status: 'completed', + dispatchedNodeId: 'node_1', output: { spawned: true, ready: true, name: 'worker_1' } } }) }, + commands: { getInvocation: vi.fn() }, + }); + await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', + idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)).resolves.toMatchObject({ + nodeId: 'node_1', nodeName: 'desktop-1', agentId: 'agent_1', + }); + expect(list).toHaveBeenCalledOnce(); + }); + + it('refuses an otherwise ready spawn when placement was not confirmed', async () => { + const port = new RemoteNodeSpawnPort({ + agents: { get: async name => ({ id: 'agent_1', name }) }, + nodes: { list: async () => [] }, + placement: { spawn: async () => ({ invocationId: 'ack_1', actionName: 'spawn', status: 'pending', + node: { id: 'node_1', name: 'desktop-1', status: 'online', capabilities: [] }, + placement: { node: 'desktop-1', confirmed: false, state: 'accepted' }, + confirmation: { invocationId: 'inv_1', actionName: 'spawn', status: 'completed', + output: { spawned: true, ready: true, name: 'worker_1' } } }) }, + commands: { getInvocation: vi.fn() }, + }); + await expect(port.spawn({ name: 'worker_1', harness: 'codex', task: 'work', + idempotencyKey: 'dispatch_1234567890' }, beforeDispatch)).rejects.toMatchObject({ + code: 'spawn_placement_unconfirmed', + cleanup: expect.objectContaining({ agentId: 'agent_1', nodeName: 'desktop-1' }), + }); + }); +}); diff --git a/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts b/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts new file mode 100644 index 000000000..e49325282 --- /dev/null +++ b/packages/sdk/tests/node-spawn-response-loss-proxy.test.ts @@ -0,0 +1,549 @@ +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import net from 'node:net'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +interface ProxyStats { + methodCounts: Record; + spawnInvokeTaskDigests: string[]; + spawnInvokeKeys: string[]; + droppedSpawnReply?: { + invocationId?: string; + spawnIdentityDigest: string; + responseDigest: string; + responseBytes: number; + }; + droppedSpawnReplyForwardedBytes: number; +} + +interface ProofProxy { + stats: ProxyStats; + intercepted: Promise; + close(): Promise; +} + +interface ProxyModule { + createBridgeResponseLossProxy(options: { + listenSocket: string; + upstreamSocket: string; + onIntercept?: () => void; + }): Promise; + spawnIdentityDigest(identity: Record): string; + verifyResponseLossProxyEvidence(stats: ProxyStats): Record; +} + +const moduleUrl = new URL('../scripts/node-spawn-response-loss-proxy.mjs', import.meta.url).href; +const runnerModuleUrl = new URL('../scripts/node-spawn-response-loss-e2e.mjs', import.meta.url).href; +const auditModuleUrl = new URL('../scripts/node-spawn-response-loss-audit.mjs', import.meta.url).href; +const roots: string[] = []; +const servers: net.Server[] = []; +const proxies: ProofProxy[] = []; + +afterEach(async () => { + await Promise.all(proxies.splice(0).map(proxy => proxy.close())); + await Promise.all(servers.splice(0).map(server => new Promise(resolveClose => server.close(() => resolveClose())))); + roots.splice(0).forEach(root => rmSync(root, { recursive: true, force: true })); +}); + +function lineServer(socket: string) { + const keys = new Set(); + let upstreamSpawnFrames = 0; + const server = net.createServer(connection => { + let buffered = ''; + connection.on('data', chunk => { + buffered += String(chunk); + for (;;) { + const newline = buffered.indexOf('\n'); + if (newline < 0) return; + const line = buffered.slice(0, newline); + buffered = buffered.slice(newline + 1); + const request = JSON.parse(line) as { + id: number; + method: string; + params?: Record; + }; + if (request.method === 'spawn.invoke') { + const key = String(request.params?.['idempotency_key']); + if (!keys.has(key)) { + keys.add(key); + upstreamSpawnFrames += 1; + } + connection.write(`${JSON.stringify({ + jsonrpc: '2.0', id: request.id, + result: { + id: 'invocation-1', agent_id: 'agent-1', status: 'completed', + node_id: 'node-1', name: 'worker-1', spawned: true, ready: true, + released: false, task_unconfirmed: false, error_code: null, + }, + })}\n`); + } else { + connection.write(`${JSON.stringify({ jsonrpc: '2.0', id: request.id, result: { ok: true } })}\n`); + } + } + }); + }); + server.listen(socket); + servers.push(server); + return { server, upstreamSpawnFrames: () => upstreamSpawnFrames }; +} + +async function request(socket: string, value: unknown): Promise<{ bytes: number; reply?: unknown }> { + const client = net.createConnection(socket); + await once(client, 'connect'); + let buffered = ''; + let bytes = 0; + const reply = new Promise(resolveReply => { + client.on('data', chunk => { + bytes += chunk.length; + buffered += String(chunk); + const newline = buffered.indexOf('\n'); + if (newline >= 0) resolveReply(JSON.parse(buffered.slice(0, newline))); + }); + client.once('close', () => resolveReply(undefined)); + }); + client.write(`${JSON.stringify(value)}\n`); + const answer = await reply; + client.destroy(); + return { bytes, reply: answer }; +} + +async function run(command: string, args: string[], cwd: string) { + const child = spawn(command, args, { cwd, stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', chunk => { stdout += String(chunk); }); + child.stderr.on('data', chunk => { stderr += String(chunk); }); + const [code, signal] = await once(child, 'exit') as [number | null, NodeJS.Signals | null]; + return { code, signal, stdout, stderr }; +} + +function writeConsumerFixture(root: string, cliSource: string) { + const consumerRoot = join(root, 'consumer'); + const sdkRoot = join(consumerRoot, 'node_modules', '@relayflows', 'sdk'); + const bridgeRoot = join(consumerRoot, 'node_modules', '@agent-relay', 'core-bridge'); + const cliPath = join(sdkRoot, 'dist', 'cli.js'); + mkdirSync(join(sdkRoot, 'dist'), { recursive: true }); + mkdirSync(bridgeRoot, { recursive: true }); + chmodSync(consumerRoot, 0o700); + writeFileSync(join(consumerRoot, 'package.json'), JSON.stringify({ + name: 'response-loss-consumer', private: true, + })); + writeFileSync(join(consumerRoot, 'package-lock.json'), JSON.stringify({ + name: 'response-loss-consumer', lockfileVersion: 3, + })); + writeFileSync(join(sdkRoot, 'package.json'), JSON.stringify({ + name: '@relayflows/sdk', version: '2.0.66', type: 'module', + main: './dist/index.js', bin: { flows: './dist/cli.js' }, + })); + writeFileSync(join(sdkRoot, 'dist', 'index.js'), 'export {};\n'); + writeFileSync(cliPath, cliSource); + writeFileSync(join(bridgeRoot, 'package.json'), JSON.stringify({ + name: '@agent-relay/core-bridge', version: '18.1.0', type: 'module', main: './index.js', + })); + writeFileSync(join(bridgeRoot, 'index.js'), ` +import { readFile } from 'node:fs/promises'; +export async function readCallerDescriptor(path) { + const value = JSON.parse(await readFile(path, 'utf8')); + return { socket: value.socket }; +} +`); + return { consumerRoot, cliPath }; +} + +async function localEvidenceFixture() { + const proxy = await import(moduleUrl) as ProxyModule; + const identity = { + invocationId: 'invocation-1', nodeId: 'node-1', nodeName: 'desktop-1', + name: 'worker-1', agentId: 'agent-1', + }; + const key = 'a'.repeat(32); + return { + runId: 'run_fixture', + claims: [{ path: '/private/claim.json', claim: { + version: 3, + baseUrl: 'https://cast.agentrelay.com', publisherId: 'publisher-1', + publisherName: 'flows-e2e', runId: 'run_fixture', stepId: 'step-1', + idempotencyKey: key, agentName: 'worker-1', timeoutMs: 600000, + request: { cli: 'codex', task: 'task', task_identity: 'task' }, startedAt: 1, + spawn: { id: identity.invocationId, action: 'spawn', status: 'completed', + nodeId: identity.nodeId, nodeName: identity.nodeName, name: identity.name, + agentId: identity.agentId, spawned: true, ready: true }, + result: { inboxItemId: 'item-1', messageId: 'message-1', status: 'completed', + envelope: { version: 1, run_id: 'run_fixture', step_id: 'step-1', + dispatch_id: key, status: 'completed', output: { ok: true } }, + settlementAction: 'ack', receivedAt: 150, + settlement: { attempt: 1, completionReason: 'success', preparedAt: 175, + settledAt: 200, settledAs: 'ack' } }, + release: { invocationId: 'release-1', status: 'completed' }, + } }], + journalEntries: [{ seq: 10, segment_id: 1, entry_type: 'step.completed', + run_id: 'run_fixture', step_id: 'step-1', attempt: 1, at_ms: 100, + payload: { completionReason: 'success', trajectory_tail: { relay_task: { + task_execution: { run_id: 'run_fixture', step_id: 'step-1', dispatch_id: key, + delivery_id: 'item-1', message_id: 'message-1' }, + } } } }], + proxyStats: { + methodCounts: { 'spawn.invoke': 2, 'spawn.release': 1, 'spawn_callers.inbox.ack': 1 }, + spawnInvokeTaskDigests: ['task-digest', 'task-digest'], + spawnInvokeKeys: [key, key], + droppedSpawnReply: { invocationId: identity.invocationId, + spawnIdentityDigest: proxy.spawnIdentityDigest(identity), + responseDigest: 'response-digest', responseBytes: 64 }, + droppedSpawnReplyForwardedBytes: 0, + }, + processEvidence: { + initial: { pid: 101, code: null, signal: 'SIGKILL' }, + replays: [{ pid: 102, code: 0, signal: null }, { pid: 103, code: 3, signal: null }], + }, + }; +} + +describe('Core Bridge response-loss proof proxy', () => { + it('drops every byte of the committed reply and lets two same-key replays converge on one frame', async () => { + const root = mkdtempSync(join(tmpdir(), 'flows-response-loss-proxy-')); + roots.push(root); + const upstreamSocket = join(root, 'upstream.sock'); + const proxySocket = join(root, 'proxy.sock'); + const upstream = lineServer(upstreamSocket); + await once(upstream.server, 'listening'); + let intercepted = 0; + const module = await import(moduleUrl) as ProxyModule; + const proxy = await module.createBridgeResponseLossProxy({ + listenSocket: proxySocket, + upstreamSocket, + onIntercept: () => { intercepted += 1; }, + }); + proxies.push(proxy); + const spawn = { + jsonrpc: '2.0', id: 1, method: 'spawn.invoke', + params: { + idempotency_key: 'a'.repeat(32), task: 'exact original task bytes', + node_id: 'node-1', node_name: 'desktop-1', name: 'worker-1', + }, + }; + + const first = await request(proxySocket, spawn); + await proxy.intercepted; + expect(first).toEqual({ bytes: 0, reply: undefined }); + expect(intercepted).toBe(1); + + const [left, right] = await Promise.all([ + request(proxySocket, spawn), + request(proxySocket, spawn), + ]); + expect(left.reply).toMatchObject({ result: { id: 'invocation-1' } }); + expect(right.reply).toMatchObject({ result: { id: 'invocation-1' } }); + expect(upstream.upstreamSpawnFrames()).toBe(1); + + await request(proxySocket, { jsonrpc: '2.0', id: 2, method: 'spawn.release', params: {} }); + await request(proxySocket, { jsonrpc: '2.0', id: 3, method: 'spawn_callers.inbox.ack', params: {} }); + expect(module.verifyResponseLossProxyEvidence(proxy.stats)).toMatchObject({ + dropped_response_forwarded_bytes: 0, + spawn_invoke_calls: 3, + replay_attempts: 2, + unique_spawn_keys: 1, + release_calls: 1, + inbox_settlement_calls: 1, + }); + }); + + it('rejects changed task bytes even when the fixture reports a clean lifecycle', async () => { + const module = await import(moduleUrl) as ProxyModule; + const stats: ProxyStats = { + methodCounts: { 'spawn.invoke': 3, 'spawn.release': 1, 'spawn_callers.inbox.ack': 1 }, + spawnInvokeTaskDigests: ['one', 'two', 'two'], + spawnInvokeKeys: ['key', 'key', 'key'], + droppedSpawnReply: { spawnIdentityDigest: 'identity', responseDigest: 'reply', responseBytes: 12 }, + droppedSpawnReplyForwardedBytes: 0, + }; + expect(() => module.verifyResponseLossProxyEvidence(stats)).toThrow('original task bytes'); + }); + + it('fails closed when the final claim or journal ordering is not durable', async () => { + const audit = await import(auditModuleUrl) as { + verifyLocalResponseLossEvidence(input: Record): Record; + }; + const fixture = await localEvidenceFixture(); + expect(audit.verifyLocalResponseLossEvidence(fixture)).toMatchObject({ + schema: 'relayflows-node-spawn-response-loss-local/1', + final_claim: { result_present: true, release_status: 'completed', cleanup_pending: false }, + journal: { terminal_before_settlement: true }, + }); + const missingResult = structuredClone(fixture); + delete missingResult.claims[0].claim.result; + expect(() => audit.verifyLocalResponseLossEvidence(missingResult)) + .toThrow('retain its correlated result'); + const settledBeforeJournal = structuredClone(fixture); + settledBeforeJournal.claims[0].claim.result.settlement.settledAt = 50; + expect(() => audit.verifyLocalResponseLossEvidence(settledBeforeJournal)) + .toThrow('precede claim settlement'); + }); + + it('rejects ar_ credentials in captured proof logs', async () => { + const root = mkdtempSync(join(tmpdir(), 'flows-response-loss-secret-')); + roots.push(root); + const log = join(root, 'resume.log'); + writeFileSync(log, `unexpected credential ar_${'A'.repeat(24)}\n`, { mode: 0o600 }); + const runner = await import(runnerModuleUrl) as { + assertSecretFree(paths: string[]): Promise; + }; + await expect(runner.assertSecretFree([log])).rejects.toThrow('credential-shaped material'); + }); + + it('binds both commands to the exact SDK CLI resolved from consumerRoot', async () => { + const root = mkdtempSync(join(tmpdir(), 'flows-response-loss-consumer-')); + roots.push(root); + const { consumerRoot, cliPath } = writeConsumerFixture(root, 'export {};\n'); + const outside = join(root, 'outside.mjs'); + writeFileSync(outside, 'export {};\n'); + const runner = await import(runnerModuleUrl) as { + resolveConsumerArtifacts(root: string): Promise<{ + sdkCliPath: string; + evidence: Record; + }>; + assertConsumerCommand( + spec: { command: string; args: string[] }, + label: string, + artifacts: { sdkCliPath: string }, + ): Promise; + }; + const artifacts = await runner.resolveConsumerArtifacts(consumerRoot); + await expect(runner.assertConsumerCommand({ + command: process.execPath, args: [cliPath, 'run'], + }, 'start', artifacts)).resolves.toBeUndefined(); + await expect(runner.assertConsumerCommand({ + command: process.execPath, args: [outside, 'run'], + }, 'start', artifacts)).rejects.toThrow( + 'must launch the exact @relayflows/sdk CLI from consumerRoot', + ); + expect(artifacts.evidence).toMatchObject({ + consumer_lock_sha256: expect.stringMatching(/^[a-f0-9]{64}$/), + relayflows_sdk: { + version: '2.0.66', + tree_sha256: expect.stringMatching(/^[a-f0-9]{64}$/), + cli_sha256: expect.stringMatching(/^[a-f0-9]{64}$/), + }, + core_bridge: { + version: '18.1.0', + tree_sha256: expect.stringMatching(/^[a-f0-9]{64}$/), + }, + }); + }); + + it('uses the default timeout for killed-child and two-resume acceptance orchestration', async () => { + const root = mkdtempSync(join(tmpdir(), 'flows-response-loss-runner-')); + roots.push(root); + const upstreamSocket = join(root, 'upstream.sock'); + const upstream = lineServer(upstreamSocket); + await once(upstream.server, 'listening'); + const dataDir = join(root, 'data'); + const evidenceDir = join(root, 'evidence'); + const homeDir = join(root, 'home'); + const workDir = join(root, 'work'); + mkdirSync(workDir, { recursive: true }); + const callerFile = join(root, 'caller.json'); + writeFileSync(callerFile, `${JSON.stringify({ + schema: 'agent-relay-service-caller/2', + auth: { + scheme: 'challenge-hmac-sha256/v1', + challenge: 'spawn_callers.challenge', + authenticate: 'spawn_callers.authenticate', + }, + caller_id: 'sc_0123456789abcdef', + caller_name: 'flows_e2e', + secret: `rsc_${'A'.repeat(43)}`, + publisher: { id: 'publisher_1', name: 'flows_e2e' }, + workspace: { + site_url: 'https://agentrelay.com', + base_url: 'https://cast.agentrelay.com', + workspace_id: 'workspace_1', + relaycast_workspace_id: 'relaycast_1', + }, + account: { user_id: 'user_1' }, + created_at: '2026-11-01T00:00:00.000Z', + expires_at: '2026-12-01T00:00:00.000Z', + harnesses: ['codex'], + directory_scope: [workDir], + owns_only: true, + socket: upstreamSocket, + inbox: { + transport: 'core-bridge', + open: 'spawn_callers.inbox.open', + next: 'spawn_callers.inbox.next', + ack: 'spawn_callers.inbox.ack', + fail: 'spawn_callers.inbox.fail', + close: 'spawn_callers.inbox.close', + disconnect: 'spawn_callers.disconnect', + }, + desktop_version: 'fixture', + })}\n`, { mode: 0o600 }); + chmodSync(callerFile, 0o600); + + const childSource = ` +import { closeSync, mkdirSync, openSync, readFileSync, writeFileSync } from 'node:fs'; +import net from 'node:net'; +import { join } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; +const [mode, dataDir, winner] = process.argv.slice(2); +const runId = 'run_fixture'; +const stepId = 'step_fixture'; +const key = 'a'.repeat(32); +const caller = JSON.parse(readFileSync(process.env.RELAYFLOW_SERVICE_CALLER_FILE, 'utf8')); +const rpc = (method, id) => new Promise((resolve, reject) => { + const socket = net.createConnection(caller.socket); + let buffered = ''; + socket.once('error', reject); + socket.once('connect', () => socket.write(JSON.stringify({ + jsonrpc: '2.0', id, method, params: method === 'spawn.invoke' + ? { idempotency_key: key, task: 'exact original task bytes', + node_id: 'node-1', node_name: 'desktop-1', name: 'worker-1' } + : {}, + }) + '\\n')); + socket.on('data', chunk => { + buffered += String(chunk); + const newline = buffered.indexOf('\\n'); + if (newline < 0) return; + socket.destroy(); + resolve(JSON.parse(buffered.slice(0, newline))); + }); +}); +if (mode === 'start') { + mkdirSync(join(dataDir, 'runs'), { recursive: true }); + const db = new DatabaseSync(join(dataDir, 'runs', runId + '.sqlite3')); + db.exec(\` + CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID; + INSERT INTO meta VALUES ('run_id', 'run_fixture'), ('journal_version', '1'); + CREATE TABLE segments (segment_id INTEGER PRIMARY KEY, journal_version INTEGER NOT NULL, opened_seq INTEGER NOT NULL); + INSERT INTO segments VALUES (1, 1, 1); + CREATE TABLE entries (seq INTEGER PRIMARY KEY, segment_id INTEGER NOT NULL REFERENCES segments(segment_id), + entry_type TEXT NOT NULL, step_id TEXT, attempt INTEGER, at_ms INTEGER NOT NULL, payload TEXT NOT NULL); + \`); + db.close(); + await rpc('spawn.invoke', 1); + process.exit(99); +} +let won = false; +try { closeSync(openSync(winner, 'wx')); won = true; } catch (error) { + if (error.code !== 'EEXIST') throw error; +} +if (!won) process.exit(3); +await rpc('spawn.invoke', 1); +await rpc('spawn.release', 2); +await rpc('spawn_callers.inbox.ack', 3); +const journal = new DatabaseSync(join(dataDir, 'runs', runId + '.sqlite3')); +journal.prepare('INSERT INTO entries VALUES (?, ?, ?, ?, ?, ?, ?)').run( + 10, 1, 'step.completed', stepId, 1, 100, JSON.stringify({ + completionReason: 'success', + trajectory_tail: { relay_task: { task_execution: { + run_id: runId, step_id: stepId, dispatch_id: key, + delivery_id: 'item-1', message_id: 'message-1', + } } }, + }), +); +journal.close(); +const claim = { + version: 3, + baseUrl: 'https://cast.agentrelay.com', + publisherId: 'publisher-1', + publisherName: 'flows-e2e', + runId, + stepId, + idempotencyKey: key, + agentName: 'worker-1', + timeoutMs: 600000, + request: { cli: 'codex', task: 'exact original task bytes', task_identity: 'proof-task' }, + startedAt: 1, + spawn: { + id: 'invocation-1', action: 'spawn', status: 'completed', + nodeId: 'node-1', nodeName: 'desktop-1', name: 'worker-1', agentId: 'agent-1', + spawned: true, ready: true, + }, + result: { + inboxItemId: 'item-1', messageId: 'message-1', status: 'completed', + envelope: { version: 1, run_id: runId, step_id: stepId, dispatch_id: key, + status: 'completed', output: { ok: true } }, + settlementAction: 'ack', receivedAt: 150, + settlement: { attempt: 1, completionReason: 'success', preparedAt: 175, + settledAt: 200, settledAs: 'ack' }, + }, + release: { invocationId: 'release-1', status: 'completed' }, +}; +mkdirSync(join(dataDir, 'node-spawns'), { recursive: true }); +writeFileSync(join(dataDir, 'node-spawns', 'b'.repeat(64) + '.json'), JSON.stringify(claim), { mode: 0o600 }); +`; + const { consumerRoot, cliPath } = writeConsumerFixture(root, childSource); + const config = join(root, 'config.json'); + writeFileSync(config, `${JSON.stringify({ + callerFile, + consumerRoot, + dataDir, + evidenceDir, + homeDir, + workDir, + start: { command: process.execPath, args: [cliPath, 'start', dataDir, join(root, 'winner')] }, + resume: { command: process.execPath, args: [cliPath, 'resume', dataDir, join(root, 'winner'), '{RUN_ID}'] }, + })}\n`, { mode: 0o600 }); + + const result = await run(process.execPath, [ + new URL('../scripts/node-spawn-response-loss-e2e.mjs', import.meta.url).pathname, + '--config', config, + ], resolveRoot()); + expect(result, result.stderr).toMatchObject({ code: 0, signal: null }); + expect(JSON.parse(result.stdout)).toMatchObject({ ok: true, run_id: 'run_fixture' }); + const evidence = JSON.parse(readFileSync(join(evidenceDir, 'response-loss-proof.json'), 'utf8')); + expect(evidence).toMatchObject({ + schema: 'relayflows-node-spawn-response-loss-local/1', + dropped_response_forwarded_bytes: 0, + spawn_invoke_calls: 2, + replay_attempts: 1, + unique_spawn_keys: 1, + release_calls: 1, + inbox_settlement_calls: 1, + consumer_artifacts: { + consumer_lock_sha256: expect.stringMatching(/^[a-f0-9]{64}$/), + relayflows_sdk: { + version: '2.0.66', + tree_sha256: expect.stringMatching(/^[a-f0-9]{64}$/), + cli_sha256: expect.stringMatching(/^[a-f0-9]{64}$/), + }, + core_bridge: { + version: '18.1.0', + tree_sha256: expect.stringMatching(/^[a-f0-9]{64}$/), + }, + }, + final_claim: { + version: 3, + result_present: true, + release_status: 'completed', + cleanup_pending: false, + settlement_action: 'ack', + }, + journal: { terminal_sequence: 10, terminal_before_settlement: true }, + process_generations: { + initial_pid: expect.any(Number), + replay_pids: [expect.any(Number), expect.any(Number)], + }, + deferred_assertions: [ + 'stored_relaycast_contract_and_capability', + 'provider_frame_and_worker_counts', + 'target_capacity', + 'unrelated_reservation_preservation', + 'remote_fleet_lifecycle', + ], + }); + expect(evidence).not.toHaveProperty('unrelated_reservation'); + expect(new Set(evidence.process_generations.replay_pids).size).toBe(2); + expect(evidence.process_generations.replay_pids).not.toContain( + evidence.process_generations.initial_pid, + ); + expect(upstream.upstreamSpawnFrames()).toBe(1); + }, 20_000); +}); + +function resolveRoot() { + return new URL('../../..', import.meta.url).pathname; +} diff --git a/packages/sdk/tests/node-spawn-runtime.test.ts b/packages/sdk/tests/node-spawn-runtime.test.ts new file mode 100644 index 000000000..39655fc2c --- /dev/null +++ b/packages/sdk/tests/node-spawn-runtime.test.ts @@ -0,0 +1,507 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const relay = vi.hoisted(() => ({ + constructorOptions: [] as Array>, + disconnect: vi.fn(async () => {}), + me: vi.fn<() => Promise<{ id: string; name: string }>>(), +})); + +vi.mock('@agent-relay/sdk', () => ({ + RelaycastMessagingClient: class { + readonly agents = { me: relay.me, get: vi.fn() }; + readonly commands = { getInvocation: vi.fn() }; + readonly events = { disconnect: relay.disconnect }; + readonly inbox = { subscribe: vi.fn(), ack: vi.fn(), fail: vi.fn() }; + readonly nodes = { list: vi.fn() }; + readonly placement = { spawn: vi.fn() }; + + constructor(options: Record) { + relay.constructorOptions.push(options); + } + }, +})); + +import { createLocalNodeSpawnRuntime, createNodeSpawnRuntime, createNodeSpawnRuntimeOwner, createRemoteNodeSpawnRuntime, + type NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; +import type { CallerDescriptor, CoreBridgeApi, CoreBridgeClient } from '../src/node-spawn-local.js'; +import { waitForNodeResult } from '../src/node-spawn-result.js'; +import { runAgentRelayTask } from '../src/agent-relay-transport.js'; +import { hasReplayableFencedDispatch } from '../src/node-spawn-dispatch.js'; + +describe('remote node-spawn runtime', () => { + beforeEach(() => { + relay.constructorOptions.length = 0; + relay.disconnect.mockClear(); + relay.me.mockReset(); + }); + + it('rejects a non-HTTPS credential origin before constructing a Relay client', async () => { + await expect(createRemoteNodeSpawnRuntime({ + RELAY_AGENT_TOKEN: 'agent-token', + RELAY_BASE_URL: 'http://relay.example', + })).rejects.toMatchObject({ code: 'spawn_invalid_input' }); + expect(relay.constructorOptions).toHaveLength(0); + }); + + it('allows a literal loopback HTTP credential origin', async () => { + relay.me.mockResolvedValue({ id: 'publisher_1', name: 'publisher' }); + const runtime = await createRemoteNodeSpawnRuntime({ + RELAY_AGENT_TOKEN: 'agent-token', + RELAY_BASE_URL: 'http://127.0.0.1:8787', + }); + expect(runtime).toMatchObject({ + baseUrl: 'http://127.0.0.1:8787', + publisher: { id: 'publisher_1', name: 'publisher' }, + }); + expect(relay.constructorOptions).toEqual([{ + agentToken: 'agent-token', + baseUrl: 'http://127.0.0.1:8787', + }]); + await runtime.disconnect?.(); + expect(relay.disconnect).toHaveBeenCalledOnce(); + }); + + it('shares one publisher runtime across concurrent tasks and closes it only with its owner', async () => { + let resolve!: (runtime: NodeSpawnRuntime) => void; + const disconnect = vi.fn(async () => { throw new Error('disconnect failed'); }); + const runtime = { port: {} as NodeSpawnRuntime['port'], inbox: {} as NodeSpawnRuntime['inbox'], + publisher: { id: 'publisher_1', name: 'publisher' }, baseUrl: 'https://cast.agentrelay.com', + disconnect } satisfies NodeSpawnRuntime; + const factory = vi.fn((_signal: AbortSignal) => new Promise(done => { resolve = done; })); + const owner = createNodeSpawnRuntimeOwner(undefined, factory); + expect(owner.peek()).toBeUndefined(); + const cancelled = new AbortController(); + const first = owner.get(cancelled.signal); + const second = owner.get(); + cancelled.abort(new Error('first task lost its lease')); + await expect(first).rejects.toThrow('first task lost its lease'); + expect(factory).toHaveBeenCalledOnce(); + expect(factory.mock.calls[0]![0].aborted).toBe(false); + resolve(runtime); + await expect(second).resolves.toBe(runtime); + expect(owner.peek()).toBe(runtime); + await expect(owner.get()).resolves.toBe(runtime); + expect(disconnect).not.toHaveBeenCalled(); + await expect(owner.close()).resolves.toBeUndefined(); + expect(disconnect).toHaveBeenCalledOnce(); + }); + + it('shares one pending close across concurrent owner shutdown callers', async () => { + let releaseDisconnect!: () => void; + const disconnect = vi.fn(() => new Promise(resolve => { releaseDisconnect = resolve; })); + const runtime = { port: {} as NodeSpawnRuntime['port'], inbox: {} as NodeSpawnRuntime['inbox'], + publisher: { id: 'publisher_1', name: 'publisher' }, baseUrl: 'https://cast.agentrelay.com', + disconnect } satisfies NodeSpawnRuntime; + const owner = createNodeSpawnRuntimeOwner(undefined, async () => runtime); + await expect(owner.get()).resolves.toBe(runtime); + + let firstSettled = false; + let secondSettled = false; + const first = owner.close(); + const second = owner.close(); + void first.finally(() => { firstSettled = true; }); + void second.finally(() => { secondSettled = true; }); + + expect(first).toBe(second); + await vi.waitFor(() => expect(disconnect).toHaveBeenCalledOnce()); + expect(firstSettled).toBe(false); + expect(secondSettled).toBe(false); + + releaseDisconnect(); + await expect(Promise.all([first, second])).resolves.toEqual([undefined, undefined]); + }); + + it('honors cancellation during publisher discovery and cleans up the client', async () => { + relay.me.mockImplementation(() => new Promise(() => {})); + const controller = new AbortController(); + const pending = createRemoteNodeSpawnRuntime({ RELAY_AGENT_TOKEN: 'agent-token' }, { + signal: controller.signal, + }); + await vi.waitFor(() => expect(relay.me).toHaveBeenCalledOnce()); + controller.abort(new Error('lease lost')); + await expect(pending).rejects.toThrow('lease lost'); + expect(relay.disconnect).toHaveBeenCalledOnce(); + }); + + it('bounds publisher discovery and cleans up the client on timeout', async () => { + relay.me.mockImplementation(() => new Promise(() => {})); + const pending = createRemoteNodeSpawnRuntime({ RELAY_AGENT_TOKEN: 'agent-token' }); + await expect(pending).rejects.toMatchObject({ name: 'TimeoutError' }); + expect(relay.disconnect).toHaveBeenCalledOnce(); + }, 20_000); + + it('forwards the worker lease cancellation into publisher discovery', async () => { + relay.me.mockImplementation(() => new Promise(() => {})); + const controller = new AbortController(); + const pending = runAgentRelayTask({ + cli: 'codex', task: 'work', runId: 'run_1', stepId: 'step_1', + idempotencyKey: 'dispatch_1234567890', attempt: 1, dataDir: '/tmp/unused', + }, { agentToken: 'agent-token', signal: controller.signal }); + await vi.waitFor(() => expect(relay.me).toHaveBeenCalledOnce()); + controller.abort(new Error('lease lost')); + await expect(pending).rejects.toThrow('lease lost'); + expect(relay.disconnect).toHaveBeenCalledOnce(); + }); + + it('bounds a runtime provider that ignores cancellation by the task deadline', async () => { + const runtimeProvider = vi.fn((_signal?: AbortSignal) => new Promise(() => undefined)); + await expect(runAgentRelayTask({ + cli: 'codex', task: 'work', runId: 'run_1', stepId: 'step_1', + idempotencyKey: 'dispatch_1234567890', attempt: 1, dataDir: '/tmp/unused', timeoutMs: 100, + }, { runtimeProvider })).rejects.toMatchObject({ + cause: expect.objectContaining({ code: 'task_timeout' }), + }); + expect(runtimeProvider).toHaveBeenCalledOnce(); + expect(runtimeProvider.mock.calls[0]![0]?.aborted).toBe(true); + }); +}); + +describe('Desktop service-caller runtime', () => { + const descriptor: CallerDescriptor = { + schema: 'agent-relay-service-caller/2', + schemaVersion: 2, + auth: { type: 'challenge' }, + credential: { callerId: 'sc_0123456789abcdef' }, + callerId: 'sc_0123456789abcdef', + callerName: 'flows-project', + publisher: { id: 'agent_publisher', name: 'flows-project' }, + workspace: { + site_url: 'https://agentrelay.com', + base_url: 'https://cast.agentrelay.com', + workspace_id: 'workspace_1', + relaycast_workspace_id: 'relay_workspace_1', + }, + account: { user_id: 'user_1' }, + createdAt: '2026-11-08T00:00:00.000Z', + expiresAt: '2026-12-08T00:00:00.000Z', + harnesses: ['codex'], + directoryScope: ['/workspace'], + ownsOnly: true, + socket: '/tmp/core.sock', + inbox: { + transport: 'core-bridge', + open: 'caller.inbox.open', + next: 'caller.inbox.next', + ack: 'caller.inbox.ack', + fail: 'caller.inbox.fail', + close: 'caller.inbox.close', + disconnect: 'caller.disconnect', + }, + desktopVersion: '18.0.0', + }; + + function fixture() { + const clients: CoreBridgeClient[] = []; + const close = vi.fn(); + const bridge: CoreBridgeApi = { + connectCore: vi.fn(async () => { + const client = { + hello: { bridge_version: 2, daemon_version: 'test', pid: 1, methods: [], topics: [] }, + close, + } satisfies CoreBridgeClient; + clients.push(client); + return client; + }), + readCallerDescriptor: vi.fn(async () => descriptor), + authenticate: vi.fn(async () => ({ caller_id: descriptor.callerId })), + requireServiceCallerMethods: vi.fn(), + assertServiceCallerIdentity: vi.fn((caller, identity) => { + if (caller.publisher.id !== identity.publisher.id + || caller.publisher.name !== identity.publisher.name + || caller.workspace.base_url !== identity.base_url + || caller.workspace.workspace_id !== identity.workspace_id + || caller.workspace.relaycast_workspace_id !== identity.relaycast_workspace_id) { + throw Object.assign(new Error('identity mismatch'), { code: 'caller_identity_mismatch' }); + } + }), + serviceCallerIdentity: vi.fn(async () => ({ + publisher: descriptor.publisher, + node: { id: 'node_1', name: 'desktop-1' }, + base_url: descriptor.workspace.base_url, + workspace_id: descriptor.workspace.workspace_id, + relaycast_workspace_id: descriptor.workspace.relaycast_workspace_id, + })), + spawnTargets: vi.fn(async () => ({ targets: [] })), + spawnInvoke: vi.fn(), + spawnStatus: vi.fn(), + spawnRelease: vi.fn(), + serviceCallerInboxOpen: vi.fn(async () => ({ iterator_id: 'it_1' })), + serviceCallerInboxNext: vi.fn(async () => ({ delivery: { + item_id: 'item_1', message_id: 'message_1', invocation_id: 'inv_1', sent_at: 'now', + sender: { id: 'agent_1', name: 'worker_1' }, text: 'RELAYFLOWS_RESULT_V1 {}', + } })), + serviceCallerInboxAck: vi.fn(async () => ({})), + serviceCallerInboxFail: vi.fn(async () => ({})), + serviceCallerInboxClose: vi.fn(async () => ({})), + serviceCallerDisconnect: vi.fn(async () => ({})), + }; + return { bridge, clients, close }; + } + + it('refuses ambiguous local and remote authority before loading either runtime', async () => { + const { bridge } = fixture(); + await expect(createNodeSpawnRuntime({ + RELAYFLOW_SERVICE_CALLER_FILE: '/tmp/caller.json', + RELAY_AGENT_TOKEN: 'agent-token', + }, { bridge })).rejects.toMatchObject({ code: 'ambiguous_publisher_authority' }); + expect(bridge.readCallerDescriptor).not.toHaveBeenCalled(); + expect(relay.constructorOptions).toHaveLength(0); + }); + + it('passes the exact caller capability path without trimming it', async () => { + const { bridge } = fixture(); + const exactPath = '/tmp/caller.json '; + const runtime = await createNodeSpawnRuntime({ + RELAYFLOW_SERVICE_CALLER_FILE: exactPath, + }, { bridge }); + expect(bridge.readCallerDescriptor).toHaveBeenCalledWith(exactPath); + await runtime.disconnect?.(); + }); + + it('refuses whitespace-only caller authority instead of selecting the remote path', async () => { + const { bridge } = fixture(); + await expect(createNodeSpawnRuntime({ + RELAYFLOW_SERVICE_CALLER_FILE: ' ', + }, { bridge })).rejects.toMatchObject({ code: 'caller_file_invalid' }); + await expect(createNodeSpawnRuntime({ + RELAYFLOW_SERVICE_CALLER_FILE: ' ', + RELAY_AGENT_TOKEN: 'agent-token', + }, { bridge })).rejects.toMatchObject({ code: 'ambiguous_publisher_authority' }); + expect(bridge.readCallerDescriptor).not.toHaveBeenCalled(); + expect(relay.constructorOptions).toHaveLength(0); + }); + + it('does not scan for a Desktop caller when neither authority was explicit', async () => { + const { bridge } = fixture(); + await expect(createNodeSpawnRuntime({}, { bridge })) + .rejects.toMatchObject({ code: 'publisher_token_required' }); + expect(bridge.readCallerDescriptor).not.toHaveBeenCalled(); + expect(bridge.connectCore).not.toHaveBeenCalled(); + }); + + it('refuses a relative caller file before loading or connecting the optional bridge', async () => { + const { bridge } = fixture(); + await expect(createLocalNodeSpawnRuntime('caller.json', { bridge })) + .rejects.toMatchObject({ code: 'caller_file_invalid' }); + expect(bridge.readCallerDescriptor).not.toHaveBeenCalled(); + expect(bridge.connectCore).not.toHaveBeenCalled(); + }); + + it('binds schema-v2 authority and checks the live identity against caller.json', async () => { + const { bridge, close } = fixture(); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge }); + expect(bridge.readCallerDescriptor).toHaveBeenCalledWith('/tmp/caller.json'); + expect(bridge.readCallerDescriptor).toHaveBeenCalledOnce(); + expect(bridge.connectCore).toHaveBeenCalledWith({ socket: '/tmp/core.sock', signal: undefined }); + expect(bridge.requireServiceCallerMethods).toHaveBeenCalledOnce(); + expect(bridge.authenticate).toHaveBeenCalledWith(expect.anything(), descriptor.credential, { signal: undefined }); + expect(runtime).toMatchObject({ + publisher: descriptor.publisher, + baseUrl: descriptor.workspace.base_url, + node: 'desktop-1', + }); + await runtime.disconnect?.(); + expect(bridge.serviceCallerDisconnect).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + }); + + it('never grants fenced replay authority to an injected structural bridge', async () => { + const { bridge } = fixture(); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge }); + expect(hasReplayableFencedDispatch(runtime.port)).toBe(false); + runtime.port.status = vi.fn(runtime.port.status.bind(runtime.port)); + expect(hasReplayableFencedDispatch(runtime.port)).toBe(false); + await runtime.disconnect?.(); + }); + + it('fails closed when the live publisher or workspace differs from caller.json', async () => { + const { bridge, close } = fixture(); + vi.mocked(bridge.serviceCallerIdentity).mockResolvedValue({ + publisher: { ...descriptor.publisher, id: 'agent_other' }, + node: { id: 'node_1', name: 'desktop-1' }, + base_url: descriptor.workspace.base_url, + workspace_id: descriptor.workspace.workspace_id, + relaycast_workspace_id: descriptor.workspace.relaycast_workspace_id, + }); + await expect(createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge })) + .rejects.toMatchObject({ code: 'caller_identity_mismatch' }); + expect(close).toHaveBeenCalledOnce(); + }); + + it('fails closed when Desktop omits the exact live node identity', async () => { + const { bridge, close } = fixture(); + vi.mocked(bridge.serviceCallerIdentity).mockResolvedValue({ + publisher: descriptor.publisher, + node: { id: '', name: '' }, + base_url: descriptor.workspace.base_url, + workspace_id: descriptor.workspace.workspace_id, + relaycast_workspace_id: descriptor.workspace.relaycast_workspace_id, + }); + await expect(createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge })) + .rejects.toMatchObject({ code: 'caller_identity_mismatch' }); + expect(bridge.serviceCallerDisconnect).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + }); + + it('opens the exact invocation and settles after iterator teardown on a fresh binding', async () => { + const { bridge, clients, close } = fixture(); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge }); + const iterator = runtime.inbox.inbox.subscribe({ invocationId: 'inv_1' })[Symbol.asyncIterator](); + await expect(iterator.next()).resolves.toMatchObject({ + done: false, + value: { id: 'item_1', message: { id: 'message_1', from: { id: 'agent_1', name: 'worker_1' } } }, + }); + await iterator.return?.(); + await runtime.inbox.inbox.ack({ inboxItemId: 'item_1', metadata: { relayflows_result: 'journaled' } }); + expect(bridge.serviceCallerInboxOpen).toHaveBeenCalledWith(clients[1], 'inv_1', { signal: undefined }); + expect(bridge.serviceCallerInboxClose).toHaveBeenCalledWith(clients[1], 'it_1'); + expect(bridge.serviceCallerInboxAck).toHaveBeenCalledWith(clients[2], 'item_1'); + expect(bridge.connectCore).toHaveBeenCalledTimes(3); + expect(bridge.authenticate).toHaveBeenCalledTimes(3); + expect(bridge.serviceCallerDisconnect).toHaveBeenCalledTimes(2); + expect(close).toHaveBeenCalledTimes(2); + await runtime.disconnect?.(); + expect(bridge.serviceCallerDisconnect).toHaveBeenCalledTimes(3); + expect(close).toHaveBeenCalledTimes(3); + }); + + it('does not open the fresh settlement binding until the invocation iterator is closed', async () => { + const { bridge } = fixture(); + let finishClose!: () => void; + vi.mocked(bridge.serviceCallerInboxClose).mockImplementation( + () => new Promise(resolve => { finishClose = () => resolve({}); }), + ); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge }); + const delivery = await waitForNodeResult(runtime.inbox, { + agentId: 'agent_1', agentName: 'worker_1', invocationId: 'inv_1', + runId: 'run_1', stepId: 'step_1', dispatchId: 'dispatch_1', + }); + await vi.waitFor(() => expect(bridge.serviceCallerInboxClose).toHaveBeenCalledOnce()); + const settlement = delivery.ack(); + expect(bridge.connectCore).toHaveBeenCalledTimes(2); + finishClose(); + await settlement; + expect(bridge.connectCore).toHaveBeenCalledTimes(3); + }); + + it('keeps polling the same invocation after empty long polls without opening another binding', async () => { + const { bridge } = fixture(); + vi.mocked(bridge.serviceCallerInboxNext) + .mockResolvedValueOnce({ delivery: null }) + .mockResolvedValueOnce({ delivery: null }); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge }); + const iterator = runtime.inbox.inbox.subscribe({ invocationId: 'inv_1' })[Symbol.asyncIterator](); + await expect(iterator.next()).resolves.toMatchObject({ done: false, value: { id: 'item_1' } }); + expect(bridge.serviceCallerInboxOpen).toHaveBeenCalledOnce(); + expect(bridge.serviceCallerInboxNext).toHaveBeenCalledTimes(3); + await iterator.return?.(); + }); + + it('disconnects a late inbox binding without opening an iterator after close', async () => { + const { bridge, close } = fixture(); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge }); + let resolveBinding!: (client: CoreBridgeClient) => void; + vi.mocked(bridge.connectCore).mockImplementationOnce(() => new Promise(resolve => { + resolveBinding = resolve; + })); + const iterator = runtime.inbox.inbox.subscribe({ invocationId: 'inv_1' })[Symbol.asyncIterator](); + const next = iterator.next(); + await vi.waitFor(() => expect(bridge.connectCore).toHaveBeenCalledTimes(2)); + const closing = iterator.return!(); + resolveBinding({ + hello: { bridge_version: 2, daemon_version: 'test', pid: 1, methods: [], topics: [] }, + close, + }); + await expect(next).resolves.toEqual({ done: true, value: undefined }); + await expect(closing).resolves.toEqual({ done: true, value: undefined }); + expect(bridge.serviceCallerInboxOpen).not.toHaveBeenCalled(); + expect(bridge.serviceCallerInboxClose).not.toHaveBeenCalled(); + expect(bridge.serviceCallerDisconnect).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + }); + + it('closes an iterator whose uncooperative open resolves after close', async () => { + const { bridge, close } = fixture(); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge }); + let finishOpen!: () => void; + vi.mocked(bridge.serviceCallerInboxOpen).mockImplementationOnce( + () => new Promise(resolve => { finishOpen = () => resolve({ iterator_id: 'it_late' }); }), + ); + const iterator = runtime.inbox.inbox.subscribe({ invocationId: 'inv_1' })[Symbol.asyncIterator](); + const next = iterator.next(); + await vi.waitFor(() => expect(bridge.serviceCallerInboxOpen).toHaveBeenCalledOnce()); + const closing = iterator.return!(); + expect(bridge.serviceCallerInboxNext).not.toHaveBeenCalled(); + expect(bridge.serviceCallerDisconnect).not.toHaveBeenCalled(); + finishOpen(); + await expect(next).resolves.toEqual({ done: true, value: undefined }); + await expect(closing).resolves.toEqual({ done: true, value: undefined }); + expect(bridge.serviceCallerInboxNext).not.toHaveBeenCalled(); + expect(bridge.serviceCallerInboxClose).toHaveBeenCalledWith(expect.anything(), 'it_late'); + expect(bridge.serviceCallerDisconnect).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + }); + + it('disconnects an authenticated binding when inbox open rejects', async () => { + const { bridge, close } = fixture(); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge }); + const openFailure = new Error('open rejected'); + vi.mocked(bridge.serviceCallerInboxOpen).mockRejectedValueOnce(openFailure); + const iterator = runtime.inbox.inbox.subscribe({ invocationId: 'inv_1' })[Symbol.asyncIterator](); + await expect(iterator.next()).rejects.toBe(openFailure); + await expect(iterator.return!()).rejects.toBe(openFailure); + expect(bridge.serviceCallerInboxNext).not.toHaveBeenCalled(); + expect(bridge.serviceCallerInboxClose).not.toHaveBeenCalled(); + expect(bridge.serviceCallerDisconnect).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + }); + + it.each(['result_too_large', 'result_write_failed'] as const)( + 'preserves typed %s details with exact correlation and settles it as failed on a fresh binding', + async code => { + const { bridge, clients } = fixture(); + vi.mocked(bridge.serviceCallerInboxNext).mockResolvedValue({ delivery: { + item_id: 'item_failure', message_id: 'message_failure', invocation_id: 'inv_1', sent_at: 'now', + sender: { id: 'agent_1', name: 'worker_1' }, failure: { code, bytes: 1025, limit: 1024 }, + } }); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge }); + const delivery = await waitForNodeResult(runtime.inbox, { + agentId: 'agent_1', agentName: 'worker_1', invocationId: 'inv_1', + runId: 'run_1', stepId: 'step_1', dispatchId: 'dispatch_1', + }); + expect(delivery).toMatchObject({ + settlementAction: 'fail', + envelope: { + version: 1, run_id: 'run_1', step_id: 'step_1', dispatch_id: 'dispatch_1', + status: 'failed', error: `${code} (bytes=1025, limit=1024)`, + }, + }); + await delivery.ack(); + await vi.waitFor(() => expect(bridge.serviceCallerInboxClose).toHaveBeenCalledOnce()); + expect(bridge.serviceCallerInboxFail).toHaveBeenCalledWith(clients[2], 'item_failure'); + expect(bridge.connectCore).toHaveBeenCalledTimes(3); + }, + ); + + it('requires an invocation id before opening a local inbox connection', async () => { + const { bridge } = fixture(); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge }); + expect(() => runtime.inbox.inbox.subscribe({})).toThrow( + expect.objectContaining({ code: 'result_identity_missing' }), + ); + expect(bridge.connectCore).toHaveBeenCalledOnce(); + }); + + it('refuses a delivery outside the exact opened invocation', async () => { + const { bridge } = fixture(); + vi.mocked(bridge.serviceCallerInboxNext).mockResolvedValue({ delivery: { + item_id: 'item_1', message_id: 'message_1', invocation_id: 'inv_other', sent_at: 'now', + sender: { id: 'agent_1', name: 'worker_1' }, text: 'RELAYFLOWS_RESULT_V1 {}', + } }); + const runtime = await createLocalNodeSpawnRuntime('/tmp/caller.json', { bridge }); + const iterator = runtime.inbox.inbox.subscribe({ invocationId: 'inv_1' })[Symbol.asyncIterator](); + await expect(iterator.next()).rejects.toMatchObject({ code: 'result_identity_mismatch' }); + await iterator.return?.(); + expect(bridge.serviceCallerInboxClose).toHaveBeenCalledOnce(); + }); +}); diff --git a/packages/sdk/tests/node-spawn-settlement.test.ts b/packages/sdk/tests/node-spawn-settlement.test.ts new file mode 100644 index 000000000..4a8ea3e23 --- /dev/null +++ b/packages/sdk/tests/node-spawn-settlement.test.ts @@ -0,0 +1,851 @@ +import { EventEmitter } from 'node:events'; +import { mkdtemp, rename, rm, symlink } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { runAgentRelayTask, type AgentRelayTaskRequest } from '../src/agent-relay-transport.js'; +import type { JournalClient } from '../src/journal-client.js'; +import { NODE_RESULT_PREFIX, type NodeResultDelivery } from '../src/node-spawn-result.js'; +import { createNodeSpawnRuntimeOwner, type NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; +import { durableNodeResultDelivery, reconcileNodeResultSettlements } from '../src/node-spawn-settlement.js'; +import { readNodeSpawnClaims, recordNodeSpawn } from '../src/node-spawn-state.js'; +import { NodeSpawnError, type NodeSpawnPort, type NodeSpawnReceipt } from '../src/node-spawn.js'; +import type { StepDispatchEvent } from '../src/protocol.js'; +import { AgentWorker } from '../src/worker.js'; + +const request = (dataDir: string): AgentRelayTaskRequest => ({ + cli: 'codex', task: 'Implement the change', runId: 'run_1', stepId: 'edit', + idempotencyKey: 'dispatch_1234567890', attempt: 1, dataDir, timeoutMs: 10_000, +}); + +const roots: string[] = []; +afterEach(async () => { + await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); +}); + +async function dataDirectory(label: string): Promise { + const directory = await mkdtemp(join(tmpdir(), label)); + roots.push(directory); + return directory; +} + +async function waitForSettlementTimer(): Promise { + for (let attempt = 0; attempt < 500; attempt += 1) { + if (vi.getTimerCount() > 0) return; + await new Promise(resolve => { + const handle = setInterval(() => { clearInterval(handle); resolve(); }, 1); + }); + } + throw new Error('Settlement deadline was not armed'); +} + +function fixture(): { runtime: NodeSpawnRuntime; spawn: ReturnType; + release: ReturnType; ack: ReturnType; fail: ReturnType } { + const receipt: NodeSpawnReceipt = { + id: 'inv_spawn_1', action: 'spawn', status: 'completed', nodeId: 'node_1', nodeName: 'desktop-1', + name: 'placeholder', agentId: 'agent_1', spawned: true, ready: true, + }; + const spawn = vi.fn(async (input, beforeDispatch) => { + await beforeDispatch(); + return { ...receipt, name: input.name }; + }); + const release = vi.fn(async () => ({ id: 'inv_release_1', action: 'release' as const, + status: 'completed' as const, spawned: false, ready: false, released: true })); + const ack = vi.fn(async () => ({})); + const fail = vi.fn(async () => ({})); + const port: NodeSpawnPort = { + dispatchFenceVersion: 1, targets: async () => [], spawn, + status: vi.fn(async () => ({ ...receipt })), + release, + }; + const runtime: NodeSpawnRuntime = { + port, + publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + inbox: { inbox: { + subscribe: async function* () { + const name = (await spawn.mock.results[0]!.value).name; + yield { id: 'delivery_1', message: { id: 'message_1', + from: { id: 'agent_1', name }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_1', + step_id: 'edit', dispatch_id: 'dispatch_1234567890', status: 'completed', + output: { summary: 'done' } }) } }; + }, + ack, + fail, + } }, + }; + return { runtime, spawn, release, ack, fail }; +} + +function journaledRelayResult(): Record { + return { + completionReason: 'success', + trajectory_tail: { relay_task: { task_execution: { + run_id: 'run_1', step_id: 'edit', dispatch_id: 'dispatch_1234567890', + delivery_id: 'delivery_1', message_id: 'message_1', + } } }, + }; +} + +function terminalEntry(payload: Record = journaledRelayResult()) { + return { + seq: 2, segment_id: 0, entry_type: 'step.completed', run_id: 'run_1', + step_id: 'edit', attempt: 1, at_ms: Date.now(), payload, + }; +} + +describe('durable node-result settlement', () => { + it('carries the observed teardown barrier through durable settlement', async () => { + const dataDir = await dataDirectory('flows-node-settlement-teardown-barrier-'); + const f = fixture(); + const order: string[] = []; + let finishClose!: () => void; + const closeGate = new Promise(resolve => { finishClose = resolve; }); + let reportClosed!: (value: { failure?: unknown }) => void; + const settlementReady = new Promise<{ failure?: unknown }>(resolve => { reportClosed = resolve; }); + let delivered = false; + f.runtime.inbox.inbox.subscribe = () => ({ [Symbol.asyncIterator]() { + return { + async next() { + if (delivered) return new Promise>(() => undefined); + delivered = true; + const name = (await f.spawn.mock.results[0]!.value).name; + return { done: false as const, value: { + id: 'delivery_1', settlementReady, + message: { id: 'message_1', from: { id: 'agent_1', name }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_1', + step_id: 'edit', dispatch_id: 'dispatch_1234567890', status: 'completed', + output: { summary: 'done' } }) }, + } }; + }, + async return() { + order.push('inbox.close'); + await closeGate; + order.push('caller.disconnect'); + reportClosed({}); + return { done: true as const, value: undefined }; + }, + }; + } }); + f.ack.mockImplementation(async () => { order.push('settle'); return {}; }); + + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await vi.waitFor(() => expect(order).toEqual(['inbox.close'])); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + const settling = received.delivery.ack(); + await Promise.resolve(); + expect(f.ack).not.toHaveBeenCalled(); + finishClose(); + await settling; + expect(order).toEqual(['inbox.close', 'caller.disconnect', 'settle']); + }); + + it('bounds an observed teardown that never settles before opening settlement', async () => { + const dataDir = await dataDirectory('flows-node-settlement-teardown-timeout-'); + const f = fixture(); + let delivered = false; + let finishTeardown!: () => void; + const settlementReady = new Promise<{ failure?: unknown }>(resolve => { + finishTeardown = () => resolve({}); + }); + f.runtime.inbox.inbox.subscribe = () => ({ [Symbol.asyncIterator]() { + return { + async next() { + if (delivered) return new Promise>(() => undefined); + delivered = true; + const name = (await f.spawn.mock.results[0]!.value).name; + return { done: false as const, value: { + id: 'delivery_1', settlementReady, + message: { id: 'message_1', from: { id: 'agent_1', name }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_1', + step_id: 'edit', dispatch_id: 'dispatch_1234567890', status: 'completed', + output: { summary: 'done' } }) }, + } }; + }, + return: () => new Promise>(() => undefined), + }; + } }); + + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + try { + const settlement = expect(received.delivery.ack()) + .rejects.toMatchObject({ code: 'result_settlement_unconfirmed' }); + await waitForSettlementTimer(); + await vi.advanceTimersByTimeAsync(10_001); + await settlement; + expect(f.ack).not.toHaveBeenCalled(); + expect(f.fail).not.toHaveBeenCalled(); + const claims = await readNodeSpawnClaims(dataDir); + expect(claims[0]!.claim.result?.settlement).not.toHaveProperty('settledAt'); + finishTeardown(); + await Promise.resolve(); + } finally { + vi.useRealTimers(); + } + }); + + it('retains every live teardown registered for the same durable claim', async () => { + const dataDir = await dataDirectory('flows-node-settlement-multiple-teardowns-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + const [entry] = await readNodeSpawnClaims(dataDir); + let finishFirst!: () => void; + let finishSecond!: () => void; + const firstReady = new Promise<{ failure?: unknown }>(resolve => { finishFirst = () => resolve({}); }); + const secondReady = new Promise<{ failure?: unknown }>(resolve => { finishSecond = () => resolve({}); }); + const observed = (settlementReady: Promise<{ failure?: unknown }>): NodeResultDelivery => ({ + ...received.delivery, settlementReady, + }); + const delivery = durableNodeResultDelivery(f.runtime, entry!.path, entry!.claim, observed(firstReady)); + await delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + const settling = delivery.ack(); + durableNodeResultDelivery(f.runtime, entry!.path, entry!.claim, observed(secondReady)); + finishFirst(); + await Promise.resolve(); + expect(f.ack).not.toHaveBeenCalled(); + finishSecond(); + await Promise.resolve(); + await expect(settling).resolves.toBeUndefined(); + expect(f.ack).toHaveBeenCalledOnce(); + }); + + it.each([ + { label: 'cancelled', timeoutMs: 10_000 }, + { label: 'deadline-expired', timeoutMs: 50 }, + ])('skips a $label queued execution without opening another result iterator', async testCase => { + const dataDir = await dataDirectory('flows-node-settlement-concurrent-observation-'); + const aliasedDataDir = `${dataDir}-alias`; + await symlink(dataDir, aliasedDataDir, process.platform === 'win32' ? 'junction' : 'dir'); + roots.push(aliasedDataDir); + const f = fixture(); + let reportResult!: () => void; + const resultGate = new Promise(resolve => { reportResult = resolve; }); + const subscribe = vi.fn(async function* () { + await resultGate; + const name = (await f.spawn.mock.results[0]!.value).name; + yield { id: 'delivery_1', message: { id: 'message_1', + from: { id: 'agent_1', name }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_1', + step_id: 'edit', dispatch_id: 'dispatch_1234567890', status: 'completed', + output: { summary: 'done' } }) } }; + }); + f.runtime.inbox.inbox.subscribe = subscribe; + + const first = runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await vi.waitFor(() => expect(subscribe).toHaveBeenCalledOnce()); + const aliasedRequest = request(aliasedDataDir); + const cancelled = new AbortController(); + const cancellation = new Error('worker lease lost'); + const queuedRequest = { ...aliasedRequest, timeoutMs: testCase.timeoutMs }; + const second = runAgentRelayTask(queuedRequest, { runtime: f.runtime, + ...(testCase.label === 'cancelled' ? { signal: cancelled.signal } : {}) }); + const third = runAgentRelayTask(aliasedRequest, { runtime: f.runtime }); + await Promise.resolve(); + expect(subscribe).toHaveBeenCalledOnce(); + expect(f.spawn).toHaveBeenCalledOnce(); + if (testCase.label === 'cancelled') { + cancelled.abort(cancellation); + await expect(second).rejects.toBe(cancellation); + } else { + await expect(second).rejects.toMatchObject({ + message: 'The node-spawn task deadline elapsed', + cause: { code: 'task_timeout' }, + }); + } + expect(subscribe).toHaveBeenCalledOnce(); + expect(f.spawn).toHaveBeenCalledOnce(); + + reportResult(); + const [firstResult, thirdResult] = await Promise.all([first, third]); + expect(subscribe).toHaveBeenCalledOnce(); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).toHaveBeenCalledOnce(); + expect(thirdResult.delivery).toMatchObject({ + inboxItemId: firstResult.delivery.inboxItemId, + messageId: firstResult.delivery.messageId, + }); + await Promise.all([ + firstResult.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }), + thirdResult.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }), + ]); + await Promise.all([firstResult.delivery.ack(), thirdResult.delivery.ack()]); + expect(f.ack).toHaveBeenCalledOnce(); + const [settled] = await readNodeSpawnClaims(dataDir); + expect(settled!.claim.result?.settlement).toMatchObject({ + attempt: 1, completionReason: 'success', settledAs: 'ack', settledAt: expect.any(Number), + }); + await expect(thirdResult.delivery.prepareSettlement!({ + attempt: 2, completionReason: 'success', + })).rejects.toMatchObject({ code: 'result_identity_mismatch' }); + const [unchanged] = await readNodeSpawnClaims(dataDir); + expect(unchanged!.claim.result?.settlement).toEqual(settled!.claim.result?.settlement); + }); + + it('releases the process settlement lock when persisting settledAt fails', async () => { + const dataDir = await dataDirectory('flows-node-settlement-record-retry-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + let confirmAck!: () => void; + f.ack.mockImplementationOnce(() => new Promise(resolve => { confirmAck = resolve; })); + const attempt = received.delivery.ack(); + await vi.waitFor(() => expect(f.ack).toHaveBeenCalledOnce()); + const claimsDirectory = join(dataDir, 'node-spawns'); + const unavailableDirectory = `${claimsDirectory}.unavailable`; + await rename(claimsDirectory, unavailableDirectory); + let restored = false; + try { + confirmAck(); + await expect(attempt).rejects.toMatchObject({ code: 'ENOENT' }); + expect(f.ack).toHaveBeenCalledOnce(); + await rename(unavailableDirectory, claimsDirectory); + restored = true; + } finally { + if (!restored) await rename(unavailableDirectory, claimsDirectory).catch(() => undefined); + } + + await expect(received.delivery.ack()).resolves.toBeUndefined(); + expect(f.ack).toHaveBeenCalledTimes(2); + const [settled] = await readNodeSpawnClaims(dataDir); + expect(settled!.claim.result?.settlement).toMatchObject({ settledAs: 'ack' }); + }); + + it.each([ + { label: 'successful', disconnectFails: false }, + { label: 'failed', disconnectFails: true }, + ])('$label publisher disconnect leaves a separate live inbox teardown blocked', async testCase => { + const dataDir = await dataDirectory(`flows-node-settlement-owner-${testCase.label}-`); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + const [entry] = await readNodeSpawnClaims(dataDir); + let finishTeardown!: () => void; + const settlementReady = new Promise<{ failure?: unknown }>(resolve => { + finishTeardown = () => resolve({}); + }); + const delivery = durableNodeResultDelivery(f.runtime, entry!.path, entry!.claim, { + ...received.delivery, settlementReady, + }); + await delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + const disconnect = vi.fn(async () => { + if (testCase.disconnectFails) throw new Error('disconnect failed'); + }); + f.runtime.disconnect = disconnect; + const owner = createNodeSpawnRuntimeOwner(undefined, async () => f.runtime); + await owner.get(); + await owner.close(); + const pending = await reconcileNodeResultSettlements(dataDir, f.runtime, { + journalRead: vi.fn(async () => ({ entries: [terminalEntry()] })), + }); + expect(pending).toBe(1); + expect(f.ack).not.toHaveBeenCalled(); + expect(disconnect).toHaveBeenCalledOnce(); + + const freshRuntime = { ...f.runtime } satisfies NodeSpawnRuntime; + await expect(reconcileNodeResultSettlements(dataDir, freshRuntime, { + journalRead: vi.fn(async () => ({ entries: [terminalEntry()] })), + })).resolves.toBe(1); + expect(f.ack).not.toHaveBeenCalled(); + finishTeardown(); + await Promise.resolve(); + await expect(reconcileNodeResultSettlements(dataDir, freshRuntime, { + journalRead: vi.fn(async () => ({ entries: [terminalEntry()] })), + })).resolves.toBe(0); + expect(f.ack).toHaveBeenCalledOnce(); + }); + + it('rejects an invalid resolved spawn receipt before reading the inbox and releases exact identity', async () => { + const dataDir = await dataDirectory('flows-node-invalid-resolved-spawn-'); + const f = fixture(); + f.spawn.mockImplementationOnce(async (input, beforeDispatch) => { + await beforeDispatch(); + return { + id: 'inv_spawn_1', action: 'spawn', status: 'completed', nodeId: 'node_1', nodeName: 'desktop-1', + name: input.name, agentId: 'agent_1', spawned: true, ready: false, + } as never; + }); + f.runtime.inbox.inbox.subscribe = async function* () { throw new Error('inbox must not be read'); }; + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('valid ready identity'); + expect(f.release).toHaveBeenCalledOnce(); + }); + + it('refuses malformed cleanup identity from a custom port without issuing a release', async () => { + const dataDir = await dataDirectory('flows-node-malformed-cleanup-'); + const f = fixture(); + f.spawn.mockImplementationOnce(async (_input, beforeDispatch) => { + await beforeDispatch(); + throw new NodeSpawnError('custom port failed', + 'spawn_unconfirmed', undefined, { id: 'inv_spawn_1' } as never); + }); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('custom port failed'); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('ambiguous pre-ack'); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).not.toHaveBeenCalled(); + }); + + it('rejects a resolved but unconfirmed release and keeps the result unacked', async () => { + const dataDir = await dataDirectory('flows-node-cleanup-negative-receipt-'); + const f = fixture(); + f.release.mockResolvedValueOnce({ id: 'inv_release_1', action: 'release', status: 'failed', + spawned: true, ready: true, released: false, errorCode: 'release_refused' }); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toMatchObject({ cleanupPending: true }); + expect(f.ack).not.toHaveBeenCalled(); + const [claim] = await readNodeSpawnClaims(dataDir); + expect(claim!.claim.cleanupPending).toMatchObject({ attempts: 1, lastError: expect.any(String) }); + expect(claim!.claim.release).toBeUndefined(); + }); + + it('fails closed on an incomplete durable result instead of replaying or spawning', async () => { + const dataDir = await dataDirectory('flows-node-invalid-claim-'); + const f = fixture(); + await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + const [entry] = await readNodeSpawnClaims(dataDir); + const result = entry!.claim.result!; + const { envelope: _envelope, ...incomplete } = result; + await recordNodeSpawn(entry!.path, { ...entry!.claim, + result: incomplete as unknown as typeof result }); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('Invalid durable node-spawn claim'); + expect(f.spawn).toHaveBeenCalledOnce(); + }); + + it('fails closed on a corrupt durable spawn identity before attempting release', async () => { + const dataDir = await dataDirectory('flows-node-invalid-spawn-'); + const f = fixture(); + await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + const [entry] = await readNodeSpawnClaims(dataDir); + const spawn = entry!.claim.spawn!; + const { agentId: _agentId, ...incomplete } = spawn; + await recordNodeSpawn(entry!.path, { ...entry!.claim, + spawn: incomplete as unknown as typeof spawn, release: undefined }); + f.release.mockClear(); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('Invalid durable node-spawn claim'); + expect(f.release).not.toHaveBeenCalled(); + }); + + it('fails closed on an unconfirmed durable release before replaying a result', async () => { + const dataDir = await dataDirectory('flows-node-invalid-release-'); + const f = fixture(); + await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + const [entry] = await readNodeSpawnClaims(dataDir); + await recordNodeSpawn(entry!.path, { ...entry!.claim, + release: { ...entry!.claim.release!, status: 'failed' } }); + f.release.mockClear(); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toThrow('Invalid durable node-spawn claim'); + expect(f.release).not.toHaveBeenCalled(); + }); + + it('reconciles a journaled result left unacked by a crashed worker', async () => { + const dataDir = await dataDirectory('flows-node-settlement-crash-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + journalRead: vi.fn(async () => ({ entries: [terminalEntry()] })), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_recovery', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + await worker.attach(); + expect(f.ack).toHaveBeenCalledOnce(); + const claims = await readNodeSpawnClaims(dataDir); + expect(claims[0]!.claim.result?.settlement).toMatchObject({ settledAs: 'ack' }); + await worker.close(); + }); + + it('reconciles failure plus exact identity when a crash precedes cleanupPending', async () => { + const dataDir = await dataDirectory('flows-node-cleanup-only-crash-'); + const f = fixture(); + f.runtime.inbox.inbox.subscribe = async function* () { return; }; + f.release.mockRejectedValueOnce(new Error('release response lost during crash')); + await expect(runAgentRelayTask(request(dataDir), { runtime: f.runtime })) + .rejects.toMatchObject({ cleanupPending: true }); + const [pending] = await readNodeSpawnClaims(dataDir); + const { cleanupPending: _cleanupPending, ...crashedBeforeCleanupPending } = pending!.claim; + await recordNodeSpawn(pending!.path, crashedBeforeCleanupPending); + const [crashed] = await readNodeSpawnClaims(dataDir); + expect(crashed!.claim.cleanupPending).toBeUndefined(); + expect(crashed!.claim.failure).toBeDefined(); + expect(crashed!.claim.result).toBeUndefined(); + expect(crashed!.claim.release).toBeUndefined(); + + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + journalRead: vi.fn(async () => { + throw new Error('cleanup-only reconciliation must not read the journal'); + }), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_cleanup_recovery', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + await worker.attach(); + expect(f.release).toHaveBeenCalledTimes(2); + expect(client.journalRead).not.toHaveBeenCalled(); + const [reconciled] = await readNodeSpawnClaims(dataDir); + expect(reconciled!.claim.release).toEqual({ + invocationId: 'inv_release_1', status: 'completed', + }); + expect(reconciled!.claim.cleanupPending).toBeUndefined(); + expect(reconciled!.claim.result).toBeUndefined(); + await worker.close(); + }); + + it('releases an unreleased durable result before reading its terminal journal settlement', async () => { + const dataDir = await dataDirectory('flows-node-result-before-release-crash-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + const [persisted] = await readNodeSpawnClaims(dataDir); + const { release: _release, ...crashedBeforeRelease } = persisted!.claim; + await recordNodeSpawn(persisted!.path, crashedBeforeRelease); + + const order: string[] = []; + f.release.mockClear(); + f.release.mockImplementation(async () => { + order.push('release'); + return { id: 'inv_release_reconciled', action: 'release', status: 'completed', + spawned: false, ready: false, released: true }; + }); + f.ack.mockImplementation(async () => { order.push('ack'); return {}; }); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + journalRead: vi.fn(async () => { + order.push('journal'); + expect(order).toEqual(['release', 'journal']); + return { entries: [terminalEntry()] }; + }), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_result_release_recovery', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + await worker.attach(); + expect(order).toEqual(['release', 'journal', 'ack']); + const [reconciled] = await readNodeSpawnClaims(dataDir); + expect(reconciled!.claim.release).toEqual({ + invocationId: 'inv_release_reconciled', status: 'completed', + }); + expect(reconciled!.claim.result?.settlement).toMatchObject({ settledAs: 'ack' }); + await worker.close(); + }); + + it('does not release an active spawn without durable terminal evidence', async () => { + const dataDir = await dataDirectory('flows-node-active-spawn-no-cleanup-'); + const f = fixture(); + await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + const [persisted] = await readNodeSpawnClaims(dataDir); + const { result: _result, release: _release, failure: _failure, + cleanupPending: _cleanupPending, ...active } = persisted!.claim; + await recordNodeSpawn(persisted!.path, active); + f.release.mockClear(); + const journalRead = vi.fn(async () => { + throw new Error('active spawn reconciliation must not read the journal'); + }); + + await expect(reconcileNodeResultSettlements(dataDir, f.runtime, { journalRead })) + .resolves.toBe(0); + expect(f.release).not.toHaveBeenCalled(); + expect(journalRead).not.toHaveBeenCalled(); + }); + + it('skips a foreign pending settlement without blocking or rearming this runtime', async () => { + const dataDir = await dataDirectory('flows-node-settlement-foreign-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + const [own] = await readNodeSpawnClaims(dataDir); + const foreignPath = join(dataDir, 'node-spawns', `${'0'.repeat(64)}.json`); + await recordNodeSpawn(foreignPath, { + ...own!.claim, + publisherId: 'publisher_foreign', + publisherName: 'foreign-publisher', + }); + const journal = { journalRead: vi.fn(async () => ({ entries: [terminalEntry()] })) }; + + await expect(reconcileNodeResultSettlements(dataDir, f.runtime, journal)).resolves.toBe(0); + expect(f.ack).toHaveBeenCalledOnce(); + const afterOwn = await readNodeSpawnClaims(dataDir); + expect(afterOwn.find(entry => entry.claim.publisherId === 'publisher_1')! + .claim.result?.settlement).toMatchObject({ settledAs: 'ack' }); + expect(afterOwn.find(entry => entry.claim.publisherId === 'publisher_foreign')! + .claim.result?.settlement?.settledAt).toBeUndefined(); + + const foreignAck = vi.fn(async () => ({})); + const foreignRuntime: NodeSpawnRuntime = { + ...f.runtime, + publisher: { id: 'publisher_foreign', name: 'foreign-publisher' }, + inbox: { inbox: { ...f.runtime.inbox.inbox, ack: foreignAck } }, + }; + await expect(reconcileNodeResultSettlements(dataDir, foreignRuntime, journal)).resolves.toBe(0); + expect(foreignAck).toHaveBeenCalledOnce(); + }); + + it('finds a terminal relay result beyond the first journal page before acknowledging', async () => { + const dataDir = await dataDirectory('flows-node-settlement-paged-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + const filler = Array.from({ length: 256 }, (_, index) => ({ + seq: index + 1, segment_id: 0, entry_type: 'channel.appended', run_id: 'run_1', + step_id: null, attempt: null, at_ms: Date.now(), payload: {}, + })); + const journalRead = vi.fn(async (_runId: string, fromSeq: number, limit?: number) => ({ + entries: fromSeq === 1 ? filler.slice(0, limit) : [{ ...terminalEntry(), seq: 257 }], + })); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + journalRead, + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_recovery', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + await worker.attach(); + expect(journalRead).toHaveBeenNthCalledWith(1, 'run_1', 1, 256); + expect(journalRead).toHaveBeenNthCalledWith(2, 'run_1', 257, 256); + expect(f.ack).toHaveBeenCalledOnce(); + const claims = await readNodeSpawnClaims(dataDir); + expect(claims[0]!.claim.result?.settlement).toMatchObject({ settledAs: 'ack' }); + await worker.close(); + }); + + it('fails a delivery when the journal terminal belongs to different relay evidence', async () => { + const dataDir = await dataDirectory('flows-node-settlement-mismatch-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + const mismatch = { + completionReason: 'success', + trajectory_tail: { relay_task: { task_execution: { + delivery_id: 'different_delivery', message_id: 'different_message', + } } }, + }; + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + journalRead: vi.fn(async () => ({ entries: [terminalEntry(mismatch)] })), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_recovery', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + await worker.attach(); + expect(f.ack).not.toHaveBeenCalled(); + expect(f.fail).toHaveBeenCalledWith(expect.objectContaining({ inboxItemId: 'delivery_1', retry: false })); + const claims = await readNodeSpawnClaims(dataDir); + expect(claims[0]!.claim.result?.settlement).toMatchObject({ settledAs: 'fail' }); + await worker.close(); + }); + + it('dead-letters matching delivery ids attached to a different dispatch', async () => { + const dataDir = await dataDirectory('flows-node-settlement-dispatch-mismatch-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + const mismatch = journaledRelayResult(); + const trajectory = mismatch.trajectory_tail as { relay_task: { task_execution: Record } }; + trajectory.relay_task.task_execution.dispatch_id = 'dispatch_different1'; + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + journalRead: vi.fn(async () => ({ entries: [terminalEntry(mismatch)] })), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_recovery', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + await worker.attach(); + expect(f.ack).not.toHaveBeenCalled(); + expect(f.fail).toHaveBeenCalledOnce(); + const claims = await readNodeSpawnClaims(dataDir); + expect(claims[0]!.claim.result?.settlement).toMatchObject({ settledAs: 'fail' }); + await worker.close(); + }); + + it('retries an idempotent inbox settlement when ack preceded its durable record', async () => { + const dataDir = await dataDirectory('flows-node-settlement-record-crash-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + await received.delivery.ack(); + const [entry] = await readNodeSpawnClaims(dataDir); + const result = entry!.claim.result!; + await recordNodeSpawn(entry!.path, { ...entry!.claim, result: { ...result, + settlement: { ...result.settlement!, settledAt: undefined, settledAs: undefined } } }); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + journalRead: vi.fn(async () => ({ entries: [terminalEntry()] })), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_recovery', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + await worker.attach(); + expect(f.ack).toHaveBeenCalledTimes(2); + await worker.close(); + }); + + it('replays an unjournaled result on a later kernel attempt without another spawn', async () => { + const dataDir = await dataDirectory('flows-node-settlement-redispatch-'); + const f = fixture(); + const first = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await first.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + const replayed = await runAgentRelayTask({ ...request(dataDir), attempt: 2 }, { runtime: f.runtime }); + await replayed.delivery.prepareSettlement!({ attempt: 2, completionReason: 'success' }); + const claims = await readNodeSpawnClaims(dataDir); + expect(claims[0]!.claim.result?.settlement).toMatchObject({ attempt: 2, completionReason: 'success' }); + expect(f.spawn).toHaveBeenCalledOnce(); + expect(f.release).toHaveBeenCalledOnce(); + }); + + it('bounds a stalled inbox settlement and leaves its durable intent pending', async () => { + const dataDir = await dataDirectory('flows-node-settlement-stall-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + let confirmLate!: () => void; + let confirmRetry!: () => void; + f.ack + .mockImplementationOnce(() => new Promise(resolve => { confirmLate = resolve; })) + .mockImplementationOnce(() => new Promise(resolve => { confirmRetry = resolve; })); + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + try { + const pending = expect(received.delivery.ack()) + .rejects.toMatchObject({ code: 'result_settlement_unconfirmed' }); + await waitForSettlementTimer(); + expect(f.ack).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(10_001); + await pending; + const claims = await readNodeSpawnClaims(dataDir); + expect(claims[0]!.claim.result?.settlement).not.toHaveProperty('settledAt'); + await Promise.resolve(); + const retry = received.delivery.ack(); + await vi.waitFor(() => expect(f.ack).toHaveBeenCalledTimes(2)); + confirmLate(); + await Promise.resolve(); + await Promise.resolve(); + const afterLateSuccess = await readNodeSpawnClaims(dataDir); + expect(afterLateSuccess[0]!.claim.result?.settlement).not.toHaveProperty('settledAt'); + expect(() => durableNodeResultDelivery(f.runtime, claims[0]!.path, afterLateSuccess[0]!.claim, { + ...received.delivery, settlementReady: Promise.resolve({}), + })).toThrow('gained a live teardown after settlement started'); + confirmRetry(); + await retry; + const settled = await readNodeSpawnClaims(dataDir); + expect(settled[0]!.claim.result?.settlement).toMatchObject({ + settledAt: expect.any(Number), settledAs: 'ack', + }); + } finally { + vi.useRealTimers(); + } + }); + + it('retries after a timed-out inbox settlement that never resolves', async () => { + const dataDir = await dataDirectory('flows-node-settlement-permanent-stall-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + f.ack.mockImplementationOnce(() => new Promise(() => undefined)); + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + try { + const stalled = expect(received.delivery.ack()) + .rejects.toMatchObject({ code: 'result_settlement_unconfirmed' }); + await waitForSettlementTimer(); + await vi.advanceTimersByTimeAsync(10_001); + await stalled; + await Promise.resolve(); + await expect(received.delivery.ack()).resolves.toBeUndefined(); + expect(f.ack).toHaveBeenCalledTimes(2); + const settled = await readNodeSpawnClaims(dataDir); + expect(settled[0]!.claim.result?.settlement).toMatchObject({ + settledAt: expect.any(Number), settledAs: 'ack', + }); + } finally { + vi.useRealTimers(); + } + }); + + it('coalesces concurrent settlement callers onto one inbox operation', async () => { + const dataDir = await dataDirectory('flows-node-settlement-coalesce-'); + const f = fixture(); + const received = await runAgentRelayTask(request(dataDir), { runtime: f.runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + let confirm!: () => void; + f.ack.mockImplementationOnce(() => new Promise(resolve => { confirm = resolve; })); + const first = received.delivery.ack(); + const second = received.delivery.ack(); + expect(first).toBe(second); + await vi.waitFor(() => expect(f.ack).toHaveBeenCalledOnce()); + confirm(); + await expect(Promise.all([first, second])).resolves.toEqual([undefined, undefined]); + }); + + it('retries a rejected post-journal ack through the durable settlement outbox', async () => { + const dataDir = await dataDirectory('flows-node-settlement-reject-'); + const f = fixture(); + f.ack.mockRejectedValueOnce(new Error('temporary inbox outage')); + let completed = false; + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), + stepComplete: vi.fn(async () => { completed = true; return {}; }), + journalRead: vi.fn(async () => ({ entries: completed ? [terminalEntry()] : [] })), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_1', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + const errors: unknown[] = []; + worker.on('error', error => errors.push(error)); + await worker.attach(); + client.emit('step.dispatch', dispatch()); + await worker.close(); + expect(errors).toEqual([expect.objectContaining({ message: 'temporary inbox outage' })]); + expect(f.ack).toHaveBeenCalledTimes(2); + const claims = await readNodeSpawnClaims(dataDir); + expect(claims[0]!.claim.result?.settlement).toMatchObject({ settledAs: 'ack' }); + }); + + it('keeps retrying a journaled settlement after repeated inbox outages', async () => { + vi.useFakeTimers(); + try { + const dataDir = await dataDirectory('flows-node-settlement-repeat-'); + const f = fixture(); + f.ack.mockRejectedValueOnce(new Error('first inbox outage')) + .mockRejectedValueOnce(new Error('second inbox outage')); + let completed = false; + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), + stepComplete: vi.fn(async () => { completed = true; return {}; }), + journalRead: vi.fn(async () => ({ entries: completed ? [terminalEntry()] : [] })), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_1', pins: {}, dataDir, nodeSpawnRuntime: f.runtime, + }); + const errors: unknown[] = []; + worker.on('error', error => errors.push(error)); + await worker.attach(); + client.emit('step.dispatch', dispatch()); + await vi.waitFor(() => expect(f.ack).toHaveBeenCalledTimes(2)); + expect(errors).toHaveLength(2); + await vi.advanceTimersByTimeAsync(5_001); + await vi.waitFor(() => expect(f.ack).toHaveBeenCalledTimes(3)); + const claims = await readNodeSpawnClaims(dataDir); + expect(claims[0]!.claim.result?.settlement).toMatchObject({ settledAs: 'ack' }); + await worker.close(); + } finally { + vi.useRealTimers(); + } + }); +}); + +function dispatch(): StepDispatchEvent { + return { + run_id: 'run_1', step_id: 'edit', attempt: 1, step_type: 'agent', + spec: { cli: 'codex', instruction: 'Implement the change', transport: 'relay' }, + idempotency_key: 'dispatch_1234567890', lease_id: 'lease_1', + lease_deadline_ms: Date.now() + 30_000, pins: {}, + }; +} diff --git a/packages/sdk/tests/relay-cli-surface.test.ts b/packages/sdk/tests/relay-cli-surface.test.ts index 6f76038ff..9a5e9317d 100644 --- a/packages/sdk/tests/relay-cli-surface.test.ts +++ b/packages/sdk/tests/relay-cli-surface.test.ts @@ -142,14 +142,16 @@ const INVOCATIONS: readonly { verb: string; argv: readonly string[]; variant: Pa { verb: 'resume', argv: ['resume', RUN_ID], variant: 'resume' }, { verb: 'resume', - argv: ['resume', '--json', '--data-dir', '.relayflowd', '--local-agent', '--agent-capacity', '8', '--no-spawn', + argv: ['resume', '--json', '--data-dir', '.relayflowd', '--local-agent', '--agent-capacity', '8', + '--service-caller-file', '/tmp/caller.json', '--no-spawn', '--no-observer-link', '--cloud-mirror', '--detach', '--allow-human-influenced', RUN_ID], variant: 'resume', }, { verb: 'run', argv: ['run', 'flow.yaml'], variant: 'run' }, { verb: 'run', - argv: ['run', '--json', '--data-dir', '.relayflowd', '--local-agent', '--agent-capacity', '8', '--no-spawn', + argv: ['run', '--json', '--data-dir', '.relayflowd', '--local-agent', '--agent-capacity', '8', + '--service-caller-file', '/tmp/caller.json', '--no-spawn', '--no-observer-link', '--cloud-mirror', '--detach', '--allow-human-influenced', '--input', '{"a":1}', 'review.flow.ts'], variant: 'run', }, diff --git a/packages/sdk/tests/worker-node-runtime-sharing.test.ts b/packages/sdk/tests/worker-node-runtime-sharing.test.ts new file mode 100644 index 000000000..da6e361cc --- /dev/null +++ b/packages/sdk/tests/worker-node-runtime-sharing.test.ts @@ -0,0 +1,823 @@ +import { EventEmitter } from 'node:events'; +import { mkdir, mkdtemp, realpath, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { NODE_RESULT_PREFIX } from '../src/node-spawn-result.js'; +import { runAgentRelayTask } from '../src/agent-relay-transport.js'; +import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; +import type { NodeSpawnPort } from '../src/node-spawn.js'; +import { claimNodeSpawn, readNodeSpawnClaims, type NodeSpawnClaim } from '../src/node-spawn-state.js'; +import type { JournalClient } from '../src/journal-client.js'; +import type { StepDispatchEvent } from '../src/protocol.js'; +import { WorkerLeaseLostError } from '../src/worker-lease.js'; + +const owner = vi.hoisted(() => ({ get: vi.fn(), peek: vi.fn(), close: vi.fn(async () => {}) })); +vi.mock('../src/node-spawn-runtime.js', async importOriginal => ({ + ...await importOriginal(), + createNodeSpawnRuntimeOwner: () => ({ + get: async (signal?: AbortSignal) => { + const runtime = await owner.get(signal); + owner.peek.mockReturnValue(runtime); + return runtime; + }, + peek: () => owner.peek(), + close: () => owner.close(), + }), +})); + +import { AgentWorker } from '../src/worker.js'; + +const roots: string[] = []; +afterEach(async () => { + vi.unstubAllEnvs(); + owner.get.mockReset(); + owner.peek.mockReset(); + owner.close.mockClear(); + await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); +}); + +describe('AgentWorker node-spawn runtime ownership', () => { + it('does not acquire authority or block attach for a foreign pending claim', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-foreign-claim-')); + roots.push(dataDir); + await claimNodeSpawn(dataDir, pendingCleanupClaim( + 'publisher_foreign', 'foreign-publisher', 'dispatch_foreign_claim', 'agent_foreign')); + owner.get.mockRejectedValue(new Error('publisher_token_required')); + const journalRead = vi.fn(async () => { + throw new Error('foreign claim must not read this worker journal'); + }); + const workerAttach = vi.fn(async () => ({})); + const client = Object.assign(new EventEmitter(), { workerAttach, journalRead }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_foreign_claim', pins: {}, dataDir, + }); + + await expect(worker.attach()).resolves.toBeUndefined(); + await expect(worker.close()).resolves.toBeUndefined(); + expect(owner.peek).toHaveBeenCalled(); + expect(owner.get).not.toHaveBeenCalled(); + expect(journalRead).not.toHaveBeenCalled(); + expect(workerAttach).toHaveBeenCalledOnce(); + const [unchanged] = await readNodeSpawnClaims(dataDir); + expect(unchanged?.claim).toMatchObject({ + publisherId: 'publisher_foreign', cleanup: { agentId: 'agent_foreign' }, + }); + expect(unchanged?.claim.release).toBeUndefined(); + }); + + it('reconciles only matching old claims after a real dispatch establishes authority', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-lazy-authority-')); + roots.push(dataDir); + await claimNodeSpawn(dataDir, pendingCleanupClaim( + 'publisher_1', 'flows-publisher', 'dispatch_matching_old', 'agent_matching_old')); + await claimNodeSpawn(dataDir, pendingCleanupClaim( + 'publisher_foreign', 'foreign-publisher', 'dispatch_foreign_old', 'agent_foreign_old')); + let spawnedName = ''; + const release = vi.fn(async input => ({ + id: `release_${input.receipt.agentId}`, action: 'release', status: 'completed', + spawned: false, ready: false, released: true, + })); + const runtime: NodeSpawnRuntime = { + publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + port: { + dispatchFenceVersion: 1, + targets: async () => [], + spawn: async (input, beforeDispatch) => { + await beforeDispatch(); + spawnedName = input.name; + return { id: 'spawn_lazy_authority', action: 'spawn', status: 'completed', + nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, + agentId: 'agent_lazy_authority', spawned: true, ready: true }; + }, + status: vi.fn(), + release, + }, + inbox: { inbox: { + subscribe: async function* () { + yield { id: 'delivery_lazy_authority', message: { id: 'message_lazy_authority', + from: { id: 'agent_lazy_authority', name: spawnedName }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_parallel', + step_id: 'edit-lazy-authority', dispatch_id: 'dispatch_lazyauthority', + status: 'completed', output: { summary: 'done' } }) } }; + }, + ack: vi.fn(async () => ({})), + fail: vi.fn(async () => ({})), + } }, + }; + owner.get.mockResolvedValue(runtime); + const stepComplete = vi.fn(async () => + ({} as Awaited>)); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), + stepComplete, + journalRead: vi.fn(async () => ({ entries: [] })), + supportsFeature: vi.fn(() => true), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_lazy_authority', pins: {}, dataDir, + }); + const errors: unknown[] = []; + worker.on('error', error => errors.push(error)); + + await worker.attach(); + expect(owner.get).not.toHaveBeenCalled(); + client.emit('step.dispatch', dispatch('edit-lazy-authority', 'dispatch_lazyauthority')); + await worker.close(); + + expect(errors).toEqual([]); + expect(owner.get).toHaveBeenCalledOnce(); + expect(stepComplete).toHaveBeenCalledOnce(); + expect(release.mock.calls.map(call => call[0].receipt.agentId)).toContain('agent_matching_old'); + expect(release.mock.calls.map(call => call[0].receipt.agentId)).not.toContain('agent_foreign_old'); + const claims = await readNodeSpawnClaims(dataDir); + expect(claims.find(entry => entry.claim.publisherId === 'publisher_1' + && entry.claim.cleanup?.agentId === 'agent_matching_old')?.claim.release).toBeDefined(); + expect(claims.find(entry => entry.claim.publisherId === 'publisher_foreign')?.claim.release) + .toBeUndefined(); + }); + + it('validates an explicit Desktop caller before registering the worker', async () => { + const callerFailure = new Error('Desktop caller is expired'); + owner.get.mockRejectedValue(callerFailure); + const workerAttach = vi.fn(async () => ({})); + const client = Object.assign(new EventEmitter(), { workerAttach }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_invalid_desktop_caller', + pins: {}, + serviceCallerFile: '/tmp/caller.json', + }); + + await expect(worker.attach()).rejects.toBe(callerFailure); + expect(owner.get).toHaveBeenCalledOnce(); + expect(workerAttach).not.toHaveBeenCalled(); + expect(owner.close).toHaveBeenCalledOnce(); + }); + + it.each([ + { label: 'whitespace-only caller', environment: { RELAYFLOW_SERVICE_CALLER_FILE: ' ' } }, + { label: 'whitespace caller plus remote token', environment: { + RELAYFLOW_SERVICE_CALLER_FILE: ' ', RELAY_AGENT_TOKEN: 'agent-token', + } }, + ])('validates a $label before registering the worker', async ({ environment }) => { + const callerFailure = new Error('Desktop caller authority is invalid'); + owner.get.mockRejectedValue(callerFailure); + const workerAttach = vi.fn(async () => ({})); + const client = Object.assign(new EventEmitter(), { workerAttach }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_invalid_whitespace_caller', + pins: {}, + environment, + }); + + await expect(worker.attach()).rejects.toBe(callerFailure); + expect(owner.get).toHaveBeenCalledOnce(); + expect(workerAttach).not.toHaveBeenCalled(); + expect(owner.close).toHaveBeenCalledOnce(); + }); + + it.each([ + { label: 'Desktop caller', serviceCallerFile: '/tmp/caller.json', authority: undefined, + provideRuntime: false, expectedDirectory: 'run-root', expectedSpawn: true, expectedModel: 'codex-medium' }, + { label: 'Desktop caller with adapter default', serviceCallerFile: '/tmp/caller.json', authority: undefined, + provideRuntime: false, expectedDirectory: 'run-root', expectedSpawn: true, expectedModel: undefined, + omitModel: true }, + { label: 'provided Desktop runtime', serviceCallerFile: undefined, authority: 'desktop' as const, + provideRuntime: true, expectedDirectory: 'run-root', expectedSpawn: true, expectedModel: 'codex-medium' }, + { label: 'remote publisher', serviceCallerFile: undefined, authority: 'remote' as const, + provideRuntime: true, inheritedCaller: true, expectedDirectory: undefined, expectedSpawn: true, + expectedModel: 'codex-medium' }, + { label: 'inherited caller cleared by worker environment', serviceCallerFile: undefined, + authority: undefined, provideRuntime: false, inheritedCaller: true, environmentCaller: '', + expectedDirectory: undefined, expectedSpawn: true, expectedModel: 'codex-medium' }, + { label: 'Desktop caller with declared cwd', serviceCallerFile: '/tmp/caller.json', authority: undefined, + provideRuntime: false, cwd: 'checkout', expectedDirectory: 'declared', expectedSpawn: true, + expectedModel: 'codex-medium' }, + { label: 'remote publisher with declared cwd', serviceCallerFile: undefined, authority: 'remote' as const, + provideRuntime: true, cwd: 'checkout', expectedDirectory: undefined, expectedSpawn: false, + expectedModel: 'codex-medium' }, + ])('routes the run directory only for the $label', async testCase => { + const runRoot = await mkdtemp(join(tmpdir(), 'flows-node-worker-run-root-')); + roots.push(runRoot); + if (testCase.inheritedCaller) { + vi.stubEnv('RELAYFLOW_SERVICE_CALLER_FILE', '/tmp/inherited-caller.json'); + } + const declaredDirectory = join(runRoot, 'checkout'); + await mkdir(declaredDirectory); + const exactDeclaredDirectory = await realpath(declaredDirectory); + let spawnedName = ''; + const spawn = vi.fn(async (input, beforeDispatch) => { + await beforeDispatch(); + spawnedName = input.name; + return { id: 'spawn_worker_path', action: 'spawn', status: 'completed', + nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, agentId: 'agent_worker_path', + spawned: true, ready: true }; + }); + const runtime: NodeSpawnRuntime = { + publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + ...(testCase.authority === undefined ? {} : { authority: testCase.authority }), + port: { dispatchFenceVersion: 1, targets: async () => [], spawn, status: vi.fn(), + release: vi.fn(async () => ({ id: 'release_worker_path', action: 'release' as const, + status: 'completed' as const, spawned: false, ready: false, released: true })) }, + inbox: { inbox: { + subscribe: async function* () { + yield { id: 'delivery_worker_path', message: { id: 'message_worker_path', + from: { id: 'agent_worker_path', name: spawnedName }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_parallel', + step_id: 'edit-local-path', dispatch_id: 'dispatch_localpath', status: 'completed', + output: { summary: 'done' }, accounting: { tokens_input: 10, tokens_output: 5 } }) } }; + }, + ack: vi.fn(async () => ({})), fail: vi.fn(async () => ({})), + } }, + }; + owner.get.mockResolvedValue(runtime); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), + stepComplete: vi.fn(async () => ({} as Awaited< + ReturnType>)), + supportsFeature: vi.fn(() => true), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: `worker_${testCase.label}`, pins: {}, dataDir: join(runRoot, 'data'), runRoot, + ...(testCase.serviceCallerFile === undefined ? {} : { serviceCallerFile: testCase.serviceCallerFile }), + ...(testCase.provideRuntime ? { nodeSpawnRuntime: runtime } : {}), + ...(testCase.environmentCaller === undefined + ? {} + : { environment: { RELAYFLOW_SERVICE_CALLER_FILE: testCase.environmentCaller } }), + }); + await worker.attach(); + const value = dispatch('edit-local-path', 'dispatch_localpath'); + if ('omitModel' in testCase && testCase.omitModel) { + value.spec = { ...(value.spec as Record), cli: 'claude', cli_identity: 'claude' }; + delete (value.spec as { model?: string }).model; + } + if (testCase.cwd !== undefined) { + value.spec = { ...(value.spec as Record), cwd: testCase.cwd }; + } + client.emit('step.dispatch', value); + await worker.close(); + expect(spawn).toHaveBeenCalledTimes(testCase.expectedSpawn ? 1 : 0); + if (testCase.expectedSpawn) { + expect(spawn.mock.calls[0]![0].directory).toBe(testCase.expectedDirectory === 'run-root' + ? runRoot : testCase.expectedDirectory === 'declared' ? exactDeclaredDirectory : undefined); + expect(spawn.mock.calls[0]![0].model).toBe(testCase.expectedModel); + } + if ('omitModel' in testCase && testCase.omitModel) { + expect(client.stepComplete.mock.calls[0]?.[5]).toMatchObject({ + usage: { tokens_in: 10, tokens_out: 5, dollars_unmetered: true }, + }); + expect(client.stepComplete.mock.calls[0]?.[5]).not.toHaveProperty('reported_cost'); + } + }); + + it('does not reconcile a journaled delivery while its live iterator teardown is unresolved', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-live-teardown-reconcile-')); + roots.push(dataDir); + let finishTeardown!: () => void; + const settlementReady = new Promise<{ failure?: unknown }>(resolve => { + finishTeardown = () => resolve({}); + }); + let spawnedName = ''; + let delivered = false; + const ack = vi.fn(async () => ({})); + const fail = vi.fn(async () => ({})); + const runtime: NodeSpawnRuntime = { + authority: 'desktop', publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + port: { + dispatchFenceVersion: 1, + targets: async () => [], + spawn: async (input, beforeDispatch) => { + await beforeDispatch(); + spawnedName = input.name; + return { id: 'spawn_live_teardown', action: 'spawn', status: 'completed', + nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, + agentId: 'agent_live_teardown', spawned: true, ready: true }; + }, + status: vi.fn(), + release: async () => ({ id: 'release_live_teardown', action: 'release', + status: 'completed', spawned: false, ready: false, released: true }), + }, + inbox: { inbox: { + subscribe: () => ({ [Symbol.asyncIterator]() { return { + async next() { + if (delivered) return new Promise>(() => undefined); + delivered = true; + return { done: false as const, value: { + id: 'delivery_live_teardown', settlementReady, + message: { id: 'message_live_teardown', + from: { id: 'agent_live_teardown', name: spawnedName }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_parallel', + step_id: 'edit-live-teardown', dispatch_id: 'dispatch_liveteardown', + status: 'completed', output: { summary: 'done' } }) }, + } }; + }, + async return() { await settlementReady; return { done: true as const, value: undefined }; }, + }; } }), + ack, + fail, + } }, + }; + owner.get.mockResolvedValue(runtime); + let completed = false; + let terminalPayload: Record = {}; + const stepComplete = vi.fn(async ( + _runId, _stepId, _attempt, _idempotencyKey, completionReason, payload, + ) => { + completed = true; + terminalPayload = { ...payload, completionReason }; + return {} as Awaited>; + }); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), + stepComplete, + journalRead: vi.fn(async () => ({ entries: completed ? [{ + seq: 2, segment_id: 0, entry_type: 'step.completed', run_id: 'run_parallel', + step_id: 'edit-live-teardown', attempt: 1, at_ms: Date.now(), payload: terminalPayload, + }] : [] })), + supportsFeature: vi.fn(() => true), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_live_teardown', pins: {}, dataDir, nodeSpawnRuntime: runtime, + }); + const errors: unknown[] = []; + worker.on('error', error => errors.push(error)); + await worker.attach(); + vi.useFakeTimers(); + try { + client.emit('step.dispatch', dispatch('edit-live-teardown', 'dispatch_liveteardown')); + await vi.waitFor(() => expect(client.stepComplete).toHaveBeenCalledOnce()); + expect(ack).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(10_001); + await vi.waitFor(() => expect(errors).toHaveLength(1)); + const state = worker as unknown as { reconcileRelaySettlementsAfterDrain(): Promise }; + await state.reconcileRelaySettlementsAfterDrain(); + expect(ack).not.toHaveBeenCalled(); + expect(fail).not.toHaveBeenCalled(); + + finishTeardown(); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(5_001); + await vi.waitFor(() => expect(ack).toHaveBeenCalledOnce()); + expect(ack).toHaveBeenCalledOnce(); + expect(fail).not.toHaveBeenCalled(); + await worker.close(); + } finally { + vi.useRealTimers(); + } + }); + + it('closes the shared runtime even when final durable inbox settlement still fails', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-shared-node-runtime-settlement-failure-')); + roots.push(dataDir); + const ack = vi.fn(async () => { throw new Error('inbox unavailable'); }); + let spawnedName = ''; + const runtime: NodeSpawnRuntime = { + publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + port: { + dispatchFenceVersion: 1, + targets: async () => [], + spawn: async (input, beforeDispatch) => { + await beforeDispatch(); + spawnedName = input.name; + return { id: 'spawn_1', action: 'spawn', status: 'completed', + nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, agentId: 'agent_1', + spawned: true, ready: true }; + }, + status: vi.fn(), + release: async () => ({ id: 'release_1', action: 'release', status: 'completed', + spawned: false, ready: false, released: true }), + }, + inbox: { inbox: { + subscribe: async function* () { + yield { id: 'delivery_1', message: { id: 'message_1', + from: { id: 'agent_1', name: spawnedName }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: 'run_1', + step_id: 'edit', dispatch_id: 'dispatch_1234567890', status: 'completed', + output: { summary: 'done' } }) } }; + }, + ack, + fail: vi.fn(async () => ({})), + } }, + }; + const received = await runAgentRelayTask({ + cli: 'codex', task: 'Implement the change', runId: 'run_1', stepId: 'edit', + idempotencyKey: 'dispatch_1234567890', attempt: 1, dataDir, + }, { runtime }); + await received.delivery.prepareSettlement!({ attempt: 1, completionReason: 'success' }); + owner.get.mockResolvedValue(runtime); + owner.peek.mockReturnValue(runtime); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + journalRead: vi.fn(async () => ({ entries: [{ + seq: 2, segment_id: 0, entry_type: 'step.completed', run_id: 'run_1', + step_id: 'edit', attempt: 1, at_ms: Date.now(), payload: { + completionReason: 'success', trajectory_tail: { relay_task: { task_execution: { + run_id: 'run_1', step_id: 'edit', dispatch_id: 'dispatch_1234567890', + delivery_id: 'delivery_1', message_id: 'message_1', + } } }, + }, + }] })), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_recovery', pins: {}, dataDir, + }); + await expect(worker.attach()).rejects.toThrow('inbox unavailable'); + expect(owner.close).toHaveBeenCalledOnce(); + await expect(worker.close()).resolves.toBeUndefined(); + expect(ack).toHaveBeenCalledOnce(); + expect(owner.close).toHaveBeenCalledOnce(); + }); + + it('shares one runtime and one concurrent close promise through dispatch drain', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-shared-node-runtime-')); + roots.push(dataDir); + const dispatches = [dispatch('edit-a', 'dispatch_aaaaaaaaaa'), dispatch('edit-b', 'dispatch_bbbbbbbbbb')]; + const spawned: Array<{ name: string; agentId: string; key: string }> = []; + const order: string[] = []; + let bothSpawned!: () => void; + const ready = new Promise(resolve => { bothSpawned = resolve; }); + let bothJournalsStarted!: () => void; + const journalsStarted = new Promise(resolve => { bothJournalsStarted = resolve; }); + let allowJournals!: () => void; + const journalGate = new Promise(resolve => { allowJournals = resolve; }); + let ownerCloseStarted!: () => void; + const closeStarted = new Promise(resolve => { ownerCloseStarted = resolve; }); + let allowOwnerClose!: () => void; + const ownerCloseGate = new Promise(resolve => { allowOwnerClose = resolve; }); + const ack = vi.fn(async ({ inboxItemId }: { inboxItemId: string }) => { + order.push(`ack:${inboxItemId}`); + return {}; + }); + const spawn = vi.fn(async (input, beforeDispatch) => { + await beforeDispatch(); + const agentId = `agent_${spawned.length + 1}`; + spawned.push({ name: input.name, agentId, key: input.idempotencyKey }); + if (spawned.length === 2) bothSpawned(); + return { id: `spawn_${agentId}`, action: 'spawn', status: 'completed', + nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, agentId, + spawned: true, ready: true }; + }); + const release = vi.fn(async input => { + order.push(`release:${input.receipt.agentId}`); + return { id: `release_${input.receipt.agentId}`, action: 'release', status: 'completed', + spawned: false, ready: false, released: true }; + }); + const runtime: NodeSpawnRuntime = { + publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + port: { + dispatchFenceVersion: 1, + targets: async () => [], + spawn, + status: vi.fn(), + release, + }, + inbox: { inbox: { + subscribe: async function* () { + await ready; + for (const item of spawned) { + const expected = dispatches.find(value => value.idempotency_key === item.key)!; + yield { id: `delivery_${item.agentId}`, message: { id: `message_${item.agentId}`, + from: { id: item.agentId, name: item.name }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: expected.run_id, + step_id: expected.step_id, dispatch_id: expected.idempotency_key, + status: 'completed', output: { step: expected.step_id }, + accounting: { tokens_input: 1, tokens_output: 1 } }) } }; + } + }, + ack, + fail: vi.fn(async () => ({})), + } }, + }; + owner.get.mockResolvedValue(runtime); + owner.close.mockImplementation(async () => { + order.push('close'); + ownerCloseStarted(); + await ownerCloseGate; + }); + let journalCalls = 0; + const stepComplete = vi.fn(async (_runId, stepId) => { + order.push(`journal:${stepId}`); + journalCalls += 1; + if (journalCalls === dispatches.length) bothJournalsStarted(); + await journalGate; + return {} as Awaited>; + }); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), + stepComplete, + supportsFeature: vi.fn(() => true), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_1', pins: {}, capacity: 2, dataDir, + }); + const errors: unknown[] = []; + worker.on('error', error => errors.push(error)); + await worker.attach(); + for (const value of dispatches) client.emit('step.dispatch', value); + const firstClose = worker.close(); + const secondClose = worker.close(); + expect(secondClose).toBe(firstClose); + let firstSettled = false; + let secondSettled = false; + void firstClose.then(() => { firstSettled = true; }); + void secondClose.then(() => { secondSettled = true; }); + + await journalsStarted; + await Promise.resolve(); + expect([firstSettled, secondSettled]).toEqual([false, false]); + expect(owner.close).not.toHaveBeenCalled(); + + allowJournals(); + await closeStarted; + await Promise.resolve(); + expect(owner.close).toHaveBeenCalledOnce(); + expect([firstSettled, secondSettled]).toEqual([false, false]); + + allowOwnerClose(); + await Promise.all([firstClose, secondClose]); + expect([firstSettled, secondSettled]).toEqual([true, true]); + + expect(errors).toEqual([]); + expect(owner.get).toHaveBeenCalledTimes(2); + expect(spawn).toHaveBeenCalledTimes(2); + expect(stepComplete).toHaveBeenCalledTimes(2); + expect(stepComplete.mock.calls.map(call => [call[1], call[4], call[5]?.output])) + .toEqual(expect.arrayContaining([ + ['edit-a', 'success', { step: 'edit-a' }], + ['edit-b', 'success', { step: 'edit-b' }], + ])); + expect(ack).toHaveBeenCalledTimes(2); + expect(owner.close).toHaveBeenCalledOnce(); + for (const item of spawned) { + const stepId = dispatches.find(value => value.idempotency_key === item.key)!.step_id; + const releaseIndex = order.indexOf(`release:${item.agentId}`); + const journalIndex = order.indexOf(`journal:${stepId}`); + const ackIndex = order.indexOf(`ack:delivery_${item.agentId}`); + const closeIndex = order.indexOf('close'); + expect([releaseIndex, journalIndex, ackIndex, closeIndex].every(index => index >= 0)).toBe(true); + expect(releaseIndex).toBeLessThan(journalIndex); + expect(journalIndex).toBeLessThan(ackIndex); + expect(ackIndex).toBeLessThan(closeIndex); + } + expect(order.at(-1)).toBe('close'); + }); + + it('closes admission while a due settlement retry drains its current generation', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-shared-node-runtime-reconcile-barrier-')); + roots.push(dataDir); + const pending = dispatch('edit-after-drain', 'dispatch_afterdrain'); + let spawnedName = ''; + const spawn = vi.fn(async (input, beforeDispatch) => { + await beforeDispatch(); + spawnedName = input.name; + return { id: 'spawn_after_drain', action: 'spawn', status: 'completed', + nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, agentId: 'agent_after_drain', + spawned: true, ready: true }; + }); + const runtime: NodeSpawnRuntime = { + publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + port: { + dispatchFenceVersion: 1, + targets: async () => [], + spawn, + status: vi.fn(), + release: vi.fn(async () => ({ + id: 'release_after_drain', action: 'release', status: 'completed', + spawned: false, ready: false, released: true, + })), + }, + inbox: { inbox: { + subscribe: async function* () { + yield { id: 'delivery_after_drain', message: { id: 'message_after_drain', + from: { id: 'agent_after_drain', name: spawnedName }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: pending.run_id, + step_id: pending.step_id, dispatch_id: pending.idempotency_key, + status: 'completed', output: { step: pending.step_id } }) } }; + }, + ack: vi.fn(async () => ({})), + fail: vi.fn(async () => ({})), + } }, + }; + owner.get.mockResolvedValue(runtime); + const stepHeartbeat = vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat, + stepComplete: vi.fn(async () => ({})), + supportsFeature: vi.fn(() => true), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_reconcile_barrier', pins: {}, dataDir, + }); + const errors: unknown[] = []; + worker.on('error', error => errors.push(error)); + await worker.attach(); + + const state = worker as unknown as { + inFlight: Set>; + reconcileRelaySettlementsAfterDrain(): Promise; + }; + let finishCurrentGeneration!: () => void; + const currentGeneration = new Promise(resolve => { finishCurrentGeneration = resolve; }); + state.inFlight.add(currentGeneration); + const reconciliation = state.reconcileRelaySettlementsAfterDrain(); + client.emit('step.dispatch', pending); + await Promise.resolve(); + await Promise.resolve(); + expect(stepHeartbeat).not.toHaveBeenCalled(); + expect(spawn).not.toHaveBeenCalled(); + + const closing = worker.close(); + state.inFlight.delete(currentGeneration); + finishCurrentGeneration(); + await Promise.all([reconciliation, closing]); + expect(errors).toEqual([]); + expect(stepHeartbeat).toHaveBeenCalledOnce(); + expect(spawn).toHaveBeenCalledOnce(); + }); + + it('drains a six-task mixed-order fan with one lease cancellation before closing the shared runtime', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-shared-node-runtime-fan-')); + roots.push(dataDir); + const cancelledIndex = 2; + const dispatches = Array.from({ length: 6 }, (_, index) => dispatch( + `edit-${index}`, + `dispatch_${String(index).padStart(10, '0')}`, + index === cancelledIndex ? 300 : 30_000, + )); + const spawned: Array<{ name: string; agentId: string; key: string }> = []; + const order: string[] = []; + let allSpawned!: () => void; + const spawnedGate = new Promise(resolve => { allSpawned = resolve; }); + let allowResults!: () => void; + const resultsGate = new Promise(resolve => { allowResults = resolve; }); + const completionOrder = [5, 1, 4, 0, 3]; + const ack = vi.fn(async ({ inboxItemId }: { inboxItemId: string }) => { + order.push(`ack:${inboxItemId}`); + return {}; + }); + const spawn = vi.fn(async (input, beforeDispatch) => { + await beforeDispatch(); + const agentId = `agent_${spawned.length}`; + spawned.push({ name: input.name, agentId, key: input.idempotencyKey }); + if (spawned.length === dispatches.length) allSpawned(); + return { id: `spawn_${agentId}`, action: 'spawn', status: 'completed', + nodeId: 'node_1', nodeName: 'desktop-1', name: input.name, agentId, + spawned: true, ready: true }; + }); + const release = vi.fn(async input => { + order.push(`release:${input.receipt.agentId}`); + return { id: `release_${input.receipt.agentId}`, action: 'release', status: 'completed', + spawned: false, ready: false, released: true }; + }); + const runtime: NodeSpawnRuntime = { + publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + port: { dispatchFenceVersion: 1, targets: async () => [], spawn, status: vi.fn(), release }, + inbox: { inbox: { + subscribe: async function* () { + await spawnedGate; + await resultsGate; + for (const index of completionOrder) { + const expected = dispatches[index]!; + const item = spawned.find(value => value.key === expected.idempotency_key)!; + yield { id: `delivery_${item.agentId}`, message: { id: `message_${item.agentId}`, + from: { id: item.agentId, name: item.name }, + text: NODE_RESULT_PREFIX + JSON.stringify({ version: 1, run_id: expected.run_id, + step_id: expected.step_id, dispatch_id: expected.idempotency_key, + status: 'completed', output: { step: expected.step_id }, + accounting: { tokens_input: 1, tokens_output: 1 } }) } }; + } + await new Promise(() => {}); + }, + ack, + fail: vi.fn(async () => ({})), + } }, + }; + owner.get.mockResolvedValue(runtime); + owner.close.mockImplementation(async () => { order.push('close'); }); + const heartbeatCalls = new Map(); + const stepHeartbeat = vi.fn(async (_runId, stepId) => { + const count = (heartbeatCalls.get(stepId) ?? 0) + 1; + heartbeatCalls.set(stepId, count); + if (stepId === dispatches[cancelledIndex]!.step_id && count > 1) { + throw new WorkerLeaseLostError('renewal_expired', 'review cancellation'); + } + const ttl = stepId === dispatches[cancelledIndex]!.step_id ? 300 : 30_000; + return { lease_deadline_ms: Date.now() + ttl, lease_ttl_ms: ttl }; + }); + const stepComplete = vi.fn(async (_runId, stepId) => { + order.push(`journal:${stepId}`); + return {} as Awaited>; + }); + const client = Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat, + stepComplete, + supportsFeature: vi.fn(() => true), + }); + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker_fan', pins: {}, capacity: dispatches.length, dataDir, + }); + const errors: Array<{ error: unknown; stepId: string }> = []; + worker.on('error', (error, value: StepDispatchEvent) => { + errors.push({ error, stepId: value.step_id }); + order.push(`error:${value.step_id}`); + }); + await worker.attach(); + for (const value of dispatches) client.emit('step.dispatch', value); + await spawnedGate; + + const firstClose = worker.close(); + const secondClose = worker.close(); + expect(secondClose).toBe(firstClose); + let closed = false; + void firstClose.then(() => { closed = true; }); + await Promise.resolve(); + expect(closed).toBe(false); + expect(owner.close).not.toHaveBeenCalled(); + + allowResults(); + await firstClose; + + expect(errors).toHaveLength(1); + expect(errors[0]).toMatchObject({ + error: { name: 'WorkerLeaseLostError', reason: 'renewal_expired' }, + stepId: dispatches[cancelledIndex]!.step_id, + }); + expect(owner.get).toHaveBeenCalledTimes(dispatches.length); + expect(spawn).toHaveBeenCalledTimes(dispatches.length); + expect(release).toHaveBeenCalledTimes(dispatches.length); + expect(stepComplete).toHaveBeenCalledTimes(dispatches.length - 1); + expect(ack).toHaveBeenCalledTimes(dispatches.length - 1); + expect(owner.close).toHaveBeenCalledOnce(); + for (const index of completionOrder) { + const expected = dispatches[index]!; + const item = spawned.find(value => value.key === expected.idempotency_key)!; + const stepId = expected.step_id; + const releaseIndex = order.indexOf(`release:${item.agentId}`); + const journalIndex = order.indexOf(`journal:${stepId}`); + const ackIndex = order.indexOf(`ack:delivery_${item.agentId}`); + const closeIndex = order.indexOf('close'); + expect(releaseIndex).toBeLessThan(journalIndex); + expect(journalIndex).toBeLessThan(ackIndex); + expect(ackIndex).toBeLessThan(closeIndex); + } + const cancelled = spawned.find(value => value.key === dispatches[cancelledIndex]!.idempotency_key)!; + expect(order.indexOf(`release:${cancelled.agentId}`)) + .toBeLessThan(order.indexOf(`error:${dispatches[cancelledIndex]!.step_id}`)); + expect(order.indexOf(`error:${dispatches[cancelledIndex]!.step_id}`)) + .toBeLessThan(order.indexOf('close')); + expect(order.at(-1)).toBe('close'); + }); +}); + +function dispatch(stepId: string, idempotencyKey: string, leaseTtlMs?: number): StepDispatchEvent { + return { + run_id: 'run_parallel', step_id: stepId, attempt: 1, step_type: 'agent', + spec: { cli: 'codex', model: 'codex-medium', instruction: `Implement ${stepId}`, transport: 'relay' }, + idempotency_key: idempotencyKey, lease_id: `lease_${stepId}`, + lease_deadline_ms: Date.now() + (leaseTtlMs ?? 30_000), + ...(leaseTtlMs === undefined ? {} : { lease_ttl_ms: leaseTtlMs }), + pins: {}, + }; +} + +function pendingCleanupClaim( + publisherId: string, + publisherName: string, + idempotencyKey: string, + agentId: string, +): NodeSpawnClaim { + return { + version: 3, + baseUrl: 'https://cast.agentrelay.com', + publisherId, + publisherName, + runId: 'run_parallel', + stepId: `step_${agentId}`, + idempotencyKey, + agentName: `relayflow-${agentId}`, + timeoutMs: 30_000, + request: { cli: 'codex', task: 'Recover the prior task', task_identity: `identity_${agentId}` }, + startedAt: Date.now(), + cleanup: { + id: `spawn_${agentId}`, + nodeId: 'node_1', + nodeName: 'desktop-1', + name: `relayflow-${agentId}`, + agentId, + }, + failure: { code: 'result_timeout', message: 'worker crashed before cleanup' }, + }; +} diff --git a/packages/sdk/tests/worker-node-runtime-timeout.test.ts b/packages/sdk/tests/worker-node-runtime-timeout.test.ts new file mode 100644 index 000000000..c715e94f5 --- /dev/null +++ b/packages/sdk/tests/worker-node-runtime-timeout.test.ts @@ -0,0 +1,214 @@ +import { EventEmitter } from 'node:events'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { JournalClient } from '../src/journal-client.js'; +import type { NodeSpawnRuntime } from '../src/node-spawn-runtime.js'; +import { readNodeSpawnClaims } from '../src/node-spawn-state.js'; +import type { NodeSpawnPort, NodeSpawnReceipt } from '../src/node-spawn.js'; +import type { StepDispatchEvent } from '../src/protocol.js'; +import { AgentWorker } from '../src/worker.js'; + +const roots: string[] = []; +afterEach(async () => { + await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); +}); + +function dispatch(stepId: string, idempotencyKey: string, timeoutMs: number): StepDispatchEvent { + return { + run_id: 'run_timeout', step_id: stepId, attempt: 1, step_type: 'agent', + spec: { cli: 'codex', instruction: 'Implement the change', transport: 'relay', + timeout_ms: timeoutMs }, + idempotency_key: idempotencyKey, lease_id: `lease_${stepId}`, + lease_deadline_ms: Date.now() + 30_000, pins: {}, + }; +} + +function client(stepComplete: JournalClient['stepComplete']): JournalClient & EventEmitter { + return Object.assign(new EventEmitter(), { + workerAttach: vi.fn(async () => ({})), + stepHeartbeat: vi.fn(async () => ({ lease_deadline_ms: Date.now() + 30_000 })), + stepComplete, + supportsFeature: vi.fn(() => true), + }) as unknown as JournalClient & EventEmitter; +} + +function runtime(spawn: NodeSpawnPort['spawn']): NodeSpawnRuntime { + return { + authority: 'remote', publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + port: { dispatchFenceVersion: 1, targets: async () => [], spawn, status: vi.fn(), release: vi.fn() }, + inbox: { inbox: { + subscribe: async function* () { return; }, + ack: vi.fn(async () => ({})), fail: vi.fn(async () => ({})), + } }, + }; +} + +async function waitForImmediate(assertion: () => void | Promise): Promise { + const deadline = process.hrtime.bigint() + 10_000_000_000n; + let failure: unknown; + while (process.hrtime.bigint() < deadline) { + try { + await assertion(); + return; + } catch (error) { + failure = error; + await new Promise(resolve => setImmediate(resolve)); + } + } + throw failure; +} + +describe('AgentWorker relay deadlines', () => { + it('carries an admitted journal timeout into the native node-spawn deadline', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-worker-timeout-')); + roots.push(dataDir); + const declaredTimeoutMs = 5_000; + const spawn = vi.fn(async () => { + throw new Error('stop after observing timeout'); + }); + const stepComplete = vi.fn(async () => ( + {} as Awaited> + )); + const journal = client(stepComplete); + const worker = new AgentWorker(journal, { + workerId: 'worker_timeout', pins: {}, dataDir, nodeSpawnRuntime: runtime(spawn), + }); + await worker.attach(); + journal.emit('step.dispatch', dispatch('edit-timeout', 'dispatch_timeout000', declaredTimeoutMs)); + await waitForImmediate(() => expect(stepComplete).toHaveBeenCalledOnce()); + await worker.close(); + + expect(spawn).toHaveBeenCalledOnce(); + const observedTimeoutMs = spawn.mock.calls[0]![0].timeoutMs; + expect(observedTimeoutMs).toBeGreaterThan(0); + expect(observedTimeoutMs).toBeLessThanOrEqual(declaredTimeoutMs); + }); + + it('journals a native node-spawn deadline as timeout rather than worker_error', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-worker-timeout-reason-')); + roots.push(dataDir); + const spawn = vi.fn(async (_input, beforeDispatch) => { + await beforeDispatch(); + return new Promise(() => undefined); + }); + const stepComplete = vi.fn(async () => ( + {} as Awaited> + )); + const journal = client(stepComplete); + const worker = new AgentWorker(journal, { + workerId: 'worker_timeout_reason', pins: {}, dataDir, nodeSpawnRuntime: runtime(spawn), + }); + await worker.attach(); + journal.emit('step.dispatch', dispatch('edit-timeout-reason', 'dispatch_timeout001', 100)); + await waitForImmediate(() => expect(stepComplete).toHaveBeenCalledOnce()); + await worker.close(); + + expect(stepComplete).toHaveBeenCalledWith( + 'run_timeout', 'edit-timeout-reason', 1, 'dispatch_timeout001', 'timeout', + expect.objectContaining({ trajectory_tail: expect.objectContaining({ + transport: expect.objectContaining({ phase: 'timeout', cause: 'timeout', retryable: false }), + }) }), + ); + }); + + it('durably retries failed exact cleanup after a late confirmation and terminal timeout', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'flows-node-worker-late-cleanup-')); + roots.push(dataDir); + let confirm!: (receipt: NodeSpawnReceipt) => void; + let spawnEntered!: () => void; + const entered = new Promise(resolve => { spawnEntered = resolve; }); + const spawn = vi.fn(async (input, beforeDispatch) => { + await beforeDispatch(); + return new Promise(resolve => { + confirm = receipt => resolve({ ...receipt, name: input.name }); + spawnEntered(); + }); + }); + const release = vi.fn() + .mockRejectedValueOnce(new Error('release response temporarily unavailable')) + .mockResolvedValueOnce({ + id: 'inv_release_late', action: 'release', status: 'completed', + spawned: false, ready: false, released: true, + }); + const nodeRuntime: NodeSpawnRuntime = { + authority: 'remote', publisher: { id: 'publisher_1', name: 'flows-publisher' }, + baseUrl: 'https://cast.agentrelay.com', + port: { + dispatchFenceVersion: 1, targets: async () => [], spawn, + status: vi.fn(), release, + }, + inbox: { inbox: { + subscribe: async function* () { return; }, + ack: vi.fn(async () => ({})), fail: vi.fn(async () => ({})), + } }, + }; + const stepComplete = vi.fn(async () => ( + {} as Awaited> + )); + const journal = client(stepComplete); + journal.journalRead = vi.fn(async () => { + throw new Error('cleanup-only reconciliation must not read the journal'); + }); + const worker = new AgentWorker(journal, { + workerId: 'worker_late_cleanup', pins: {}, dataDir, nodeSpawnRuntime: nodeRuntime, + }); + const errors: unknown[] = []; + worker.on('error', error => errors.push(error)); + const taskDeadline = new AbortController(); + const nativeTimeout = AbortSignal.timeout.bind(AbortSignal); + const timeout = vi.spyOn(AbortSignal, 'timeout').mockImplementation(milliseconds => + milliseconds < 2_000 ? taskDeadline.signal : nativeTimeout(milliseconds)); + let replacement: AgentWorker | undefined; + try { + await worker.attach(); + journal.emit('step.dispatch', dispatch('edit-late-cleanup', 'dispatch_latecleanup1', 1_000)); + await entered; + taskDeadline.abort(new DOMException('The operation was aborted due to timeout', 'TimeoutError')); + await waitForImmediate(() => expect(stepComplete).toHaveBeenCalledOnce()); + expect(stepComplete).toHaveBeenCalledWith( + 'run_timeout', 'edit-late-cleanup', 1, 'dispatch_latecleanup1', 'timeout', expect.any(Object), + ); + await worker.close(); + + confirm({ + id: 'inv_spawn_late', action: 'spawn', status: 'completed', + nodeId: 'node_1', nodeName: 'desktop-1', name: 'placeholder', + agentId: 'agent_late', spawned: true, ready: true, + }); + let pending: Awaited>[number] | undefined; + await waitForImmediate(async () => { + expect(release).toHaveBeenCalledOnce(); + [pending] = await readNodeSpawnClaims(dataDir); + expect(pending!.claim).toMatchObject({ + cleanupPending: { attempts: 1, lastError: 'release response temporarily unavailable' }, + }); + }); + expect(pending!.claim.result).toBeUndefined(); + expect(pending!.claim.release).toBeUndefined(); + + replacement = new AgentWorker(journal, { + workerId: 'worker_late_cleanup_replacement', pins: {}, dataDir, nodeSpawnRuntime: nodeRuntime, + }); + replacement.on('error', error => errors.push(error)); + await replacement.attach(); + + expect(release).toHaveBeenCalledTimes(2); + const [reconciled] = await readNodeSpawnClaims(dataDir); + expect(reconciled!.claim.release).toEqual({ + invocationId: 'inv_release_late', status: 'completed', + }); + expect(reconciled!.claim.cleanupPending).toBeUndefined(); + expect(stepComplete).toHaveBeenCalledOnce(); + expect(spawn).toHaveBeenCalledOnce(); + expect(journal.journalRead).not.toHaveBeenCalled(); + expect(errors).toEqual([]); + } finally { + timeout.mockRestore(); + await replacement?.close(); + await worker.close(); + } + }); +}); diff --git a/packages/sdk/tests/worker-slots.test.ts b/packages/sdk/tests/worker-slots.test.ts index 52fb55da2..c8bfc667c 100644 --- a/packages/sdk/tests/worker-slots.test.ts +++ b/packages/sdk/tests/worker-slots.test.ts @@ -128,3 +128,26 @@ describe('--agent-capacity', () => { } }); }); + +describe('--service-caller-file', () => { + it('carries an explicit Desktop handoff only for local run and resume workers', () => { + expect(parseCliArgs(['run', '--local-agent', '--service-caller-file', '/tmp/caller.json', 'flow.yaml'])) + .toMatchObject({ command: 'run', localAgent: true, serviceCallerFile: '/tmp/caller.json' }); + expect(parseCliArgs(['resume', '--local-agent', '--service-caller-file', '/tmp/caller.json', 'run-1'])) + .toMatchObject({ command: 'resume', localAgent: true, serviceCallerFile: '/tmp/caller.json' }); + }); + + it('refuses a handoff that would be ignored or sent to Cloud', () => { + for (const argv of [ + ['run', '--service-caller-file', '/tmp/caller.json', 'flow.yaml'], + ['run', '--cloud', '--local-agent', '--service-caller-file', '/tmp/caller.json', 'flow.yaml'], + ['check', '--service-caller-file', '/tmp/caller.json', 'flow.yaml'], + ['run', '--local-agent', '--service-caller-file', 'relative/caller.json', 'flow.yaml'], + ['run', '--local-agent', '--service-caller-file', '/tmp/caller\0.json', 'flow.yaml'], + ['run', '--local-agent', '--service-caller-file', 'flow.yaml'], + ['run', '--local-agent', '--service-caller-file', '/tmp/a', '--service-caller-file', '/tmp/b', 'flow.yaml'], + ]) { + expect(parseCliArgs(argv), argv.join(' ')).toBeUndefined(); + } + }); +}); diff --git a/packages/sdk/tests/yaml-declared-streams-live.test.ts b/packages/sdk/tests/yaml-declared-streams-live.test.ts index 91ecc7afd..bb3934bd0 100644 --- a/packages/sdk/tests/yaml-declared-streams-live.test.ts +++ b/packages/sdk/tests/yaml-declared-streams-live.test.ts @@ -1,5 +1,5 @@ import { execFileSync, spawnSync } from 'node:child_process'; -import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { stringify } from 'yaml'; @@ -30,7 +30,6 @@ function fixture(source: 'step' | 'named' | 'flow' | 'project' = 'step', instruc const root = mkdtempSync(join(tmpdir(), 'flows-yaml-local-')); roots.push(root); mkdirSync(join(root, 'flows')); - chmodSync(wrapper, 0o755); symlinkSync(wrapper, join(root, 'agent.mjs')); writeFileSync(join(root, 'flows.json'), JSON.stringify({ models: ['yaml-test-model'], ...(source === 'project' ? { cli: './agent.mjs' } : {}), diff --git a/packages/sdk/tests/yaml-local-agent-live.test.ts b/packages/sdk/tests/yaml-local-agent-live.test.ts index b2cfa0dd4..50c5f5dc1 100644 --- a/packages/sdk/tests/yaml-local-agent-live.test.ts +++ b/packages/sdk/tests/yaml-local-agent-live.test.ts @@ -1,5 +1,5 @@ import { execFileSync, spawnSync } from 'node:child_process'; -import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { stringify } from 'yaml'; @@ -30,7 +30,6 @@ function fixture(source: 'step' | 'named' | 'flow' | 'project' = 'step', instruc const root = mkdtempSync(join(tmpdir(), 'flows-yaml-local-')); roots.push(root); mkdirSync(join(root, 'flows')); - chmodSync(wrapper, 0o755); symlinkSync(wrapper, join(root, 'agent.mjs')); writeFileSync(join(root, 'flows.json'), JSON.stringify({ models: ['yaml-test-model'], ...(source === 'project' ? { cli: './agent.mjs' } : {}), diff --git a/packages/sdk/tsconfig.tests.json b/packages/sdk/tsconfig.tests.json index 9af6213f8..bb165423a 100644 --- a/packages/sdk/tsconfig.tests.json +++ b/packages/sdk/tsconfig.tests.json @@ -72,7 +72,16 @@ "tests/shipped-source-worker-invocations.test.ts", "tests/worker-cli-cwd.test.ts", "tests/agent-relay-transport.test.ts", + "tests/broker-retirement.test.ts", "tests/agent-relay-hardening.test.ts", + "tests/node-spawn-core-bridge.test.ts", + "tests/node-spawn-dispatch-fence.test.ts", + "tests/node-spawn-ports.test.ts", + "tests/node-spawn-runtime.test.ts", + "tests/node-spawn-settlement.test.ts", + "tests/node-spawn-live.test.ts", + "tests/worker-node-runtime-sharing.test.ts", + "tests/worker-node-runtime-timeout.test.ts", "tests/webhook-hardening.test.ts", "tests/adapters/claude.test.ts", "tests/adapters/codex.test.ts", diff --git a/packages/surface/AUTHORING.md b/packages/surface/AUTHORING.md index f44957ecc..f12b88750 100644 --- a/packages/surface/AUTHORING.md +++ b/packages/surface/AUTHORING.md @@ -94,9 +94,9 @@ export interface Ctx extends Helpers { /** Classify evidence with Jev; the journal replays the recorded answer on resume. */ classify>(options: ClassifyOptions): Step>; agent(name: string, options: AgentOptions): Step; - /** Send work to a retained broker agent and await its next JSON completion. */ + /** Retired retained-session method; calling it is refused before admission. */ message(agentName: string, text: string, options?: { timeout?: string | number }): Step; - /** Release a retained broker agent before cleaning up its working directory. */ + /** Retired retained-session method; calling it is refused before admission. */ release(agentName: string): Step; /** Open a durable, bounded event subscription for this running body. */ on(source: TriggerSource, options: ActivityOptions): Activity; @@ -196,8 +196,8 @@ export interface RunResult { ```ts export interface AgentOptions { - /** CLI time limit: milliseconds or ms/s/m duration, at most 60m. - * Resolves with completionReason: timeout. No default for direct; broker defaults to 60m. Requires maxIterations 1. */ + /** Direct CLI time limit: milliseconds or ms/s/m duration, at most 60m. + * Resolves with completionReason: timeout. No default. Relay transport refuses this option. Requires maxIterations 1. */ timeout?: string | number; task: string; /** Run environment secrets this agent may receive by name. Default: none. */ @@ -208,9 +208,7 @@ export interface AgentOptions { cli?: string; model?: string; /** - * Broker transport accepts absolute or run-root-relative directories, including - * external worktrees. For direct transport: - * Directory this agent's CLI is spawned in, relative to the run root — the + * Directory this direct agent's CLI is spawned in, relative to the run root — the * flow-runner's working directory, and where the CLI runs when this is * absent. One flow can therefore drive agents in sibling checkouts. The * path must be relative and free of `.`, `..` and empty components, and @@ -232,16 +230,16 @@ export interface AgentOptions { */ resume?: string; /** - * Dispatch transport (flows#385). `'direct'` (default) spawns the CLI as a - * local subprocess. `'broker'` opens an interactive local Relay session. `'relay'` posts to agent-relay so the agent registers - * as a first-class workspace participant that DMs can steer. + * Dispatch transport. `'direct'` (default) spawns a local subprocess. + * `'relay'` uses native Agent Relay node spawn. `'broker'` is retained only + * for source compatibility and is refused before admission. */ transport?: 'direct' | 'relay' | 'broker'; - /** Canonical AgentWorkforce/skills names installed for this broker agent. */ + /** Legacy broker option; currently refused. */ skills?: string[]; - /** Keep the broker session available for f.message until the flow ends. */ + /** Legacy broker option; currently refused. */ keepAlive?: boolean; - /** Broker session channel; defaults to flow-. */ + /** Legacy broker option; currently refused. */ channel?: string; /** Semantic executions allowed when verification rejects output. Default 1. */ maxIterations?: number; @@ -285,7 +283,7 @@ export interface AgentResult { * continue the same conversation. */ sessionId?: string; - /** Broker session name; pass to f.message while keepAlive is enabled. */ + /** Retired broker session identifier; no native node-spawn step populates it. */ agentName?: string; } ``` diff --git a/packages/surface/src/context.ts b/packages/surface/src/context.ts index 8bae41805..3549a1b43 100644 --- a/packages/surface/src/context.ts +++ b/packages/surface/src/context.ts @@ -23,7 +23,7 @@ export interface AgentResult { * continue the same conversation. */ sessionId?: string; - /** Broker session name; pass to f.message while keepAlive is enabled. */ + /** Retired broker session identifier; no native node-spawn step populates it. */ agentName?: string; } @@ -37,8 +37,8 @@ export interface PermissionsSpec { } export interface AgentOptions { - /** CLI time limit: milliseconds or ms/s/m duration, at most 60m. - * Resolves with completionReason: timeout. No default for direct; broker defaults to 60m. Requires maxIterations 1. */ + /** Direct CLI time limit: milliseconds or ms/s/m duration, at most 60m. + * Resolves with completionReason: timeout. No default. Relay transport refuses this option. Requires maxIterations 1. */ timeout?: string | number; task: string; /** Run environment secrets this agent may receive by name. Default: none. */ @@ -49,9 +49,7 @@ export interface AgentOptions { cli?: string; model?: string; /** - * Broker transport accepts absolute or run-root-relative directories, including - * external worktrees. For direct transport: - * Directory this agent's CLI is spawned in, relative to the run root — the + * Directory this direct agent's CLI is spawned in, relative to the run root — the * flow-runner's working directory, and where the CLI runs when this is * absent. One flow can therefore drive agents in sibling checkouts. The * path must be relative and free of `.`, `..` and empty components, and @@ -73,16 +71,16 @@ export interface AgentOptions { */ resume?: string; /** - * Dispatch transport (flows#385). `'direct'` (default) spawns the CLI as a - * local subprocess. `'broker'` opens an interactive local Relay session. `'relay'` posts to agent-relay so the agent registers - * as a first-class workspace participant that DMs can steer. + * Dispatch transport. `'direct'` (default) spawns a local subprocess. + * `'relay'` uses native Agent Relay node spawn. `'broker'` is retained only + * for source compatibility and is refused before admission. */ transport?: 'direct' | 'relay' | 'broker'; - /** Canonical AgentWorkforce/skills names installed for this broker agent. */ + /** Legacy broker option; currently refused. */ skills?: string[]; - /** Keep the broker session available for f.message until the flow ends. */ + /** Legacy broker option; currently refused. */ keepAlive?: boolean; - /** Broker session channel; defaults to flow-. */ + /** Legacy broker option; currently refused. */ channel?: string; /** Semantic executions allowed when verification rejects output. Default 1. */ maxIterations?: number; @@ -214,9 +212,9 @@ export interface Ctx extends Helpers { /** Classify evidence with Jev; the journal replays the recorded answer on resume. */ classify>(options: ClassifyOptions): Step>; agent(name: string, options: AgentOptions): Step; - /** Send work to a retained broker agent and await its next JSON completion. */ + /** Retired retained-session method; calling it is refused before admission. */ message(agentName: string, text: string, options?: { timeout?: string | number }): Step; - /** Release a retained broker agent before cleaning up its working directory. */ + /** Retired retained-session method; calling it is refused before admission. */ release(agentName: string): Step; /** Open a durable, bounded event subscription for this running body. */ on(source: TriggerSource, options: ActivityOptions): Activity;