From 29f0022670a1158715dc8578a59304062e2d221f Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Thu, 22 Oct 2026 10:46:11 +0000 Subject: [PATCH] Cold-replace a checkpoint that cannot be restored --- .../agents-api/node-generation-protocol.md | 1 + .../agents-api/zh/node-generation-protocol.md | 3 +- docs/sandbox-provider.md | 6 ++- docs/zh/sandbox-provider.md | 8 ++-- .../queries/runtime_suspension_pressure.sql | 2 +- .../sqlc/runtime_suspension_pressure.sql.go | 2 +- services/core/internal/deployment/storage.go | 2 +- .../deployment/suspension_pressure.go | 14 ++++++ .../internal/execution/runtime_compute.go | 11 ++++- .../execution/runtime_restore_attempt_test.go | 33 +++++++++++++ .../internal/sandbox/microsandbox/provider.go | 2 + .../internal/sandbox/node/operations_test.go | 9 ++++ services/core/internal/sandbox/node/wire.go | 4 ++ .../core/internal/sandbox/sandbox_provider.go | 5 ++ .../tools/microsandbox-provider/README.md | 2 +- .../tools/microsandbox-provider/checkpoint.go | 47 ++++++++++++++++--- .../core/tools/microsandbox-provider/main.go | 2 + .../tools/microsandbox-provider/snapshot.go | 27 +++++++---- .../snapshot_observation_test.go | 39 +++++++++++---- 19 files changed, 185 insertions(+), 34 deletions(-) diff --git a/contracts/agents-api/node-generation-protocol.md b/contracts/agents-api/node-generation-protocol.md index 7112c9f9f..c6b2972b1 100644 --- a/contracts/agents-api/node-generation-protocol.md +++ b/contracts/agents-api/node-generation-protocol.md @@ -98,6 +98,7 @@ The `response` frame carries `id` and `connection_id`. A successful response car | --- | --- | | `invalid`, `ownership`, `exists`, `not_found` | `ErrInvalid`, `ErrOwnership`, `ErrExists`, `ErrNotFound` | | `command_unconfirmed` | `ErrCommandUnconfirmed` | +| `checkpoint_unusable` | `ErrCheckpointUnusable`, a `resume` outcome | | `observation_unavailable`, `runtime_not_running` | The observation outcomes | | `unsupported` | The operation is declared unsupported; see below | | `unconfirmed`, or any other value | The outcome is unknown | diff --git a/contracts/agents-api/zh/node-generation-protocol.md b/contracts/agents-api/zh/node-generation-protocol.md index bcec4415b..ac649c21d 100644 --- a/contracts/agents-api/zh/node-generation-protocol.md +++ b/contracts/agents-api/zh/node-generation-protocol.md @@ -1,7 +1,7 @@ --- title: "沙箱节点协议" source: contracts/agents-api/node-generation-protocol.md -source_hash: cc071a27e2a33ecbecf7d5f4a3aa7cbd5d61e0f1cac114df686eea8df0baf5e3 +source_hash: 090803f51d2e29d8900cfe8612b0eb21363a0fd2687daf3d368cc746dd65809f --- 沙箱节点协议连接 Core 与 Docker 或 microsandbox 主机:主机针对其本地提供商执行提供商操作,并报告就绪状态、主机测量值及其持有的部署代次。 @@ -100,6 +100,7 @@ Core 发送包含以下内容的 `request` 帧: | --- | --- | | `invalid`、`ownership`、`exists`、`not_found` | `ErrInvalid`、`ErrOwnership`、`ErrExists`、`ErrNotFound` | | `command_unconfirmed` | `ErrCommandUnconfirmed` | +| `checkpoint_unusable` | `ErrCheckpointUnusable`,`resume` 的结果 | | `observation_unavailable`、`runtime_not_running` | 对应的观察结果 | | `unsupported` | 该操作被声明为不支持;见下文 | | `unconfirmed` 或任何其他值 | 结果未知 | diff --git a/docs/sandbox-provider.md b/docs/sandbox-provider.md index 79f914bb2..b8db55b7e 100644 --- a/docs/sandbox-provider.md +++ b/docs/sandbox-provider.md @@ -107,7 +107,7 @@ Every call receives a bounded context. Expiry or cancellation ends the caller's | `Kill` | Owned compute and retained storage removed; repeated confirmed absence succeeds | An error keeps ownership and cleanup intent; an ownership mismatch never deletes foreign resources | Retry cleanup of the same `Reference` after outstanding creation or mutation is fenced; never release the owner early | | `RunCommand`, `RunCommandCompute` | Collected output and actual exit code; a nonzero exit is a settled command failure | Missing native completion is `ErrCommandUnconfirmed`; partial output is not success | Never replay. Keep the owner and reclaim before reuse when completion cannot be proved | -`ErrInvalid`, `ErrOwnership`, `ErrExists`, `ErrNotFound`, `ErrComputeUnconfirmed` and `ErrCommandUnconfirmed` keep their defined meanings. An unclassified native or transport error is unknown, never permission to retry a mutation. Core never reads provider diagnostics as lifecycle truth or exposes native error text or credentials; the node transport maps errors to fixed codes, and direct SDK details stay private. +`ErrInvalid`, `ErrOwnership`, `ErrExists`, `ErrNotFound`, `ErrComputeUnconfirmed`, `ErrCommandUnconfirmed` and, for `Resume` only, `ErrCheckpointUnusable` keep their defined meanings. An unclassified native or transport error is unknown, never permission to retry a mutation. Core never reads provider diagnostics as lifecycle truth or exposes native error text or credentials; the node transport maps errors to fixed codes, and direct SDK details stay private. Checkpoint support adds `Compute` generation, name and ID and `SnapshotIdentity`; persist operation IDs and the provider's snapshot provenance unchanged. `ObserveOnly` on suspend or resume observes the previous attempt and never starts another capture or restore. `ResumeCompute` only thaws the retained source and never cold-starts a stopped one. Cleanup targets the exact compute incarnation and snapshot, not whatever instance now has the same name. `GetCompute` returns `ErrNotFound` only when the owning node has verified that the exact incarnation no longer exists; an instance with the same name and another native ID is `ErrOwnership`, and an unreachable node or backend is an unknown result. A `ComputeState.Status` of `stopped` or `crashed` means the adapter verified that the exact incarnation’s process no longer exists; its record or disks can remain until `KillCompute` removes them. `paused`, `running` without a Runtime connection, and unknown states do not establish termination. Read [`runtime_compute.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute.go) and its failure tests before declaring checkpoint support. @@ -258,7 +258,7 @@ A checkpoint-capable generation reports `CheckpointCompatibility`: opaque `Artif `Suspend` may report `SourceStopped` only after publishing a verified full checkpoint durably, stopping the exact source compute and settling source-local capture and cleanup obligations. Core persists the snapshot before marking it suspended; native absence alone is insufficient. `Suspend.ObserveOnly` may finish checkpoint publication and source cleanup for an already verified snapshot of the same operation, but cannot capture another snapshot or stop a source that has resumed running. If a previously published checkpoint is missing or damaged, `ObserveOnly` may return its exact durable ownership receipt with `Status: unknown` and `SourceStopped: false` for cleanup; that receipt proves neither current checkpoint usability nor source absence. `Resume` still verifies the checkpoint before execution. After the source-settlement barrier, source-node unavailability does not prevent restoration. Deletion and ordinary expiry can transfer an idle suspended allocation's cleanup route to an online node in the same artifact domain; deletion does not require execution-class compatibility. The cleanup destination must already serve the same immutable generation and have retained capacity. Without such a node, cleanup stays pending and ownership is not released; operators must restore an eligible node or its capacity. An unknown running or restoring writer never moves merely because its node is unreachable. -Recovery uses `Resume.ObserveOnly` for the persisted target and operation. `RestoreAttemptClosed` is an exact operation ID, not a generic absence flag: the adapter may return it only after durably closing an attempt that never entered native execution and fencing every late request for that ID. An adapter may also close a durably admitted attempt when it can prove that its current invocation failed before dispatching native restoration. The microsandbox adapter does this only when the fresh request's wire deadline expires before native dispatch; checkpoint corruption, qualification errors and opaque SDK failures do not authorize an automatic retry. Core then persists a new operation ID before attempting execution. Once native restoration has been dispatched, an unknown outcome retains the same target, ownership and original retention deadline; a missing native listing, helper death or timeout does not authorize replay. Such a restore can remain unavailable until explicit deletion or ordinary retention expiry, and pending inputs can reach their public deadline while waiting. User deletion still requires exact native settlement before resources and ownership can be released. +Recovery uses `Resume.ObserveOnly` for the persisted target and operation. `RestoreAttemptClosed` is an exact operation ID, not a generic absence flag: the adapter may return it only after durably closing an attempt that never entered native execution and fencing every late request for that ID. An adapter may also close a durably admitted attempt when it can prove that its current invocation failed before dispatching native restoration. The microsandbox adapter does this for the fresh request that created the admission, on any failure before it dispatches native restoration. An expired wire deadline or an unclassified failure closes the attempt, and Core then persists a new operation ID. A snapshot that is missing, fails identity or integrity verification, or fails the adapter's qualification for this generation, such as a resource, workspace or runtime image mismatch, is also closed and reported as `ErrCheckpointUnusable`. That outcome is durable: `Resume.ObserveOnly` for the same operation reports it again, and late requests for the operation ID stay fenced. A Provider never returns it once native restoration was dispatched. On `ErrCheckpointUnusable` Core never restores that snapshot again and [ends its retention at once](#cold-replacement-after-checkpoint-retention). Once native restoration has been dispatched, an unknown outcome retains the same target, ownership and original retention deadline; a missing native listing, helper death or timeout does not authorize replay. Such a restore can remain unavailable until explicit deletion or ordinary retention expiry, and pending inputs can reach their public deadline while waiting. User deletion still requires exact native settlement before resources and ownership can be released. The microsandbox adapter requires an explicit private `checkpoint_root`, outside every guest-accessible filesystem. Its directory is mode `0700`; a private namespace marker identifies the actual checkpoint store. The default installation creates a node-local private store. Cross-node restoration requires operators to mount the same private store on each participating node; mount paths may differ. Each retained checkpoint is an uncompressed native snapshot of the guest's memory and root-disk layers, so a shared store carries that volume for every suspension and cross-node restore. Runtime state directories and workspace bindings are never used to infer this root. The adapter verifies the checkpoint, external filesystem identity and native execution class again before restoration. Restored RAM and file descriptors do not promise continuity of external TCP connections or replay safety for application side effects. @@ -266,6 +266,8 @@ The microsandbox adapter requires an explicit private `checkpoint_root`, outside Snapshot retention bounds compute resources, not the lifetime of an eligible retained Session. After the deadline, Core can clean up the allocation without terminating the Session only when its independent filesystem object is ready, initialization is complete, the Session and Environment remain nonterminal, and the previous authenticated Runtime has a persisted `retained_native_history` declaration. The capability and native recovery obligations are defined by the [Core–Runtime protocol](runtime-protocol.md). Filesystem retention alone is insufficient. +An unusable checkpoint, reported as `ErrCheckpointUnusable` by a restoring allocation's `Resume`, ends retention at once, exactly as if its deadline had passed. Core deletes the target and snapshot through the ordinary expiry cleanup, and a qualified Session continues as below, while an unqualified one ends as at ordinary expiry. + Capacity pressure ends retention early only for the node's oldest idle suspended allocation that meets these conditions, and only when waiting placement demand is blocked by the node's full retained slots. Core then treats the allocation as expired, one release per blocked demand, and every other suspended allocation keeps its snapshot and retained slot until its configured deadline. A checkpoint whose restore lacks capacity or a compatible node is not cold-replaced for that reason. Pending requests retain their original input deadline. The ordinary expiry cleanup must confirm resource deletion before another allocation can reserve the slot; files and qualified native history remain retained. Core confirms the original creation is settled and all owned compute and snapshots are cleaned up before releasing allocation and placement ownership. The old device's authority is revoked before replacement admission. Until Create, Kill or DeleteSnapshot has a confirmed outcome, it keeps ownership and admits no replacement writer. A recoverable Session and Environment remain disconnected with the same filesystem object and initialization result. Archive, reset and deletion take precedence and cannot be undone by a wake request. diff --git a/docs/zh/sandbox-provider.md b/docs/zh/sandbox-provider.md index 4c153364b..99173c5b2 100644 --- a/docs/zh/sandbox-provider.md +++ b/docs/zh/sandbox-provider.md @@ -1,7 +1,7 @@ --- title: "沙箱提供商协议" source: docs/sandbox-provider.md -source_hash: 6476bd4c551ee002c2006eba645cfd6ebe3f107b9369570d2fa386fb9c0ef132 +source_hash: 2e359fb0bd99974bd1077f15b3a529c80a4ff19b3d9c2f7a3882fe125aa9f8c8 --- 沙箱提供商协议连接 Core 与为 Core 管理的(`openai_hosted`)Environment 提供计算资源的 adapter,并由该 adapter 运行启动 Runtime daemon 的有界引导流程。本页面向 adapter 作者:说明提供商必须实现什么,以及可以依赖 Core 做什么。Docker、microsandbox 和 E2B 实现了该协议。 @@ -109,7 +109,7 @@ Microsandbox 在返回创建不存在回执前永久关闭初始 Create 准入 | `Kill` | 所属计算资源与保留存储已移除;重复已确认不存在时成功 | error 保留所有权与清理意图;所有权不匹配不删除外来资源 | 未决创建或 mutation 已隔离后重试同一 `Reference` 的清理;不提前释放 owner | | `RunCommand`, `RunCommandCompute` | 收集到输出和实际 exit code;非零退出是已结算命令失败 | 缺失原生完成为 `ErrCommandUnconfirmed`;部分输出不是成功 | 不重放。无法证明完成时保留 owner,并在复用前回收 | -`ErrInvalid`、`ErrOwnership`、`ErrExists`、`ErrNotFound`、`ErrComputeUnconfirmed` 和 `ErrCommandUnconfirmed` 保持其定义含义。未分类原生或 transport error 表示未知,不授权重试 mutation。Core 不将提供商 diagnostics 读作生命周期事实,也不暴露原生错误文本或凭据;node transport 将错误映射为固定 code,直接 SDK 细节保持私有。 +`ErrInvalid`、`ErrOwnership`、`ErrExists`、`ErrNotFound`、`ErrComputeUnconfirmed`、`ErrCommandUnconfirmed`,以及仅用于 `Resume` 的 `ErrCheckpointUnusable` 保持其定义含义。未分类原生或 transport error 表示未知,不授权重试 mutation。Core 不将提供商 diagnostics 读作生命周期事实,也不暴露原生错误文本或凭据;node transport 将错误映射为固定 code,直接 SDK 细节保持私有。 Checkpoint 支持增加 `Compute` generation、name、ID 和 `SnapshotIdentity`;原样持久化 operation ID 与提供商 snapshot provenance。suspend 或 resume 的 `ObserveOnly` 仅观察上次尝试,不启动另一 capture 或 restore。`ResumeCompute` 仅解冻保留源,不冷启动已停止源。清理针对精确 compute incarnation 和 snapshot,不针对当前同名实例。只有所属节点已验证精确 incarnation 不再存在时,`GetCompute` 才返回 `ErrNotFound`;同名但 native ID 不同的实例为 `ErrOwnership`,节点或提供商不可达属于未知结果。`ComputeState.Status` 为 `stopped` 或 `crashed` 表示 adapter 已验证该精确 incarnation 的进程不再存在;其记录或磁盘可保留到 `KillCompute` 将其移除。`paused`、没有 Runtime 连接的 `running` 和未知状态均不证明进程已终止。声明 checkpoint 支持前阅读 [`runtime_compute.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute.go) 及其失败测试。 @@ -260,7 +260,7 @@ Core 在初始化完成、没有 root 或 Subagent Turn 处于排队、进行中 `Suspend` 只有在持久发布已验证的完整检查点、停止精确 source compute,并结清 source 本地 capture 与 cleanup 义务后,才能报告 `SourceStopped`。Core 先持久化快照,再标记 suspended;仅原生资源不存在并不足够。`Suspend.ObserveOnly` 可以为同一 operation 已验证的快照完成检查点发布和源清理,但不能重新捕获快照,也不能停止已经恢复 running 的源。此前已发布的检查点缺失或损坏时,`ObserveOnly` 可以返回其精确持久 ownership receipt,标记 `Status: unknown` 和 `SourceStopped: false`,供清理使用;该 receipt 不证明检查点当前可用,也不证明源不存在。`Resume` 仍在执行前验证检查点。完成源结清屏障后,source node 不可用不再阻止恢复。删除和正常到期可以将空闲 suspended allocation 的清理路由转给同一 artifact domain 内的在线 node;删除不要求 execution class 匹配。清理目标必须已提供同一不可变 generation,且有 retained 容量。没有这样的 node 时,清理保持 pending,不释放所有权;运维需恢复符合条件的 node 或其容量。未知 running 或 restoring writer 不会仅因 node 不可达而迁移。 -恢复用 `Resume.ObserveOnly` 观察已持久化的 target 与 operation。`RestoreAttemptClosed` 是精确 operation ID,不是通用 absence 标志:adapter 只有在持久关闭从未进入原生执行的 attempt,并隔离该 ID 的所有迟到请求后才能返回。如果 adapter 能证明当前调用在派发原生恢复之前失败,也可以关闭已持久 admitted 的 attempt。microsandbox adapter 仅在 fresh 请求的 wire deadline 于原生派发之前到期时执行这种关闭;检查点损坏、资格错误和不透明 SDK 失败都不授权自动重试。Core 随后先持久化新的 operation ID,再尝试执行。一旦已派发原生恢复,结果未知的 attempt 保留相同 target、所有权与原始 retention deadline;原生列表缺失、helper 死亡或超时都不授权重放。这种恢复可能一直不可用,直到显式删除或正常保留期到期;等待中的 input 可能达到其公共 deadline。用户删除仍需精确原生 settlement,之后才能释放资源与所有权。 +恢复用 `Resume.ObserveOnly` 观察已持久化的 target 与 operation。`RestoreAttemptClosed` 是精确 operation ID,不是通用 absence 标志:adapter 只有在持久关闭从未进入原生执行的 attempt,并隔离该 ID 的所有迟到请求后才能返回。如果 adapter 能证明当前调用在派发原生恢复之前失败,也可以关闭已持久 admitted 的 attempt。microsandbox adapter 对创建该 admission 的 fresh 请求,在派发原生恢复之前的任何失败上执行这种关闭。wire deadline 到期或未分类失败会关闭该 attempt,Core 随后持久化新的 operation ID。快照缺失、未通过身份或完整性校验,或未通过 adapter 对当前 generation 的资格检查(例如资源、workspace 或 runtime image 不匹配)时,也会关闭该 attempt,并报告为 `ErrCheckpointUnusable`。该结果是持久的:同一 operation 的 `Resume.ObserveOnly` 会再次报告它,该 operation ID 的迟到请求仍被隔离。一旦已派发原生恢复,提供商不会返回它。收到 `ErrCheckpointUnusable` 后,Core 不再恢复该快照,并[立即结束其保留](#cold-replacement-after-checkpoint-retention)。一旦已派发原生恢复,结果未知的 attempt 保留相同 target、所有权与原始 retention deadline;原生列表缺失、helper 死亡或超时都不授权重放。这种恢复可能一直不可用,直到显式删除或正常保留期到期;等待中的 input 可能达到其公共 deadline。用户删除仍需精确原生 settlement,之后才能释放资源与所有权。 microsandbox adapter 要求显式的私有 `checkpoint_root`,位于所有 guest 可访问文件系统之外。目录权限为 `0700`,私有 namespace marker 标识真实检查点存储。默认安装创建 node 本地私有 store。跨 node 恢复要求运维为参与 node 挂载同一个私有 store,挂载路径可以不同。每个保留的检查点都是 guest 内存与根磁盘层的未压缩原生快照,因此共享 store 要为每次挂起和跨 node 恢复承载这部分数据量。Runtime state 目录与 workspace binding 都不用于推导该 root。adapter 在恢复前再次验证检查点、外部文件系统身份和原生 execution class。恢复 RAM 与文件描述符不保证外部 TCP 连接连续,也不保证应用副作用可安全重放。 @@ -268,6 +268,8 @@ microsandbox adapter 要求显式的私有 `checkpoint_root`,位于所有 gues 快照保留期限制计算资源,不限制符合条件的保留 Session 的生命周期。到期后,只有独立文件系统对象已就绪、初始化已完成、Session 与 Environment 均未终结,且此前已认证 Runtime 的 `retained_native_history` 声明已持久化时,Core 才能在不终结 Session 的情况下清理 allocation。能力及原生恢复义务由 [Core–Runtime 协议](runtime-protocol.md)定义。仅保留文件系统并不足够。 +不可用的检查点(恢复中的 allocation 的 `Resume` 报告 `ErrCheckpointUnusable`)会立即结束保留,如同其 deadline 已到。Core 通过正常的到期清理删除 target 与快照;符合条件的 Session 按下文继续,不符合条件的则如正常到期一样结束。 + 仅当等待中的放置需求被节点已满的 retained slot 阻塞时,容量压力才会提前结束保留期,且只针对该节点上最早挂起、空闲并满足上述条件的 allocation。Core 将其视为已过期,每个被阻塞的需求只释放一个;其他已挂起的 allocation 在配置的期限到达前保留快照并继续占用 retained slot。恢复时缺少容量或兼容节点的检查点不会因此被冷替换。等待请求保持原有的输入期限。常规 expiry cleanup 必须确认资源已删除,其他 allocation 才能预留该 slot;文件与已验证的原生历史继续保留。 Core 在释放 allocation 和 placement 归属前,确认原始创建已结算且所属计算资源与快照均已清理。替代计算资源准入前撤销旧 device 的权限。Create、Kill 或 DeleteSnapshot 的结果尚未确认时,Core 保留归属,不准入替代写入方。可恢复的 Session 与 Environment 保持断连,保留同一文件系统对象和初始化结果。归档、重置和删除优先,唤醒请求不能撤销它们。 diff --git a/services/core/internal/db/queries/runtime_suspension_pressure.sql b/services/core/internal/db/queries/runtime_suspension_pressure.sql index a2bc87869..e5006f892 100644 --- a/services/core/internal/db/queries/runtime_suspension_pressure.sql +++ b/services/core/internal/db/queries/runtime_suspension_pressure.sql @@ -33,5 +33,5 @@ ORDER BY a.compute_retained_until, a.id; -- name: ExpireRuntimeCheckpoint :one UPDATE runtime_allocations SET compute_retained_until=clock_timestamp(), compute_revision=compute_revision+1 -WHERE id=$1 AND compute_revision=$2 AND state='running' AND compute_phase='suspended' AND compute_retained_until>clock_timestamp() +WHERE id=$1 AND compute_revision=$2 AND state='running' AND compute_phase IN ('suspended','restoring') AND compute_retained_until>clock_timestamp() RETURNING *; diff --git a/services/core/internal/db/sqlc/runtime_suspension_pressure.sql.go b/services/core/internal/db/sqlc/runtime_suspension_pressure.sql.go index 689f160a7..2d851f997 100644 --- a/services/core/internal/db/sqlc/runtime_suspension_pressure.sql.go +++ b/services/core/internal/db/sqlc/runtime_suspension_pressure.sql.go @@ -14,7 +14,7 @@ import ( const expireRuntimeCheckpoint = `-- name: ExpireRuntimeCheckpoint :one UPDATE runtime_allocations SET compute_retained_until=clock_timestamp(), compute_revision=compute_revision+1 -WHERE id=$1 AND compute_revision=$2 AND state='running' AND compute_phase='suspended' AND compute_retained_until>clock_timestamp() +WHERE id=$1 AND compute_revision=$2 AND state='running' AND compute_phase IN ('suspended','restoring') AND compute_retained_until>clock_timestamp() RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation ` diff --git a/services/core/internal/deployment/storage.go b/services/core/internal/deployment/storage.go index 3a7dd0ae5..0cd4169e2 100644 --- a/services/core/internal/deployment/storage.go +++ b/services/core/internal/deployment/storage.go @@ -130,7 +130,7 @@ type AllocationTx interface { // OldestIdleCheckpoint reports whether current has the earliest retention // deadline among its node's idle suspended allocations that can be retained. OldestIdleCheckpoint(current Allocation) (bool, error) - // ExpireCheckpoint ends the suspended allocation's retention now. + // ExpireCheckpoint ends the retention of a suspended or restoring allocation now. ExpireCheckpoint(current Allocation) (Allocation, error) // PlacementDemand reads one qualified page; next advances past all inspected // rows, including ineligible receipts. An empty next.EnvironmentID ends the scan. diff --git a/services/core/internal/deployment/suspension_pressure.go b/services/core/internal/deployment/suspension_pressure.go index 05c217e43..7e5920e7c 100644 --- a/services/core/internal/deployment/suspension_pressure.go +++ b/services/core/internal/deployment/suspension_pressure.go @@ -163,6 +163,20 @@ func (e *ExecutionOperations) ExpireCheckpointForDemand(ctx context.Context, own return err } +// ExpireUnusableCheckpoint ends the retention of a restoring allocation whose +// Provider reported its checkpoint unusable, exactly as if its deadline had +// passed. The snapshot is never restored again; ordinary expiry cleanup deletes +// it and any qualified Session continues on new compute. +func (e *ExecutionOperations) ExpireUnusableCheckpoint(ctx context.Context, owner Allocation) error { + _, err := e.change(ctx, owner, false, func(tx AllocationTx, current Allocation) (Allocation, error) { + if current.State != "running" || current.ComputePhase != "restoring" || current.Expired || current.ComputeRevision != owner.ComputeRevision { + return Allocation{}, ErrAllocationConflict + } + return tx.ExpireCheckpoint(current) + }) + return err +} + // placementNeedsCapacity scans bounded pages under the caller's deployment // lock and transaction deadline. It reports whether waiting demand would place // on nodeID once release frees the slot the caller may reclaim; an ineligible diff --git a/services/core/internal/execution/runtime_compute.go b/services/core/internal/execution/runtime_compute.go index c6ca50a16..3ec7b8fa5 100644 --- a/services/core/internal/execution/runtime_compute.go +++ b/services/core/internal/execution/runtime_compute.go @@ -316,7 +316,11 @@ func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.San // early. Core decides eligibility and demand; a refusal changes nothing, and // the expired allocation is observed and cleaned up as at its deadline. func (r *runtimeLifecycle) expireForDemand(ctx context.Context, owner deployment.Allocation) error { - err := r.deployment.ExpireCheckpointForDemand(ctx, owner) + return r.expire(ctx, owner, r.deployment.ExpireCheckpointForDemand) +} + +func (r *runtimeLifecycle) expire(ctx context.Context, owner deployment.Allocation, end func(context.Context, deployment.Allocation) error) error { + err := end(ctx, owner) if errors.Is(err, deployment.ErrNotIdle) || errors.Is(err, deployment.ErrAllocationConflict) { return nil } @@ -366,6 +370,11 @@ func (r *runtimeLifecycle) restoreCompute(ctx context.Context, p sandbox.Sandbox started := time.Now() result, err := p.Resume(ctx, sandbox.ResumeRequest{Workspace: workspace, Reference: runtimeReference(owner), OperationID: state.RestoreID, Snapshot: *state.Snapshot, Target: *state.Target, ObserveOnly: observeOnly}) logComputeStep(ctx, owner, "resume", started, err) + if errors.Is(err, sandbox.ErrCheckpointUnusable) { + // The Provider closed this attempt before native dispatch. Never restore + // the snapshot again: expire it and continue by ordinary expiry cleanup. + return r.expire(ctx, owner, r.deployment.ExpireUnusableCheckpoint) + } if err != nil { return err } diff --git a/services/core/internal/execution/runtime_restore_attempt_test.go b/services/core/internal/execution/runtime_restore_attempt_test.go index fa677fbd1..9b228727b 100644 --- a/services/core/internal/execution/runtime_restore_attempt_test.go +++ b/services/core/internal/execution/runtime_restore_attempt_test.go @@ -26,6 +26,9 @@ func (p *restoreAttemptProvider) NewCompute(_ context.Context, _ sandbox.Referen } func (p *restoreAttemptProvider) Resume(_ context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { p.requests = append(p.requests, q) + if p.outcome == "unusable" { + return sandbox.ComputeState{}, sandbox.ErrCheckpointUnusable + } if !q.ObserveOnly { return sandbox.ComputeState{}, p.stop } @@ -101,6 +104,36 @@ func TestRestoreAttemptRecoveryUsesExactClosedEvidence(t *testing.T) { } } +func TestUnusableCheckpointEndsRetentionAndIsNeverRestoredAgain(t *testing.T) { + provider := &restoreAttemptProvider{outcome: "unusable"} + f := workspaceSettlementFixtureWithSetup(t, provider, workspaceNodeSetup(t, true)) + r, session := f.lifecycle, f.session + owner, err := r.provision(t.Context(), session.TenantID, session.Environment.ID, r.config.InstallationID) + if err != nil { + t.Fatal(err) + } + snapshot := &sandbox.SnapshotIdentity{ID: "snapshot", Compatibility: sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}} + state := runtimeCompute{Current: sandbox.Compute{ID: "source", Name: "source", Generation: 1}, Snapshot: snapshot, RestoreID: uuid.NewString(), Target: &sandbox.Compute{Name: "target", Generation: 2, RestoredFrom: snapshot}} + raw, err := json.Marshal(state) + if err != nil { + t.Fatal(err) + } + if _, err = f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='restoring',compute_revision=4,compute_retained_until=clock_timestamp()+interval '1 hour',compute_state=$2 WHERE id=$1`, owner.ID, raw); err != nil { + t.Fatal(err) + } + owner, err = r.reader.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil { + t.Fatal(err) + } + if err = r.observe(t.Context(), owner); err != nil { + t.Fatal(err) + } + released, err := r.reader.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil || released.State != "released" || provider.kills != 1 || provider.snapshots != 1 || len(provider.requests) != 1 { + t.Fatalf("unusable checkpoint was not cleaned up once: %v %#v kills=%d snapshots=%d requests=%d", err, released, provider.kills, provider.snapshots, len(provider.requests)) + } +} + func TestCheckpointTransferRoutesBeforeTargetIO(t *testing.T) { for _, operation := range []string{"wake", "expiry"} { t.Run(operation, func(t *testing.T) { diff --git a/services/core/internal/sandbox/microsandbox/provider.go b/services/core/internal/sandbox/microsandbox/provider.go index d39b79237..37637de66 100644 --- a/services/core/internal/sandbox/microsandbox/provider.go +++ b/services/core/internal/sandbox/microsandbox/provider.go @@ -67,6 +67,8 @@ func (p *Provider) call(ctx context.Context, q Request) (Response, error) { return out, sandbox.ErrNotFound case "command_unconfirmed": return out, sandbox.ErrCommandUnconfirmed + case "checkpoint_unusable": + return out, sandbox.ErrCheckpointUnusable case "metrics_unavailable": return out, runtimeobs.ErrUnavailable default: diff --git a/services/core/internal/sandbox/node/operations_test.go b/services/core/internal/sandbox/node/operations_test.go index b5a83f0d7..5cda07c72 100644 --- a/services/core/internal/sandbox/node/operations_test.go +++ b/services/core/internal/sandbox/node/operations_test.go @@ -55,3 +55,12 @@ func TestNodeOperationMappingCoversForwardedMethods(t *testing.T) { } } } + +func TestCheckpointUnusableCrossesTheWireAsFixedCode(t *testing.T) { + if code := errorCode(sandbox.ErrCheckpointUnusable); code != "checkpoint_unusable" { + t.Fatal(code) + } + if !errors.Is(responseError(response{ErrorCode: "checkpoint_unusable"}), sandbox.ErrCheckpointUnusable) { + t.Fatal("code not mapped back") + } +} diff --git a/services/core/internal/sandbox/node/wire.go b/services/core/internal/sandbox/node/wire.go index 087466bf4..5a48fde61 100644 --- a/services/core/internal/sandbox/node/wire.go +++ b/services/core/internal/sandbox/node/wire.go @@ -197,6 +197,8 @@ func errorCode(err error) string { return "not_found" case errors.Is(err, sandbox.ErrCommandUnconfirmed): return "command_unconfirmed" + case errors.Is(err, sandbox.ErrCheckpointUnusable): + return "checkpoint_unusable" default: return "unconfirmed" } @@ -233,6 +235,8 @@ func responseError(out response) error { return sandbox.ErrNotFound case "command_unconfirmed": return sandbox.ErrCommandUnconfirmed + case "checkpoint_unusable": + return sandbox.ErrCheckpointUnusable default: return sandbox.ErrComputeUnconfirmed } diff --git a/services/core/internal/sandbox/sandbox_provider.go b/services/core/internal/sandbox/sandbox_provider.go index 712eaf4fe..571ce6bb6 100644 --- a/services/core/internal/sandbox/sandbox_provider.go +++ b/services/core/internal/sandbox/sandbox_provider.go @@ -46,6 +46,11 @@ var ( // ErrComputeUnconfirmed requires observation of the retained operation identity; // it does not authorize another Create, capture, restore, or cold start. ErrComputeUnconfirmed = errors.New("sandbox lifecycle outcome unconfirmed") + // ErrCheckpointUnusable is a Resume outcome: the adapter durably closed this + // operation before dispatching native restoration, and the snapshot cannot be + // restored. It repeats for the same operation and is never returned once + // native restoration was dispatched. + ErrCheckpointUnusable = errors.New("sandbox checkpoint cannot be restored") ) // Reference must be persisted by the caller before Create. AllocationID is a fresh diff --git a/services/core/tools/microsandbox-provider/README.md b/services/core/tools/microsandbox-provider/README.md index c307b0fdf..a7cb64a30 100644 --- a/services/core/tools/microsandbox-provider/README.md +++ b/services/core/tools/microsandbox-provider/README.md @@ -40,7 +40,7 @@ Core persists operation IDs, source and target generations, exact identities and After a lost response Core uses `ObserveOnly`. It never starts a capture or restore. For an existing verified capture it may complete the receipt and exact paused-source cleanup, but never captures again or kills a running source. Source termination is recorded before local removal so a lost cleanup response can be settled from the receipt. A valid ownership receipt remains discoverable when the snapshot is missing or corrupt: observation returns its snapshot identity with unknown source state and `SourceStopped=false`, so ordinary explicit cleanup can proceed. Restore still requires full snapshot verification. An interrupted restore may finish the derived proof on the exact running target, but never restarts a stopped target or restores again. -Before native restore, the helper synchronizes an admission record bound to the request operation, snapshot and target. Observation of a never-admitted operation writes a permanent closed record and returns that exact `RestoreAttemptClosed`; late dispatch is rejected. A fresh dispatch checks its existing request deadline after admission and immediately before calling native Restore. If the deadline has elapsed, it durably closes its exact attempt while still holding the allocation lock and returns without invoking Restore. Native FFI waits remain uncancelled. Qualification errors keep the admission open, avoiding repeated attempts against an invalid snapshot. This proof is never inferred from a previous admission or a native Restore error. After admission, helper death or native absence alone never establishes no effect. Observation checks an admitted target before reading the snapshot; typed absence or a non-running target remains unconfirmed without verifying it. A running target still requires full snapshot verification. Successful resume requires an observable exact running target. A failed or uncertain admitted restore retains its artifact and owner until explicit deletion or the original retention deadline; the normal cleanup operation uses the native lifecycle fence and records completion. It never shortens retention or substitutes a cold restart. +Before native restore, the helper synchronizes an admission record bound to the request operation, snapshot and target. Observation of a never-admitted operation writes a permanent closed record and returns that exact `RestoreAttemptClosed`; late dispatch is rejected. A fresh dispatch checks its existing request deadline after admission and immediately before calling native Restore. If the deadline has elapsed, it durably closes its exact attempt while still holding the allocation lock and returns without invoking Restore. Native FFI waits remain uncancelled. Any other failure of that fresh dispatch before native Restore also closes its attempt. A missing snapshot, a failed receipt, identity or integrity check, or a failed resource, workspace or runtime-image qualification also marks the record unusable: every later request for the operation returns `checkpoint_unusable`, and Core [ends the checkpoint's retention](../../../../docs/sandbox-provider.md#cold-replacement-after-checkpoint-retention). This proof is never inferred from a previous admission or a native Restore error. After admission, helper death or native absence alone never establishes no effect. Observation checks an admitted target before reading the snapshot; typed absence or a non-running target remains unconfirmed without verifying it. A running target still requires full snapshot verification. Successful resume requires an observable exact running target. An uncertain admitted restore retains its artifact and owner until explicit deletion or the original retention deadline; the normal cleanup operation uses the native lifecycle fence and records completion. The helper itself never shortens retention or substitutes a cold restart. GetCompute, commands, cleanup and the next suspension verify restored provenance from the persisted VM configuration after the consumed artifact is deleted. diff --git a/services/core/tools/microsandbox-provider/checkpoint.go b/services/core/tools/microsandbox-provider/checkpoint.go index 92af874bc..4c4d64fde 100644 --- a/services/core/tools/microsandbox-provider/checkpoint.go +++ b/services/core/tools/microsandbox-provider/checkpoint.go @@ -191,6 +191,8 @@ type restoreAdmission struct { Snapshot wire.SnapshotIdentity Target wire.Compute Closed bool + // Unusable marks a closed attempt whose snapshot cannot be restored. + Unusable bool CompletedID string Cleaned bool } @@ -207,14 +209,17 @@ func (b backend) admitRestore(q wire.ResumeRequest) (bool, error) { var prior restoreAdmission e := readPrivateJSON(path, &prior) if e == nil { - closed := prior.Closed - prior.Closed = false + closed, unusable := prior.Closed, prior.Unusable + prior.Closed, prior.Unusable = false, false prior.CompletedID = "" cleaned := prior.Cleaned prior.Cleaned = false if !reflect.DeepEqual(prior, want) { return false, sandbox.ErrOwnership } + if unusable { + return false, sandbox.ErrCheckpointUnusable + } if closed { return true, nil } @@ -246,6 +251,37 @@ func (b backend) closeExpiredRestore(q wire.ResumeRequest) error { if time.Now().Before(b.q.Deadline) { return nil } + if err := b.closeRestore(q, false); err != nil { + return err + } + return context.DeadlineExceeded +} + +// closeUndispatchedRestore closes the attempt this fresh dispatch just admitted +// after a failure before RestoreSandbox. A snapshot defect is recorded so that +// every later call for the operation reports ErrCheckpointUnusable; any other +// failure is returned as it is and Core replaces the closed attempt. +func (b backend) closeUndispatchedRestore(q wire.ResumeRequest, cause error) error { + unusable := snapshotDefect(cause) + if err := b.closeRestore(q, unusable); err != nil { + return err + } + if unusable { + return sandbox.ErrCheckpointUnusable + } + return cause +} + +// snapshotDefect reports a failure that proves this snapshot cannot be restored: +// it is missing, fails identity or integrity verification, or fails this +// generation's qualification. I/O and context errors prove nothing. +func snapshotDefect(e error) bool { + return errors.Is(e, sandbox.ErrOwnership) || errors.Is(e, sandbox.ErrInvalid) || errors.Is(e, os.ErrNotExist) || + sdk.IsKind(e, sdk.ErrSnapshotNotFound) || sdk.IsKind(e, sdk.ErrSnapshotIntegrity) || + sdk.IsKind(e, sdk.ErrSnapshotImageMissing) || sdk.IsKind(e, sdk.ErrSnapshotMigration) +} + +func (b backend) closeRestore(q wire.ResumeRequest, unusable bool) error { if q.ObserveOnly || q.Target.ID != "" { return sandbox.ErrOwnership } @@ -258,11 +294,8 @@ func (b backend) closeExpiredRestore(q wire.ResumeRequest) error { if !reflect.DeepEqual(prior, want) { return sandbox.ErrOwnership } - want.Closed = true - if err := durableJSON(path, want); err != nil { - return err - } - return context.DeadlineExceeded + want.Closed, want.Unusable = true, unusable + return durableJSON(path, want) } // Completion records the incarnation whose native restore has become observable diff --git a/services/core/tools/microsandbox-provider/main.go b/services/core/tools/microsandbox-provider/main.go index f6220baae..e023f322f 100644 --- a/services/core/tools/microsandbox-provider/main.go +++ b/services/core/tools/microsandbox-provider/main.go @@ -102,6 +102,8 @@ func code(err error) string { return "not_found" case errors.Is(err, sandbox.ErrCommandUnconfirmed): return "command_unconfirmed" + case errors.Is(err, sandbox.ErrCheckpointUnusable): + return "checkpoint_unusable" case errors.Is(err, runtimeobs.ErrUnavailable), sdk.IsKind(err, sdk.ErrMetricsDisabled), sdk.IsKind(err, sdk.ErrMetricsUnavailable): return "metrics_unavailable" default: diff --git a/services/core/tools/microsandbox-provider/snapshot.go b/services/core/tools/microsandbox-provider/snapshot.go index 11f138ac6..13ade20fe 100644 --- a/services/core/tools/microsandbox-provider/snapshot.go +++ b/services/core/tools/microsandbox-provider/snapshot.go @@ -307,24 +307,35 @@ func (b backend) resume(ctx context.Context, q wire.ResumeRequest) (wire.State, } // Admission is durable before any native import/restore operation. A prior // admitted operation with no visible target remains unknown, never replayed. + // The fresh dispatch that created the admission closes it on any failure + // before RestoreSandbox, since no guest execution can have occurred. artifact, e := b.restoreSource(ctx, q.Snapshot) - if e != nil { - return wire.State{}, e + if e == nil { + e = qualifySnapshotResources(b.q.Config, artifact.Labels()) } - if e = qualifySnapshotResources(b.q.Config, artifact.Labels()); e != nil { - return wire.State{}, e + if e == nil { + e = qualifyWorkspace(artifact.Labels(), b.q.Workspace) } - if err := qualifyWorkspace(artifact.Labels(), b.q.Workspace); err != nil { - return wire.State{}, err + if e == nil { + e = qualifyRuntimeImage(ctx, b.q.Config, artifact.ImageManifestDigest()) } - if err := qualifyRuntimeImage(ctx, b.q.Config, artifact.ImageManifestDigest()); err != nil { - return wire.State{}, err + if e != nil { + if q.ObserveOnly { + return wire.State{}, e + } + return wire.State{}, b.closeUndispatchedRestore(q, e) } _, _, e = b.inspectOwned(ctx, q.Target) if e == nil { return b.completeRestore(ctx, q, q.Target) } if !sdk.IsKind(e, sdk.ErrSandboxNotFound) { + // A failed target lookup proves nothing about the snapshot: close only. + if !q.ObserveOnly { + if err := b.closeRestore(q, false); err != nil { + return wire.State{}, err + } + } return wire.State{}, e } if q.ObserveOnly || q.Target.ID != "" { diff --git a/services/core/tools/microsandbox-provider/snapshot_observation_test.go b/services/core/tools/microsandbox-provider/snapshot_observation_test.go index ee4b6b353..de907c8e9 100644 --- a/services/core/tools/microsandbox-provider/snapshot_observation_test.go +++ b/services/core/tools/microsandbox-provider/snapshot_observation_test.go @@ -232,7 +232,7 @@ func TestLostCaptureResponseStillExposesOwnedIdentityWhenCheckpointIsDamaged(t * } // These requests use the real SDK's empty local registry, without starting a VM. -func TestPermanentRestoreImportFailureKeepsOneAdmission(t *testing.T) { +func TestUnusableCheckpointClosesAdmissionDurably(t *testing.T) { if !isolatedSDKTest(t) { return } @@ -266,13 +266,13 @@ func TestPermanentRestoreImportFailureKeepsOneAdmission(t *testing.T) { t.Fatal(err) } } - if _, err = b.resume(t.Context(), q); err == nil { - t.Fatal("unusable checkpoint accepted") + if _, err = b.resume(t.Context(), q); !errors.Is(err, sandbox.ErrCheckpointUnusable) { + t.Fatal("unusable checkpoint not reported", err) } path := filepath.Join(b.storeDirectory(), "r-"+q.OperationID+".json") var record restoreAdmission - if err = readPrivateJSON(path, &record); err != nil || record.Closed { - t.Fatal("permanent failure closed", record, err) + if err = readPrivateJSON(path, &record); err != nil || !record.Closed || !record.Unusable { + t.Fatal("unusable checkpoint left admission open", record, err) } before, err := os.ReadFile(path) if err != nil { @@ -282,10 +282,11 @@ func TestPermanentRestoreImportFailureKeepsOneAdmission(t *testing.T) { if err != nil { t.Fatal(err) } - q.ObserveOnly = true - for i := 0; i < 3; i++ { + // Observation and late fresh requests report the same durable outcome. + for _, observe := range []bool{true, false, true} { + q.ObserveOnly = observe state, err := b.resume(t.Context(), q) - if !errors.Is(err, wire.ErrUnconfirmed) || state.RestoreAttemptClosed != "" { + if !errors.Is(err, sandbox.ErrCheckpointUnusable) || state.RestoreAttemptClosed != "" { t.Fatal(state, err) } } @@ -301,6 +302,28 @@ func TestPermanentRestoreImportFailureKeepsOneAdmission(t *testing.T) { } } +func TestUnclassifiedPreDispatchFailureClosesAttemptForReplacement(t *testing.T) { + request := initialInfoRequest(t) + guard, err := allocationLock(request) + if err != nil { + t.Fatal(err) + } + defer guard.Close() + b := backend{q: request} + q := wire.ResumeRequest{OperationID: "66666666-6666-4666-8666-666666666666", Target: wire.Compute{Name: "target", Generation: 1}} + if _, err = b.admitRestore(q); err != nil { + t.Fatal(err) + } + if err = b.closeUndispatchedRestore(q, context.Canceled); !errors.Is(err, context.Canceled) { + t.Fatal("unclassified failure changed", err) + } + q.ObserveOnly = true + state, err := b.resume(t.Context(), q) + if err != nil || state.RestoreAttemptClosed != q.OperationID { + t.Fatal("closed attempt not reported", state, err) + } +} + func TestFreshRestoreRequestDeadlineClosesBeforeNativeExecution(t *testing.T) { if !isolatedSDKTest(t) { return