From 977a987f84f00c20af7c4d06a6c5e72d1108644b Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Wed, 21 Oct 2026 09:05:54 -0700 Subject: [PATCH 1/3] nvx-rs: identify located artifacts by their directory's metadata Artifacts that a release or a repository checkout holds now keep its ArtifactDir, and Artifacts::runtime_identity reads their runtime identity from its install metadata, through ArtifactDir::runtime_identity, without reading or hashing an artifact. Artifacts named by path have no identity, and neither do located artifacts whose paths were changed to other files. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: aa7b9fda-3774-4817-aa98-af05f8d50215 --- nvx-rs/src/host/artifacts.rs | 139 +++++++++++++++++++++++++++++++---- 1 file changed, 123 insertions(+), 16 deletions(-) diff --git a/nvx-rs/src/host/artifacts.rs b/nvx-rs/src/host/artifacts.rs index 0de8de471..597a8a149 100644 --- a/nvx-rs/src/host/artifacts.rs +++ b/nvx-rs/src/host/artifacts.rs @@ -53,6 +53,11 @@ const BUNDLED_DIR: Option<&str> = option_env!("NVX_BUNDLED_ARTIFACTS_DIR"); /// /// Every path is used as given. Resolve relative paths with [`absolute`] before a VM runs them, /// and check that the files exist, because nothing here does. +/// +/// Artifacts located in a release or a repository checkout keep its [`ArtifactDir`], whose +/// install metadata names their [`runtime_identity`](Self::runtime_identity). Artifacts named by +/// path, through [`new`](Self::new) or the explicit variables of [`discover`](Self::discover), +/// have none: their caller supplies the runtime identity of a launch. #[derive(Debug, Clone, PartialEq, Eq)] #[non_exhaustive] pub struct Artifacts { @@ -62,6 +67,8 @@ pub struct Artifacts { pub kernel: PathBuf, /// The NVX Alpine initramfs (`initramfs.cpio.gz`): the guest userland and its control agent. pub initrd: PathBuf, + /// The release or checkout in which the artifacts were located, if they were. + directory: Option, } impl Artifacts { @@ -75,7 +82,58 @@ impl Artifacts { openvmm: openvmm.into(), kernel: kernel.into(), initrd: initrd.into(), + directory: None, + } + } + + /// Returns the sandbox artifacts of `directory`, which they remember. + fn located(directory: &ArtifactDir) -> Self { + Self { + directory: Some(directory.clone()), + ..Self::new( + directory.openvmm(), + directory.kernel(), + directory.initramfs(Guest::Alpine), + ) + } + } + + /// Returns the release or repository checkout in which the artifacts were located, or `None` + /// for artifacts named by path. + pub fn artifact_dir(&self) -> Option<&ArtifactDir> { + self.directory.as_ref() + } + + /// Returns the runtime identity of these artifacts, which the install metadata of the release + /// or checkout in which they were located names, as [`ArtifactDir::runtime_identity`] reads + /// it, without reading or hashing an artifact. + /// + /// Fails with [`ErrorKind::Unavailable`](crate::ErrorKind::Unavailable) for artifacts named by + /// path, whose caller supplies their identity instead, for artifacts whose paths no longer + /// name the ones of their directory, and as [`ArtifactDir::runtime_identity`] fails. + pub fn runtime_identity(&self) -> Result { + let Some(directory) = &self.directory else { + return Err(Error::unavailable( + "the artifacts are named by path, so no install metadata identifies them: locate \ + them in an NVX release or repository checkout, or supply their runtime identity", + )); + }; + let located = Self::located(directory); + for (path, expected, description) in [ + (&self.openvmm, &located.openvmm, "OpenVMM executable"), + (&self.kernel, &located.kernel, "guest kernel"), + (&self.initrd, &located.initrd, "guest initramfs"), + ] { + if path != expected { + return Err(Error::unavailable(format!( + "the {description} {} is not the one of {}, whose install metadata \ + identifies the artifacts", + path.display(), + directory.root().display() + ))); + } } + directory.runtime_identity() } /// Checks that this host can run NVX VMs and that the three files exist. @@ -135,11 +193,7 @@ impl Artifacts { /// [`check_manifest`] does. fn from_artifact_dir(directory: &ArtifactDir) -> Result { check_manifest(&directory.root().join(MANIFEST_NAME))?; - Ok(Self::new( - directory.openvmm(), - directory.kernel(), - directory.initramfs(Guest::Alpine), - )) + Ok(Self::located(directory)) } /// Returns the artifacts staged by the build script, if the crate was built with the @@ -573,16 +627,17 @@ impl ArtifactDir { } /// Returns the artifacts of a sandbox, as `nvx.py sandbox run` and `nvx.py sandbox start` - /// require them: the OpenVMM executable, the kernel, and the Alpine initramfs, in that order. + /// require them: the OpenVMM executable, the kernel, and the Alpine initramfs, in that order, + /// located in this directory. /// /// Fails as [`require_file`] does for the first one that is not a regular file, with the /// messages of [`require_openvmm`](Self::require_openvmm) and /// [`require_kernel`](Self::require_kernel), or `initramfs not found: PATH`. pub fn sandbox_artifacts(&self) -> Result { - let openvmm = self.require_openvmm()?; - let kernel = self.require_kernel()?; - let initrd = required(self.initramfs(Guest::Alpine), "initramfs")?; - Ok(Artifacts::new(openvmm, kernel, initrd)) + self.require_openvmm()?; + self.require_kernel()?; + required(self.initramfs(Guest::Alpine), "initramfs")?; + Ok(Artifacts::located(self)) } /// Returns the paths of every artifact, for a caller to show or record. @@ -1222,13 +1277,19 @@ mod tests { artifacts.initramfs(Guest::Alpine), ); create(artifacts.initramfs(Guest::Alpine)); + let located = artifacts.sandbox_artifacts().unwrap(); assert_eq!( - artifacts.sandbox_artifacts().unwrap(), - Artifacts::new( - artifacts.openvmm(), - artifacts.kernel(), - artifacts.initramfs(Guest::Alpine) - ) + [&located.openvmm, &located.kernel, &located.initrd], + [ + &artifacts.openvmm(), + &artifacts.kernel(), + &artifacts.initramfs(Guest::Alpine) + ] + ); + assert_eq!(located.artifact_dir(), Some(&artifacts)); + assert_eq!( + located, + Artifacts::from_repo_layout(directory.path()).unwrap() ); assert_eq!(artifacts.require_openvmm().unwrap(), artifacts.openvmm()); @@ -1502,6 +1563,52 @@ mod tests { } } + #[test] + fn located_artifacts_have_the_identity_of_their_directory() { + let directory = tempfile::tempdir().unwrap(); + release(directory.path()); + let digests = ["4", "5", "6"].map(|digit| digit.repeat(64)); + let text: String = [ + (&digests[0], format!("bin/{}", openvmm_executable())), + (&digests[1], "guest/vmlinux".to_owned()), + (&digests[2], "guest/initramfs.cpio.gz".to_owned()), + ] + .iter() + .map(|(digest, name)| format!("{digest} {name}\n")) + .collect(); + fs::write(directory.path().join(CHECKSUMS_NAME), text).unwrap(); + let located = Artifacts::from_dir(directory.path()).unwrap(); + assert_eq!( + located.artifact_dir(), + Some(&ArtifactDir::release(directory.path()).unwrap()) + ); + assert_eq!(located.runtime_identity().unwrap(), identity_of(&digests)); + + // Artifacts named by path have no install metadata. + let named = Artifacts::new(&located.openvmm, &located.kernel, &located.initrd); + assert_eq!(named.artifact_dir(), None); + let error = named.runtime_identity().unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Unavailable); + assert!( + error.message().contains("supply their runtime identity"), + "{error}" + ); + // Nor does an artifact that replaced one of the directory's. + let mut replaced = located.clone(); + replaced.kernel = directory.path().join("vmlinux"); + let error = replaced.runtime_identity().unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Unavailable); + assert_eq!( + error.message(), + format!( + "the guest kernel {} is not the one of {}, whose install metadata identifies the \ + artifacts", + replaced.kernel.display(), + located.artifact_dir().unwrap().root().display() + ) + ); + } + #[cfg(feature = "bundled")] #[test] fn the_bundle_is_a_compatible_release() { From da8366cf67978119e86b3c978bf2e04e8038cf75 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Wed, 21 Oct 2026 09:06:21 -0700 Subject: [PATCH 2/3] nvx-rs: drive host::instance through the store and supervisor host::instance kept its own runtime record, launch record, lock, and recovery. An instance is now a sandbox store of its own, which the store, the supervisor, and the control session drive, as the CLI's sandboxes are: - Instance::create and Instance::open name the store, and Instance::lock takes its lock and returns a LockedInstance. - LockedInstance::find reattaches to the recorded VM: it ends a start whose caller died, forgets a VM that exited, and reports a VM of another runtime identity as Mismatched without driving it. It asks for the caller's runtime identity only if the store records a VM or a launch. - LockedInstance::start launches OpenVMM through supervisor::launch under the caller's runtime identity, waits until the guest is ready, and commits the launch, or ends OpenVMM and abandons it. - LockedInstance::stop stops the VM, with a forced step after its timeout, or at once when no control session can reach the guest, as when the VM's capability cannot be read; exec then fails. - instance::check_launch checks that a launch fits OpenVMM's command line before anything is recorded. aci_edge_sandboxes keeps each sandbox's configuration in the sandbox's store, whose lock replaces the state root's .locks directory. The runtime identity comes from the install metadata of the configured artifacts, or from the new OpenVmmConfig::runtime_identity for artifacts named by path; without one, the backend is unavailable. A VM of another runtime identity fails every operation with backend_error and keeps running. sandbox.json keeps its format 3. The real-hypervisor test locates its artifacts in the checkout, whose build provenance identifies them. Runtime records and launch journals of instance's format 3 fail as unsupported, naming their format: stop running sandboxes with the build that started them before switching. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: aa7b9fda-3774-4817-aa98-af05f8d50215 --- aci_edge_sandboxes/README.md | 55 +- aci_edge_sandboxes/src/openvmm/config.rs | 65 +- aci_edge_sandboxes/src/openvmm/mod.rs | 159 ++- aci_edge_sandboxes/src/openvmm/state.rs | 275 ++-- aci_edge_sandboxes/tests/openvmm_e2e.rs | 38 +- aci_edge_sandboxes/tests/openvmm_fake.rs | 388 ++++-- .../tests/openvmm_unavailable.rs | 36 + aci_edge_sandboxes/tests/support/mod.rs | 20 +- nvx-rs/README.md | 33 +- nvx-rs/src/host/instance.rs | 1185 +++++++---------- nvx-rs/src/host/mod.rs | 7 +- nvx-rs/src/host/store/mod.rs | 4 +- nvx-rs/src/lib.rs | 30 +- nvx-rs/tests/fake_openvmm.rs | 270 +++- scripts/nvx_tools/aci_edge_sandboxes_tests.py | 20 +- scripts/test_nvx_tools.py | 23 +- 16 files changed, 1441 insertions(+), 1167 deletions(-) diff --git a/aci_edge_sandboxes/README.md b/aci_edge_sandboxes/README.md index 12ea74196..33b39237f 100644 --- a/aci_edge_sandboxes/README.md +++ b/aci_edge_sandboxes/README.md @@ -625,6 +625,7 @@ Defaults: | `guest_network` | `10.0.0.2/24` | | `start_timeout`, `control_timeout` | 60 s | | `stop_timeout`, `exec_response_grace` | 30 s | +| `runtime_identity` | none: the install metadata of the artifacts' release or checkout names it; see [Runtime identity](#runtime-identity) | Each timeout must be at most 30 days (`OpenVmmConfig::MAX_TIMEOUT`), and all but `exec_response_grace` must be positive. `guest_network` must be an address @@ -636,6 +637,36 @@ Choose a `state_root` that only the current user can access. Windows and `$XDG_STATE_HOME/nvx/sandboxes` (default `~/.local/state/nvx/sandboxes`) elsewhere. +### Runtime identity + +Each sandbox's state directory is the `nvx::host::store` of its VM, which +records the runtime identity of the VM's start: the identities of its +OpenVMM, kernel, and initramfs, and the boot and control contract versions of +the build that started it. Every operation finds the sandbox's VM under the +backend's runtime identity, so a backend never drives a VM that another NVX +build, or other artifacts, started. + +- Artifacts located in a release or a repository checkout, by + `OpenVmmConfig::discover` from `NVX_ARTIFACTS_DIR`, an `nvx/` directory, or + the bundle, by `from_release_dir` or `from_repo_layout`, or by + `from_artifacts` with artifacts that `nvx::Artifacts` located, take their + identity from the directory's install metadata: the digests that a + release's `SHA256SUMS` lists, or the build provenance that `nvx.py` records + beside a checkout's guest artifacts. Nothing reads or hashes an artifact. + Pointing `openvmm`, `kernel`, or `initrd` of such a configuration at other + files leaves it without an identity. +- Artifacts named by path, by `OpenVmmConfig::new` or by `discover` from + `NVX_OPENVMM`, `NVX_KERNEL`, and `NVX_INITRD`, have no install metadata. Set + `OpenVmmConfig::runtime_identity` to the identity that the deployment + records for them; it also replaces the identity of located artifacts. + +Without a runtime identity, `probe`, `provision`, and `start` fail with +`backend_unavailable`, as does any operation on a sandbox whose store records a +VM. An operation on a sandbox whose VM runs under another runtime identity, +such as one that the build before an upgrade started, fails with +`backend_error`, naming the parts that differ and OpenVMM's process ID, and +leaves the VM running: stop it with the build and artifacts that started it. + ### How it runs OpenVMM - **Provision** validates the request and records it in @@ -646,10 +677,12 @@ Windows and `$XDG_STATE_HOME/nvx/sandboxes` (default system and starts the managed agent directly. - A fresh 32-byte capability travels through OpenVMM's standard-input pipe, which is closed before OpenVMM starts. - - Before launching OpenVMM, start records the control endpoint in - `launch.json`. If the caller dies before OpenVMM's process identity is - recorded, the next operation on the sandbox finds the VM through that - endpoint and terminates it, so no VM is left untracked. + - Before launching OpenVMM, start records the control endpoint and + OpenVMM's log in the store's launch journal, `launch.json`, and then + OpenVMM's process identity. If the caller dies before the VM is ready, the + next operation on the sandbox terminates the VM, through its process + identity, or through the endpoint's server if that process holds the log, + so no VM is left untracked. - OpenVMM runs detached from the caller: in a new session on Linux, and without the caller's console on Windows. It inherits no handle of the caller except its standard input and log, so a caller whose own output is @@ -672,9 +705,13 @@ Windows and `$XDG_STATE_HOME/nvx/sandboxes` (default - **Deprovision** deletes the state directory. It refuses to remove a directory that contains unexpected files. -The backend stores durable lifecycle state and reconciles it with the OpenVMM -process before every operation. Operations can therefore run safely from -different caller processes without forgetting a live VM. On Windows, set +Before every operation, the backend reattaches to the VM that the sandbox's +store records, as NVX's supervisor does after a restart: it checks that +OpenVMM's process still runs, verifying the process rather than trusting its +ID, under the backend's [runtime identity](#runtime-identity). A sandbox whose +VM died is provisioned again. Operations can therefore run safely from +different caller processes, one after another or after a caller restarts, +without forgetting a live VM. On Windows, set `breakaway_from_job` if the caller runs inside a job object that would otherwise terminate the VM when the caller exits. @@ -950,8 +987,8 @@ egress default of `allow` and no deny rule that covers the guest network; see | `start` or `deprovision` on a running sandbox | `AlreadyStarted` | `already_started` | | `stop` on a stopped sandbox | `AlreadyStopped` | `already_stopped` | | Unsupported policy or exec feature, oversized command, relative or oversized `process.cwd` | `PolicyValidation` | `policy_validation` | -| Missing OpenVMM artifacts, inaccessible hypervisor, incompatible release or guest image, unsupported host | `BackendUnavailable` | `backend_unavailable` | -| OpenVMM launch failure, boot timeout, control-session failure, I/O errors | `BackendError` | `backend_error` | +| Missing OpenVMM artifacts, inaccessible hypervisor, incompatible release or guest image, no [runtime identity](#runtime-identity), unsupported host | `BackendUnavailable` | `backend_unavailable` | +| OpenVMM launch failure, boot timeout, control-session failure, a VM of another runtime identity, I/O errors | `BackendError` | `backend_error` | Start failures name the OpenVMM log, `OpenVmmBackend::log_path`. diff --git a/aci_edge_sandboxes/src/openvmm/config.rs b/aci_edge_sandboxes/src/openvmm/config.rs index a4fc20cad..94cc54bb9 100644 --- a/aci_edge_sandboxes/src/openvmm/config.rs +++ b/aci_edge_sandboxes/src/openvmm/config.rs @@ -4,8 +4,8 @@ use std::time::Duration; use aci_edge_sandboxes_model::spec::{is_valid_guest_network, is_valid_hostname}; use nvx::Hypervisor; use nvx::host::artifacts::{Artifacts, absolute}; -use nvx::host::instance::SOCKET_NAME; use nvx::host::launch::{self, WorkloadIdentity}; +use nvx::host::store::{RuntimeIdentity, SOCKET_NAME}; use crate::error::{Error, Result}; @@ -14,7 +14,8 @@ use crate::error::{Error, Result}; /// Construct it with [`OpenVmmConfig::discover`], [`OpenVmmConfig::from_artifacts`], /// [`OpenVmmConfig::new`], [`OpenVmmConfig::from_release_dir`], or /// [`OpenVmmConfig::from_repo_layout`], then adjust the public fields. Defaults match -/// `scripts/nvx.py sandbox`. +/// `scripts/nvx.py sandbox`. Artifacts named by path, as by [`OpenVmmConfig::new`], also need +/// their [`runtime_identity`](Self::runtime_identity). #[derive(Debug, Clone, PartialEq, Eq)] #[non_exhaustive] pub struct OpenVmmConfig { @@ -24,6 +25,19 @@ pub struct OpenVmmConfig { pub kernel: PathBuf, /// The NVX Alpine initramfs (`initramfs.cpio.gz`): the guest userland and its control agent. pub initrd: PathBuf, + /// The runtime identity of the artifacts, which the store records with each sandbox's VM: + /// the identities of OpenVMM, the kernel, and the initramfs, with the contract versions of + /// this build. A VM that runs under another runtime identity, as after an upgrade, is reported + /// and neither driven nor stopped. + /// + /// Leave it unset for artifacts located in an NVX release or repository checkout, as by + /// [`from_release_dir`](Self::from_release_dir), [`from_repo_layout`](Self::from_repo_layout), + /// [`discover`](Self::discover), or [`from_artifacts`](Self::from_artifacts): every operation + /// that needs it then reads it from the install metadata of the release or checkout, a + /// release's `SHA256SUMS` or a checkout's build provenance, without hashing an artifact, so an + /// install upgraded in place is identified anew. Artifacts named by path have no install + /// metadata: without it, the backend is unavailable. + pub runtime_identity: Option, /// Hypervisor used by OpenVMM. pub hypervisor: Hypervisor, /// Directory that holds per-sandbox state. Choose a directory private to the current user. @@ -72,6 +86,9 @@ pub struct OpenVmmConfig { /// hosts without a hypervisor. Available only with the `testing` feature. #[cfg(feature = "testing")] pub skip_hypervisor_probe: bool, + /// The artifacts as [`from_artifacts`](Self::from_artifacts) received them, with the release + /// or checkout in which they were located, whose install metadata identifies them. + located: Option, } impl OpenVmmConfig { @@ -103,6 +120,7 @@ impl OpenVmmConfig { openvmm: openvmm.into(), kernel: kernel.into(), initrd: initrd.into(), + runtime_identity: None, hypervisor, state_root: state_root.into(), memory_mib: Self::DEFAULT_MEMORY_MIB, @@ -119,22 +137,53 @@ impl OpenVmmConfig { breakaway_from_job: false, #[cfg(feature = "testing")] skip_hypervisor_probe: false, + located: None, } } - /// Creates a configuration from located [`Artifacts`]. + /// Creates a configuration from [`Artifacts`], whose release or checkout, if they were located + /// in one, identifies them; see [`runtime_identity`](Self::runtime_identity). pub fn from_artifacts( artifacts: Artifacts, hypervisor: Hypervisor, state_root: impl Into, ) -> Self { - Self::new( - artifacts.openvmm, - artifacts.kernel, - artifacts.initrd, + let mut config = Self::new( + artifacts.openvmm.clone(), + artifacts.kernel.clone(), + artifacts.initrd.clone(), hypervisor, state_root, - ) + ); + config.located = Some(artifacts); + config + } + + /// Returns the runtime identity of the configured artifacts: + /// [`runtime_identity`](Self::runtime_identity) if it is set, and otherwise the one that the + /// install metadata of the release or checkout in which they were located names, read now. + /// + /// Fails with [`nvx::ErrorKind::Unavailable`] for artifacts named by path without one, and + /// when the metadata is missing or unreadable, or does not name the configured paths. + pub(crate) fn identify(&self) -> nvx::Result { + if let Some(identity) = &self.runtime_identity { + return Ok(identity.clone()); + } + let Some(mut artifacts) = self + .located + .clone() + .filter(|artifacts| artifacts.artifact_dir().is_some()) + else { + return Err(nvx::Error::unavailable( + "the artifacts are named by path, so no install metadata identifies them: set \ + OpenVmmConfig::runtime_identity, or locate them in an NVX release or repository \ + checkout", + )); + }; + artifacts.openvmm.clone_from(&self.openvmm); + artifacts.kernel.clone_from(&self.kernel); + artifacts.initrd.clone_from(&self.initrd); + artifacts.runtime_identity() } /// Creates a configuration with no NVX-specific input: the artifacts come from diff --git a/aci_edge_sandboxes/src/openvmm/mod.rs b/aci_edge_sandboxes/src/openvmm/mod.rs index d3e84f913..3637baebb 100644 --- a/aci_edge_sandboxes/src/openvmm/mod.rs +++ b/aci_edge_sandboxes/src/openvmm/mod.rs @@ -7,8 +7,9 @@ //! [`OpenVmmBackend`] converts the contract's requests into the launch specifications, policies, //! and workloads of [`nvx`], and maps NVX's errors onto the contract's error codes. It keeps each //! sandbox in a state directory under [`OpenVmmConfig::state_root`], which holds the sandbox's -//! configuration and the files of its VM [`Instance`](nvx::host::instance::Instance), so every -//! operation can run in a different process: +//! configuration and the store of its VM [`Instance`](nvx::host::instance::Instance): the lock, +//! the launch journal, and the runtime record of NVX's sandbox store, so every operation can run +//! in a different process: //! //! - **provision** validates the request and records the configuration. No VM runs yet. //! - **start** launches a detached OpenVMM process with the managed lifecycle, passes it a fresh @@ -25,8 +26,14 @@ //! [`OpenVmmConfig::stop_timeout`]. //! - **deprovision** deletes the state directory. //! -//! Every operation first reconciles the recorded OpenVMM process with the host: a sandbox whose -//! VM died is treated as provisioned again. +//! Every operation first finds the sandbox's VM as NVX's supervisor reattaches to it after a +//! restart: it ends a start whose caller died, and a sandbox whose VM died is treated as +//! provisioned again. The store records the runtime identity of each VM: the identities of its +//! OpenVMM, kernel, and initramfs, which [`OpenVmmConfig::runtime_identity`] gives or the install +//! metadata of the artifacts' release or checkout names, and the contract versions of the build +//! that started it. A VM whose runtime identity differs from the backend's, as after an upgrade, +//! fails every operation with [`ErrorCode::BackendError`](crate::ErrorCode::BackendError), and +//! keeps running: only the build and artifacts that started it can drive or stop it. //! //! # Policy honor matrix //! @@ -104,10 +111,10 @@ //! //! # Concurrency and cancellation //! -//! Lifecycle transitions of one sandbox are serialized by a lock file. Executions are serialized -//! by the guest: a second exec waits up to [`OpenVmmConfig::control_timeout`] for the first to -//! finish. A stop that cannot reach the guest within its timeout terminates the VM, and any -//! running exec then fails. +//! Lifecycle transitions of one sandbox are serialized by the lock of its store. Executions are +//! serialized by the guest: a second exec waits up to [`OpenVmmConfig::control_timeout`] for the +//! first to finish. A stop that cannot reach the guest within its timeout terminates the VM, and +//! any running exec then fails. //! //! The guest agent contains each workload in a cgroup and kills whatever the workload leaves //! behind when its first process exits, so no workload process outlives its exec. @@ -136,9 +143,10 @@ use std::time::Duration; use nvx::Artifacts; use nvx::host::filesystem::{self, ExportPlan, ExportRequest, PathList}; -use nvx::host::instance::{Execution, RuntimeRecord, WorkloadOutcome}; +use nvx::host::instance::{self, Execution, Found, LockedInstance, WorkloadOutcome}; use nvx::host::launch::{LaunchSpec, WorkloadIdentity}; use nvx::host::report::Report; +use nvx::host::store::RuntimeIdentity; use nvx::host::{network, platform}; use nvx::protocol::{ self, MAX_ARGUMENT_BYTES, MAX_CWD_BYTES, MAX_OUTPUT_BYTES, MAX_TIMEOUT_MS, Workload, @@ -172,11 +180,6 @@ pub struct OpenVmmBackend { store: StateStore, } -enum RunState { - Provisioned, - Running(RuntimeRecord), -} - impl OpenVmmBackend { /// Creates the backend, resolving relative paths and creating the state root. /// @@ -201,16 +204,32 @@ impl OpenVmmBackend { /// Returns the path of the OpenVMM log of a sandbox's latest start, for diagnostics. pub fn log_path(&self, sandbox_id: &SandboxId) -> PathBuf { - self.store.instance(sandbox_id).log_path() + self.store.log_path(sandbox_id) } - /// Returns whether the recorded OpenVMM process still runs, clearing stale runtime state; - /// see [`Instance::reconcile`](nvx::host::instance::Instance::reconcile). - fn reconcile(&self, sandbox_id: &SandboxId) -> Result { - Ok(match self.store.instance(sandbox_id).reconcile()? { - Some(runtime) => RunState::Running(runtime), - None => RunState::Provisioned, - }) + /// Finds the VM of the sandbox that `locked` holds, under the runtime identity of the + /// configured artifacts, which it reads only if the store records a VM; see + /// [`LockedInstance::find`]. + fn find(&self, locked: &mut LockedInstance) -> Result { + Ok(locked.find(|| self.config.identify())?) + } + + /// Checks that this host can run the configured artifacts, as [`Backend::probe`] does, and + /// returns their runtime identity. + fn check_host(&self) -> Result { + Artifacts::new( + &self.config.openvmm, + &self.config.kernel, + &self.config.initrd, + ) + .probe()?; + let runtime = self.config.identify()?; + #[cfg(feature = "testing")] + if self.config.skip_hypervisor_probe { + return Ok(runtime); + } + platform::probe_hypervisor(self.config.hypervisor)?; + Ok(runtime) } /// Returns the launch specification of a provisioned sandbox. @@ -449,17 +468,7 @@ impl Backend for OpenVmmBackend { } fn probe(&self) -> Result<()> { - Artifacts::new( - &self.config.openvmm, - &self.config.kernel, - &self.config.initrd, - ) - .probe()?; - #[cfg(feature = "testing")] - if self.config.skip_hypervisor_probe { - return Ok(()); - } - Ok(platform::probe_hypervisor(self.config.hypervisor)?) + self.check_host().map(drop) } fn validate_provision(&self, request: &ProvisionRequest) -> Result<()> { @@ -551,7 +560,7 @@ impl Backend for OpenVmmBackend { spec.kernel_command_line()?; let sandbox_id = SandboxId::generate()?; // Windows names each start's control pipe afresh, but every name has the same length. - self.store.instance(&sandbox_id).check_launch(&spec)?; + instance::check_launch(&self.store.dir(&sandbox_id), &spec)?; self.store.create(&sandbox_id, &record)?; Ok(ProvisionResult { sandbox_id, @@ -560,17 +569,22 @@ impl Backend for OpenVmmBackend { } fn start(&self, sandbox_id: &SandboxId) -> Result { - let (_guard, record) = self.store.lock_and_load(sandbox_id)?; - if let RunState::Running(_) = self.reconcile(sandbox_id)? { - return Err(Error::already_started(format!( - "sandbox {sandbox_id} is already running" - ))); + let (mut locked, record) = self.store.lock_and_load(sandbox_id)?; + match self.find(&mut locked)? { + Found::Stopped => {} + Found::Running(_) => { + return Err(Error::already_started(format!( + "sandbox {sandbox_id} is already running" + ))); + } + Found::Mismatched(mismatched) => return Err(mismatched.error().into()), } - self.probe()?; - let started = self - .store - .instance(sandbox_id) - .start(&self.launch_spec(&record), self.config.start_timeout)?; + let runtime = self.check_host()?; + let started = locked.start( + &self.launch_spec(&record), + &runtime, + self.config.start_timeout, + )?; let mut metadata = Metadata::new(); let boot_ms = u64::try_from(started.boot.as_millis()).unwrap_or(u64::MAX); metadata.insert("bootMilliseconds".to_owned(), boot_ms.into()); @@ -586,25 +600,23 @@ impl Backend for OpenVmmBackend { io: ExecIo, ) -> Result> { let workload = prepare_exec(&request.process)?; - let instance = self.store.instance(sandbox_id); - let (runtime, capability) = { - let (_guard, _) = self.store.lock_and_load(sandbox_id)?; - match self.reconcile(sandbox_id)? { - RunState::Provisioned => { + // The lock is held only while the running VM is found, so a long workload cannot delay + // a stop. + let (instance, running) = { + let (mut locked, _) = self.store.lock_and_load(sandbox_id)?; + match self.find(&mut locked)? { + Found::Stopped => { return Err(Error::not_started(format!( "sandbox {sandbox_id} is not running" ))); } - RunState::Running(runtime) => { - let capability = instance.read_capability()?; - (runtime, capability) - } + Found::Running(running) => (locked.instance().clone(), running), + Found::Mismatched(mismatched) => return Err(mismatched.error().into()), } }; let execution = instance .exec( - &runtime, - &capability, + &running, &workload, self.config.control_timeout, self.config.exec_response_grace, @@ -625,16 +637,17 @@ impl Backend for OpenVmmBackend { } fn stop(&self, sandbox_id: &SandboxId) -> Result { - let (_guard, _) = self.store.lock_and_load(sandbox_id)?; - let RunState::Running(runtime) = self.reconcile(sandbox_id)? else { - return Err(Error::already_stopped(format!( - "sandbox {sandbox_id} is not running" - ))); + let (mut locked, _) = self.store.lock_and_load(sandbox_id)?; + let running = match self.find(&mut locked)? { + Found::Stopped => { + return Err(Error::already_stopped(format!( + "sandbox {sandbox_id} is not running" + ))); + } + Found::Running(running) => running, + Found::Mismatched(mismatched) => return Err(mismatched.error().into()), }; - let stopped = self - .store - .instance(sandbox_id) - .stop(&runtime, self.config.stop_timeout)?; + let stopped = locked.stop(running, self.config.stop_timeout)?; let mut metadata = Metadata::new(); metadata.insert("forced".to_owned(), stopped.forced.into()); if let Some(report) = &stopped.report { @@ -646,15 +659,17 @@ impl Backend for OpenVmmBackend { } fn deprovision(&self, sandbox_id: &SandboxId) -> Result { - let (guard, _) = self.store.lock_and_load(sandbox_id)?; - if let RunState::Running(_) = self.reconcile(sandbox_id)? { - return Err(Error::already_started(format!( - "sandbox {sandbox_id} is running; stop it before deprovisioning" - ))); + let (mut locked, _) = self.store.lock_and_load(sandbox_id)?; + match self.find(&mut locked)? { + Found::Stopped => {} + Found::Running(_) => { + return Err(Error::already_started(format!( + "sandbox {sandbox_id} is running; stop it before deprovisioning" + ))); + } + Found::Mismatched(mismatched) => return Err(mismatched.error().into()), } - self.store.remove(sandbox_id)?; - drop(guard); - self.store.remove_lock(sandbox_id); + self.store.remove(sandbox_id, locked)?; Ok(DeprovisionResult::default()) } } diff --git a/aci_edge_sandboxes/src/openvmm/state.rs b/aci_edge_sandboxes/src/openvmm/state.rs index ceacefbc2..6c2fd3431 100644 --- a/aci_edge_sandboxes/src/openvmm/state.rs +++ b/aci_edge_sandboxes/src/openvmm/state.rs @@ -2,39 +2,46 @@ //! //! ```text //! / -//! .locks/.lock serializes lifecycle transitions of one sandbox //! / //! sandbox.json provisioned configuration -//! launch.json endpoint and available process identity of a start in progress -//! runtime.json OpenVMM process identity and endpoint (running only) +//! sandbox.lock serializes lifecycle transitions of the sandbox +//! launch.json the journal of a start in progress, or of one that was interrupted +//! runtime.json the runtime record of the running VM //! control.capability launch capability (running only) //! control.sock Linux control endpoint (running only) //! openvmm.log OpenVMM output of the latest start //! outcome.json OpenVMM outcome report of the latest run //! ``` //! -//! Each sandbox's directory is the directory of its VM [`Instance`], which keeps every file but -//! `sandbox.json`. +//! Each sandbox's directory is the store of its VM [`Instance`], which keeps every file but +//! `sandbox.json`, the configuration that this crate records in a format of its own, +//! [`STATE_FORMAT`]. use std::fs; use std::io; use std::path::{Path, PathBuf}; use nvx::host::filesystem::ExportPlan; -use nvx::host::instance::{self, Instance, LockGuard}; +use nvx::host::instance::{self, Instance, LockedInstance}; +use nvx::host::platform; +use nvx::host::store::{FILE_NAMES, LOCK_NAME, LOG_NAME}; use serde::{Deserialize, Serialize}; use crate::error::{Error, ErrorCode, Result}; use crate::id::SandboxId; use crate::model::{HostLoopbackForward, NetworkPolicy}; -/// Version of the state files written by this crate, which records the instance's export plan. -pub(crate) const STATE_FORMAT: u32 = instance::STATE_FORMAT; +/// Version of `sandbox.json`, the configuration that this crate records, which holds the +/// sandbox's [`ExportPlan`] in the form of `nvx`. +/// +/// Version 1 described sandboxes assembled from image layers and a scratch disk, and version 2 +/// exported the mapped host paths as one directory whose bind mounts the kernel command line +/// listed. +pub(crate) const STATE_FORMAT: u32 = 3; /// Backend key recorded in every sandbox. pub(crate) const BACKEND_KEY: &str = "openvmm"; const RECORD_NAME: &str = "sandbox.json"; -const LOCKS_NAME: &str = ".locks"; /// Provisioned configuration of one sandbox. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -74,14 +81,32 @@ fn not_provisioned(sandbox_id: &SandboxId) -> Error { Error::stale_id(format!("sandbox {sandbox_id} is not provisioned")) } +/// Removes the directory `dir` of a sandbox whose configuration is gone, and returns whether it +/// is gone. +/// +/// The directory stays while it is not empty: a transition that waited for the sandbox's lock may +/// hold a lock file in it, and a file system without POSIX deletion keeps a removed lock file +/// there until its last handle closes. Each such transition finds no configuration, and removes +/// its lock file and the directory as it releases the lock. +fn remove_emptied(dir: &Path) -> Result { + match fs::remove_dir(dir) { + Ok(()) => Ok(true), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(true), + Err(error) if error.kind() == io::ErrorKind::DirectoryNotEmpty => Ok(false), + Err(error) => Err(io_error(format!( + "cannot remove sandbox state {}", + dir.display() + ))(error)), + } +} + impl StateStore { /// Opens `root`, creating it and restricting it to the current user if needed. pub(crate) fn open(root: &Path) -> io::Result { if let Some(parent) = root.parent() { fs::create_dir_all(parent)?; } - instance::create_private_dir(root)?; - instance::create_private_dir(&root.join(LOCKS_NAME))?; + platform::create_private_dir(root)?; Ok(Self { root: root.to_path_buf(), }) @@ -91,64 +116,64 @@ impl StateStore { self.root.join(sandbox_id.token()) } - /// Returns the VM instance of a sandbox, which messages call by the sandbox's ID. - pub(crate) fn instance(&self, sandbox_id: &SandboxId) -> Instance { - Instance::new(self.dir(sandbox_id), sandbox_id.to_string()) + /// Returns the path of the OpenVMM log of a sandbox's latest start. + pub(crate) fn log_path(&self, sandbox_id: &SandboxId) -> PathBuf { + self.dir(sandbox_id).join(LOG_NAME) } - fn lock_path(&self, sandbox_id: &SandboxId) -> PathBuf { - self.root - .join(LOCKS_NAME) - .join(format!("{}.lock", sandbox_id.token())) - } - - /// Takes the exclusive lifecycle lock of a sandbox and loads its configuration. + /// Takes the exclusive lifecycle lock of a sandbox, the lock of its instance's store, and + /// loads its configuration. /// - /// An ID without state fails with [`ErrorCode::StaleId`] and leaves no lock file behind, also - /// when its state disappears while this call waits for the lock, as it does when a - /// concurrent deprovision removes it. A stale ID never becomes valid again, so no other call - /// can still need the removed lock file. + /// An ID without configuration fails with [`ErrorCode::StaleId`], also when its state + /// disappears while this call waits for the lock, as it does when a concurrent deprovision + /// removes it. A stale ID never becomes valid again, so the call removes the lock file that + /// it may have created, and the sandbox's directory, such as one that an interrupted removal + /// left, if that empties it: a stale ID leaves nothing behind. pub(crate) fn lock_and_load( &self, sandbox_id: &SandboxId, - ) -> Result<(LockGuard, SandboxRecord)> { - if matches!(self.dir(sandbox_id).try_exists(), Ok(false)) { - return Err(not_provisioned(sandbox_id)); - } - let guard = instance::lock(&self.lock_path(sandbox_id))?; + ) -> Result<(LockedInstance, SandboxRecord)> { + let dir = self.dir(sandbox_id); + let locked = match Instance::open(&dir, sandbox_id.to_string()) + .and_then(|instance| instance.lock(None)) + { + Ok(locked) => locked, + // The directory or its lock file went away, or never existed, with the sandbox. + Err(_) if matches!(dir.join(RECORD_NAME).try_exists(), Ok(false)) => { + let _ = remove_emptied(&dir); + return Err(not_provisioned(sandbox_id)); + } + Err(error) => return Err(error.into()), + }; match self.load(sandbox_id) { - Ok(record) => Ok((guard, record)), + Ok(record) => Ok((locked, record)), Err(error) if error.code() == ErrorCode::StaleId => { - drop(guard); - self.remove_lock(sandbox_id); + let _ = fs::remove_file(dir.join(LOCK_NAME)); + drop(locked); + let _ = remove_emptied(&dir); Err(error) } Err(error) => Err(error), } } - /// Removes the lock file of a deprovisioned sandbox. - pub(crate) fn remove_lock(&self, sandbox_id: &SandboxId) { - let _ = fs::remove_file(self.lock_path(sandbox_id)); - } - - /// Creates the state directory of a new sandbox. + /// Creates the state directory of a new sandbox, the store of its instance, private to the + /// current user, and records its configuration. pub(crate) fn create(&self, sandbox_id: &SandboxId, record: &SandboxRecord) -> Result<()> { let dir = self.dir(sandbox_id); - fs::create_dir(&dir).map_err(io_error(format!( - "cannot create sandbox state {}", - dir.display() - )))?; - let created = instance::create_private_dir(&dir) - .map_err(io_error(format!( - "cannot restrict sandbox state {}", + // Sandbox IDs are random, so the directory of a new one never exists. + if !matches!(dir.try_exists(), Ok(false)) { + return Err(Error::backend_error(format!( + "cannot create sandbox state {}: it exists already", dir.display() - ))) - .and_then(|()| Ok(instance::write_json(&dir.join(RECORD_NAME), record)?)); + ))); + } + let created = Instance::create(&dir, sandbox_id.to_string()) + .and_then(|_| instance::write_json(&dir.join(RECORD_NAME), record)); if created.is_err() { let _ = fs::remove_dir_all(&dir); } - created + Ok(created?) } /// Loads the configuration of a provisioned sandbox. @@ -173,11 +198,13 @@ impl StateStore { }) } - /// Deletes the state of a stopped sandbox. + /// Deletes the state of a stopped sandbox, whose lock `locked` holds, and releases the lock. /// - /// Unknown files abort the removal before anything is deleted. The configuration is removed - /// last, so an interrupted removal leaves a sandbox that can be deprovisioned again. - pub(crate) fn remove(&self, sandbox_id: &SandboxId) -> Result<()> { + /// Unknown files abort the removal before anything is deleted. The configuration goes after + /// the instance's files, so an interrupted removal leaves a sandbox that can be deprovisioned + /// again, and the lock file goes last, while it is held, so that a transition that waits for + /// the lock finds no configuration. + pub(crate) fn remove(&self, sandbox_id: &SandboxId, locked: LockedInstance) -> Result<()> { let dir = self.dir(sandbox_id); let entries = fs::read_dir(&dir).map_err(io_error(format!( "cannot list sandbox state {}", @@ -192,9 +219,12 @@ impl StateStore { let name = entry.file_name(); let name = name.to_string_lossy(); let temporary = name.starts_with('.') && name.ends_with(".tmp"); - if instance::FILE_NAMES.contains(&name.as_ref()) || temporary { + if name == RECORD_NAME || name == LOCK_NAME { + continue; + } + if FILE_NAMES.contains(&name.as_ref()) || temporary { owned.push(entry.path()); - } else if name != RECORD_NAME { + } else { return Err(Error::backend_error(format!( "sandbox state {} contains the unexpected entry {name:?}; remove it and retry", dir.display() @@ -205,22 +235,24 @@ impl StateStore { instance::remove_if_present(&path)?; } instance::remove_if_present(&dir.join(RECORD_NAME))?; - fs::remove_dir(&dir).map_err(io_error(format!( - "cannot remove sandbox state {}", - dir.display() - ))) + instance::remove_if_present(&dir.join(LOCK_NAME))?; + // The directory goes while the lock is held where it can, and once it is released where + // the removed lock file stays until its last handle closes. + if !remove_emptied(&dir)? { + drop(locked); + remove_emptied(&dir)?; + } + Ok(()) } } #[cfg(test)] mod tests { - use std::fs::OpenOptions; use std::sync::Arc; use std::thread; - use std::time::Duration; + use std::time::{Duration, Instant}; - use nvx::host::instance::{LaunchRecord, ProcessIdentity, RuntimeRecord}; - use nvx::protocol::CAPABILITY_LEN; + use nvx::ErrorKind; use super::*; use crate::ErrorCode; @@ -240,6 +272,16 @@ mod tests { } } + /// Returns the names of the entries of `dir`, sorted. + fn entries(dir: &Path) -> Vec { + let mut names: Vec = fs::read_dir(dir) + .unwrap() + .map(|entry| entry.unwrap().file_name().into_string().unwrap()) + .collect(); + names.sort(); + names + } + #[test] fn records_round_trip_and_unknown_ids_are_stale() { let root = tempfile::tempdir().unwrap(); @@ -248,53 +290,33 @@ mod tests { assert_eq!(store.load(&id).unwrap_err().code(), ErrorCode::StaleId); store.create(&id, &record()).unwrap(); assert_eq!(store.load(&id).unwrap(), record()); - assert!(store.instance(&id).runtime().unwrap().is_none()); + let (locked, loaded) = store.lock_and_load(&id).unwrap(); + assert_eq!(loaded, record()); + assert!(locked.instance().store().runtime().unwrap().is_none()); + assert_eq!(locked.instance().dir(), store.dir(&id)); + // A sandbox's ID is random, so its directory is new. + let error = store.create(&id, &record()).unwrap_err(); + assert_eq!(error.code(), ErrorCode::BackendError); + assert!(error.message().contains("exists already"), "{error}"); } #[test] - fn runtime_files_are_cleared_and_state_removed() { + fn the_instance_files_and_the_state_are_removed() { let root = tempfile::tempdir().unwrap(); let store = StateStore::open(root.path()).unwrap(); let id = SandboxId::generate().unwrap(); store.create(&id, &record()).unwrap(); - let instance = store.instance(&id); - assert!(instance.launch().unwrap().is_none()); - let launch = LaunchRecord { - format: STATE_FORMAT, - endpoint: "endpoint".to_owned(), - process: None, - }; - instance.write_launch(&launch).unwrap(); - assert_eq!(instance.launch().unwrap(), Some(launch.clone())); - let identified = LaunchRecord { - process: Some(ProcessIdentity { - pid: 1, - start_time: 2, - }), - ..launch.clone() - }; - instance.write_launch(&identified).unwrap(); - assert_eq!(instance.launch().unwrap(), Some(identified)); - instance.remove_launch(); - assert!(instance.launch().unwrap().is_none()); - instance.write_launch(&launch).unwrap(); - let runtime = RuntimeRecord { - format: STATE_FORMAT, - pid: 1, - start_time: 2, - endpoint: "endpoint".to_owned(), - }; - instance.write_runtime(&runtime).unwrap(); - instance.write_capability(&[9; CAPABILITY_LEN]).unwrap(); - assert_eq!(instance.runtime().unwrap(), Some(runtime)); - assert_eq!(instance.read_capability().unwrap(), [9; CAPABILITY_LEN]); - instance.clear_runtime().unwrap(); - assert!(instance.runtime().unwrap().is_none()); - assert!(instance.launch().unwrap().is_none()); - assert!(instance.read_capability().is_err()); - drop(instance.create_log().unwrap()); - store.remove(&id).unwrap(); - assert!(!store.dir(&id).exists()); + let (locked, _) = store.lock_and_load(&id).unwrap(); + let dir = store.dir(&id); + // A stopped VM leaves its log and outcome report, and an interrupted replacement of a + // record its temporary file. + for name in ["openvmm.log", "outcome.json", ".runtime.json.12345.tmp"] { + fs::write(dir.join(name), b"left").unwrap(); + } + assert_eq!(store.log_path(&id), dir.join("openvmm.log")); + store.remove(&id, locked).unwrap(); + assert!(!dir.exists()); + assert!(entries(root.path()).is_empty()); } #[test] @@ -304,7 +326,8 @@ mod tests { let id = SandboxId::generate().unwrap(); store.create(&id, &record()).unwrap(); fs::write(store.dir(&id).join("keep.txt"), b"user data").unwrap(); - assert!(store.remove(&id).is_err()); + let (locked, _) = store.lock_and_load(&id).unwrap(); + assert!(store.remove(&id, locked).is_err()); assert!(store.dir(&id).join("keep.txt").exists()); assert_eq!(store.load(&id).unwrap(), record()); } @@ -405,58 +428,58 @@ mod tests { let store = StateStore::open(root.path()).unwrap(); let id = SandboxId::generate().unwrap(); store.create(&id, &record()).unwrap(); - let (guard, loaded) = store.lock_and_load(&id).unwrap(); + let (locked, loaded) = store.lock_and_load(&id).unwrap(); assert_eq!(loaded, record()); - let path = store.lock_path(&id); - let contender = OpenOptions::new().write(true).open(&path).unwrap(); - assert!(contender.try_lock().is_err()); - drop(guard); - contender.try_lock().unwrap(); - } - - fn lock_files(root: &Path) -> usize { - fs::read_dir(root.join(LOCKS_NAME)).unwrap().count() + // The lock is the lock of the instance's store. + let contender = Instance::open(store.dir(&id), id.to_string()).unwrap(); + let error = contender.lock(Some(Instant::now())).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::TimedOut, "{error}"); + drop(locked); + drop(contender.lock(Some(Instant::now())).unwrap()); + assert!(store.dir(&id).join(LOCK_NAME).is_file()); } #[test] - fn stale_ids_leave_no_lock_files() { + fn stale_ids_leave_nothing_behind() { let root = tempfile::tempdir().unwrap(); let store = StateStore::open(root.path()).unwrap(); let never_provisioned = SandboxId::generate().unwrap(); - // An interrupted removal can leave an empty directory, which is stale too. + // An interrupted removal can leave an empty directory, or one with only its lock file, + // which are stale too. let emptied = SandboxId::generate().unwrap(); fs::create_dir(store.dir(&emptied)).unwrap(); - for id in [&never_provisioned, &emptied] { + let locked_only = SandboxId::generate().unwrap(); + Instance::create(store.dir(&locked_only), locked_only.to_string()).unwrap(); + fs::write(store.dir(&locked_only).join(LOCK_NAME), b"").unwrap(); + for id in [&never_provisioned, &emptied, &locked_only] { let error = store.lock_and_load(id).unwrap_err(); assert_eq!(error.code(), ErrorCode::StaleId, "{error}"); } - assert_eq!(lock_files(root.path()), 0); + assert!(entries(root.path()).is_empty()); let id = SandboxId::generate().unwrap(); store.create(&id, &record()).unwrap(); drop(store.lock_and_load(&id).unwrap()); - assert_eq!(lock_files(root.path()), 1); + assert_eq!(entries(&store.dir(&id)), [RECORD_NAME, LOCK_NAME]); } #[test] - fn state_removed_while_waiting_for_the_lock_leaves_no_lock_file() { + fn state_removed_while_waiting_for_the_lock_leaves_nothing_behind() { let root = tempfile::tempdir().unwrap(); let store = Arc::new(StateStore::open(root.path()).unwrap()); let id = SandboxId::generate().unwrap(); store.create(&id, &record()).unwrap(); - let (guard, _) = store.lock_and_load(&id).unwrap(); + let (locked, _) = store.lock_and_load(&id).unwrap(); let waiter = thread::spawn({ let (store, id) = (Arc::clone(&store), id.clone()); move || store.lock_and_load(&id).map(drop) }); // Let the waiter block on the lock, then deprovision as the backend does. A waiter that - // starts late finds no state and must leave no lock file either. + // starts late finds no state and must leave nothing behind either. thread::sleep(Duration::from_millis(100)); - store.remove(&id).unwrap(); - drop(guard); - store.remove_lock(&id); + store.remove(&id, locked).unwrap(); let error = waiter.join().unwrap().unwrap_err(); assert_eq!(error.code(), ErrorCode::StaleId, "{error}"); - assert_eq!(lock_files(root.path()), 0); + assert!(entries(root.path()).is_empty()); } } diff --git a/aci_edge_sandboxes/tests/openvmm_e2e.rs b/aci_edge_sandboxes/tests/openvmm_e2e.rs index 97f5cadd9..ee850dcfd 100644 --- a/aci_edge_sandboxes/tests/openvmm_e2e.rs +++ b/aci_edge_sandboxes/tests/openvmm_e2e.rs @@ -3,7 +3,8 @@ //! Run it through `scripts/nvx.py test-aci-edge-sandboxes --backend `, which resolves the //! artifacts and sets these variables: //! -//! - `ACI_EDGE_SANDBOXES_E2E_OPENVMM`, `ACI_EDGE_SANDBOXES_E2E_KERNEL`, `ACI_EDGE_SANDBOXES_E2E_INITRD`: OpenVMM and the Alpine guest. +//! - `ACI_EDGE_SANDBOXES_E2E_ARTIFACTS_DIR`: the NVX repository checkout or release that holds +//! OpenVMM and the Alpine guest, whose install metadata identifies them. //! - `ACI_EDGE_SANDBOXES_E2E_HYPERVISOR`: `kvm`, `mshv`, or `whp`. //! - `ACI_EDGE_SANDBOXES_E2E_STATE_ROOT` (optional): sandbox state directory. //! - `ACI_EDGE_SANDBOXES_E2E_OUTPUT_DIR` (optional): receives `openvmm-.log`, the OpenVMM log @@ -17,6 +18,7 @@ use std::sync::Arc; use std::thread; use std::time::{Duration, Instant}; +use aci_edge_sandboxes::nvx::Artifacts; use aci_edge_sandboxes::nvx::resolve_guest_path as guest_path; use aci_edge_sandboxes::openvmm::{OpenVmmBackend, OpenVmmConfig}; use aci_edge_sandboxes::{ @@ -33,6 +35,18 @@ fn variable(name: &str) -> String { }) } +/// Returns the configuration of the artifacts that the environment names, with `state_root`. +fn config(state_root: PathBuf) -> OpenVmmConfig { + let artifacts = Artifacts::from_dir(variable("ACI_EDGE_SANDBOXES_E2E_ARTIFACTS_DIR")).unwrap(); + OpenVmmConfig::from_artifacts( + artifacts, + variable("ACI_EDGE_SANDBOXES_E2E_HYPERVISOR") + .parse() + .unwrap(), + state_root, + ) +} + /// Stops and deprovisions the sandbox and keeps the OpenVMM log when the test fails. struct Cleanup { nvx: AciEdgeSandbox, @@ -87,16 +101,7 @@ fn client(name: &str) -> (AciEdgeSandbox, Arc) { env::temp_dir().join(format!("aci-edge-sandboxes-e2e-{}", std::process::id())) }) .join(name); - let config = OpenVmmConfig::new( - variable("ACI_EDGE_SANDBOXES_E2E_OPENVMM"), - variable("ACI_EDGE_SANDBOXES_E2E_KERNEL"), - variable("ACI_EDGE_SANDBOXES_E2E_INITRD"), - variable("ACI_EDGE_SANDBOXES_E2E_HYPERVISOR") - .parse() - .unwrap(), - state_root, - ); - let backend = Arc::new(OpenVmmBackend::new(config).unwrap()); + let backend = Arc::new(OpenVmmBackend::new(config(state_root)).unwrap()); (AciEdgeSandbox::from_shared(backend.clone()), backend) } @@ -1553,16 +1558,7 @@ fn openvmm_lifecycle_on_a_real_hypervisor() { .unwrap_or_else(|_| { env::temp_dir().join(format!("aci-edge-sandboxes-e2e-{}", std::process::id())) }); - let config = OpenVmmConfig::new( - variable("ACI_EDGE_SANDBOXES_E2E_OPENVMM"), - variable("ACI_EDGE_SANDBOXES_E2E_KERNEL"), - variable("ACI_EDGE_SANDBOXES_E2E_INITRD"), - variable("ACI_EDGE_SANDBOXES_E2E_HYPERVISOR") - .parse() - .unwrap(), - &state_root, - ); - let backend = Arc::new(OpenVmmBackend::new(config).unwrap()); + let backend = Arc::new(OpenVmmBackend::new(config(state_root.clone())).unwrap()); let nvx = AciEdgeSandbox::from_shared(backend.clone()); nvx.probe().unwrap(); diff --git a/aci_edge_sandboxes/tests/openvmm_fake.rs b/aci_edge_sandboxes/tests/openvmm_fake.rs index 602cf5b29..b5d960eef 100644 --- a/aci_edge_sandboxes/tests/openvmm_fake.rs +++ b/aci_edge_sandboxes/tests/openvmm_fake.rs @@ -6,16 +6,19 @@ any(target_os = "linux", windows) ))] +use std::ffi::OsString; use std::fs; -use std::io::{Read, Write}; +use std::io::Read; use std::path::{Path, PathBuf}; -use std::process::{Child, Command, Stdio}; use std::sync::mpsc; use std::thread; -use std::time::{Duration, Instant, SystemTime}; +use std::time::{Duration, Instant}; use aci_edge_sandboxes::nvx::Hypervisor; use aci_edge_sandboxes::nvx::host::platform; +use aci_edge_sandboxes::nvx::host::store::{ArtifactIdentity, RuntimeIdentity, SandboxStore}; +use aci_edge_sandboxes::nvx::host::supervisor::{self, Launched, VmmCommand}; +use aci_edge_sandboxes::nvx::protocol::CAPABILITY_LEN; use aci_edge_sandboxes::openvmm::OpenVmmConfig; use aci_edge_sandboxes::{ Access, AciEdgeSandbox, EgressPolicy, ErrorCode, ExecFailure, ExecOutcome, ExecOutput, @@ -32,78 +35,37 @@ struct Fixture { state_root: PathBuf, } -struct PendingVm { - child: Child, -} - -impl PendingVm { - fn spawn(arguments: &[String], capability: &[u8], directory: &Path) -> Self { - let (reader, mut writer) = std::io::pipe().unwrap(); - writer.write_all(capability).unwrap(); - drop(writer); - Self { - child: Command::new(env!("CARGO_BIN_EXE_aci-edge-sandboxes-fake-openvmm")) - .args(arguments) - .current_dir(directory) - .stdin(Stdio::from(reader)) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .unwrap(), - } - } - - #[cfg(target_os = "linux")] - fn start_time(&self) -> u64 { - let stat = fs::read_to_string(format!("/proc/{}/stat", self.child.id())).unwrap(); - stat.rsplit_once(')') - .unwrap() - .1 - .split_whitespace() - .nth(19) - .unwrap() - .parse() - .unwrap() - } +/// The capability of the launches that tests begin themselves. +const CAPABILITY: [u8; CAPABILITY_LEN] = [7; CAPABILITY_LEN]; - #[cfg(windows)] - fn start_time(&self) -> u64 { - use std::os::windows::io::AsRawHandle; +/// The OpenVMM process of a start that a test interrupted, which is killed when the test ends if +/// it still runs. +struct Orphan(Option); - use windows_sys::Win32::Foundation::FILETIME; - use windows_sys::Win32::System::Threading::GetProcessTimes; - - let mut created = FILETIME { - dwLowDateTime: 0, - dwHighDateTime: 0, - }; - let mut exited = created; - let mut kernel = created; - let mut user = created; - // SAFETY: Child owns the live process handle and all outputs point to valid storage. - let succeeded = unsafe { - GetProcessTimes( - self.child.as_raw_handle().cast(), - &mut created, - &mut exited, - &mut kernel, - &mut user, - ) - }; - assert_ne!(succeeded, 0, "{}", std::io::Error::last_os_error()); - (u64::from(created.dwHighDateTime) << 32) | u64::from(created.dwLowDateTime) +impl Orphan { + /// Returns the process. + fn launched(&self) -> &Launched { + self.0 + .as_ref() + .expect("the process is held until the test ends") } } -impl Drop for PendingVm { +impl Drop for Orphan { fn drop(&mut self) { - if self.child.try_wait().unwrap().is_none() { - self.child.kill().unwrap(); + if let Some(launched) = self.0.take() { + launched.kill_and_wait(); } - self.child.wait().unwrap(); } } +/// Returns the runtime identity of the fake artifacts, whose OpenVMM identity is the digit +/// `openvmm` repeated. +fn runtime_with(openvmm: char) -> RuntimeIdentity { + let artifact = |digit: char| ArtifactIdentity::sha256(&digit.to_string().repeat(64)).unwrap(); + RuntimeIdentity::new(artifact(openvmm), artifact('2'), artifact('3')) +} + impl Fixture { fn new() -> Self { let directory = tempfile::tempdir().unwrap(); @@ -124,6 +86,7 @@ impl Fixture { Hypervisor::platform_default().expect("tests run on Linux or Windows"), &self.state_root, ); + config.runtime_identity = Some(runtime_with('1')); config.skip_hypervisor_probe = true; config.start_timeout = Duration::from_secs(30); config.control_timeout = Duration::from_secs(15); @@ -172,9 +135,18 @@ impl Fixture { .count() } + /// Returns the number of sandbox directories that hold a lock file. fn lock_files(&self) -> usize { - fs::read_dir(self.state_root.join(".locks")) + fs::read_dir(&self.state_root) .unwrap() + .filter(|entry| { + entry + .as_ref() + .unwrap() + .path() + .join("sandbox.lock") + .is_file() + }) .count() } } @@ -627,6 +599,39 @@ fn state_machine_violations_use_contract_codes() { ); } +#[test] +fn a_sandbox_whose_capability_is_lost_is_still_stopped() { + // Only control sessions need the capability: without it, exec fails, start and deprovision + // still find the sandbox running, and stop ends its VM through its verified process identity. + let fixture = Fixture::new(); + let nvx = fixture.nvx(); + let sandbox_id = nvx.provision(&ProvisionRequest::new()).unwrap().sandbox_id; + nvx.start(&sandbox_id).unwrap(); + let capability = fixture + .state_root + .join(sandbox_id.token()) + .join("control.capability"); + fs::remove_file(&capability).unwrap(); + let echo = ExecRequest::command_line("echo"); + assert_eq!( + nvx.exec(&sandbox_id, &echo).unwrap_err().code(), + ErrorCode::BackendError + ); + assert_eq!( + nvx.start(&sandbox_id).unwrap_err().code(), + ErrorCode::AlreadyStarted + ); + assert_eq!( + nvx.deprovision(&sandbox_id).unwrap_err().code(), + ErrorCode::AlreadyStarted + ); + let stopped = nvx.stop(&sandbox_id).unwrap().metadata.unwrap(); + assert_eq!(stopped["forced"], true); + assert!(!capability.exists()); + nvx.deprovision(&sandbox_id).unwrap(); + assert!(!fixture.state_root.join(sandbox_id.token()).exists()); +} + #[test] fn calls_with_stale_ids_leave_no_lock_files() { let fixture = Fixture::new(); @@ -985,8 +990,8 @@ fn crashed_vms_fall_back_to_provisioned() { // Wait until the VM is gone, as the backend tells it: an exec that the crash interrupts fails // as a broken control session instead, with `BackendError`. let runtime = state_json(&fixture, &sandbox_id, "runtime.json"); - let pid = u32::try_from(runtime["pid"].as_u64().unwrap()).unwrap(); - let start_time = runtime["startTime"].as_u64().unwrap(); + let pid = support::recorded_pid(&runtime); + let start_time = runtime["process"]["startTime"].as_u64().unwrap(); let deadline = Instant::now() + Duration::from_secs(30); while platform::process_start_time(pid).unwrap() == Some(start_time) { assert!(Instant::now() < deadline, "the crash was never observed"); @@ -1146,7 +1151,7 @@ fn interrupted_starts_do_not_leak_vms() { let dir = fixture.state_root.join(sandbox_id.token()); assert!(!dir.join("launch.json").exists()); let restarted = state_json(&fixture, &sandbox_id, "runtime.json"); - assert_ne!(restarted["pid"], runtime["pid"]); + assert_ne!(restarted["process"], runtime["process"]); assert_eq!( run(&nvx, &sandbox_id, "echo recovered").stdout, b"recovered\n" @@ -1161,39 +1166,46 @@ fn interrupted_starts_before_an_endpoint_recover_or_retain_the_child() { let fixture = Fixture::new(); let nvx = fixture.nvx(); let sandbox_id = nvx.provision(&ProvisionRequest::new()).unwrap().sandbox_id; + // OpenVMM's arguments, as a start of the sandbox recorded them. nvx.start(&sandbox_id).unwrap(); - let runtime = state_json(&fixture, &sandbox_id, "runtime.json"); - let dir = fixture.state_root.join(sandbox_id.token()); - let capability = fs::read(dir.join("control.capability")).unwrap(); let mut arguments = fixture.launch_arguments(&sandbox_id); + nvx.stop(&sandbox_id).unwrap(); let command_line = arguments .iter() .position(|argument| argument == "--cmdline") .unwrap() + 1; arguments[command_line].push_str(" fake_boot_delay_ms=30000"); - nvx.stop(&sandbox_id).unwrap(); - fs::remove_file(dir.join("outcome.json")).unwrap(); - let child = PendingVm::spawn(&arguments, &capability, &dir); - let pid = child.child.id(); - let mut launch = serde_json::json!({ - "format": runtime["format"], - "endpoint": runtime["endpoint"], - }); + // A caller begins a start in the sandbox's store, spawns OpenVMM, which serves no + // endpoint for a while, records its process identity or not, and dies. + let dir = fixture.state_root.join(sandbox_id.token()); + let store = SandboxStore::open(&dir).unwrap(); + let mut locked = store.lock(None).unwrap(); + let mut launch = locked + .begin_launch(&runtime_with('1'), &CAPABILITY) + .unwrap(); + let listen = arguments + .iter() + .position(|argument| argument.starts_with("listen=")) + .unwrap(); + arguments[listen] = format!("listen={}", launch.endpoint()); + let command = VmmCommand::new( + env!("CARGO_BIN_EXE_aci-edge-sandboxes-fake-openvmm"), + arguments.iter().map(OsString::from).collect(), + &dir, + ); + let log = launch.take_log().unwrap(); + let child = Orphan(Some(supervisor::spawn(&command, &CAPABILITY, log).unwrap())); if identified { - launch["process"] = serde_json::json!({ "pid": pid, "startTime": child.start_time() }); + launch.record_process(child.launched()).unwrap(); } - let marker = dir.join("launch.json"); - fs::write(&marker, launch.to_string()).unwrap(); - fs::File::options() - .write(true) - .open(&marker) - .unwrap() - .set_modified(SystemTime::now() - Duration::from_secs(120)) - .unwrap(); + drop(launch); + drop(locked); + let pid = child.launched().id(); + let journal = dir.join("launch.json"); assert!(support::process_running(pid)); - assert!(!dir.join("runtime.json").exists()); + assert!(journal.exists()); let result = nvx.start(&sandbox_id); if identified { @@ -1202,9 +1214,9 @@ fn interrupted_starts_before_an_endpoint_recover_or_retain_the_child() { !support::process_running(pid), "the pre-endpoint child survived recovery" ); - assert!(!marker.exists()); + assert!(!journal.exists()); assert_ne!( - state_json(&fixture, &sandbox_id, "runtime.json")["pid"], + support::recorded_pid(&state_json(&fixture, &sandbox_id, "runtime.json")), pid ); assert_eq!( @@ -1217,9 +1229,16 @@ fn interrupted_starts_before_an_endpoint_recover_or_retain_the_child() { if result.is_ok() { nvx.stop(&sandbox_id).unwrap(); } - assert_eq!(result.unwrap_err().code(), ErrorCode::BackendError); + // OpenVMM holds the launch's log, and serves no endpoint that would name it, so + // nothing verifies it, and it is neither signaled nor forgotten. + let error = result.unwrap_err(); + assert_eq!(error.code(), ErrorCode::BackendError, "{error}"); + assert!( + error.message().contains("cannot determine whether"), + "{error}" + ); assert!(support::process_running(pid)); - assert!(marker.exists()); + assert!(journal.exists()); assert!(!dir.join("runtime.json").exists()); assert_eq!( nvx.deprovision(&sandbox_id).unwrap_err().code(), @@ -1230,46 +1249,39 @@ fn interrupted_starts_before_an_endpoint_recover_or_retain_the_child() { } #[test] -fn launch_markers_without_process_identity_do_not_expire() { +fn unrecorded_launches_end_once_their_log_is_released() { let fixture = Fixture::new(); let nvx = fixture.nvx(); let sandbox_id = nvx.provision(&ProvisionRequest::new()).unwrap().sandbox_id; + // A caller begins a start and dies before it records OpenVMM, which would hold the launch's + // log, as this test does. let dir = fixture.state_root.join(sandbox_id.token()); - let endpoint = if cfg!(windows) { - format!("//./pipe/openvmm-microvm-{}", sandbox_id.token()) - } else { - dir.join("control.sock").to_string_lossy().into_owned() - }; - let marker = dir.join("launch.json"); - let format = state_json(&fixture, &sandbox_id, "sandbox.json")["format"].clone(); - let launch = serde_json::json!({ "format": format, "endpoint": endpoint }); - fs::write(&marker, launch.to_string()).unwrap(); + let store = SandboxStore::open(&dir).unwrap(); + let mut locked = store.lock(None).unwrap(); + let mut launch = locked + .begin_launch(&runtime_with('1'), &CAPABILITY) + .unwrap(); + let log = launch.take_log().unwrap(); + drop(launch); + drop(locked); + let journal = dir.join("launch.json"); - // An absent endpoint does not prove that the interrupted launch has no surviving child. + // While the log is held, nothing proves that the launch ended. for error in [ nvx.start(&sandbox_id).unwrap_err(), nvx.deprovision(&sandbox_id).unwrap_err(), ] { - assert_eq!(error.code(), ErrorCode::BackendError); + assert_eq!(error.code(), ErrorCode::BackendError, "{error}"); } - assert!(marker.exists()); + assert!(journal.exists()); - fs::File::options() - .write(true) - .open(&marker) - .unwrap() - .set_modified(SystemTime::now() - Duration::from_secs(120)) - .unwrap(); - let result = nvx.start(&sandbox_id); - if result.is_ok() { - nvx.stop(&sandbox_id).unwrap(); - } - assert_eq!(result.unwrap_err().code(), ErrorCode::BackendError); - assert!(marker.exists()); - assert_eq!( - nvx.deprovision(&sandbox_id).unwrap_err().code(), - ErrorCode::BackendError - ); + // Its release proves it. + drop(log); + nvx.start(&sandbox_id).unwrap(); + assert!(!journal.exists()); + assert_eq!(run(&nvx, &sandbox_id, "echo started").stdout, b"started\n"); + nvx.stop(&sandbox_id).unwrap(); + nvx.deprovision(&sandbox_id).unwrap(); } #[test] @@ -2104,6 +2116,136 @@ fn sandboxes_of_the_previous_state_format_are_refused() { assert!(error.message().contains("unsupported format"), "{error}"); } +#[test] +fn runtime_records_of_the_previous_format_are_refused() { + let fixture = Fixture::new(); + let nvx = fixture.nvx(); + let sandbox_id = nvx.provision(&ProvisionRequest::new()).unwrap().sandbox_id; + // Before NVX's sandbox store, the runtime record of a running VM had format 3. Nothing reads + // or migrates it: the VM must be stopped with the build that started it. + let record = fixture + .state_root + .join(sandbox_id.token()) + .join("runtime.json"); + let earlier = serde_json::json!({ + "format": 3, "pid": 4242, "startTime": 1, "endpoint": "endpoint", + }); + fs::write(&record, earlier.to_string()).unwrap(); + for error in [ + nvx.start(&sandbox_id).unwrap_err(), + nvx.exec(&sandbox_id, &ExecRequest::command_line("echo")) + .unwrap_err(), + nvx.stop(&sandbox_id).unwrap_err(), + nvx.deprovision(&sandbox_id).unwrap_err(), + ] { + assert_eq!(error.code(), ErrorCode::BackendError, "{error}"); + assert!( + error.message().contains("has the unsupported format 3"), + "{error}" + ); + } + assert!(record.is_file()); +} + +/// Writes the checksum list of `release`, in which `openvmm` is the digit of the OpenVMM +/// executable's digest. +fn write_checksums(release: &Path, openvmm: char) { + let executable = if cfg!(windows) { + "openvmm.exe" + } else { + "openvmm" + }; + let text = format!( + "{} bin/{executable}\n{} guest/vmlinux\n{} guest/initramfs.cpio.gz\n", + openvmm.to_string().repeat(64), + "2".repeat(64), + "3".repeat(64), + ); + fs::write(release.join("SHA256SUMS"), text).unwrap(); +} + +#[test] +fn runtime_identities_come_from_the_install_metadata() { + use aci_edge_sandboxes::nvx::contract; + + let fixture = Fixture::new(); + // A release that holds the fake OpenVMM and guest artifacts. + let release = fixture.directory.path().join("release"); + let executable = if cfg!(windows) { + "openvmm.exe" + } else { + "openvmm" + }; + fs::create_dir_all(release.join("bin")).unwrap(); + fs::create_dir_all(release.join("guest")).unwrap(); + fs::copy( + env!("CARGO_BIN_EXE_aci-edge-sandboxes-fake-openvmm"), + release.join("bin").join(executable), + ) + .unwrap(); + fs::write(release.join("guest").join("vmlinux"), b"kernel").unwrap(); + fs::write(release.join("guest").join("initramfs.cpio.gz"), b"initrd").unwrap(); + let manifest = serde_json::json!({ "openvmm": { + "microvm_abi_version": contract::MICROVM_ABI_VERSION, + "control_session_protocol_version": contract::CONTROL_SESSION_PROTOCOL_VERSION, + "control_contract_revision": contract::CONTROL_CONTRACT_REVISION, + }}); + fs::write(release.join("SOURCE-MANIFEST.json"), manifest.to_string()).unwrap(); + write_checksums(&release, '1'); + let mut config = OpenVmmConfig::from_release_dir( + &release, + Hypervisor::platform_default().expect("tests run on Linux or Windows"), + &fixture.state_root, + ) + .unwrap(); + assert_eq!(config.runtime_identity, None); + config.skip_hypervisor_probe = true; + config.stop_timeout = Duration::from_secs(15); + let nvx = AciEdgeSandbox::openvmm(config).unwrap(); + let sandbox_id = nvx.provision(&ProvisionRequest::new()).unwrap().sandbox_id; + nvx.start(&sandbox_id).unwrap(); + // The store records the identity that the release's checksum list names. + let recorded = state_json(&fixture, &sandbox_id, "runtime.json")["runtime"].clone(); + assert_eq!( + recorded["openvmm"], + format!("sha256:{}", "1".repeat(64)).as_str() + ); + assert_eq!(run(&nvx, &sandbox_id, "echo before").stdout, b"before\n"); + + // The release is upgraded in place: the backend reads the new identity, and reports the VM + // that the earlier OpenVMM started rather than driving or stopping it. + write_checksums(&release, '4'); + for error in [ + nvx.exec(&sandbox_id, &ExecRequest::command_line("echo")) + .unwrap_err(), + nvx.stop(&sandbox_id).unwrap_err(), + nvx.start(&sandbox_id).unwrap_err(), + nvx.deprovision(&sandbox_id).unwrap_err(), + ] { + assert_eq!(error.code(), ErrorCode::BackendError, "{error}"); + assert!( + error + .message() + .contains("runs under another runtime identity, which differs in OpenVMM"), + "{error}" + ); + } + let pid = support::recorded_pid(&state_json(&fixture, &sandbox_id, "runtime.json")); + assert!(support::process_running(pid)); + + // Back on the artifacts that started it, the backend drives the VM again. + write_checksums(&release, '1'); + assert_eq!(run(&nvx, &sandbox_id, "echo after").stdout, b"after\n"); + nvx.stop(&sandbox_id).unwrap(); + nvx.deprovision(&sandbox_id).unwrap(); + + // A release without its checksum list cannot be identified. + fs::remove_file(release.join("SHA256SUMS")).unwrap(); + let error = nvx.probe().unwrap_err(); + assert_eq!(error.code(), ErrorCode::BackendUnavailable, "{error}"); + assert!(error.message().contains("SHA256SUMS"), "{error}"); +} + #[test] fn concurrent_execs_are_serialized() { let fixture = Fixture::new(); diff --git a/aci_edge_sandboxes/tests/openvmm_unavailable.rs b/aci_edge_sandboxes/tests/openvmm_unavailable.rs index bdcbc6759..367c30e88 100644 --- a/aci_edge_sandboxes/tests/openvmm_unavailable.rs +++ b/aci_edge_sandboxes/tests/openvmm_unavailable.rs @@ -57,3 +57,39 @@ fn incompatible_releases_are_reported_as_backend_unavailable() { .unwrap_err(); assert_eq!(error.code(), ErrorCode::BackendUnavailable); } + +#[test] +fn artifacts_named_by_path_need_a_runtime_identity() { + let directory = tempfile::tempdir().unwrap(); + let file = |name: &str| { + let path = directory.path().join(name); + fs::write(&path, b"").unwrap(); + path + }; + let config = OpenVmmConfig::new( + file("openvmm"), + file("vmlinux"), + file("initramfs.cpio.gz"), + hypervisor(), + directory.path().join("state"), + ); + assert_eq!(config.runtime_identity, None); + let Ok(nvx) = AciEdgeSandbox::openvmm(config) else { + // Hosts that cannot run the backend at all reject the configuration up front. + return; + }; + // The files exist, but no install metadata names their identity, which the store records + // with each VM. + for error in [ + nvx.probe().unwrap_err(), + nvx.provision(&ProvisionRequest::new()).unwrap_err(), + ] { + assert_eq!(error.code(), ErrorCode::BackendUnavailable, "{error}"); + assert!( + error + .message() + .contains("set OpenVmmConfig::runtime_identity"), + "{error}" + ); + } +} diff --git a/aci_edge_sandboxes/tests/support/mod.rs b/aci_edge_sandboxes/tests/support/mod.rs index f511d6a07..fdefa0cef 100644 --- a/aci_edge_sandboxes/tests/support/mod.rs +++ b/aci_edge_sandboxes/tests/support/mod.rs @@ -4,20 +4,32 @@ use std::fs; use std::path::Path; use aci_edge_sandboxes::SandboxId; +use aci_edge_sandboxes::nvx::host::platform; /// Turns the state of a running sandbox into the state left by a caller that died between -/// launching OpenVMM and recording its process identity. Returns the OpenVMM process ID. +/// launching OpenVMM and recording its process identity: the launch journal of the VM's launch, +/// without the identity, whose claimed log OpenVMM still holds as its standard output. Returns the +/// OpenVMM process ID. pub(crate) fn interrupt_start(state_root: &Path, sandbox_id: &SandboxId) -> u32 { let dir = state_root.join(sandbox_id.token()); let runtime: serde_json::Value = serde_json::from_slice(&fs::read(dir.join("runtime.json")).unwrap()).unwrap(); - let marker = serde_json::json!({ + let (device, index) = platform::file_identity(&dir.join("openvmm.log")).unwrap(); + let journal = serde_json::json!({ "format": runtime["format"], + "launch": runtime["launch"], "endpoint": runtime["endpoint"], + "runtime": runtime["runtime"], + "log": { "identity": { "device": device, "index": index } }, }); - fs::write(dir.join("launch.json"), marker.to_string()).unwrap(); + fs::write(dir.join("launch.json"), journal.to_string()).unwrap(); fs::remove_file(dir.join("runtime.json")).unwrap(); - u32::try_from(runtime["pid"].as_u64().unwrap()).unwrap() + recorded_pid(&runtime) +} + +/// Returns the OpenVMM process ID that a runtime record names. +pub(crate) fn recorded_pid(runtime: &serde_json::Value) -> u32 { + u32::try_from(runtime["process"]["pid"].as_u64().unwrap()).unwrap() } /// Returns whether a process runs. Zombies have exited; only their parent has yet to reap them. diff --git a/nvx-rs/README.md b/nvx-rs/README.md index 88d4d6da6..5a6736425 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -69,13 +69,13 @@ nvx-rs/ | `nvx::gpt` | `Disk::read`: the protective MBR, primary header, and primary partition entry array of a GPT disk, read through a `ReadAt` source that the caller gives and checked strictly, as UEFI specifies them and as Linux requires them, without ever reading the backup: 128-byte entries, at most 1 MiB of them, used entries first and without holes, a first partition and 1 to 254 layers, extents in the usable blocks without overlap, and unique GUIDs, with checked arithmetic in the disk's logical blocks. `Expected` adds the disk GUID and the partitions' unique GUIDs and sizes that a caller knows, and `Disk::read_first_partition` returns the first partition's opaque bytes, up to a bound and without trailing NUL bytes and whitespace. GUIDs are `nvx::uuid::Uuid`s. It has no dependencies, so the guest and the host's checks of the images that it stages share it | | `nvx::uuid` | `Uuid`: a UUID, such as the GUID of a GPT disk or partition or the UUID of a filesystem, in the order of its canonical text, the one text that it displays and parses: 32 lowercase hexadecimal digits in groups of 8, 4, 4, 4, and 12, joined by hyphens. `from_bytes_le` and `to_bytes_le` convert the layout in which GPT and other EFI GUIDs store their first three groups little-endian. It has no dependencies, so the host and the guest share it | | `nvx::Hypervisor` | KVM, MSHV, or WHP, with its serde form behind the `serde` feature | -| `nvx::host::artifacts` | `Artifacts` (also `nvx::Artifacts`): the OpenVMM executable, guest kernel, and Alpine initramfs, from a release, a repository checkout, the environment, a directory next to the executable, or the bundle. `ArtifactDir` and `Guest`: every artifact of the guests that `nvx.py` runs, the OpenVMM executable, the kernel, the Alpine, Ubuntu, and Azure Linux initramfs images and their package manifests, and the Ubuntu distro layer and its manifest, in a checkout's `build/` and `openvmm/target/release/` or a release's `guest/` and `bin/`, checked in the order and with the messages of `nvx.py`. `ArtifactPaths` lists them all and serializes as JSON. `ArtifactDir::runtime_identity` reads the `RuntimeIdentity` of the OpenVMM executable, kernel, and Alpine initramfs from the install metadata, a release's `SHA256SUMS` or a checkout's build provenance, without hashing an artifact | +| `nvx::host::artifacts` | `Artifacts` (also `nvx::Artifacts`): the OpenVMM executable, guest kernel, and Alpine initramfs, from a release, a repository checkout, the environment, a directory next to the executable, or the bundle. Artifacts located in a release or checkout keep its `ArtifactDir`, and `Artifacts::runtime_identity` reads their `RuntimeIdentity` from its install metadata; artifacts named by path have none. `ArtifactDir` and `Guest`: every artifact of the guests that `nvx.py` runs, the OpenVMM executable, the kernel, the Alpine, Ubuntu, and Azure Linux initramfs images and their package manifests, and the Ubuntu distro layer and its manifest, in a checkout's `build/` and `openvmm/target/release/` or a release's `guest/` and `bin/`, checked in the order and with the messages of `nvx.py`. `ArtifactPaths` lists them all and serializes as JSON. `ArtifactDir::runtime_identity` reads the `RuntimeIdentity` of the OpenVMM executable, kernel, and Alpine initramfs from the install metadata, a release's `SHA256SUMS` or a checkout's build provenance, without hashing an artifact | | `nvx::host::launch` | `LaunchSpec` and OpenVMM's command line: the managed lifecycle, the `WorkloadIdentity`, the 1024-byte kernel command line, its reserved parameters, the 32,767-character limit of a Windows command line, and `LaunchSpec::check`, which every command line applies to the memory size, the non-root identity, the hostname, the guest network, and the kernel parameters | | `nvx::host::network` | `NetworkPolicy` and `openvmm_arguments`: the egress default and rules, ingress, host-loopback access and forwards as OpenVMM options. Egress rules are lowered exactly into OpenVMM's IPv4 and IPv6 rules by the lowering of egress policy files, with at most 256 rules for each action | | `nvx::host::network::policy` | Egress policy files: `compile_policy_file`, `compile_policy_bytes`, and `compile_policy` compile one to a `CompiledEgressPolicy` of native rules; see [Egress policy files](#egress-policy-files) | | `nvx::host::filesystem` | The `ExportPlan` of host paths through OpenVMM's aggregate virtio-fs device, the host-mapping table, the reserved guest targets, and `guest_path` and `resolve_guest_path` (also at `nvx::`) | | `nvx::host::store` | `SandboxStore`: the runtime records of one sandbox in a private directory with one lock, which every host that supervises the sandbox shares. The `RuntimeRecord` names the running VM: the `ProcessRef` of its VMM, its control endpoint, and the `RuntimeIdentity` that it was launched with, the identities of its OpenVMM, kernel, and initramfs from install or staging metadata and its boot and control contract versions, which decide whether another build of the host may reattach to it. The `LaunchJournal` of a launch holds a marker written before the spawn, the VMM's `ProcessRef` right after it, and an inherited claim on the VMM's log, which proves that a launch that never published its endpoint has exited, so that recovery ends an interrupted launch and signals only a VMM that it holds through a verified `Process`. Records are JSON in one versioned format, and records in any other format fail rather than being migrated | -| `nvx::host::instance` | `Instance`: a VM in a directory of its own. It starts OpenVMM with a fresh capability and waits until the guest is ready and has mounted the mapped paths, reconciles recorded state, recovers interrupted launches, runs workloads and pumps their output, and stops the VM with a forced fallback | +| `nvx::host::instance` | `Instance`: a VM in a sandbox store of its own, which `nvx::host::store`, `nvx::host::supervisor`, and `nvx::host::session` drive. Under the store's lock, `LockedInstance::find` reattaches to the VM, ends a start that was interrupted, and reports a VM of another runtime identity as `Mismatched` rather than driving it; `LockedInstance::start` launches OpenVMM through the supervisor with a fresh capability, under the caller's `RuntimeIdentity`, waits until the guest is ready and has mounted the mapped paths, and commits the launch; and `LockedInstance::stop` stops the VM with a forced fallback. `Instance::exec` runs workloads without the lock and pumps their output. `write_json` and `read_json` keep the configuration records that callers store beside an instance | | `nvx::host::nvxs` | `HostConnection`: the host's role of the NVXS session over a platform transport, on which `nvx::host::session` and other host applications run their protocols. It attaches with the launch capability, waiting while the broker asks it to, sends application bytes in as many DATA records as they take, and returns each DATA payload from the guest, with deadlines; a receive that times out keeps a partial record | | `nvx::host::process` | Verified process references, and the launch of OpenVMM for `nvx::host::instance`, through `nvx::host::supervisor`. A `ProcessRef` names one process by its ID, its start time, and the `BootId` of the host boot during which it started. `Process::open` checks a reference and holds its process through a pidfd on Linux or a process handle on Windows, whether or not the caller started it, so that waits sleep until the process exits and termination never reaches a process that reuses its ID. A reference from an earlier boot names an exited process, and a process whose identity this user cannot check, such as another user's, makes `Process::open` fail rather than report an exit. Linux hosts without pidfds poll the reference instead, and cannot terminate the process | | `nvx::host::supervisor` | The supervisor of a sandbox's VMM, beneath the CLI, `Instance`, and external orchestrators, which keep the configuration, the authorization, and the application protocol. `launch` runs the store's launch steps under the sandbox's lock: it writes the journal's marker, spawns OpenVMM with the claimed log, records its `ProcessRef` right after, and returns a `Starting` launch that the caller commits once the VMM serves its control endpoint, or abandons. A launch that fails never leaves an unrecorded VMM running. `VmmCommand::inherit` lists the descriptors (Linux) or handles (Windows) that the VMM inherits under their numbers, besides its standard streams, and it inherits nothing else of the caller's: on Linux, they must be close-on-exec, a flag that only the VMM's own process clears; on Windows, they join the VMM's explicit handle list. `Vmm::stop` runs one graceful step of the caller's and waits for the VMM's exit event until a deadline, and takes a bounded forced step through the VMM's verified identity only on request. `reattach` recovers an interrupted launch, and holds the recorded VMM again after the orchestrator restarts within the same host boot: `Live`, `Mismatched` with another runtime identity, `Exited`, or `Indeterminate` | @@ -570,17 +570,25 @@ when `serde` is enabled and `nix` and `getrandom` when `guest` is; use std::sync::atomic::AtomicBool; use std::time::Duration; -use nvx::host::instance::{self, Instance}; +use nvx::host::instance::{Found, Instance}; use nvx::host::launch::LaunchSpec; use nvx::protocol::{Workload, WorkloadEnvironment}; use nvx::{Artifacts, Hypervisor}; fn main() -> Result<(), Box> { - let spec = LaunchSpec::new(Artifacts::discover()?, Hypervisor::from_env_or_default()?); - let directory = std::env::temp_dir().join("nvx-example"); - instance::create_private_dir(&directory)?; - let instance = Instance::new(&directory, "example"); - let started = instance.start(&spec, Duration::from_secs(60))?; + let artifacts = Artifacts::discover()?; + let runtime = artifacts.runtime_identity()?; + let spec = LaunchSpec::new(artifacts, Hypervisor::from_env_or_default()?); + let instance = Instance::create(std::env::temp_dir().join("nvx-example"), "example")?; + // Lifecycle transitions hold the instance's lock; executions need none. + let mut locked = instance.lock(None)?; + let Found::Stopped = locked.find(|| Ok(runtime.clone()))? else { + return Err("the example's VM runs already".into()); + }; + locked.start(&spec, &runtime, Duration::from_secs(60))?; + let Found::Running(running) = locked.find(|| Ok(runtime.clone()))? else { + return Err("the example's VM stopped".into()); + }; let workload = Workload { argv: vec!["/bin/echo".to_owned(), "hello".to_owned()], @@ -589,8 +597,7 @@ fn main() -> Result<(), Box> { environment: WorkloadEnvironment::Default, }; let timeout = Duration::from_secs(60); - let capability = instance.read_capability()?; - let execution = instance.exec(&started.runtime, &capability, &workload, timeout, timeout)?; + let execution = instance.exec(&running, &workload, timeout, timeout)?; let mut stdout = Vec::new(); let outcome = execution.pump( &mut |chunk: &[u8]| { @@ -601,7 +608,7 @@ fn main() -> Result<(), Box> { &AtomicBool::new(false), )?; println!("{outcome:?}: {}", String::from_utf8_lossy(&stdout)); - instance.stop(&started.runtime, Duration::from_secs(30))?; + locked.stop(running, Duration::from_secs(30))?; Ok(()) } ``` @@ -621,8 +628,8 @@ as is, an `ErrorKind`, and an optional cause. | `TimedOut` | A deadline that passed | | `Other` | Any other failure | -Messages call an instance `sandbox NAME`, after the name that `Instance::new` -gives it. `aci_edge_sandboxes` maps the kinds onto its error codes; see its +Messages call an instance `sandbox NAME`, after the name that `Instance::create` +or `Instance::open` gives it. `aci_edge_sandboxes` maps the kinds onto its error codes; see its [error mapping](../aci_edge_sandboxes/README.md#error-mapping). ## Bundled artifacts diff --git a/nvx-rs/src/host/instance.rs b/nvx-rs/src/host/instance.rs index cf8939b33..d5909e600 100644 --- a/nvx-rs/src/host/instance.rs +++ b/nvx-rs/src/host/instance.rs @@ -1,156 +1,77 @@ -//! Supervision of one VM instance: its OpenVMM process, its control endpoint, and the files that -//! record them. +//! Supervision of one VM instance: its OpenVMM process, its control endpoint, and the records +//! that name them, through the [`store`](super::store), the [`supervisor`], and the control +//! [`session`]. //! -//! An instance lives in a directory of its own, which the caller creates and keeps private to the -//! current user: +//! An instance lives in a [`SandboxStore`] of its own: a directory private to the current user, +//! in which the store keeps the instance's lock, the journal of its launch, and the runtime +//! record of its VM, and in which the caller may keep configuration records of its own under +//! other names: //! //! ```text //! / -//! launch.json endpoint and available process identity of a start in progress -//! runtime.json OpenVMM process identity and endpoint (running only) +//! sandbox.lock the lock that serializes the instance's lifecycle transitions +//! launch.json the journal of a start in progress, or of one that was interrupted +//! runtime.json the runtime record of the running VM //! control.capability launch capability (running only) //! control.sock Linux control endpoint (running only) //! openvmm.log OpenVMM output of the latest start //! outcome.json OpenVMM outcome report of the latest run //! ``` //! -//! - [`Instance::start`] launches a detached OpenVMM process with the managed lifecycle, passes it -//! a fresh 32-byte capability through standard input, and waits until the guest agent answers -//! on the authenticated control console. The agent must also advertise the control features in -//! [`GuestFeatures::REQUIRED`]; a guest image that lacks one is terminated, because it would -//! silently ignore the policy or request that needs the feature. Start then hands the agent the -//! table of mapped host paths, and fails if the agent cannot mount them all. -//! - [`Instance::exec`] runs a workload through the control console, and [`Execution::pump`] -//! streams its output live. -//! - [`Instance::stop`] asks the guest to shut down and falls back to terminating OpenVMM. -//! - [`Instance::reconcile`] compares the recorded OpenVMM process with the host, so that every -//! operation can run in a different process. A VM that died leaves the instance stopped, and a -//! start whose caller died before recording OpenVMM's identity is recovered through its launch -//! marker. +//! Lifecycle transitions run under the instance's lock, through the [`LockedInstance`] that +//! [`Instance::lock`] returns, so that every operation can run in a different process: //! -//! Callers serialize the lifecycle transitions of one instance, for example with [`lock`]. The -//! guest serializes executions: a second execution waits until the first finishes. Messages call -//! the instance `sandbox NAME`, after the name that [`Instance::new`] gives it. +//! - [`LockedInstance::find`] finds the instance's VM as an orchestrator does after it restarts +//! within the same host boot, with [`supervisor::reattach`]: it ends a start whose caller died, +//! forgets a VM that exited, and compares the caller's runtime identity with the one that a +//! running VM was launched with. A VM of another runtime identity is [`Found::Mismatched`]: it +//! runs through artifacts or contracts that the caller does not have, so the instance does not +//! drive it. +//! - [`LockedInstance::start`] launches a detached OpenVMM process with the managed lifecycle +//! through [`supervisor::launch`], passes it a fresh 32-byte capability through standard input, +//! and waits until the guest agent answers on the authenticated control console. The agent must +//! also advertise the control features in [`GuestFeatures::REQUIRED`]; a guest image that lacks +//! one is terminated, because it would silently ignore the policy or request that needs the +//! feature. Start then hands the agent the table of mapped host paths, and fails if the agent +//! cannot mount them all. +//! - [`LockedInstance::stop`] asks the guest to shut down and falls back to terminating OpenVMM +//! through its verified process identity. +//! +//! [`Instance::exec`] runs a workload through the control console of a [`Running`] VM without the +//! lock, and [`Execution::pump`] streams its output live. The guest serializes executions: a +//! second execution waits until the first finishes. Messages call the instance `sandbox NAME`, +//! after the name that [`Instance::create`] or [`Instance::open`] gives it. -use std::fs::{self, File, OpenOptions}; +use std::fmt; +use std::fs::{self, OpenOptions}; use std::io::{self, Write}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; +use serde::Serialize; use serde::de::DeserializeOwned; -use serde::{Deserialize, Serialize}; use super::artifacts::Artifacts; use super::launch::{LaunchSpec, timeout_problem}; +use super::platform; +use super::process::KILL_TIMEOUT; use super::report::{self, Report}; use super::session::{self, ControlSession, ExecEvent, SessionError}; -use super::{platform, process}; +use super::store::{ + IdentityPart, LockedStore, OUTCOME_NAME, Recovery, RuntimeIdentity, RuntimeRecord, SOCKET_NAME, + SandboxStore, +}; +use super::supervisor::{self, Reattached, Starting, StopOutcome, Vmm, VmmCommand}; use crate::error::{Error, ErrorKind, Result}; use crate::protocol::{ CAPABILITY_LEN, CWD_FAILED, ExitCategory, GuestFeatures, LAUNCH_FAILED, MAPPINGS_INCOMPLETE, MAX_OUTPUT_BYTES, Workload, }; -/// Version of the state files written by this crate: the launch marker and the runtime record, -/// and the [`ExportPlan`](super::filesystem::ExportPlan) that callers record beside them. -/// -/// Version 1 described sandboxes assembled from image layers and a scratch disk, and version 2 -/// exported the mapped host paths as one directory whose bind mounts the kernel command line -/// listed. -pub const STATE_FORMAT: u32 = 3; -/// Linux control socket name. -pub const SOCKET_NAME: &str = "control.sock"; -/// Name of the launch marker. -pub const LAUNCH_NAME: &str = "launch.json"; -/// Name of the runtime record. -pub const RUNTIME_NAME: &str = "runtime.json"; -/// Name of the file that holds the launch capability. -pub const CAPABILITY_NAME: &str = "control.capability"; -/// Name of OpenVMM's log. -pub const LOG_NAME: &str = "openvmm.log"; -/// Name of OpenVMM's outcome report. -pub const REPORT_NAME: &str = "outcome.json"; -/// Every file that an instance keeps in its directory. -pub const FILE_NAMES: [&str; 6] = [ - RUNTIME_NAME, - LAUNCH_NAME, - CAPABILITY_NAME, - SOCKET_NAME, - REPORT_NAME, - LOG_NAME, -]; - /// Longest delay between a cancellation request and its delivery to the guest. const CANCEL_POLL_INTERVAL: Duration = Duration::from_millis(20); -/// Identity of an OpenVMM process: its ID and start time, which together never repeat. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct ProcessIdentity { - /// Process ID. - pub pid: u32, - /// Start time, in the units of [`platform::process_start_time`]. - pub start_time: u64, -} - -/// Identity of the OpenVMM process of a running instance, and its control endpoint. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct RuntimeRecord { - /// [`STATE_FORMAT`]. - pub format: u32, - /// Process ID. - pub pid: u32, - /// Start time, in the units of [`platform::process_start_time`]. - pub start_time: u64, - /// Control endpoint: a Unix socket path on Linux, and a named pipe on Windows. - pub endpoint: String, -} - -/// Marker written before OpenVMM is launched and replaced by the runtime record afterwards. -/// -/// Process identity is added as soon as the child is identified, before writing runtime state. -/// A marker without identity is never expired: its child may be alive without an endpoint. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct LaunchRecord { - /// [`STATE_FORMAT`]. - pub format: u32, - /// Control endpoint that the launch chose. - pub endpoint: String, - /// Identity of the launched process, once known. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub process: Option, -} - -/// Held advisory lock. Dropping it releases the lock. -#[derive(Debug)] -pub struct LockGuard { - /// Open locked file whose lifetime holds the operating-system lock. - _file: File, -} - -/// Takes the exclusive advisory lock of the file at `path`, creating the file if needed, and -/// waits until no other holder has it. -pub fn lock(path: &Path) -> Result { - let file = OpenOptions::new() - .read(true) - .write(true) - .create(true) - .truncate(false) - .open(path) - .map_err(io_error(format!("cannot open lock {}", path.display())))?; - file.lock() - .map_err(io_error(format!("cannot lock {}", path.display())))?; - Ok(LockGuard { _file: file }) -} - -/// Creates the directory at `path` if needed and restricts it to the current user. -pub fn create_private_dir(path: &Path) -> io::Result<()> { - platform::create_private_dir(path) -} - /// Builds an error mapper that preserves an I/O source under `context`. fn io_error(context: String) -> impl FnOnce(io::Error) -> Error { move |error| Error::io(context).with_source(error) @@ -165,7 +86,8 @@ fn private_options() -> OpenOptions { options } -/// Atomically replaces `path` with `contents`, readable only by the current user. +/// Atomically replaces `path` with `contents`, readable only by the current user, as callers do +/// with the configuration records that they keep beside an instance's store. pub fn write_private(path: &Path, contents: &[u8]) -> Result<()> { let mut suffix = [0u8; 8]; getrandom::fill(&mut suffix) @@ -230,6 +152,29 @@ pub fn generate_capability() -> Result<[u8; CAPABILITY_LEN]> { Ok(capability) } +/// Returns a control endpoint of an instance whose Linux control socket is `socket`: the socket +/// on Linux, and a fresh named pipe, whose name always has the same length, on Windows, as a +/// launch chooses it. +fn control_endpoint(socket: &Path) -> Result { + platform::control_endpoint(socket) + .map_err(|error| Error::io("cannot choose a control endpoint").with_source(error)) +} + +/// Fails with [`ErrorKind::Unsupported`] if OpenVMM's command line for `spec`, launched for an +/// instance in the directory `dir`, would not fit the limit of a Windows process command line; +/// see [`LaunchSpec::check_process_command_line`]. +/// +/// The directory need not exist yet, so that a caller can check a launch before it creates the +/// instance. Every control endpoint of an instance has the same length. +pub fn check_launch(dir: &Path, spec: &LaunchSpec) -> Result<()> { + let dir = std::path::absolute(dir).map_err(io_error(format!( + "cannot resolve the instance directory {}", + dir.display() + )))?; + let endpoint = control_endpoint(&dir.join(SOCKET_NAME))?; + spec.check_process_command_line(&endpoint, &dir.join(OUTCOME_NAME)) +} + /// Classifies a session failure. fn session_kind(error: &SessionError) -> ErrorKind { match error { @@ -260,16 +205,17 @@ fn readiness_failure(error: &SessionError, runs_as_host_user: bool) -> String { reason } -/// A VM that [`Instance::start`] launched and the guest agent of which is ready. +/// A VM that [`LockedInstance::start`] launched and the guest agent of which is ready. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Started { - /// The recorded identity and endpoint of its OpenVMM process. + /// The runtime record of its launch: its OpenVMM process, its control endpoint, and the + /// runtime identity that it was launched with. pub runtime: RuntimeRecord, /// Time from launch to guest readiness. pub boot: Duration, } -/// A VM that [`Instance::stop`] stopped. +/// A VM that [`LockedInstance::stop`] stopped. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Stopped { /// Whether OpenVMM had to be terminated because the guest did not shut down in time. @@ -278,226 +224,162 @@ pub struct Stopped { pub report: Option, } -/// One VM instance in a directory of its own. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Instance { - /// Private directory containing runtime state, logs, and the control endpoint. - dir: PathBuf, - /// Sandbox name used in diagnostics. - name: String, +/// The running VM of an instance, which [`LockedInstance::find`] found under the caller's runtime +/// identity, and the capability with which its control sessions attach. +pub struct Running { + /// The VM, held through its verified process identity. + vmm: Vmm, + /// The launch capability of the VM, or the error with which reading it failed: only the + /// VM's control sessions need it, and a stop ends the VM without one. + capability: Result<[u8; CAPABILITY_LEN]>, } -impl Instance { - /// Supervises the instance in `dir`, which messages call `sandbox NAME`. - pub fn new(dir: impl Into, name: impl Into) -> Self { - Self { - dir: dir.into(), - name: name.into(), - } - } - - /// Returns the instance's directory. - pub fn dir(&self) -> &Path { - &self.dir +impl Running { + /// Returns the VM, held through its verified process identity. + pub fn vmm(&self) -> &Vmm { + &self.vmm } - /// Returns the instance's name in messages. - pub fn name(&self) -> &str { - &self.name + /// Returns the runtime record of the VM's launch. + pub fn runtime(&self) -> &RuntimeRecord { + self.vmm.record() } - /// Returns the path of the OpenVMM log of the latest start. - pub fn log_path(&self) -> PathBuf { - self.dir.join(LOG_NAME) + /// Returns the launch capability of the VM, or fails as reading it failed, with the same + /// kind, description, and the description of its cause. + fn capability(&self) -> Result<&[u8; CAPABILITY_LEN]> { + self.capability.as_ref().map_err(|error| { + let copy = Error::new(error.kind(), error.message()); + match std::error::Error::source(error) { + Some(cause) => copy.with_source(io::Error::other(cause.to_string())), + None => copy, + } + }) } +} - /// Returns the path of OpenVMM's outcome report of the latest run. - pub fn report_path(&self) -> PathBuf { - self.dir.join(REPORT_NAME) +impl fmt::Debug for Running { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + // The capability authenticates control sessions, so it is never shown. + formatter + .debug_struct("Running") + .field("vmm", &self.vmm) + .finish_non_exhaustive() } +} - /// Returns the path of the Linux control socket. - pub fn socket_path(&self) -> PathBuf { - self.dir.join(SOCKET_NAME) - } +/// The running VM of an instance, which [`LockedInstance::find`] found under another runtime +/// identity than the caller's. +#[derive(Debug)] +pub struct Mismatched { + /// The instance's name in messages. + name: String, + /// The VM, held through its verified process identity. + vmm: Vmm, + /// The parts of the runtime identity that differ. + differences: Vec, +} - /// Returns a control endpoint for the instance: its socket on Linux, and a fresh named pipe, - /// whose name always has the same length, on Windows. - fn endpoint(&self) -> Result { - platform::control_endpoint(&self.socket_path()) - .map_err(|error| Error::io("cannot choose a control endpoint").with_source(error)) +impl Mismatched { + /// Returns the VM, held through its verified process identity. + pub fn vmm(&self) -> &Vmm { + &self.vmm } - /// Reads the runtime record, if the instance runs. - pub fn runtime(&self) -> Result> { - let path = self.dir.join(RUNTIME_NAME); - let runtime: Option = read_json(&path)?; - match runtime { - Some(runtime) if runtime.format != STATE_FORMAT => Err(Error::io(format!( - "sandbox runtime state {} has an unsupported format", - path.display() - ))), - runtime => Ok(runtime), - } + /// Returns the parts of the runtime identity that differ, as + /// [`RuntimeIdentity::differences`] returns them. + pub fn differences(&self) -> &[IdentityPart] { + &self.differences } - /// Records the OpenVMM process of a running instance. - pub fn write_runtime(&self, runtime: &RuntimeRecord) -> Result<()> { - write_json(&self.dir.join(RUNTIME_NAME), runtime) + /// Returns the error that reports the VM, which the instance does not drive: only the build + /// and artifacts that started it speak its contracts. + pub fn error(&self) -> Error { + let parts: Vec<&str> = self.differences.iter().map(|part| part.name()).collect(); + Error::other(format!( + "sandbox {} runs under another runtime identity, which differs in {}, so it is not \ + driven: stop it with the NVX build and artifacts that started it (OpenVMM process {})", + self.name, + parts.join(", "), + self.vmm.record().process().pid + )) } +} - /// Records a launch in progress. - pub fn write_launch(&self, launch: &LaunchRecord) -> Result<()> { - write_json(&self.dir.join(LAUNCH_NAME), launch) - } +/// What [`LockedInstance::find`] found of an instance's VM. +#[derive(Debug)] +pub enum Found { + /// No VM of the instance runs: none was started, the last one stopped or exited, or a start + /// that was interrupted is over. + Stopped, + /// The VM runs under the caller's runtime identity. + Running(Running), + /// The VM runs under another runtime identity, which the caller reports, as + /// [`Mismatched::error`] does. + Mismatched(Mismatched), +} - /// Reads the marker of a launch in progress, if any. - pub fn launch(&self) -> Result> { - let path = self.dir.join(LAUNCH_NAME); - let Some(launch) = read_json::(&path)? else { - return Ok(None); - }; - if launch.format != STATE_FORMAT { - return Err(Error::io(format!( - "sandbox launch marker {} has an unsupported format", - path.display() - ))); - } - Ok(Some(launch)) - } +/// One VM instance in a sandbox store of its own. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Instance { + /// The store of the instance's runtime records, logs, and control endpoint. + store: SandboxStore, + /// Sandbox name used in diagnostics. + name: String, +} - /// Removes the launch marker once the runtime record replaces it. - pub fn remove_launch(&self) { - let _ = fs::remove_file(self.dir.join(LAUNCH_NAME)); +impl Instance { + /// Creates the directory `dir` of a new instance, whose parent must exist, private to the + /// current user, as [`SandboxStore::create`] does, and supervises the instance in it, which + /// messages call `sandbox NAME`. + pub fn create(dir: impl Into, name: impl Into) -> Result { + Ok(Self { + store: SandboxStore::create(dir)?, + name: name.into(), + }) } - /// Stores the launch capability, readable only by the current user. - pub fn write_capability(&self, capability: &[u8; CAPABILITY_LEN]) -> Result<()> { - write_private(&self.dir.join(CAPABILITY_NAME), capability) + /// Supervises the instance in the existing directory `dir`, which must be private to the + /// current user, as [`SandboxStore::open`] requires, and which messages call `sandbox NAME`. + pub fn open(dir: impl Into, name: impl Into) -> Result { + Ok(Self { + store: SandboxStore::open(dir)?, + name: name.into(), + }) } - /// Reads the launch capability of a running instance. - pub fn read_capability(&self) -> Result<[u8; CAPABILITY_LEN]> { - let path = self.dir.join(CAPABILITY_NAME); - let bytes = fs::read(&path).map_err(io_error(format!( - "cannot read the sandbox control capability {}", - path.display() - )))?; - match <[u8; CAPABILITY_LEN]>::try_from(bytes.as_slice()) { - Ok(capability) if capability != [0; CAPABILITY_LEN] => Ok(capability), - _ => Err(Error::io(format!( - "sandbox control capability {} is invalid", - path.display() - ))), - } + /// Returns the store of the instance's runtime records. + pub fn store(&self) -> &SandboxStore { + &self.store } - /// Removes the files that exist only while the instance starts or runs. - pub fn clear_runtime(&self) -> Result<()> { - for name in [RUNTIME_NAME, LAUNCH_NAME, CAPABILITY_NAME, SOCKET_NAME] { - remove_if_present(&self.dir.join(name))?; - } - Ok(()) + /// Returns the instance's directory, as an absolute path. + pub fn dir(&self) -> &Path { + self.store.dir() } - /// Opens a fresh OpenVMM log, replacing the previous one. - pub fn create_log(&self) -> Result { - let path = self.log_path(); - private_options() - .truncate(true) - .open(&path) - .map_err(io_error(format!("cannot create {}", path.display()))) + /// Returns the instance's name in messages. + pub fn name(&self) -> &str { + &self.name } - /// Fails with [`ErrorKind::Unsupported`] if OpenVMM's command line for `spec` would not fit - /// the limit of a Windows process command line; see - /// [`LaunchSpec::check_process_command_line`]. - pub fn check_launch(&self, spec: &LaunchSpec) -> Result<()> { - spec.check_process_command_line(&self.endpoint()?, &self.report_path()) + /// Returns the path of the OpenVMM log of the latest start. + pub fn log_path(&self) -> PathBuf { + self.store.log_path() } - /// Returns the runtime record of the instance's OpenVMM process if it still runs, clearing - /// stale runtime state otherwise. - /// - /// State is cleared only when the process is definitely gone; an inconclusive check fails - /// instead, so a live VM is never forgotten. A start whose caller died before recording - /// OpenVMM's identity is recovered through its launch marker. Callers hold the instance's - /// lifecycle lock. - pub fn reconcile(&self) -> Result> { - if let Some(runtime) = self.runtime()? { - let current = platform::process_start_time(runtime.pid).map_err(|error| { - Error::io(format!( - "cannot determine whether OpenVMM process {} of sandbox {} is running", - runtime.pid, self.name - )) - .with_source(error) - })?; - if current == Some(runtime.start_time) { - self.remove_launch(); - return Ok(Some(runtime)); - } - self.clear_runtime()?; - return Ok(None); - } - if let Some(launch) = self.launch()? { - self.recover_launch(&launch)?; - self.clear_runtime()?; - } - Ok(None) + /// Returns the path of OpenVMM's outcome report of the latest run. + pub fn report_path(&self) -> PathBuf { + self.store.outcome_path() } - /// Terminates the OpenVMM process of a start whose caller died before recording it. - /// - /// Lifecycle locks serialize starts, so a launch marker seen under the lock always belongs - /// to an interrupted start. Recorded identity is authoritative even before an endpoint - /// exists; legacy markers can be recovered only when their endpoint identifies the child. - fn recover_launch(&self, launch: &LaunchRecord) -> Result<()> { - let interrupted = |kind: ErrorKind, detail: &str| { - Error::new( - kind, - format!("an interrupted start of sandbox {} {detail}", self.name), - ) - }; - let identity = match &launch.process { - Some(identity) => identity.clone(), - None => { - let pid = platform::endpoint_server_pid(&launch.endpoint).map_err(|error| { - interrupted(ErrorKind::Io, "cannot be recovered yet").with_source(error) - })?; - let Some(pid) = pid else { - return Err(interrupted( - ErrorKind::Other, - "has no recorded process identity or observable endpoint; cannot verify \ - that OpenVMM exited, so the launch marker is retained", - )); - }; - let Some(start_time) = platform::process_start_time(pid).map_err(|error| { - interrupted(ErrorKind::Io, "cannot be recovered yet").with_source(error) - })? - else { - return Ok(()); - }; - ProcessIdentity { pid, start_time } - } - }; - match process::kill(identity.pid, identity.start_time) { - Ok(true) => Ok(()), - Ok(false) => Err(interrupted( - ErrorKind::Other, - &format!( - "left OpenVMM process {} running, and it did not exit", - identity.pid - ), - )), - Err(error) => Err(interrupted( - ErrorKind::Io, - &format!( - "left OpenVMM process {} running, and it could not be terminated", - identity.pid - ), - ) - .with_source(error)), - } + /// Takes the instance's lock, which serializes its lifecycle transitions, waiting for it + /// until `deadline`, or for as long as it takes without one, as [`SandboxStore::lock`] does. + pub fn lock(&self, deadline: Option) -> Result { + Ok(LockedInstance { + locked: self.store.lock(deadline)?, + instance: self.clone(), + }) } /// Describes why this instance did not start and points the caller to its log. @@ -512,179 +394,37 @@ impl Instance { ) } - /// Terminates a launch that did not become ready. + /// Ends a launch that did not become ready, as [`Starting::abandon`] does, and describes why + /// the instance did not start. /// - /// Runtime state is cleared only once the process has exited. Otherwise the instance stays - /// marked as running so that a stop can retry instead of a second VM being started. - fn abort_start(&self, runtime: &RuntimeRecord, kind: ErrorKind, reason: &str) -> Error { - let cleanup = match process::kill(runtime.pid, runtime.start_time) { - Ok(true) => { - let _ = self.clear_runtime(); - return self.start_failure(kind, reason); - } - Ok(false) => "it did not exit".to_owned(), + /// A launch whose OpenVMM cannot be ended keeps its journal, and the next transition of the + /// instance ends it, when [`LockedInstance::find`] recovers it. + fn abort_start(&self, starting: Starting<'_>, kind: ErrorKind, reason: &str) -> Error { + let unended = match starting.abandon() { + Ok(Recovery::Indeterminate(unended)) => unended, + Ok(_) => return self.start_failure(kind, reason), Err(error) => error.to_string(), }; self.start_failure( kind, &format!( - "{reason}; OpenVMM process {} could not be terminated ({cleanup}), so the \ - sandbox stays marked as running until it is stopped", - runtime.pid + "{reason}; the launch could not be ended, and remains for recovery: {unended}" ), ) } - /// Terminates a launch whose guest image lacks control features that NVX needs. + /// Ends a launch whose guest image lacks control features that NVX needs. /// /// An older image would boot and answer, but it would silently ignore host path mappings and /// cancellation requests, so the VM must not run at all. - fn reject_guest(&self, runtime: &RuntimeRecord, missing: &[&str]) -> Error { + fn reject_guest(&self, starting: Starting<'_>, missing: &[&str]) -> Error { let reason = format!( "the guest image lacks control features that the openvmm backend needs ({}); use an \ NVX release or build that includes them, for example through {}", missing.join(", "), Artifacts::ENV_DIR ); - self.abort_start(runtime, ErrorKind::Unavailable, &reason) - } - - /// Launches the VM that `spec` describes and waits up to `timeout` until its guest is ready. - /// - /// The instance's directory must exist, and the caller holds its lifecycle lock and has found - /// it stopped with [`reconcile`](Self::reconcile). Start first checks the export plan with - /// [`ExportPlan::verify`](super::filesystem::ExportPlan::verify): that the host keeps its - /// read-only paths read-only, and that the mapped host paths still name the objects they named - /// when they were planned. - /// - /// The capability travels through OpenVMM's standard-input pipe, which is closed before - /// OpenVMM starts, and never appears in arguments, the environment, or the log. Before - /// launching OpenVMM, start records the control endpoint in the launch marker, so that - /// [`reconcile`](Self::reconcile) can terminate an OpenVMM whose start was interrupted. - /// - /// Fails with [`ErrorKind::Unavailable`] when the guest image lacks a feature in - /// [`GuestFeatures::REQUIRED`], and with [`ErrorKind::Unsupported`] when `timeout` is zero or - /// longer than [`MAX_TIMEOUT`](super::launch::MAX_TIMEOUT), when the export plan maps a - /// read-only path that the host cannot keep read-only, when the specification fails - /// [`LaunchSpec::check`] or exceeds a limit, or when OpenVMM's command line would not fit - /// [`PROCESS_COMMAND_LINE_LIMIT`](super::launch::PROCESS_COMMAND_LINE_LIMIT), which holds on - /// every host. Unlike [`check_launch`](Self::check_launch), start keeps no reserve. A launch - /// that does not become ready is terminated; if it cannot be, the instance stays marked as - /// running until it is stopped. - pub fn start(&self, spec: &LaunchSpec, timeout: Duration) -> Result { - check_timeout("start timeout", timeout, true)?; - if let Some(plan) = &spec.filesystem { - plan.verify()?; - } - let capability = generate_capability()?; - let socket = self.socket_path(); - remove_if_present(&socket)?; - let endpoint = self.endpoint()?; - let report = self.report_path(); - remove_if_present(&report)?; - let arguments = spec.openvmm_arguments(&endpoint, &report)?; - spec.fit_process_command_line(&arguments, 0)?; - - self.write_capability(&capability)?; - let log = self.create_log()?; - // Until the child is identified, recovery must retain this marker unless the endpoint - // identifies the child. An absent endpoint never proves that a launch has exited. - self.write_launch(&LaunchRecord { - format: STATE_FORMAT, - endpoint: endpoint.clone(), - process: None, - })?; - let started = Instant::now(); - let child = match process::spawn( - &spec.openvmm, - spec.breakaway_from_job, - &arguments, - &capability, - log, - &self.dir, - ) { - Ok(child) => child, - Err(error) => { - let _ = self.clear_runtime(); - return Err( - Error::io(format!("cannot launch {}", spec.openvmm.display())) - .with_source(error), - ); - } - }; - let pid = child.id(); - let start_time = match platform::process_start_time(pid) { - Ok(Some(start_time)) => start_time, - outcome => { - // Keep the launch marker unless OpenVMM is gone, so recovery can retry. - if process::kill_child(child) { - let _ = self.clear_runtime(); - } - let (kind, reason) = match outcome { - Err(error) => ( - ErrorKind::Io, - format!("cannot identify the OpenVMM process: {error}"), - ), - _ => ( - ErrorKind::Exited, - "OpenVMM exited during startup".to_owned(), - ), - }; - return Err(self.start_failure(kind, &reason)); - } - }; - let runtime = RuntimeRecord { - format: STATE_FORMAT, - pid, - start_time, - endpoint: endpoint.clone(), - }; - let identified = LaunchRecord { - format: STATE_FORMAT, - endpoint: endpoint.clone(), - process: Some(ProcessIdentity { pid, start_time }), - }; - if let Err(error) = self - .write_launch(&identified) - .and_then(|()| self.write_runtime(&runtime)) - { - if process::kill_child(child) { - let _ = self.clear_runtime(); - } - return Err(error); - } - process::detach_child(child); - self.remove_launch(); - let deadline = started + timeout; - let ready = session::connect(&endpoint, pid, start_time, deadline) - .and_then(|transport| ControlSession::attach(transport, &capability, deadline)) - .and_then(|mut session| { - session.ping(deadline)?; - let features = session.features(deadline)?; - Ok((session, features)) - }); - let (mut session, features) = match ready { - Ok(ready) => ready, - Err(error) => { - let reason = readiness_failure(&error, spec.identity.create_account); - return Err(self.abort_start(&runtime, session_kind(&error), &reason)); - } - }; - let missing = features.missing(GuestFeatures::REQUIRED); - if !missing.is_empty() { - return Err(self.reject_guest(&runtime, &missing)); - } - // Workloads run only once the guest has mounted every mapped host path. - if let Some(plan) = &spec.filesystem - && let Err(error) = session.map_host_paths(&plan.guest_table(), deadline) - { - let reason = format!("the host paths were not mapped: {error}"); - return Err(self.abort_start(&runtime, session_kind(&error), &reason)); - } - Ok(Started { - runtime, - boot: started.elapsed(), - }) + self.abort_start(starting, ErrorKind::Unavailable, &reason) } /// Maps a control-session failure onto an instance-facing error and message. @@ -710,30 +450,35 @@ impl Instance { } } - /// Starts `workload` in the running instance whose OpenVMM process `runtime` records, with - /// the `capability` that it was launched with. + /// Starts `workload` in the instance's running VM, `running`, without the instance's lock. /// /// It waits up to `control_timeout` for the control session, which the guest grants one /// execution at a time. [`Execution::pump`] then waits for the outcome up to the workload's /// own timeout plus `response_grace`, and without a limit when the workload has no timeout. /// Fails with [`ErrorKind::Exited`] when the VM is gone, and with /// [`ErrorKind::Unsupported`] when `control_timeout` is zero or either duration is longer - /// than [`MAX_TIMEOUT`](super::launch::MAX_TIMEOUT). + /// than [`MAX_TIMEOUT`](super::launch::MAX_TIMEOUT). Fails as reading the capability failed + /// when [`find`](LockedInstance::find) could not read it. pub fn exec( &self, - runtime: &RuntimeRecord, - capability: &[u8; CAPABILITY_LEN], + running: &Running, workload: &Workload<'_>, control_timeout: Duration, response_grace: Duration, ) -> Result { check_timeout("control timeout", control_timeout, true)?; check_timeout("response grace", response_grace, false)?; + let capability = running.capability()?; let deadline = Instant::now() + control_timeout; - let mut session = - session::connect(&runtime.endpoint, runtime.pid, runtime.start_time, deadline) - .and_then(|transport| ControlSession::attach(transport, capability, deadline)) - .map_err(|error| self.session_error(error, control_timeout))?; + let process = running.vmm.record().process(); + let mut session = session::connect( + running.vmm.endpoint(), + process.pid, + process.start_time, + deadline, + ) + .and_then(|transport| ControlSession::attach(transport, capability, deadline)) + .map_err(|error| self.session_error(error, control_timeout))?; let request_id = session .start_exec(workload, deadline) .map_err(|error| self.session_error(error, control_timeout))?; @@ -747,48 +492,197 @@ impl Instance { control_timeout, }) } +} + +/// An instance whose lock this holds, through which its lifecycle transitions change its +/// records. Dropping it releases the lock. +#[derive(Debug)] +pub struct LockedInstance { + /// The instance. + instance: Instance, + /// Its locked store. + locked: LockedStore, +} + +impl LockedInstance { + /// Returns the instance. + pub fn instance(&self) -> &Instance { + &self.instance + } + + /// Finds the instance's VM with [`supervisor::reattach`], under the runtime identity that + /// `runtime` returns: the one with which the caller would launch it now. + /// + /// `runtime` is asked only when the store records a VM or a start, so that an instance that + /// never started, or that stopped, needs no runtime identity. Reattach ends a start that was + /// interrupted, signaling only an OpenVMM process that it verified, and forgets a VM that + /// exited, as one that a record of an earlier host boot names has. Fails as `runtime` fails, + /// for records that the store does not read, such as those of an earlier format, whose + /// message names it, and when whether the VM runs cannot be determined, because an + /// interrupted start cannot be ended yet or the recorded process cannot be examined: the + /// records then remain for a later attempt. + /// + /// A VM whose capability cannot be read, as when its file is missing or damaged, still runs, + /// and is found [`Found::Running`]: [`Instance::exec`] then fails as reading it failed, and + /// [`stop`](Self::stop) ends the VM through its verified process identity. + pub fn find(&mut self, runtime: impl FnOnce() -> Result) -> Result { + if self.locked.journal()?.is_none() && self.locked.runtime()?.is_none() { + return Ok(Found::Stopped); + } + let runtime = runtime()?; + Ok(match supervisor::reattach(&mut self.locked, &runtime)? { + Reattached::Live(vmm) => Found::Running(Running { + vmm, + capability: self.locked.read_capability(), + }), + Reattached::Mismatched { vmm, differences } => Found::Mismatched(Mismatched { + name: self.instance.name.clone(), + vmm, + differences, + }), + Reattached::Exited { .. } => Found::Stopped, + Reattached::Indeterminate(reason) => { + return Err(Error::other(format!( + "cannot determine whether the OpenVMM process of sandbox {} runs: {reason}", + self.instance.name + ))); + } + }) + } - /// Asks the guest of the running instance that `runtime` records to shut down, and - /// terminates OpenVMM if it has not exited within `timeout`. + /// Launches the VM that `spec` describes under the runtime identity `runtime`, and waits up + /// to `timeout` until its guest is ready. + /// + /// The caller has found the instance stopped with [`find`](Self::find). Start first checks the + /// export plan with [`ExportPlan::verify`](super::filesystem::ExportPlan::verify): that the + /// host keeps its read-only paths read-only, and that the mapped host paths still name the + /// objects they named when they were planned. It launches OpenVMM with + /// [`supervisor::launch`], whose journal records the launch before OpenVMM is spawned and its + /// process identity right after, so that [`find`](Self::find) ends a start whose caller died. + /// The capability travels through OpenVMM's standard-input pipe, which is closed before + /// OpenVMM starts, and never appears in arguments, the environment, or the log. Once the guest + /// is ready, has the required features, and has mounted the mapped host paths, start commits + /// the launch: the runtime record names the VM from then on. + /// + /// Fails with [`ErrorKind::Unavailable`] when the guest image lacks a feature in + /// [`GuestFeatures::REQUIRED`], and with [`ErrorKind::Unsupported`] when `timeout` is zero or + /// longer than [`MAX_TIMEOUT`](super::launch::MAX_TIMEOUT), when the export plan maps a + /// read-only path that the host cannot keep read-only, when the specification fails + /// [`LaunchSpec::check`] or exceeds a limit, or when OpenVMM's command line would not fit + /// [`PROCESS_COMMAND_LINE_LIMIT`](super::launch::PROCESS_COMMAND_LINE_LIMIT), which holds on + /// every host; such a launch records nothing. Unlike [`check_launch`], start keeps no reserve. + /// A launch that does not become ready is terminated through its verified process identity; + /// one that cannot be ended keeps its journal, and the next transition ends it. + pub fn start( + &mut self, + spec: &LaunchSpec, + runtime: &RuntimeIdentity, + timeout: Duration, + ) -> Result { + check_timeout("start timeout", timeout, true)?; + if let Some(plan) = &spec.filesystem { + plan.verify()?; + } + let instance = &self.instance; + let report = instance.report_path(); + // Every control endpoint of the instance has the same length, so a launch that would not + // fit is refused before the store records it. + let endpoint = control_endpoint(&instance.store.socket_path())?; + spec.fit_process_command_line(&spec.openvmm_arguments(&endpoint, &report)?, 0)?; + let capability = generate_capability()?; + let started = Instant::now(); + let starting = supervisor::launch(&mut self.locked, runtime, &capability, |endpoint| { + let arguments = spec.openvmm_arguments(endpoint, &report)?; + spec.fit_process_command_line(&arguments, 0)?; + let mut command = VmmCommand::new(&spec.openvmm, arguments, instance.dir()); + command.breakaway_from_job = spec.breakaway_from_job; + Ok(command) + })?; + let (pid, start_time) = { + let reference = starting.process().reference(); + (reference.pid, reference.start_time) + }; + let deadline = started + timeout; + let ready = session::connect(starting.endpoint(), pid, start_time, deadline) + .and_then(|transport| ControlSession::attach(transport, &capability, deadline)) + .and_then(|mut session| { + session.ping(deadline)?; + let features = session.features(deadline)?; + Ok((session, features)) + }); + let (mut session, features) = match ready { + Ok(ready) => ready, + Err(error) => { + let reason = readiness_failure(&error, spec.identity.create_account); + return Err(instance.abort_start(starting, session_kind(&error), &reason)); + } + }; + let missing = features.missing(GuestFeatures::REQUIRED); + if !missing.is_empty() { + drop(session); + return Err(instance.reject_guest(starting, &missing)); + } + // Workloads run only once the guest has mounted every mapped host path. + if let Some(plan) = &spec.filesystem + && let Err(error) = session.map_host_paths(&plan.guest_table(), deadline) + { + drop(session); + let reason = format!("the host paths were not mapped: {error}"); + return Err(instance.abort_start(starting, session_kind(&error), &reason)); + } + drop(session); + let vmm = starting.commit()?; + Ok(Started { + runtime: vmm.record().clone(), + boot: started.elapsed(), + }) + } + + /// Asks the guest of the running VM `running` to shut down, and terminates OpenVMM through + /// its verified process identity if it has not exited within `timeout`, as [`Vmm::stop`] + /// does with a forced step, or at once when the guest cannot be asked, as without the VM's + /// capability. /// /// A running execution keeps the control session until it ends, so a stop that cannot reach - /// the guest in time terminates the VM, and the execution then fails. Stop reads OpenVMM's - /// outcome report and clears the runtime state. Fails with [`ErrorKind::Unsupported`] when - /// `timeout` is zero or longer than [`MAX_TIMEOUT`](super::launch::MAX_TIMEOUT). - pub fn stop(&self, runtime: &RuntimeRecord, timeout: Duration) -> Result { + /// the guest in time terminates the VM, and the execution then fails. Once OpenVMM has + /// exited, the stop clears the runtime record and reads OpenVMM's outcome report. Fails with + /// [`ErrorKind::Unsupported`] when `timeout` is zero or longer than + /// [`MAX_TIMEOUT`](super::launch::MAX_TIMEOUT), and when OpenVMM cannot be terminated, or does + /// not exit after it was, which keeps its runtime record. + pub fn stop(&mut self, running: Running, timeout: Duration) -> Result { check_timeout("stop timeout", timeout, true)?; let deadline = Instant::now() + timeout; - let graceful = (|| -> std::result::Result<(), SessionError> { - let capability = self - .read_capability() - .map_err(|error| SessionError::Protocol(error.to_string()))?; - let transport = - session::connect(&runtime.endpoint, runtime.pid, runtime.start_time, deadline)?; - let mut session = ControlSession::attach(transport, &capability, deadline)?; - session.stop(deadline)?; - drop(session); - if process::wait_for_exit(runtime.pid, runtime.start_time, deadline) { - Ok(()) - } else { - Err(SessionError::TimedOut) - } - })(); - let forced = !matches!(graceful, Ok(()) | Err(SessionError::ProcessExited)); - if forced { - let exited = process::kill(runtime.pid, runtime.start_time).map_err(|error| { - Error::io(format!("cannot terminate OpenVMM process {}", runtime.pid)) - .with_source(error) - })?; - if !exited { + let Running { vmm, capability } = running; + let stopped = vmm.stop( + &mut self.locked, + deadline, + Some(KILL_TIMEOUT), + |vmm, deadline| { + // Without its capability, no session can reach the guest, and the forced step + // ends the VM. + let capability = + capability.map_err(|error| SessionError::Protocol(error.to_string()))?; + let process = vmm.record().process(); + let transport = + session::connect(vmm.endpoint(), process.pid, process.start_time, deadline)?; + ControlSession::attach(transport, &capability, deadline)?.stop(deadline) + }, + )?; + let forced = match stopped.outcome { + StopOutcome::Graceful => false, + StopOutcome::Forced => true, + StopOutcome::TimedOut => { return Err(Error::other(format!( - "OpenVMM process {} did not terminate", - runtime.pid + "OpenVMM process {} of sandbox {} did not exit after it was terminated", + vmm.record().process().pid, + self.instance.name ))); } - } - let report = report::read(&self.report_path()); - self.clear_runtime()?; - Ok(Stopped { forced, report }) + }; + Ok(Stopped { + forced, + report: report::read(&self.instance.report_path()), + }) } } @@ -964,19 +858,7 @@ impl Execution { #[cfg(test)] mod tests { - use std::thread; - use super::*; - use crate::host::test_support::released; - - /// A process ID that fits every platform's type and that no process has. - const GONE_PID: u32 = 0x7fff_fff0; - - fn instance() -> (tempfile::TempDir, Instance) { - let directory = tempfile::tempdir().unwrap(); - let instance = Instance::new(directory.path(), "aci-edge-sandboxes:0123"); - (directory, instance) - } #[test] fn readiness_failures_explain_images_without_workload_accounts() { @@ -994,195 +876,90 @@ mod tests { } } - #[test] - fn runtime_files_round_trip_and_are_cleared() { - let (_directory, instance) = instance(); - assert!(instance.launch().unwrap().is_none()); - assert!(instance.runtime().unwrap().is_none()); - let launch = LaunchRecord { - format: STATE_FORMAT, - endpoint: "endpoint".to_owned(), - process: None, - }; - instance.write_launch(&launch).unwrap(); - assert_eq!(instance.launch().unwrap(), Some(launch.clone())); - let identified = LaunchRecord { - process: Some(ProcessIdentity { - pid: 1, - start_time: 2, - }), - ..launch.clone() - }; - instance.write_launch(&identified).unwrap(); - assert_eq!(instance.launch().unwrap(), Some(identified)); - instance.remove_launch(); - assert!(instance.launch().unwrap().is_none()); - instance.write_launch(&launch).unwrap(); - let runtime = RuntimeRecord { - format: STATE_FORMAT, - pid: 1, - start_time: 2, - endpoint: "endpoint".to_owned(), - }; - instance.write_runtime(&runtime).unwrap(); - instance.write_capability(&[9; CAPABILITY_LEN]).unwrap(); - assert_eq!(instance.runtime().unwrap(), Some(runtime)); - assert_eq!(instance.read_capability().unwrap(), [9; CAPABILITY_LEN]); - drop(instance.create_log().unwrap()); - instance.clear_runtime().unwrap(); - assert!(instance.runtime().unwrap().is_none()); - assert!(instance.launch().unwrap().is_none()); - assert!(instance.read_capability().is_err()); - // The log and the report outlive the run. - assert!(instance.log_path().is_file()); - } + /// The instance's store, which exists on Linux and Windows hosts. + #[cfg(any(target_os = "linux", windows))] + mod store { + use serde_json::json; - #[test] - fn records_keep_their_json_form() { - let launch = LaunchRecord { - format: STATE_FORMAT, - endpoint: "/state/t/control.sock".to_owned(), - process: Some(ProcessIdentity { - pid: 7, - start_time: 8, - }), - }; - assert_eq!( - serde_json::to_value(&launch).unwrap(), - serde_json::json!({ - "format": 3, - "endpoint": "/state/t/control.sock", - "process": { "pid": 7, "startTime": 8 }, - }) - ); - let runtime: RuntimeRecord = serde_json::from_value(serde_json::json!({ - "format": 3, "pid": 7, "startTime": 8, "endpoint": "e", - })) - .unwrap(); - assert_eq!(runtime.start_time, 8); - // A marker of an interrupted start that predates process identities has none. - let legacy: LaunchRecord = - serde_json::from_value(serde_json::json!({ "format": 3, "endpoint": "e" })).unwrap(); - assert_eq!(legacy.process, None); - } + use super::super::*; + use crate::host::launch::PROCESS_COMMAND_LINE_LIMIT; + use crate::host::store::{ArtifactIdentity, JOURNAL_NAME, RUNTIME_NAME}; - #[test] - fn records_of_other_formats_are_rejected() { - let (directory, instance) = instance(); - let write = |name: &str, value: serde_json::Value| { - fs::write(directory.path().join(name), value.to_string()).unwrap(); - }; - write( - RUNTIME_NAME, - serde_json::json!({ "format": 2, "pid": 1, "startTime": 2, "endpoint": "e" }), - ); - let error = instance.runtime().unwrap_err(); - assert_eq!(error.kind(), ErrorKind::Io); - assert!(error.message().contains("unsupported format"), "{error}"); - write( - LAUNCH_NAME, - serde_json::json!({ "format": 2, "endpoint": "e" }), - ); - assert!(instance.launch().is_err()); - fs::write(directory.path().join(RUNTIME_NAME), b"{ truncated").unwrap(); - assert!( - instance - .runtime() - .unwrap_err() - .message() - .contains("malformed") - ); - fs::write(directory.path().join(CAPABILITY_NAME), [0; CAPABILITY_LEN]).unwrap(); - assert!(instance.read_capability().is_err()); - } + fn instance() -> (tempfile::TempDir, Instance) { + let directory = tempfile::tempdir().unwrap(); + let instance = + Instance::create(directory.path().join("vm"), "aci-edge-sandboxes:0123").unwrap(); + (directory, instance) + } - #[test] - fn reconciliation_forgets_vms_that_are_gone() { - let (_directory, instance) = instance(); - let runtime = RuntimeRecord { - format: STATE_FORMAT, - pid: GONE_PID, - start_time: 0, - endpoint: "endpoint".to_owned(), - }; - instance.write_runtime(&runtime).unwrap(); - instance.write_capability(&[1; CAPABILITY_LEN]).unwrap(); - assert_eq!(instance.reconcile().unwrap(), None); - assert!(instance.runtime().unwrap().is_none()); - assert!(instance.read_capability().is_err()); - // The marker of an interrupted start whose process is gone is cleared too. - instance - .write_launch(&LaunchRecord { - format: STATE_FORMAT, - endpoint: "endpoint".to_owned(), - process: Some(ProcessIdentity { - pid: GONE_PID, - start_time: 0, - }), - }) - .unwrap(); - assert_eq!(instance.reconcile().unwrap(), None); - assert!(instance.launch().unwrap().is_none()); - } + /// Returns the runtime identity of the launches of these tests. + fn runtime() -> RuntimeIdentity { + let artifact = |digit: &str| ArtifactIdentity::sha256(&digit.repeat(64)).unwrap(); + RuntimeIdentity::new(artifact("1"), artifact("2"), artifact("3")) + } - #[test] - fn capabilities_are_never_all_zero() { - let first = generate_capability().unwrap(); - assert_ne!(first, [0; CAPABILITY_LEN]); - assert_ne!(first, generate_capability().unwrap()); - } + #[test] + fn instances_that_record_no_vm_need_no_runtime_identity() { + let (_directory, instance) = instance(); + let mut locked = instance.lock(None).unwrap(); + let found = locked + .find(|| panic!("an instance that records no VM needs no runtime identity")) + .unwrap(); + assert!(matches!(found, Found::Stopped), "{found:?}"); + drop(locked); + let reopened = Instance::open(instance.dir(), instance.name()).unwrap(); + assert_eq!(reopened, instance); + } - #[test] - fn locks_are_exclusive() { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("instance.lock"); - let guard = lock(&path).unwrap(); - let contender = OpenOptions::new().write(true).open(&path).unwrap(); - assert!(contender.try_lock().is_err()); - drop(guard); - assert!(released(|| match contender.try_lock() { - Ok(()) => true, - Err(std::fs::TryLockError::WouldBlock) => false, - Err(error) => panic!("{error}"), - })); - drop(contender); - // A waiter takes the lock once the holder releases it. - let guard = lock(&path).unwrap(); - let waiter = thread::spawn({ - let path = path.clone(); - move || lock(&path).map(drop) - }); - thread::sleep(Duration::from_millis(50)); - drop(guard); - waiter.join().unwrap().unwrap(); - } + #[test] + fn records_of_the_earlier_format_fail_naming_it() { + // Before the store, instances kept runtime records and launch markers of format 3. + for (name, record) in [ + ( + RUNTIME_NAME, + json!({ "format": 3, "pid": 7, "startTime": 8, "endpoint": "e" }), + ), + ( + JOURNAL_NAME, + json!({ "format": 3, "endpoint": "e", "process": { "pid": 7, "startTime": 8 } }), + ), + ] { + let (_directory, instance) = instance(); + let path = instance.dir().join(name); + fs::write(&path, record.to_string()).unwrap(); + let mut locked = instance.lock(None).unwrap(); + let error = locked.find(|| Ok(runtime())).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Io); + assert!( + error.message().contains("has the unsupported format 3"), + "{error}" + ); + assert!( + error + .message() + .contains("stop the sandbox with the build that started it"), + "{error}" + ); + // Nothing is migrated or removed. + assert!(path.is_file()); + } + } - #[test] - fn private_files_are_replaced_atomically() { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("value.json"); - write_json(&path, &serde_json::json!({ "a": 1 })).unwrap(); - write_json(&path, &serde_json::json!({ "a": 2 })).unwrap(); - assert_eq!( - read_json::(&path).unwrap(), - Some(serde_json::json!({ "a": 2 })) - ); - assert_eq!(fs::read_to_string(&path).unwrap(), "{\n \"a\": 2\n}\n"); - // Nothing but the file remains, and a missing file reads as absent. - assert_eq!(fs::read_dir(directory.path()).unwrap().count(), 1); - assert!( - read_json::(&directory.path().join("missing")) - .unwrap() - .is_none() - ); - remove_if_present(&path).unwrap(); - remove_if_present(&path).unwrap(); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - write_private(&path, b"secret").unwrap(); - let mode = fs::metadata(&path).unwrap().permissions().mode(); - assert_eq!(mode & 0o777, 0o600); + #[test] + fn launches_are_checked_before_the_instance_exists() { + let directory = tempfile::tempdir().unwrap(); + let mut spec = LaunchSpec::new( + Artifacts::new("openvmm", "vmlinux", "initramfs.cpio.gz"), + crate::Hypervisor::Kvm, + ); + let dir = directory.path().join("vm"); + check_launch(&dir, &spec).unwrap(); + spec.openvmm = directory + .path() + .join("o".repeat(PROCESS_COMMAND_LINE_LIMIT)); + let error = check_launch(&dir, &spec).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Unsupported, "{error}"); + assert!(!dir.exists()); } } } diff --git a/nvx-rs/src/host/mod.rs b/nvx-rs/src/host/mod.rs index 92eaa7111..0685cb868 100644 --- a/nvx-rs/src/host/mod.rs +++ b/nvx-rs/src/host/mod.rs @@ -8,9 +8,10 @@ //! - [`network`] lowers egress rules, and the egress policy files of `nvx.py`, into OpenVMM's //! rules, and [`filesystem`] plans the export of host paths through OpenVMM's aggregate //! virtio-fs device. -//! - [`instance`] supervises one VM instance in a directory of its own: it starts OpenVMM and -//! waits until the guest is ready, reconciles recorded state with the host, runs workloads, -//! and stops the VM. [`report`] reads OpenVMM's outcome report. +//! - [`instance`] drives one VM instance in a directory of its own, the [`store`] of its VM, +//! through the [`supervisor`] and the [`session`]: it starts OpenVMM and waits until the guest +//! is ready, reattaches to the VM after a restart, runs workloads, and stops the VM. +//! [`report`] reads OpenVMM's outcome report. //! - [`store`] keeps the runtime records of a sandbox in a private directory with one lock: the //! runtime record of its VM, with the VM's runtime identity, and the journal of its launch, //! through which an interrupted launch is recovered without signaling an unverified process. diff --git a/nvx-rs/src/host/store/mod.rs b/nvx-rs/src/host/store/mod.rs index d8272751b..de7ae264b 100644 --- a/nvx-rs/src/host/store/mod.rs +++ b/nvx-rs/src/host/store/mod.rs @@ -178,8 +178,8 @@ use crate::protocol::CAPABILITY_LEN; /// Version of the store's records. /// /// It follows the versions of the runtime state that the CLI kept on its own, 1, and that -/// [`instance`](super::instance) keeps, up to 3, so that a record of theirs fails as unsupported -/// rather than being read as one of the store's. +/// [`instance`](super::instance) kept, up to 3, before both moved to the store, so that a record +/// of theirs fails as unsupported rather than being read as one of the store's. pub const FORMAT: u32 = 4; /// Name of the sandbox's lock. pub const LOCK_NAME: &str = "sandbox.lock"; diff --git a/nvx-rs/src/lib.rs b/nvx-rs/src/lib.rs index 04d6751a5..a71154ebd 100644 --- a/nvx-rs/src/lib.rs +++ b/nvx-rs/src/lib.rs @@ -22,7 +22,7 @@ //! - [`Hypervisor`]: the hypervisors that run the VMs. //! - [`host`] (feature `host`): artifact discovery, the OpenVMM command line, the lowering of //! network and filesystem policies, the host's NVXS connection, control sessions, process and -//! platform primitives, the supervision of VM instances: starting, reconciling, executing +//! platform primitives, the supervision of VM instances: starting, reattaching, executing //! workloads, and stopping, and the store of a sandbox's runtime records and launch journal. //! - `testing` (feature `testing`): a fake OpenVMM that serves the control console without //! running a VM. @@ -47,17 +47,26 @@ //! use std::sync::atomic::AtomicBool; //! use std::time::Duration; //! -//! use nvx::host::instance::{self, Instance}; +//! use nvx::host::instance::{Found, Instance}; //! use nvx::host::launch::LaunchSpec; //! use nvx::protocol::{Workload, WorkloadEnvironment}; //! use nvx::{Artifacts, Hypervisor}; //! -//! // Locates OpenVMM, the guest kernel, and the Alpine initramfs. -//! let spec = LaunchSpec::new(Artifacts::discover()?, Hypervisor::from_env_or_default()?); -//! let directory = std::env::temp_dir().join("nvx-example"); -//! instance::create_private_dir(&directory)?; -//! let instance = Instance::new(&directory, "example"); -//! let started = instance.start(&spec, Duration::from_secs(60))?; +//! // Locates OpenVMM, the guest kernel, and the Alpine initramfs, which the install metadata of +//! // their release or checkout identifies. +//! let artifacts = Artifacts::discover()?; +//! let runtime = artifacts.runtime_identity()?; +//! let spec = LaunchSpec::new(artifacts, Hypervisor::from_env_or_default()?); +//! let instance = Instance::create(std::env::temp_dir().join("nvx-example"), "example")?; +//! // Lifecycle transitions hold the instance's lock; executions need none. +//! let mut locked = instance.lock(None)?; +//! let Found::Stopped = locked.find(|| Ok(runtime.clone()))? else { +//! return Err("the example's VM runs already".into()); +//! }; +//! locked.start(&spec, &runtime, Duration::from_secs(60))?; +//! let Found::Running(running) = locked.find(|| Ok(runtime.clone()))? else { +//! return Err("the example's VM stopped".into()); +//! }; //! //! let workload = Workload { //! argv: vec!["/bin/echo".to_owned(), "hello".to_owned()], @@ -65,9 +74,8 @@ //! cwd: None, //! environment: WorkloadEnvironment::Default, //! }; -//! let capability = instance.read_capability()?; //! let timeout = Duration::from_secs(60); -//! let execution = instance.exec(&started.runtime, &capability, &workload, timeout, timeout)?; +//! let execution = instance.exec(&running, &workload, timeout, timeout)?; //! let mut stdout = Vec::new(); //! let outcome = execution.pump( //! &mut |chunk: &[u8]| { @@ -79,7 +87,7 @@ //! )?; //! println!("{outcome:?}: {}", String::from_utf8_lossy(&stdout)); //! -//! instance.stop(&started.runtime, Duration::from_secs(30))?; +//! locked.stop(running, Duration::from_secs(30))?; //! # Ok(()) //! # } //! # #[cfg(not(feature = "host"))] diff --git a/nvx-rs/tests/fake_openvmm.rs b/nvx-rs/tests/fake_openvmm.rs index 596809252..365539324 100644 --- a/nvx-rs/tests/fake_openvmm.rs +++ b/nvx-rs/tests/fake_openvmm.rs @@ -1,27 +1,48 @@ //! Drives VM instances end to end against the `nvx-fake-openvmm` test double, which serves the //! real control protocol without running a VM. +//! +//! An orchestrator that starts an instance in a process of its own, which then exits, lets a test +//! restart the orchestrator within the same host boot. #![cfg(all( feature = "host", feature = "testing", any(target_os = "linux", windows) ))] +use std::env; use std::fs; -use std::path::Path; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::thread; use std::time::Duration; -use nvx::host::instance::{self, Instance, WorkloadOutcome}; +use nvx::host::instance::{Found, Instance, Running, Started, Stopped, WorkloadOutcome}; use nvx::host::launch::{LaunchSpec, PROCESS_COMMAND_LINE_LIMIT}; +use nvx::host::store::{ArtifactIdentity, IdentityPart, RuntimeIdentity}; use nvx::protocol::{Workload, WorkloadEnvironment}; use nvx::{Artifacts, ErrorKind, Hypervisor}; const TIMEOUT: Duration = Duration::from_secs(30); +/// The instance directory in which [`orchestrator_role`] starts a VM. +const ORCHESTRATOR_INSTANCE: &str = "NVX_FAKE_OPENVMM_TEST_INSTANCE"; -/// Creates the fake guest artifacts and an instance directory below `directory`. -fn setup(directory: &Path) -> (LaunchSpec, Instance) { +/// Returns the runtime identity of these tests' launches, with the OpenVMM identity of the digit +/// `openvmm`. +fn runtime_with(openvmm: char) -> RuntimeIdentity { + let artifact = |digit: char| ArtifactIdentity::sha256(&digit.to_string().repeat(64)).unwrap(); + RuntimeIdentity::new(artifact(openvmm), artifact('2'), artifact('3')) +} + +/// Returns the runtime identity of these tests' launches. +fn runtime() -> RuntimeIdentity { + runtime_with('1') +} + +/// Creates the fake guest artifacts in `directory`, and returns the specification of a VM that +/// runs them. +fn spec(directory: &Path) -> LaunchSpec { fs::write(directory.join("vmlinux"), b"kernel").unwrap(); fs::write(directory.join("initramfs.cpio.gz"), b"initrd").unwrap(); let artifacts = Artifacts::new( @@ -30,12 +51,47 @@ fn setup(directory: &Path) -> (LaunchSpec, Instance) { directory.join("initramfs.cpio.gz"), ); let hypervisor = Hypervisor::platform_default().expect("tests run on Linux or Windows"); - let state = directory.join("vm"); - instance::create_private_dir(&state).unwrap(); - ( - LaunchSpec::new(artifacts, hypervisor), - Instance::new(state, "test"), - ) + LaunchSpec::new(artifacts, hypervisor) +} + +/// Creates the fake guest artifacts and an instance directory below `directory`. +fn setup(directory: &Path) -> (LaunchSpec, Instance) { + let instance = Instance::create(directory.join("vm"), "test").unwrap(); + (spec(directory), instance) +} + +/// Finds the instance's VM under these tests' runtime identity. +fn find(instance: &Instance) -> Found { + instance.lock(None).unwrap().find(|| Ok(runtime())).unwrap() +} + +/// Returns the instance's running VM. +fn running(instance: &Instance) -> Running { + match find(instance) { + Found::Running(running) => running, + other => panic!("the instance does not run: {other:?}"), + } +} + +/// Starts the instance's VM, which does not run, with `timeout`. +fn start_within(instance: &Instance, spec: &LaunchSpec, timeout: Duration) -> nvx::Result { + let mut locked = instance.lock(None).unwrap(); + assert!(matches!(locked.find(|| Ok(runtime()))?, Found::Stopped)); + locked.start(spec, &runtime(), timeout) +} + +/// Starts the instance's VM, which does not run. +fn start(instance: &Instance, spec: &LaunchSpec) -> nvx::Result { + start_within(instance, spec, TIMEOUT) +} + +/// Stops the instance's running VM with `timeout`. +fn stop(instance: &Instance, timeout: Duration) -> nvx::Result { + let mut locked = instance.lock(None).unwrap(); + let Found::Running(running) = locked.find(|| Ok(runtime()))? else { + panic!("the instance does not run"); + }; + locked.stop(running, timeout) } fn workload(argv: &[&str]) -> Workload<'static> { @@ -53,12 +109,10 @@ struct Output { stderr: Vec, } +/// Runs `workload` in the instance's running VM, which it finds as every exec does. fn run(instance: &Instance, workload: &Workload<'_>, cancel: &AtomicBool) -> Output { - let runtime = instance.runtime().unwrap().expect("the instance runs"); - let capability = instance.read_capability().unwrap(); - let execution = instance - .exec(&runtime, &capability, workload, TIMEOUT, TIMEOUT) - .unwrap(); + let running = running(instance); + let execution = instance.exec(&running, workload, TIMEOUT, TIMEOUT).unwrap(); let (mut stdout, mut stderr) = (Vec::new(), Vec::new()); let outcome = execution .pump( @@ -80,15 +134,24 @@ fn run(instance: &Instance, workload: &Workload<'_>, cancel: &AtomicBool) -> Out } } +/// Returns whether the store of `instance` records neither a launch nor a VM. +fn records_nothing(instance: &Instance) -> bool { + let store = instance.store(); + store.journal().unwrap().is_none() + && store.runtime().unwrap().is_none() + && store.read_capability().is_err() +} + #[test] fn instances_start_run_workloads_and_stop() { let directory = tempfile::tempdir().unwrap(); let (spec, instance) = setup(directory.path()); - assert_eq!(instance.reconcile().unwrap(), None); - let started = instance.start(&spec, TIMEOUT).unwrap(); - assert_eq!(instance.reconcile().unwrap(), Some(started.runtime.clone())); - // Only a running instance keeps a launch capability. - assert!(instance.launch().unwrap().is_none()); + assert!(matches!(find(&instance), Found::Stopped)); + let started = start(&instance, &spec).unwrap(); + assert_eq!(started.runtime.runtime(), &runtime()); + assert_eq!(running(&instance).runtime(), &started.runtime); + // Only a launch in progress keeps a journal. + assert!(instance.store().journal().unwrap().is_none()); let idle = AtomicBool::new(false); let output = run( @@ -117,13 +180,16 @@ fn instances_start_run_workloads_and_stop() { refused.stderr ); - let stopped = instance.stop(&started.runtime, TIMEOUT).unwrap(); + let stopped = stop(&instance, TIMEOUT).unwrap(); assert!(!stopped.forced); let report = stopped.report.expect("the VM reports its outcome"); assert_eq!(report.category.as_deref(), Some("success")); assert_eq!(report.teardown_complete, Some(true)); - assert_eq!(instance.reconcile().unwrap(), None); - assert!(instance.read_capability().is_err()); + assert!(matches!(find(&instance), Found::Stopped)); + assert!(records_nothing(&instance)); + // The log and the report outlive the run. + assert!(instance.log_path().is_file()); + assert!(instance.report_path().is_file()); } #[test] @@ -131,33 +197,27 @@ fn timeouts_are_checked_before_they_are_used() { let directory = tempfile::tempdir().unwrap(); let (spec, instance) = setup(directory.path()); for timeout in [Duration::ZERO, Duration::MAX] { - let error = instance.start(&spec, timeout).unwrap_err(); + let error = start_within(&instance, &spec, timeout).unwrap_err(); assert_eq!(error.kind(), ErrorKind::Unsupported, "{error}"); // Nothing was launched. - assert!(instance.launch().unwrap().is_none()); - assert!(instance.runtime().unwrap().is_none()); + assert!(records_nothing(&instance)); } - let started = instance.start(&spec, TIMEOUT).unwrap(); - let capability = instance.read_capability().unwrap(); + start(&instance, &spec).unwrap(); let echo = workload(&["/bin/echo", "hello"]); + let vm = running(&instance); for (control_timeout, response_grace) in [ (Duration::ZERO, TIMEOUT), (Duration::MAX, TIMEOUT), (TIMEOUT, Duration::MAX), ] { - let Err(error) = instance.exec( - &started.runtime, - &capability, - &echo, - control_timeout, - response_grace, - ) else { + let Err(error) = instance.exec(&vm, &echo, control_timeout, response_grace) else { panic!("exec accepted {control_timeout:?} and {response_grace:?}"); }; assert_eq!(error.kind(), ErrorKind::Unsupported, "{error}"); } + drop(vm); for timeout in [Duration::ZERO, Duration::MAX] { - let error = instance.stop(&started.runtime, timeout).unwrap_err(); + let error = stop(&instance, timeout).unwrap_err(); assert_eq!(error.kind(), ErrorKind::Unsupported, "{error}"); } // The refused requests left the VM running. @@ -165,30 +225,33 @@ fn timeouts_are_checked_before_they_are_used() { let output = run(&instance, &echo, &idle); assert_eq!(output.outcome, WorkloadOutcome::Exited(0)); assert_eq!(output.stdout, b"hello\n"); - instance.stop(&started.runtime, TIMEOUT).unwrap(); + stop(&instance, TIMEOUT).unwrap(); } #[test] fn launches_beyond_the_process_command_line_limit_are_refused() { let directory = tempfile::tempdir().unwrap(); let (mut spec, instance) = setup(directory.path()); + // The log of an earlier start outlives a launch that is refused. + start(&instance, &spec).unwrap(); + stop(&instance, TIMEOUT).unwrap(); + let log = fs::read(instance.log_path()).unwrap(); // The program's path counts toward the command line, which every host holds to the limit. spec.openvmm = directory .path() .join("o".repeat(PROCESS_COMMAND_LINE_LIMIT)); - let error = instance.start(&spec, TIMEOUT).unwrap_err(); + let error = start(&instance, &spec).unwrap_err(); assert_eq!(error.kind(), ErrorKind::Unsupported, "{error}"); // Nothing was launched or recorded. - assert!(instance.launch().unwrap().is_none()); - assert!(instance.runtime().unwrap().is_none()); - assert!(instance.read_capability().is_err()); + assert!(records_nothing(&instance)); + assert_eq!(fs::read(instance.log_path()).unwrap(), log); } #[test] fn cancellation_ends_a_running_workload() { let directory = tempfile::tempdir().unwrap(); let (spec, instance) = setup(directory.path()); - let started = instance.start(&spec, TIMEOUT).unwrap(); + start(&instance, &spec).unwrap(); let cancel = Arc::new(AtomicBool::new(false)); let canceller = thread::spawn({ let cancel = Arc::clone(&cancel); @@ -205,7 +268,51 @@ fn cancellation_ends_a_running_workload() { canceller.join().unwrap(); assert_eq!(output.outcome, WorkloadOutcome::Cancelled); assert_eq!(output.stdout, b"started\n"); - instance.stop(&started.runtime, TIMEOUT).unwrap(); + stop(&instance, TIMEOUT).unwrap(); +} + +#[test] +fn unresponsive_guests_are_terminated() { + let directory = tempfile::tempdir().unwrap(); + let (mut spec, instance) = setup(directory.path()); + spec.kernel_parameters = "fake_ignore_stop=1".to_owned(); + start(&instance, &spec).unwrap(); + let stopped = stop(&instance, Duration::from_millis(500)).unwrap(); + assert!(stopped.forced); + assert!(matches!(find(&instance), Found::Stopped)); + assert!(records_nothing(&instance)); +} + +#[test] +fn a_vm_whose_capability_is_lost_is_still_stopped() { + // Only control sessions need the capability: without it, exec fails as reading it fails, and + // stop ends the VM through its verified process identity. + let directory = tempfile::tempdir().unwrap(); + let (spec, instance) = setup(directory.path()); + let capability = instance.store().capability_path(); + for damage in ["missing", "truncated"] { + start(&instance, &spec).unwrap(); + match damage { + "missing" => fs::remove_file(&capability).unwrap(), + _ => fs::write(&capability, b"short").unwrap(), + } + let running = running(&instance); + let Err(error) = instance.exec(&running, &workload(&["/bin/true"]), TIMEOUT, TIMEOUT) + else { + panic!("{damage}: exec attached without the capability"); + }; + assert_eq!(error.kind(), ErrorKind::Io, "{damage}: {error}"); + assert!( + error.message().contains("sandbox control capability"), + "{damage}: {error}" + ); + drop(running); + let stopped = stop(&instance, TIMEOUT).unwrap(); + assert!(stopped.forced, "{damage}"); + assert!(matches!(find(&instance), Found::Stopped), "{damage}"); + assert!(records_nothing(&instance), "{damage}"); + assert!(!capability.exists(), "{damage}"); + } } #[test] @@ -214,7 +321,7 @@ fn guests_without_required_features_are_refused() { let (mut spec, instance) = setup(directory.path()); // Cancellation alone. spec.kernel_parameters = "fake_guest_features=1".to_owned(); - let error = instance.start(&spec, TIMEOUT).unwrap_err(); + let error = start(&instance, &spec).unwrap_err(); assert_eq!(error.kind(), ErrorKind::Unavailable); assert!( error.message().contains("lacks control features"), @@ -222,8 +329,8 @@ fn guests_without_required_features_are_refused() { ); assert!(error.message().contains("sandbox test failed to start")); // The rejected VM is gone and leaves no runtime state behind. - assert_eq!(instance.reconcile().unwrap(), None); - assert!(instance.runtime().unwrap().is_none()); + assert!(matches!(find(&instance), Found::Stopped)); + assert!(records_nothing(&instance)); } #[test] @@ -231,7 +338,7 @@ fn launches_that_exit_fail_to_start() { let directory = tempfile::tempdir().unwrap(); let (mut spec, instance) = setup(directory.path()); spec.kernel_parameters = "fake_exit_on_start=4".to_owned(); - let error = instance.start(&spec, TIMEOUT).unwrap_err(); + let error = start(&instance, &spec).unwrap_err(); assert_eq!(error.kind(), ErrorKind::Exited, "{error}"); assert!( error @@ -239,5 +346,72 @@ fn launches_that_exit_fail_to_start() { .contains(&instance.log_path().display().to_string()), "{error}" ); - assert_eq!(instance.reconcile().unwrap(), None); + assert!(matches!(find(&instance), Found::Stopped)); + assert!(records_nothing(&instance)); +} + +/// The orchestrator that [`a_restarted_orchestrator_reattaches_to_its_vm`] runs in a process of +/// its own: it starts the instance in the directory that [`ORCHESTRATOR_INSTANCE`] names, with the +/// fake guest artifacts of its working directory, and exits, leaving the VM running. +#[test] +#[ignore = "an orchestrator that the tests run in a process of its own"] +fn orchestrator_role() { + let Some(dir) = env::var_os(ORCHESTRATOR_INSTANCE) else { + return; + }; + let spec = spec(&env::current_dir().unwrap()); + let instance = Instance::create(PathBuf::from(dir), "test").unwrap(); + start(&instance, &spec).unwrap(); +} + +#[test] +fn a_restarted_orchestrator_reattaches_to_its_vm() { + let directory = tempfile::tempdir().unwrap(); + let dir = directory.path().join("vm"); + let status = Command::new(env::current_exe().unwrap()) + .args([ + "--exact", + "orchestrator_role", + "--ignored", + "--nocapture", + "--test-threads=1", + ]) + .current_dir(directory.path()) + .env(ORCHESTRATOR_INSTANCE, &dir) + .stdin(Stdio::null()) + .status() + .unwrap(); + assert!(status.success(), "the orchestrator failed: {status}"); + + // The orchestrator restarts within the same host boot: it finds the VM that its earlier run + // started, through the records of the store, and drives it. + let instance = Instance::open(&dir, "test").unwrap(); + let idle = AtomicBool::new(false); + let output = run(&instance, &workload(&["/bin/echo", "reattached"]), &idle); + assert_eq!(output.outcome, WorkloadOutcome::Exited(0)); + assert_eq!(output.stdout, b"reattached\n"); + + // Under another runtime identity, as after an upgrade of OpenVMM, the VM is mismatched: it is + // reported, and neither driven nor stopped. + { + let mut locked = instance.lock(None).unwrap(); + let Found::Mismatched(mismatched) = locked.find(|| Ok(runtime_with('9'))).unwrap() else { + panic!("a VM of another runtime identity was not reported"); + }; + assert_eq!(mismatched.differences(), [IdentityPart::OpenVmm]); + let error = mismatched.error(); + assert!( + error.message().starts_with( + "sandbox test runs under another runtime identity, which differs in OpenVMM" + ), + "{error}" + ); + } + assert_eq!( + run(&instance, &workload(&["/bin/echo", "still"]), &idle).stdout, + b"still\n" + ); + let stopped = stop(&instance, TIMEOUT).unwrap(); + assert!(!stopped.forced); + assert!(records_nothing(&instance)); } diff --git a/scripts/nvx_tools/aci_edge_sandboxes_tests.py b/scripts/nvx_tools/aci_edge_sandboxes_tests.py index e252d92e7..0fad09ced 100644 --- a/scripts/nvx_tools/aci_edge_sandboxes_tests.py +++ b/scripts/nvx_tools/aci_edge_sandboxes_tests.py @@ -46,16 +46,12 @@ def configure_parser(parser: argparse.ArgumentParser) -> None: def e2e_environment(backend: str, state_root: Path, output_dir: Path) -> dict[str, str]: + # The test locates the artifacts in the checkout, whose build provenance identifies them. + require_file(openvmm_binary_path(), "OpenVMM release binary") + require_file(artifact_path(KernelBuildConstants.BINARY_NAME), "Linux direct kernel") + require_file(artifact_path(AlpineBuildConstants.INITRAMFS_NAME), "Alpine initramfs") paths = { - "ACI_EDGE_SANDBOXES_E2E_OPENVMM": require_file( - openvmm_binary_path(), "OpenVMM release binary" - ), - "ACI_EDGE_SANDBOXES_E2E_KERNEL": require_file( - artifact_path(KernelBuildConstants.BINARY_NAME), "Linux direct kernel" - ), - "ACI_EDGE_SANDBOXES_E2E_INITRD": require_file( - artifact_path(AlpineBuildConstants.INITRAMFS_NAME), "Alpine initramfs" - ), + "ACI_EDGE_SANDBOXES_E2E_ARTIFACTS_DIR": BuildConstants.REPO_ROOT, "ACI_EDGE_SANDBOXES_E2E_STATE_ROOT": state_root, "ACI_EDGE_SANDBOXES_E2E_OUTPUT_DIR": output_dir, } @@ -91,13 +87,13 @@ def remaining_sandboxes(state_root: Path) -> list[Path]: def recorded_pid(sandbox: Path) -> int | None: - """Returns the OpenVMM process ID in a sandbox's runtime or launch record.""" - for name, keys in (("runtime.json", ("pid",)), ("launch.json", ("process", "pid"))): + """Returns the OpenVMM process ID in a sandbox's runtime record or launch journal.""" + for name in ("runtime.json", "launch.json"): try: value: object = json.loads((sandbox / name).read_text(encoding="utf-8")) except (OSError, ValueError): continue - for key in keys: + for key in ("process", "pid"): if not isinstance(value, dict): value = None break diff --git a/scripts/test_nvx_tools.py b/scripts/test_nvx_tools.py index 9ebb4f5ba..b696029e2 100644 --- a/scripts/test_nvx_tools.py +++ b/scripts/test_nvx_tools.py @@ -17858,14 +17858,13 @@ def test_environment_resolves_repository_artifacts(self): environment = self._environment(root, paths) self.assertEqual(environment["ACI_EDGE_SANDBOXES_E2E_HYPERVISOR"], "kvm") + # The test locates the artifacts in the checkout, whose build provenance + # identifies them, rather than by their paths. self.assertEqual( - Path(environment["ACI_EDGE_SANDBOXES_E2E_OPENVMM"]), - paths["openvmm"].resolve(), - ) - self.assertEqual( - Path(environment["ACI_EDGE_SANDBOXES_E2E_INITRD"]), - paths[AlpineBuildConstants.INITRAMFS_NAME].resolve(), + Path(environment["ACI_EDGE_SANDBOXES_E2E_ARTIFACTS_DIR"]), + BuildConstants.REPO_ROOT.resolve(), ) + self.assertNotIn("ACI_EDGE_SANDBOXES_E2E_OPENVMM", environment) self.assertEqual( Path(environment["ACI_EDGE_SANDBOXES_E2E_STATE_ROOT"]), (root / "state").resolve(), @@ -17941,9 +17940,9 @@ def _run_leaving( ) -> tuple[Path, str]: """Runs the handler with a test command that leaves one sandbox provisioned. - The sandbox gets `records` as extra state files; `None` leaves only the lock - directories that deprovisioning keeps. Returns the state root that the command - used and the handler's standard error. + The sandbox gets `records` as extra state files; `None` leaves only an empty + backend directory. Returns the state root that the command used and the handler's + standard error. """ state_roots: list[Path] = [] @@ -17956,7 +17955,7 @@ def fake_environment( def fake_run( command: list[str], *, env: dict[str, str], check: bool ) -> subprocess.CompletedProcess[bytes]: - (state_roots[0] / "filesystem" / ".locks").mkdir(parents=True) + (state_roots[0] / "filesystem").mkdir(parents=True) if records is not None: sandbox = state_roots[0] / "network" / ("a" * 32) sandbox.mkdir(parents=True) @@ -17999,7 +17998,9 @@ def test_handler_deletes_state_once_every_sandbox_is_deprovisioned(self): self.assertEqual(stderr, "") def test_handler_keeps_state_of_sandboxes_left_provisioned(self): - state_root, stderr = self._run_leaving({"runtime.json": {"pid": 4242}}) + state_root, stderr = self._run_leaving( + {"runtime.json": {"process": {"pid": 4242}}} + ) sandbox = state_root / "network" / ("a" * 32) self.assertTrue((sandbox / "runtime.json").is_file()) From a869a3e802bf021968394dae7377f1d704e1d359 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Wed, 21 Oct 2026 09:06:27 -0700 Subject: [PATCH 3/3] nvx-rs: remove the process helpers that only the old instance used host::instance launches, waits for, and ends OpenVMM through the supervisor now, so nothing uses host::process's spawn, detach_child, kill_child, wait_for_exit, kill, open_identity, and open_identity_with, or its re-export of supervisor::Launched. The store and supervisor tests use Process::open and the supervisor instead. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: aa7b9fda-3774-4817-aa98-af05f8d50215 --- nvx-rs/README.md | 2 +- nvx-rs/src/host/process.rs | 176 ++++------------------------------- nvx-rs/src/host/store/mod.rs | 3 +- nvx-rs/tests/store.rs | 42 ++++++--- nvx-rs/tests/supervisor.rs | 17 +--- 5 files changed, 53 insertions(+), 187 deletions(-) diff --git a/nvx-rs/README.md b/nvx-rs/README.md index 5a6736425..ec1500309 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -77,7 +77,7 @@ nvx-rs/ | `nvx::host::store` | `SandboxStore`: the runtime records of one sandbox in a private directory with one lock, which every host that supervises the sandbox shares. The `RuntimeRecord` names the running VM: the `ProcessRef` of its VMM, its control endpoint, and the `RuntimeIdentity` that it was launched with, the identities of its OpenVMM, kernel, and initramfs from install or staging metadata and its boot and control contract versions, which decide whether another build of the host may reattach to it. The `LaunchJournal` of a launch holds a marker written before the spawn, the VMM's `ProcessRef` right after it, and an inherited claim on the VMM's log, which proves that a launch that never published its endpoint has exited, so that recovery ends an interrupted launch and signals only a VMM that it holds through a verified `Process`. Records are JSON in one versioned format, and records in any other format fail rather than being migrated | | `nvx::host::instance` | `Instance`: a VM in a sandbox store of its own, which `nvx::host::store`, `nvx::host::supervisor`, and `nvx::host::session` drive. Under the store's lock, `LockedInstance::find` reattaches to the VM, ends a start that was interrupted, and reports a VM of another runtime identity as `Mismatched` rather than driving it; `LockedInstance::start` launches OpenVMM through the supervisor with a fresh capability, under the caller's `RuntimeIdentity`, waits until the guest is ready and has mounted the mapped paths, and commits the launch; and `LockedInstance::stop` stops the VM with a forced fallback. `Instance::exec` runs workloads without the lock and pumps their output. `write_json` and `read_json` keep the configuration records that callers store beside an instance | | `nvx::host::nvxs` | `HostConnection`: the host's role of the NVXS session over a platform transport, on which `nvx::host::session` and other host applications run their protocols. It attaches with the launch capability, waiting while the broker asks it to, sends application bytes in as many DATA records as they take, and returns each DATA payload from the guest, with deadlines; a receive that times out keeps a partial record | -| `nvx::host::process` | Verified process references, and the launch of OpenVMM for `nvx::host::instance`, through `nvx::host::supervisor`. A `ProcessRef` names one process by its ID, its start time, and the `BootId` of the host boot during which it started. `Process::open` checks a reference and holds its process through a pidfd on Linux or a process handle on Windows, whether or not the caller started it, so that waits sleep until the process exits and termination never reaches a process that reuses its ID. A reference from an earlier boot names an exited process, and a process whose identity this user cannot check, such as another user's, makes `Process::open` fail rather than report an exit. Linux hosts without pidfds poll the reference instead, and cannot terminate the process | +| `nvx::host::process` | Verified process references. A `ProcessRef` names one process by its ID, its start time, and the `BootId` of the host boot during which it started. `Process::open` checks a reference and holds its process through a pidfd on Linux or a process handle on Windows, whether or not the caller started it, so that waits sleep until the process exits and termination never reaches a process that reuses its ID. A reference from an earlier boot names an exited process, and a process whose identity this user cannot check, such as another user's, makes `Process::open` fail rather than report an exit. Linux hosts without pidfds poll the reference instead, and cannot terminate the process | | `nvx::host::supervisor` | The supervisor of a sandbox's VMM, beneath the CLI, `Instance`, and external orchestrators, which keep the configuration, the authorization, and the application protocol. `launch` runs the store's launch steps under the sandbox's lock: it writes the journal's marker, spawns OpenVMM with the claimed log, records its `ProcessRef` right after, and returns a `Starting` launch that the caller commits once the VMM serves its control endpoint, or abandons. A launch that fails never leaves an unrecorded VMM running. `VmmCommand::inherit` lists the descriptors (Linux) or handles (Windows) that the VMM inherits under their numbers, besides its standard streams, and it inherits nothing else of the caller's: on Linux, they must be close-on-exec, a flag that only the VMM's own process clears; on Windows, they join the VMM's explicit handle list. `Vmm::stop` runs one graceful step of the caller's and waits for the VMM's exit event until a deadline, and takes a bounded forced step through the VMM's verified identity only on request. `reattach` recovers an interrupted launch, and holds the recorded VMM again after the orchestrator restarts within the same host boot: `Live`, `Mismatched` with another runtime identity, `Exited`, or `Indeterminate` | | `nvx::host::seal` | `FileSeal`: what the host reports about an open file from its metadata alone, without reading it: the platform scheme, the device or volume serial number, the file ID, the kind, the length, the modification time, and on Linux the change time, with a versioned serde form. A seal detects accidental change, not tampering; see [Distro bindings](#distro-bindings) | | `nvx::host::session`, `platform`, `report` | The authenticated control session, which carries NVX's control protocol, `NVXC`, one frame per DATA record of a `HostConnection`; Linux and Windows primitives; and OpenVMM's outcome report | diff --git a/nvx-rs/src/host/process.rs b/nvx-rs/src/host/process.rs index 97e560bc1..b45f2ae34 100644 --- a/nvx-rs/src/host/process.rs +++ b/nvx-rs/src/host/process.rs @@ -15,16 +15,12 @@ //! Linux 5.3 or in sandboxes that refuse them, check the reference at a short interval instead, //! until the deadline of the wait, and cannot terminate a process that they cannot hold. //! -//! The [`supervisor`] launches OpenVMM, and its [`Launched::process`] holds the child that it -//! started through a handle of its own. [`spawn`], [`detach_child`], and [`kill_child`] launch and -//! release OpenVMM for [`instance`](super::instance), which keeps records of its own: they go -//! through the supervisor, and remain until it moves to it, as do [`wait_for_exit`] and [`kill`]. +//! The [`supervisor`](super::supervisor) launches OpenVMM, and its +//! [`Launched::process`](super::supervisor::Launched::process) holds the child that it started +//! through a handle of its own. -use std::ffi::OsString; use std::fmt; -use std::fs::File; use std::io; -use std::path::Path; use std::str::FromStr; use std::sync::OnceLock; use std::thread; @@ -33,18 +29,15 @@ use std::time::{Duration, Instant}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use super::platform; -pub use super::supervisor::Launched; -use super::supervisor::{self, VmmCommand}; use crate::error::Error; -use crate::protocol::CAPABILITY_LEN; /// Longest wait for a terminated OpenVMM process to exit. pub(crate) const KILL_TIMEOUT: Duration = Duration::from_secs(10); -/// Interval at which the exit or identity of a process is polled where no exit event serves: the -/// exit of any process on Linux hosts without pidfds, and the exit of a child that a [`Launched`] -/// waits for with a timeout on Linux, since [`std::process::Child`] can wait only without one. -/// [`wait_for_exit`] also checks again at this interval an identity that it could not read. An -/// exit is noticed at most this late, for about 40 checks a second. +/// Interval at which the exit of a process is polled where no exit event serves: the exit of any +/// process on Linux hosts without pidfds, and the exit of a child that a +/// [`Launched`](super::supervisor::Launched) waits for with a timeout on Linux, since +/// [`std::process::Child`] can wait only without one. An exit is noticed at most this late, for +/// about 40 checks a second. pub(crate) const POLL_INTERVAL: Duration = Duration::from_millis(25); /// The identity of one boot of the host, which changes whenever the host restarts. @@ -310,107 +303,6 @@ fn poll_exit(deadline: Instant, mut runs: impl FnMut() -> io::Result) -> i } } -/// Starts OpenVMM detached from the caller, with `capability` on its standard input, `log` as -/// its standard output and error, and nothing else of the caller's, as [`supervisor::spawn`] -/// starts a VMM that inherits no descriptor or handle beyond its standard streams. -/// -/// On Windows, `breakaway_from_job` starts OpenVMM outside the caller's job object, which must -/// permit breakaway, so that the VM survives the caller. -pub fn spawn( - openvmm: &Path, - breakaway_from_job: bool, - arguments: &[OsString], - capability: &[u8; CAPABILITY_LEN], - log: File, - working_dir: &Path, -) -> io::Result { - let mut command = VmmCommand::new(openvmm, arguments.to_vec(), working_dir); - command.breakaway_from_job = breakaway_from_job; - supervisor::spawn(&command, capability, log) -} - -/// Releases a launched process without waiting for it, as [`Launched::detach`] does. -pub fn detach_child(launched: Launched) { - launched.detach(); -} - -/// Kills a launched process that was never recorded, through its handle, which cannot race with -/// process ID reuse, as [`Launched::kill_and_wait`] does. Returns whether it exited within ten -/// seconds. -pub fn kill_child(launched: Launched) -> bool { - launched.kill_and_wait() -} - -/// Waits until the process with the given identity is gone, returning whether it exited before -/// `deadline`. Inconclusive checks count as still running. -/// -/// The process is held and waited for as [`Process::open`] and [`Process::wait`] do, within the -/// current boot, except that a process whose identity this user may not check counts as exited -/// wherever [`platform::process_start_time`] says that it has. -pub fn wait_for_exit(pid: u32, start_time: u64, deadline: Instant) -> bool { - loop { - match open_identity(pid, start_time) { - Ok(None) => return true, - Ok(Some(handle)) => { - if let Ok(exited) = wait_handle(&handle, pid, start_time, deadline) { - return exited; - } - } - Err(_) => {} - } - let now = Instant::now(); - if now >= deadline { - return false; - } - thread::sleep(POLL_INTERVAL.min(deadline - now)); - } -} - -/// Kills the process and waits up to ten seconds for it to disappear. -/// -/// The process is held as [`wait_for_exit`] holds it, so it is signaled only if it still has the -/// identity, and [`Process::kill`] ends it. -pub fn kill(pid: u32, start_time: u64) -> io::Result { - let Some(handle) = open_identity(pid, start_time)? else { - return Ok(true); - }; - handle.kill(pid)?; - wait_handle(&handle, pid, start_time, Instant::now() + KILL_TIMEOUT) -} - -/// Opens the process with the identity `pid` and `start_time` for [`wait_for_exit`] and -/// [`kill`], as [`Process::open`] does within the current boot. -/// -/// Unlike [`Process::open`], it keeps the rule of [`platform::process_start_time`], which -/// `host::instance` and the CLI follow: OpenVMM runs as the user that starts it, so a process -/// that this user may not query on Windows, or that `/proc` hides from it on Linux, has exited. -fn open_identity(pid: u32, start_time: u64) -> io::Result> { - open_identity_with( - pid, - start_time, - platform::open_process, - platform::process_start_time, - ) -} - -/// Opens the process with the identity `pid` and `start_time` with `open`, as [`open_identity`] -/// does, and when its identity cannot be checked, asks `current` for the start time that -/// [`platform::process_start_time`] reports. -fn open_identity_with( - pid: u32, - start_time: u64, - open: impl FnOnce(u32, u64) -> io::Result>, - current: impl FnOnce(u32) -> io::Result>, -) -> io::Result> { - match open(pid, start_time) { - Err(error) if error.kind() == io::ErrorKind::PermissionDenied => match current(pid)? { - Some(current) if current == start_time => Err(error), - _ => Ok(None), - }, - result => result, - } -} - #[cfg(test)] mod reference_tests { use serde_json::json; @@ -542,43 +434,6 @@ mod reference_tests { assert_eq!(opened.unwrap(), Some((7, 42))); } - #[test] - fn identities_that_cannot_be_checked_have_exited_where_start_times_say_so() { - let hidden = |_: u32, _: u64| { - Err::, _>(io::Error::new(io::ErrorKind::PermissionDenied, "hidden")) - }; - // The start time shows no process with the ID, or another one, so the process exited. - let gone = open_identity_with(7, 42, hidden, |_| Ok(None)); - assert!(gone.unwrap().is_none()); - let reused = open_identity_with(7, 42, hidden, |_| Ok(Some(43))); - assert!(reused.unwrap().is_none()); - // The start time shows the process, or cannot be read, so its state is unknown. - let running = open_identity_with(7, 42, hidden, |_| Ok(Some(42))); - assert_eq!(running.unwrap_err().to_string(), "hidden"); - let unreadable = open_identity_with(7, 42, hidden, |_| Err(io::Error::other("unreadable"))); - assert_eq!(unreadable.unwrap_err().to_string(), "unreadable"); - // Any other outcome of the open stands. - let unasked = |_: u32| -> io::Result> { - panic!("only an identity that cannot be checked asks for the start time") - }; - let opened = open_identity_with( - 7, - 42, - |pid, start_time| Ok(Some((pid, start_time))), - unasked, - ); - assert_eq!(opened.unwrap(), Some((7, 42))); - let exited = open_identity_with(7, 42, |_, _| Ok(None::<()>), unasked); - assert!(exited.unwrap().is_none()); - let failed = open_identity_with( - 7, - 42, - |_, _| Err::, _>(io::Error::other("failed")), - unasked, - ); - assert_eq!(failed.unwrap_err().to_string(), "failed"); - } - #[test] fn polling_checks_the_identity_until_the_deadline() { let later = Instant::now() + Duration::from_secs(60); @@ -792,13 +647,20 @@ mod tests { thread::sleep(POLL_INTERVAL); } assert_eq!(platform::process_start_time(pid).unwrap(), Some(start_time)); - assert!(!wait_for_exit( + let process = Process::open(&ProcessRef { pid, start_time, - Instant::now() + Duration::from_millis(100) - )); + boot: BootId::current().unwrap(), + }) + .unwrap() + .expect("a process whose main thread exited still runs"); + assert!( + !process + .wait(Instant::now() + Duration::from_millis(100)) + .unwrap() + ); child.0.kill().unwrap(); child.0.wait().unwrap(); - assert!(wait_for_exit(pid, start_time, Instant::now())); + assert!(process.wait(Instant::now()).unwrap()); } } diff --git a/nvx-rs/src/host/store/mod.rs b/nvx-rs/src/host/store/mod.rs index de7ae264b..f7d051762 100644 --- a/nvx-rs/src/host/store/mod.rs +++ b/nvx-rs/src/host/store/mod.rs @@ -171,7 +171,8 @@ pub use self::record::{ }; use super::filesystem::FileIdentity; use super::platform; -use super::process::{BootId, Launched, Process, ProcessRef}; +use super::process::{BootId, Process, ProcessRef}; +use super::supervisor::Launched; use crate::error::{Error, Result}; use crate::protocol::CAPABILITY_LEN; diff --git a/nvx-rs/tests/store.rs b/nvx-rs/tests/store.rs index d1d5c2e98..ad9587fd3 100644 --- a/nvx-rs/tests/store.rs +++ b/nvx-rs/tests/store.rs @@ -20,11 +20,12 @@ use std::time::{Duration, Instant}; use nvx::ErrorKind; use nvx::host::platform; -use nvx::host::process::{self, Launched, Process}; +use nvx::host::process::{BootId, Process, ProcessRef}; use nvx::host::store::{ ArtifactIdentity, IdentityCheck, IdentityPart, JOURNAL_NAME, Launch, LaunchState, LockedStore, RUNTIME_NAME, Recovery, RuntimeIdentity, SandboxStore, }; +use nvx::host::supervisor::{self, Launched, VmmCommand}; use nvx::protocol::CAPABILITY_LEN; /// The capability of every launch of these tests. @@ -110,7 +111,12 @@ fn role(name: &str, extra: &[&str]) -> (PathBuf, Vec) { fn spawn_vmm(launch: &mut Launch<'_>, store: &SandboxStore) -> Launched { let (program, arguments) = role("vmm_role", &[VMM_MARKER]); let log = launch.take_log().unwrap(); - process::spawn(&program, false, &arguments, &CAPABILITY, log, store.dir()).unwrap() + supervisor::spawn( + &VmmCommand::new(program, arguments, store.dir()), + &CAPABILITY, + log, + ) + .unwrap() } /// Spawns the fake OpenVMM, which serves the endpoint of `launch` after `boot_delay_ms`, with the @@ -134,13 +140,10 @@ fn spawn_fake(launch: &mut Launch<'_>, store: &SandboxStore, boot_delay_ms: u64) .openvmm_arguments(launch.endpoint(), &store.outcome_path()) .unwrap(); let log = launch.take_log().unwrap(); - process::spawn( - &spec.openvmm, - false, - &arguments, + supervisor::spawn( + &VmmCommand::new(&spec.openvmm, arguments, store.dir()), &CAPABILITY, log, - store.dir(), ) .unwrap() } @@ -158,7 +161,7 @@ impl Vmm { impl Drop for Vmm { fn drop(&mut self) { if let Some(launched) = self.0.take() { - process::kill_child(launched); + launched.kill_and_wait(); } } } @@ -168,6 +171,21 @@ fn runs(pid: u32, start_time: u64) -> bool { platform::process_start_time(pid).unwrap() == Some(start_time) } +/// Ends the process `pid` that started at `start_time` in this boot, through its verified +/// handle, if it still runs, and returns whether it exited within [`RECOVERY_TIMEOUT`]. +fn kill(pid: u32, start_time: u64) -> io::Result { + let reference = ProcessRef { + pid, + start_time, + boot: BootId::current()?, + }; + let Some(process) = Process::open(&reference)? else { + return Ok(true); + }; + process.kill()?; + process.wait(Instant::now() + RECOVERY_TIMEOUT) +} + /// A VMM that a crashed orchestrator left behind, which is killed when the test ends if it still /// runs. struct Orphan { @@ -186,7 +204,7 @@ impl Orphan { impl Drop for Orphan { fn drop(&mut self) { - let _ = process::kill(self.pid, self.start_time); + let _ = kill(self.pid, self.start_time); } } @@ -403,7 +421,7 @@ fn the_log_claim_proves_that_an_unrecorded_vmm_exited() { assert!(store.read_capability().is_ok()); assert!(vmm.runs()); // Once the VMM exits, its released claim proves it. - assert!(process::kill(vmm.pid, vmm.start_time).unwrap()); + assert!(kill(vmm.pid, vmm.start_time).unwrap()); assert_eq!(recover_until_determinate(&mut locked), Recovery::Exited); assert!(store.journal().unwrap().is_none()); } @@ -529,7 +547,7 @@ fn an_abandoned_launch_ends_its_recorded_vmm() { assert!(!runs(process.pid, process.start_time)); assert!(store.journal().unwrap().is_none()); // The caller still releases the child that it holds. - process::detach_child(vmm.0.take().unwrap()); + vmm.0.take().unwrap().detach(); } #[cfg(feature = "testing")] @@ -591,7 +609,7 @@ fn an_endpoint_served_without_the_claimed_log_is_not_signaled() { assert!(reason.contains("does not hold its log"), "{reason}"); assert!(runs(fake_process.pid, fake_process.start_time)); assert!(vmm.runs()); - assert!(process::kill(vmm.pid, vmm.start_time).unwrap()); + assert!(kill(vmm.pid, vmm.start_time).unwrap()); assert_eq!(recover_until_determinate(&mut locked), Recovery::Exited); drop(fake); } diff --git a/nvx-rs/tests/supervisor.rs b/nvx-rs/tests/supervisor.rs index 2cd1244b9..0ee95ac47 100644 --- a/nvx-rs/tests/supervisor.rs +++ b/nvx-rs/tests/supervisor.rs @@ -28,7 +28,7 @@ use std::sync::{Mutex, MutexGuard, PoisonError, mpsc}; use std::thread; use std::time::{Duration, Instant}; -use nvx::host::process::{self, BootId, Process, ProcessRef, WaitMethod}; +use nvx::host::process::{BootId, Process, ProcessRef, WaitMethod}; #[cfg(windows)] use nvx::host::store::FORMAT; use nvx::host::store::{ @@ -681,13 +681,6 @@ fn a_reused_process_id_is_never_signaled() { "{impostor:?} names a running process" ); } - // The functions that take a process ID and a start time do not signal it either. - assert!(process::kill(reference.pid, reference.start_time + 1).unwrap()); - assert!(process::wait_for_exit( - reference.pid, - reference.start_time + 1, - Instant::now() - )); // The process that has the ID still runs. assert!(!process.has_exited().unwrap()); @@ -1423,14 +1416,6 @@ fn a_process_that_this_user_may_not_query_has_an_unknown_state() { // Its identity cannot be checked, so it may still run. let error = Process::open(&reference).unwrap_err(); assert_eq!(error.kind(), io::ErrorKind::PermissionDenied); - // The entry points by identity count it as exited, as `process_start_time` does, and leave - // it running. - assert!(process::wait_for_exit( - reference.pid, - reference.start_time, - Instant::now() - )); - assert!(process::kill(reference.pid, reference.start_time).unwrap()); assert!( child.0.try_wait().unwrap().is_none(), "the process was ended"