diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 663c63204..1547fcbc8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -505,6 +505,30 @@ jobs: if-no-files-found: error retention-days: 1 + # The Windows guest and sandbox job runs the runtime adapter's scenarios + # with its guest agent, a static musl executable that Linux builds, as + # it builds the images that the job reads. The Linux jobs build their + # own. + - name: Build the runtime adapter's guest agent + shell: bash + working-directory: tests/runtime-adapter + run: | + set -euo pipefail + toolchain=$(sed -n 's/^channel = "\([^"]*\)".*/\1/p' ../../rust-toolchain.toml) + test -n "${toolchain}" + rustup toolchain install "${toolchain}" --profile minimal --no-self-update \ + --target x86_64-unknown-linux-musl + cargo "+${toolchain}" build --locked --release --features guest \ + --target x86_64-unknown-linux-musl --bin runtime-adapter-guest + + - name: Upload the runtime adapter's guest agent + uses: actions/upload-artifact@v7 + with: + name: runtime-adapter-guest + path: tests/runtime-adapter/target/x86_64-unknown-linux-musl/release/runtime-adapter-guest + if-no-files-found: error + retention-days: 1 + # A dev push reuses the guest artifact components that the pull request it # merges built (see build-guest-artifacts). A same-repository pull request # rehearses that selection and check on the artifacts that its own run diff --git a/.github/workflows/run-nvx-microvm-tests.yml b/.github/workflows/run-nvx-microvm-tests.yml index e79a811ee..14faa9eef 100644 --- a/.github/workflows/run-nvx-microvm-tests.yml +++ b/.github/workflows/run-nvx-microvm-tests.yml @@ -1248,24 +1248,49 @@ jobs: --backend "${{ inputs.backend }}" --output-dir "build/test-results/aci-edge-sandboxes-${{ inputs.backend }}" - # The runtime adapter's hypervisor scenarios on this backend: any ignored + # The runtime adapter's hypervisor scenarios on this backend: the ignored # tests of its host-only configuration, which read the backend through - # `nvx::Hypervisor::from_env_or_default`. The step builds the adapter - # whether or not it declares any. + # `nvx::Hypervisor::from_env_or_default`. Its host program launches a VM + # of the artifacts above, with the runtime identity that their build + # provenance gives, whose initramfs runs the adapter's guest agent in + # place of NVX's managed agent. Later processes of the host program + # reattach to the VM, as after a restart of the orchestrator, and stop + # it. The guest agent is a static musl executable: Linux builds it, and + # Windows takes the one that distro-artifacts builds on Linux, as it + # takes the Ubuntu and Azure Linux images. + - name: Build the runtime adapter's guest agent on Linux + if: runner.os != 'Windows' && inputs.guests-and-sandboxes && !inputs.debug-kernel + shell: bash + working-directory: tests/runtime-adapter + run: >- + cargo build --locked --release --features guest + --target x86_64-unknown-linux-musl --bin runtime-adapter-guest + - name: Run runtime adapter scenarios on Linux if: runner.os != 'Windows' && inputs.guests-and-sandboxes && !inputs.debug-kernel shell: bash working-directory: tests/runtime-adapter env: NVX_HYPERVISOR: ${{ inputs.backend }} + NVX_ARTIFACTS_DIR: ${{ github.workspace }} + RUNTIME_ADAPTER_GUEST: ${{ github.workspace }}/tests/runtime-adapter/target/x86_64-unknown-linux-musl/release/runtime-adapter-guest run: cargo test --locked --features host -- --ignored + - name: Download the runtime adapter's guest agent on Windows + if: runner.os == 'Windows' && inputs.guests-and-sandboxes && !inputs.debug-kernel + uses: actions/download-artifact@v8 + with: + name: runtime-adapter-guest + path: build/runtime-adapter + - name: Run runtime adapter scenarios on Windows if: runner.os == 'Windows' && inputs.guests-and-sandboxes && !inputs.debug-kernel shell: powershell working-directory: tests/runtime-adapter env: NVX_HYPERVISOR: ${{ inputs.backend }} + NVX_ARTIFACTS_DIR: ${{ github.workspace }} + RUNTIME_ADAPTER_GUEST: ${{ github.workspace }}\build\runtime-adapter\runtime-adapter-guest run: cargo test --locked --features host -- --ignored - name: Upload NVX microVM failure logs diff --git a/doc/ci.md b/doc/ci.md index 618913dc7..61e1ce6a4 100644 --- a/doc/ci.md +++ b/doc/ci.md @@ -739,6 +739,12 @@ reads, or waits for a guest artifact job that uploads none of its components. It also checks that each guest artifact job builds exactly the components that it uploads, and that no component builds in two jobs. +`distro-artifacts` also builds the runtime adapter's guest agent, which is no +guest component: a static musl executable that it uploads as +`runtime-adapter-guest` for the Windows guest and sandbox job, which downloads +it into `build/runtime-adapter/`. The Linux guest and sandbox jobs build their +own (see [Rust crates](#rust-crates)). + Each job restores its components from their caches, and builds, verifies, and saves those that miss, as one job did before. The Ubuntu determinism check, the provenance that each build records, and the cache keys are unchanged. A @@ -1295,9 +1301,12 @@ guest binary is a static executable. They also check the adapter's own framing over the control session between the host and the guest: in the host-only configuration, on Linux and Windows, that the adapter's host program attaches to the session and exchanges the framing with the guest agent, and in the -guest-only configuration, that its guest agent serves the session. Each runs -against a test broker that sends it messages that span records, resets, -disconnects, and malformed records. +guest-only configuration, that its guest agent serves the session, also on a +tty that it puts in raw mode. Each runs against a test broker that sends it +messages that span records, resets, disconnects, and malformed records. The +host-only tests also check the initramfs that runs the guest agent in place of +NVX's managed agent, and the runtime identity that the host program takes from +NVX's identity of its artifacts. The development release job depends on `aci-edge-sandboxes` and `nvx-rs` and requires their combined Linux/Windows results to be successful; failed, @@ -1317,12 +1326,19 @@ it keeps the OpenVMM log under `build/test-results/aci-edge-sandboxes-` uploaded with the other microVM logs. Changes under `aci_edge_sandboxes/` or `nvx-rs/` therefore trigger the NVX microVM test matrix. -The same jobs build the runtime adapter's -host-only configuration on their backend and run any ignored tests that it -declares, its hypervisor scenarios, with `NVX_HYPERVISOR` naming the backend. -The adapter's other tests, including those of its guest agent, run in the -`nvx-rs` job. Changes under `tests/runtime-adapter/` therefore trigger the NVX -microVM test matrix too. +The same jobs build the runtime adapter's host-only configuration on their +backend and run its ignored tests, its VM scenarios, with `NVX_HYPERVISOR` +naming the backend. Through NVX's supervisor, the adapter's host program +launches a VM of the job's artifacts, whose initramfs runs the adapter's guest +agent in place of NVX's managed agent, with the runtime identity that the +artifacts' build provenance gives. Later processes reattach to the VM, as after +a restart of the orchestrator, find it mismatched under another runtime +identity without signaling it, and stop it. The Linux jobs build the guest +agent, a static musl executable, first, and the Windows job takes the one that +`distro-artifacts` builds on Linux, as it takes the Ubuntu and Azure Linux +images. The adapter's other tests, including those of its guest agent, run in +the `nvx-rs` job. Changes under `tests/runtime-adapter/` therefore trigger the +NVX microVM test matrix too. ## Adversarial campaigns diff --git a/doc/project-structure.md b/doc/project-structure.md index 6c77b4c54..c69980da4 100644 --- a/doc/project-structure.md +++ b/doc/project-structure.md @@ -106,7 +106,7 @@ nvx/ | | |-- gpt/ Strict reading of GPT disks whose first partition configures an adapter and whose other partitions hold a root's read-only layers, shared by the host and the guest | | |-- host/ Artifacts, launch, network and egress-policy lowering, sessions, VM instances, sandbox stores, the sandbox launch profile, and the `nvx-rs` CLI (feature `cli`) | | |-- testing/ Fake OpenVMM (feature `testing`) -| | |-- guest/ Programs that run in Linux guests, the guest's NVXS connection, the libraries through which any guest init or agent starts a program, collects its children's statuses, settles its executions' cgroups, drives each execution, bind-mounts host mappings below the roots that it names, assembles and tears down a layered root, and reads a GPT disk's partition table through its block device, and the typed calls through which it runs the time ABI's steps and the reseed (feature `guest`) +| | |-- guest/ Programs that run in Linux guests, the guest's NVXS connection, the libraries through which any guest init or agent opens the terminal on which it serves its session, starts a program, collects its children's statuses, settles its executions' cgroups, drives each execution, bind-mounts host mappings below the roots that it names, assembles and tears down a layered root, and reads a GPT disk's partition table through its block device, and the typed calls through which it runs the time ABI's steps and the reseed (feature `guest`) | | `-- bin/ nvx-rs CLI, nvx-fake-openvmm test double, and the guest binaries | |-- tests/ Fake-OpenVMM instance lifecycle, sandbox store, CLI, NVXS session, guest-binary, and hypervisor managed-lifecycle tests, the store's record fixtures, and the golden fixtures that Python recorded | |-- build.rs Stages the bundled artifacts (feature `bundled`) @@ -342,7 +342,10 @@ off and uses its public API only. Its protocol-only, host-only, and guest-only configurations build in CI, and its tests check that each one builds as an independent adapter's does. Its host program and its guest agent exchange the adapter's own framing over the control session between the host and the -guest, and its tests check each of them against a test broker. See its +guest, and its tests check each of them against a test broker. Its host +program also runs the guest agent in VMs of NVX's artifacts through NVX's +supervisor, and reattaches to them from other processes, which its ignored +tests check on each backend in CI. See its [README](../tests/runtime-adapter/README.md). ## Root files diff --git a/nvx-rs/README.md b/nvx-rs/README.md index d7449df10..8364a4e8f 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -43,12 +43,12 @@ nvx-rs/ │ │ with feature `cli`, the `nvx-rs` CLI │ ├── testing/ feature `testing`: the fake OpenVMM │ ├── guest/ feature `guest`: the programs that run in Linux guests, the guest's NVXS -│ │ connection, the libraries through which a guest init or agent starts a -│ │ program, collects its children's statuses, settles its executions' -│ │ cgroups, bind-mounts host mappings below the roots that it names, -│ │ assembles and tears down a layered root, and reads a GPT disk's partition -│ │ table through its block device, and the typed calls through which it runs -│ │ the time ABI's steps and the reseed +│ │ connection, the libraries through which a guest init or agent opens the +│ │ terminal on which it serves its session, starts a program, collects its +│ │ children's statuses, settles its executions' cgroups, bind-mounts host +│ │ mappings below the roots that it names, assembles and tears down a layered +│ │ root, and reads a GPT disk's partition table through its block device, and +│ │ the typed calls through which it runs the time ABI's steps and the reseed │ └── bin/ nvx-rs, nvx-fake-openvmm, and the guest binaries ├── tests/ fake-OpenVMM, CLI, NVXS session, process-supervision, store, and guest-binary │ tests, the store's record fixtures, and the golden fixtures that Python @@ -87,6 +87,7 @@ nvx-rs/ | `nvx::guest` | The programs that run in Linux guests; see [Guest binaries](#guest-binaries) | | `nvx::guest::mappings` | `Mapping`: one host mapping for any guest init or agent, a relative source below an export root, where the host's shares appear, and a target below a target root, the tree that the mappings go into, with its `Access`. `Mapping::apply` takes both roots explicitly: it creates the target as the kind of its source, a directory or a regular file, bind-mounts the source on it, and remounts the bind `nosuid` and `nodev`, and read-only when asked. A `MappingFailure` tells which `MappingStage` failed and its `errno`. The caller keeps the order of its mappings and what a failure means for the rest. NVX's managed agent applies its host mapping table through it, and a layered `Root` binds the children of its aggregate live share through it | | `nvx::guest::nvxs` | `GuestConnection`: the guest's role of the NVXS session over a byte stream, such as the control console, on which NVX's managed agent and other guest agents run their sessions. It never reads past the record in progress, so a caller that polls the stream between records sees every record that it has not returned | +| `nvx::guest::terminal` | `Terminal`: the terminal on which a guest agent serves its session, such as the VM's control console, as a byte stream. It opens the terminal, waiting while the kernel sets it up during boot, puts it in raw mode, so that it passes the session's bytes as they are, and drains it, so that it sends what the agent wrote before the agent ends the VM. NVX's managed agent and other guest agents serve their sessions on it | | `nvx::guest::spawn` | `Command`: starts a program in a new process for any guest init or agent, with its arguments, environment, root and working directories, standard streams, inherited descriptors, cgroup, and process group, without NVX's control protocol, managed agent, or workload identity. Its `LaunchReport` tells whether the program started, or which `LaunchStage` failed and why. NVX's managed agent is one caller | | `nvx::guest::process` | `Child`: the status of one child that the caller names, by its pidfd or its process ID. It waits for that child alone, so another child's status stays for the caller; a caller that collects every status itself, such as a PID 1 that adopts orphans, hands each one to its `Child` with `observe`. Statuses are `std::process::ExitStatus`, real-time signals included | | `nvx::guest::cgroup` | `Layout`: the agent's cgroup and the workload cgroup, which the caller names, and which may not hold the agent's. `ExecutionCgroup`: the cgroup of one execution, the workload cgroup itself or a child of it, which it prepares with the caller's resource values, places processes in, checks for occupancy through `pids.current`, kills with `cgroup.kill`, and settles, over the execution's cgroup or the whole workload subtree, as the caller's `Scope` chooses. It holds its cgroups' directories open, and the process's removals and preparations of cgroups take turns, so that it never reaches another cgroup that takes one of their paths. Settling calls the caller's status collector between its checks, and collects no status itself | @@ -591,14 +592,15 @@ as `nvx/`. The `guest` feature adds the programs that run in NVX's Linux guests: the `nvx::guest` module and one binary per program. The Alpine, Ubuntu, and Azure Linux images ship all nine at their established `/sbin/nvx-*` paths. The -module's libraries are not programs: `spawn` starts a program in a new -process, `process` collects the status of a child that the caller names, -`cgroup` places, kills, and settles the processes of an execution, `exec` -drives one program as an execution, `mappings` bind-mounts host mappings below -the roots that its caller names, and `root` bootstraps the guest for a layered -root through `cgroup`, assembles the root, mounts live shares in it, binding -the children of an aggregate through `mappings`, and tears it down. -`nvx-managed-agent` launches, drives, collects, and settles its workloads +module's libraries are not programs: `terminal` opens the terminal on which an +agent serves its session, `spawn` starts a program in a new process, `process` +collects the status of a child that the caller names, `cgroup` places, kills, +and settles the processes of an execution, `exec` drives one program as an +execution, `mappings` bind-mounts host mappings below the roots that its caller +names, and `root` bootstraps the guest for a layered root through `cgroup`, +assembles the root, mounts live shares in it, binding the children of an +aggregate through `mappings`, and tears it down. `nvx-managed-agent` opens its +control terminal, and launches, drives, collects, and settles its workloads, through them, and applies its host mapping table through `mappings`, with its mount of the host's export and `/` as the roots, and any other guest init or agent can use them too. Any guest init or agent can also run the steps of diff --git a/nvx-rs/src/guest/managed_agent/tty.rs b/nvx-rs/src/guest/managed_agent/tty.rs index c106a5bee..7f47de795 100644 --- a/nvx-rs/src/guest/managed_agent/tty.rs +++ b/nvx-rs/src/guest/managed_agent/tty.rs @@ -1,32 +1,14 @@ use std::ffi::CStr; -use std::time::Duration; -use crate::guest::sys::{self, Errno, FileDescriptor, Mode, OpenFlags}; +use crate::guest::sys::{Errno, FileDescriptor}; +use crate::guest::terminal::Terminal; use super::seam::System; -/// Number of attempts allowed while the control terminal is still appearing. -const OPEN_ATTEMPTS: usize = 600; -/// Delay between retryable control-terminal open failures. -const OPEN_DELAY: Duration = Duration::from_millis(50); - -/// Opens the control terminal, retrying transient boot-time failures. +/// Opens the control terminal, waiting while the kernel sets it up during boot, as +/// [`Terminal::open`] does. pub(crate) fn open(path: &CStr) -> Result { - for attempt in 0..OPEN_ATTEMPTS { - match sys::open( - path, - OpenFlags::O_RDWR | OpenFlags::O_CLOEXEC, - Mode::empty(), - ) { - Ok(descriptor) => return Ok(descriptor), - Err(Errno::ENOENT | Errno::ENXIO | Errno::ENODEV | Errno::EIO) => {} - Err(error) => return Err(error), - } - if attempt + 1 != OPEN_ATTEMPTS { - std::thread::sleep(OPEN_DELAY); - } - } - Err(Errno::ETIMEDOUT) + Terminal::open_c(path).map(Terminal::into_descriptor) } /// Applies raw local-terminal settings through the replaceable seam. diff --git a/nvx-rs/src/guest/mod.rs b/nvx-rs/src/guest/mod.rs index 2a8a3c6f8..4c94dcc45 100644 --- a/nvx-rs/src/guest/mod.rs +++ b/nvx-rs/src/guest/mod.rs @@ -1,13 +1,14 @@ //! The programs that run in NVX's Linux guests, which ship as `/sbin/nvx-*`, and the libraries //! on which any guest init or agent builds: [`nvxs`], the guest's I/O driver of the NVXS session; -//! [`spawn`], through which it starts a program; [`process`], through which it collects the -//! status of each child that it names, and of no other; [`cgroup`], through which it places, -//! accounts, kills, and settles the processes of its executions; [`mappings`], through which it -//! bind-mounts host mappings below the roots that it names; [`exec`], which drives each program -//! as one execution, with its launch, its bounded output and input, its ending, and its outcome; -//! [`root`], through which it bootstraps the guest for a layered root, assembles the root, mounts -//! live shares in it, and tears it down; and [`gpt`], through which it reads the partition table -//! of a GPT disk that the kernel attached as a block device. +//! [`terminal`], the terminal on which it serves that session; [`spawn`], through which it starts +//! a program; [`process`], through which it collects the status of each child that it names, and +//! of no other; [`cgroup`], through which it places, accounts, kills, and settles the processes of +//! its executions; [`mappings`], through which it bind-mounts host mappings below the roots that +//! it names; [`exec`], which drives each program as one execution, with its launch, its bounded +//! output and input, its ending, and its outcome; [`root`], through which it bootstraps the guest +//! for a layered root, assembles the root, mounts live shares in it, and tears it down; and +//! [`gpt`], through which it reads the partition table of a GPT disk that the kernel attached as a +//! block device. //! //! Each other module holds one program, and the crate's binary of the same name runs its `main`. //! Two of them also offer their steps as typed calls, which any guest init or agent runs in its @@ -15,8 +16,8 @@ //! cold-boot step, capture, and restore finish, and [`reseed`] reseeds the kernel's random number //! generator with a restore's fresh entropy, which starts with the new [`GenerationId`]. The //! libraries need none of NVX's control protocol, managed agent, or workload identity, and NVX's -//! managed agent applies its host mapping table and starts, drives, collects, and settles its -//! workloads through them. +//! managed agent opens its control terminal, applies its host mapping table, and starts, drives, +//! collects, and settles its workloads through them. //! //! The programs keep the command lines, exit statuses, and output of the C programs that they //! replace, except that numeric arguments are plain decimal, with a `-` sign where `nvx-time`'s @@ -95,6 +96,7 @@ pub mod process; pub mod reseed; pub mod root; pub mod spawn; +pub mod terminal; #[cfg(target_arch = "x86_64")] pub mod time; #[cfg(target_arch = "x86_64")] diff --git a/nvx-rs/src/guest/terminal.rs b/nvx-rs/src/guest/terminal.rs new file mode 100644 index 000000000..84e01c900 --- /dev/null +++ b/nvx-rs/src/guest/terminal.rs @@ -0,0 +1,199 @@ +//! The terminal on which a guest agent serves its control session, such as the VM's control +//! console. +//! +//! [`Terminal::open`] opens the terminal for reading and writing, waiting while the kernel sets it +//! up during boot. [`Terminal::configure_raw`] puts it in raw mode, so that it passes the +//! session's bytes as they are, and [`Terminal::drain`] waits until it has sent what the agent +//! wrote, such as the confirmation of a stop, before the agent ends the VM. NVX's managed agent +//! serves its session on its control terminal through them, and any other guest agent can too: +//! +//! ```no_run +//! use std::io::Write; +//! use std::path::Path; +//! +//! use nvx::guest::terminal::Terminal; +//! +//! # fn main() -> std::io::Result<()> { +//! let mut terminal = Terminal::open(Path::new("/dev/hvc2"))?; +//! terminal.configure_raw()?; +//! // Serve the session on `terminal`, which reads and writes the terminal as a byte stream. +//! terminal.write_all(b"the session's bytes")?; +//! terminal.drain()?; +//! # Ok(()) +//! # } +//! ``` + +use std::ffi::CStr; +use std::io; +use std::os::unix::ffi::OsStrExt; +use std::path::Path; +use std::thread; +use std::time::Duration; + +use super::sys::{self, Errno, FileDescriptor, Mode, OpenFlags}; + +/// How many times [`Terminal::open`] tries to open a terminal that the kernel is still setting +/// up. +const OPEN_ATTEMPTS: usize = 600; +/// The delay between those attempts, which span 30 seconds. +const OPEN_DELAY: Duration = Duration::from_millis(50); + +/// A terminal, open for reading and writing. +/// +/// It reads and writes as a byte stream through [`io::Read`] and [`io::Write`], with one system +/// call for each call and no buffer, so that a caller that serves a session on it, such as a +/// [`GuestConnection`](super::nvxs::GuestConnection), sees each byte as soon as it arrives. +/// Dropping it closes the terminal. +#[derive(Debug)] +pub struct Terminal(FileDescriptor); + +impl Terminal { + /// Opens the terminal at `path` for reading and writing, closed on `exec`. + /// + /// A terminal that the kernel has not set up yet during boot fails to open with `ENOENT`, + /// `ENXIO`, `ENODEV`, or `EIO`, so `open` tries again every 50 milliseconds while it does, and + /// fails with `ETIMEDOUT` once 30 seconds have passed. It fails at once with any other error, + /// such as `EISDIR` for a directory, or `EINVAL` for a path that holds a NUL byte. + pub fn open(path: &Path) -> io::Result { + let path = sys::c_string(path.as_os_str().as_bytes())?; + Self::open_c(&path).map_err(io::Error::from) + } + + /// Opens the terminal at `path` as [`open`](Self::open) does, and fails with the `errno` + /// that NVX's managed agent reports. + pub(crate) fn open_c(path: &CStr) -> Result { + open_with(path, OPEN_ATTEMPTS, OPEN_DELAY).map(Self) + } + + /// Puts the terminal in raw mode, as `cfmakeraw` defines it: without echo, line editing, or + /// signal characters, without translation or flow control, and with 8-bit characters. The + /// terminal also ignores the modem's status lines (`CLOCAL`), which a console does not have. + /// The settings apply at once. + /// + /// Fails with `ENOTTY` if the file is no terminal. + pub fn configure_raw(&self) -> io::Result<()> { + sys::configure_raw(&self.0).map_err(io::Error::from) + } + + /// Waits until the terminal has sent everything written to it. + pub fn drain(&self) -> io::Result<()> { + sys::drain(&self.0).map_err(io::Error::from) + } + + /// Returns the terminal's descriptor, on which NVX's managed agent serves its session. + pub(crate) fn into_descriptor(self) -> FileDescriptor { + self.0 + } +} + +impl io::Read for Terminal { + fn read(&mut self, bytes: &mut [u8]) -> io::Result { + io::Read::read(&mut self.0, bytes) + } +} + +impl io::Write for Terminal { + fn write(&mut self, bytes: &[u8]) -> io::Result { + io::Write::write(&mut self.0, bytes) + } + + fn flush(&mut self) -> io::Result<()> { + io::Write::flush(&mut self.0) + } +} + +/// Opens the terminal at `path` for reading and writing, closed on `exec`, and tries `attempts` +/// times, `delay` apart, while it fails as a terminal that the kernel is still setting up does. +fn open_with(path: &CStr, attempts: usize, delay: Duration) -> Result { + for attempt in 0..attempts { + match sys::open( + path, + OpenFlags::O_RDWR | OpenFlags::O_CLOEXEC, + Mode::empty(), + ) { + Ok(descriptor) => return Ok(descriptor), + Err(Errno::ENOENT | Errno::ENXIO | Errno::ENODEV | Errno::EIO) => {} + Err(error) => return Err(error), + } + if attempt + 1 != attempts { + thread::sleep(delay); + } + } + Err(Errno::ETIMEDOUT) +} + +#[cfg(test)] +mod tests { + use std::io::{Read, Write}; + use std::os::fd::OwnedFd; + use std::path::PathBuf; + + use nix::pty::{self, PtyMaster}; + + use super::*; + + /// Opens the master end of a pseudo-terminal, closed on `exec` so that no process that + /// another test starts holds it, and returns it with the path of its terminal, which stays + /// locked, and fails to open with `EIO`, until [`pty::unlockpt`] unlocks it. + fn locked_pseudo_terminal() -> (PtyMaster, PathBuf) { + let flags = OpenFlags::O_RDWR | OpenFlags::O_NOCTTY | OpenFlags::O_CLOEXEC; + let master = pty::posix_openpt(flags).unwrap(); + pty::grantpt(&master).unwrap(); + let path = PathBuf::from(pty::ptsname_r(&master).unwrap()); + (master, path) + } + + #[test] + fn a_terminal_that_the_kernel_is_setting_up_opens_once_it_is_set_up() { + let (master, path) = locked_pseudo_terminal(); + let opening = thread::spawn(move || Terminal::open(&path)); + thread::sleep(Duration::from_millis(200)); + assert!(!opening.is_finished(), "a locked terminal opened"); + pty::unlockpt(&master).unwrap(); + let terminal = opening.join().unwrap().unwrap(); + terminal.configure_raw().unwrap(); + } + + #[test] + fn opening_fails_at_once_for_other_errors_and_after_its_attempts() { + let error = Terminal::open(Path::new("/")).unwrap_err(); + assert_eq!(error.raw_os_error(), Some(Errno::EISDIR as i32), "{error}"); + let error = Terminal::open(Path::new("/dev/\0null")).unwrap_err(); + assert_eq!(error.raw_os_error(), Some(Errno::EINVAL as i32), "{error}"); + let missing = c"/nonexistent/nvx-terminal"; + let error = open_with(missing, 3, Duration::from_millis(1)).unwrap_err(); + assert_eq!(error, Errno::ETIMEDOUT); + assert_eq!(io::Error::from(error).kind(), io::ErrorKind::TimedOut); + } + + #[test] + fn a_file_that_is_no_terminal_cannot_be_put_in_raw_mode() { + let terminal = Terminal::open(Path::new("/dev/null")).unwrap(); + let error = terminal.configure_raw().unwrap_err(); + assert_eq!(error.raw_os_error(), Some(Errno::ENOTTY as i32), "{error}"); + } + + #[test] + fn a_terminal_in_raw_mode_passes_every_byte_as_it_is() { + let (master, path) = locked_pseudo_terminal(); + pty::unlockpt(&master).unwrap(); + let master = FileDescriptor::from_owned(OwnedFd::from(master)); + let mut terminal = Terminal::open(&path).unwrap(); + terminal.configure_raw().unwrap(); + // Every byte value, which a terminal that was not in raw mode would translate, echo, or + // act on: carriage returns and newlines, and the characters of signals, flow control, + // line editing, and the end of a file. + let forward = (0..=u8::MAX).collect::>(); + let backward = forward.iter().rev().copied().collect::>(); + master.write_all(&forward).unwrap(); + let mut received = vec![0; forward.len()]; + terminal.read_exact(&mut received).unwrap(); + assert_eq!(received, forward); + // An echo of the bytes that the terminal received would arrive first. + terminal.write_all(&backward).unwrap(); + terminal.drain().unwrap(); + let mut sent = vec![0; backward.len()]; + master.read_exact(&mut sent).unwrap(); + assert_eq!(sent, backward); + } +} diff --git a/nvx-rs/src/lib.rs b/nvx-rs/src/lib.rs index b7be79b0f..04d6751a5 100644 --- a/nvx-rs/src/lib.rs +++ b/nvx-rs/src/lib.rs @@ -28,10 +28,11 @@ //! running a VM. //! - `guest` (feature `guest`, Linux): the programs that run in NVX guests, such as //! `nvx-console-pending`, the guest's NVXS connection, the libraries through which any guest -//! init or agent starts a program in a new process (`guest::spawn`), collects the status of -//! each child that it names (`guest::process`), places, kills, and settles the processes of -//! its executions in cgroups (`guest::cgroup`), bind-mounts host mappings below the roots that -//! it names (`guest::mappings`), drives each program as one execution, with its launch, bounded +//! init or agent opens the terminal on which it serves its session (`guest::terminal`), starts +//! a program in a new process (`guest::spawn`), collects the status of each child that it names +//! (`guest::process`), places, kills, and settles the processes of its executions in cgroups +//! (`guest::cgroup`), bind-mounts host mappings below the roots that it names +//! (`guest::mappings`), drives each program as one execution, with its launch, bounded //! output, input, ending, and outcome (`guest::exec`), bootstraps the guest for a layered root, //! assembles the root, mounts live shares in it, and tears it down (`guest::root`), and reads //! the partition table of a GPT disk's block device (`guest::gpt`), and the typed calls through diff --git a/scripts/test_nvx_tools.py b/scripts/test_nvx_tools.py index 4466a82d9..9ebb4f5ba 100644 --- a/scripts/test_nvx_tools.py +++ b/scripts/test_nvx_tools.py @@ -7386,6 +7386,91 @@ def test_ci_change_classification_runs_microvm_tests_for_the_runtime_adapter(sel ) self.assertIn("### Platform benchmarks: skip\nNo input changed.\n", summary) + def test_runtime_adapter_scenarios_take_a_guest_agent_that_linux_builds(self): + # The scenarios boot VMs of the artifacts that the microVM guest and + # sandbox jobs download into the checkout, with the adapter's guest + # agent, a static musl executable. The Linux jobs build it before the + # scenarios run, and the Windows job takes the one that + # distro-artifacts builds on Linux, as it takes the Ubuntu and Azure + # Linux images. The default suites neither build nor download it. + github = BuildConstants.REPO_ROOT / ".github" + workflow = (github / "workflows" / "ci.yml").read_text(encoding="utf-8") + microvm_workflow = ( + github / "workflows" / "run-nvx-microvm-tests.yml" + ).read_text(encoding="utf-8") + build = ( + "cargo build --locked --release --features guest " + "--target x86_64-unknown-linux-musl --bin runtime-adapter-guest" + ) + agent = ( + "tests/runtime-adapter/target/x86_64-unknown-linux-musl/release/" + "runtime-adapter-guest" + ) + + def step(job: str, name: str) -> str: + return _workflow_step( + microvm_workflow if job == "test" else workflow, job, name + ) + + linux_build = step("test", "Build the runtime adapter's guest agent on Linux") + self.assertIn( + "if: runner.os != 'Windows' && inputs.guests-and-sandboxes " + "&& !inputs.debug-kernel\n", + linux_build, + ) + self.assertIn("working-directory: tests/runtime-adapter\n", linux_build) + self.assertIn(build, " ".join(linux_build.split())) + download = step("test", "Download the runtime adapter's guest agent on Windows") + self.assertIn( + "if: runner.os == 'Windows' && inputs.guests-and-sandboxes " + "&& !inputs.debug-kernel\n", + download, + ) + self.assertIn("uses: actions/download-artifact@v8\n", download) + self.assertIn("name: runtime-adapter-guest\n", download) + self.assertIn("path: build/runtime-adapter\n", download) + for system, setup, guest_agent in ( + ("Linux", linux_build, f"${{{{ github.workspace }}}}/{agent}"), + ( + "Windows", + download, + "${{ github.workspace }}\\build\\runtime-adapter\\runtime-adapter-guest", + ), + ): + with self.subTest(system=system): + name = f"Run runtime adapter scenarios on {system}" + hook = step("test", name) + self.assertIn("NVX_ARTIFACTS_DIR: ${{ github.workspace }}\n", hook) + self.assertIn(f"RUNTIME_ADAPTER_GUEST: {guest_agent}\n", hook) + self.assertLess( + microvm_workflow.index(setup), + microvm_workflow.index(f" - name: {name}\n"), + ) + + shared = " ".join( + step("distro-artifacts", "Build the runtime adapter's guest agent") + .replace("\\\n", " ") + .split() + ) + self.assertIn("working-directory: tests/runtime-adapter", shared) + self.assertIn( + 'rustup toolchain install "${toolchain}" --profile minimal ' + "--no-self-update --target x86_64-unknown-linux-musl", + shared, + ) + self.assertIn( + build.replace("cargo build", 'cargo "+${toolchain}" build'), shared + ) + upload = step("distro-artifacts", "Upload the runtime adapter's guest agent") + self.assertIn("uses: actions/upload-artifact@v7\n", upload) + self.assertIn("name: runtime-adapter-guest\n", upload) + self.assertIn(f"path: {agent}\n", upload) + self.assertIn( + "needs: [artifacts, distro-artifacts, build-openvmm-windows-msvc, " + "nvx-cli-windows, openvmm-changes]", + _workflow_job(workflow, "nvx-microvm-guests-whp"), + ) + def test_ci_change_classification_covers_each_suite_input(self): github = BuildConstants.REPO_ROOT / ".github" tools = BuildConstants.REPO_ROOT / "scripts" / "nvx_tools" @@ -8599,10 +8684,11 @@ def test_ci_shares_openvmm_inputs_and_binary_artifacts(self): self.assertEqual(build_workflow.count("overwrite: true"), 2) self.assertEqual(build_workflow.count("retention-days: 1"), 2) # The microVM workflow also downloads nvx-rs, which the benchmarks - # don't run, and the Ubuntu sandbox scratch template and the debug - # kernel, which the platform jobs don't read. + # don't run, the Ubuntu sandbox scratch template and the debug kernel, + # which the platform jobs don't read, and on Windows the runtime + # adapter's guest agent. for consumer_workflow, download_count in ( - (microvm_workflow, 10), + (microvm_workflow, 11), (platform_workflow, 7), ): self.assertIn("path: openvmm/target/release", consumer_workflow) diff --git a/tests/runtime-adapter/Cargo.lock b/tests/runtime-adapter/Cargo.lock index e15decf52..779ab6055 100644 --- a/tests/runtime-adapter/Cargo.lock +++ b/tests/runtime-adapter/Cargo.lock @@ -165,6 +165,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" name = "runtime-adapter" version = "0.1.0" dependencies = [ + "nix", "nvx", ] diff --git a/tests/runtime-adapter/Cargo.toml b/tests/runtime-adapter/Cargo.toml index 076a8c1d2..d3af2a071 100644 --- a/tests/runtime-adapter/Cargo.toml +++ b/tests/runtime-adapter/Cargo.toml @@ -23,6 +23,11 @@ guest = ["nvx/guest"] [dependencies] nvx = { path = "../../nvx-rs", default-features = false } +# The guest agent's tests open pseudo-terminals. The guest side of `nvx` depends on the same +# release, with this feature among its own. +[target.'cfg(target_os = "linux")'.dev-dependencies] +nix = { version = "0.31.3", default-features = false, features = ["term"] } + # The binaries have no unit tests; tests/independent_build.rs runs them. [[bin]] name = "runtime-adapter-host" diff --git a/tests/runtime-adapter/README.md b/tests/runtime-adapter/README.md index 838b34ec1..c3c9a0deb 100644 --- a/tests/runtime-adapter/README.md +++ b/tests/runtime-adapter/README.md @@ -12,8 +12,10 @@ an adapter outside the repository does: The adapter brings its own application framing to the control session between the host and the guest, in place of NVX's `NVXC` frames. Its host program sends -requests in that framing, and its guest agent serves them. The host program's -binary reports the contract that it was built against. +requests in that framing, and its guest agent serves them. The host program +also runs the guest agent in VMs of NVX's artifacts, in place of NVX's managed +agent, through NVX's supervisor, and finds a VM again after it restarts, as an +orchestrator does. ## Configurations @@ -22,7 +24,7 @@ Each configuration enables one side of `nvx`: | Configuration | Cargo options | `nvx` features | Modules | Binary | | --- | --- | --- | --- | --- | | Protocol-only | none | none | `frame` | none | -| Host-only | `--features host` | `host` | `frame`, `host` | `runtime-adapter-host` | +| Host-only | `--features host` | `host` | `frame`, `host`, `runtime`, `initramfs` | `runtime-adapter-host` | | Guest-only | `--features guest --target x86_64-unknown-linux-musl` | `guest` | `frame`, `guest` | `runtime-adapter-guest`, a static executable | ## Framing @@ -69,6 +71,20 @@ closed the stream, and with 1, after a diagnostic on standard error, once serving failed. Without arguments, it reports the contract that it was built against. +`guest::serve_terminal` serves a tty, such as a VM's control tty, as `serve` +does. It opens the tty with `nvx::guest::terminal`, as NVX's managed agent +does: it waits up to 30 seconds while the kernel sets the tty up, and puts it +in raw mode, so that the tty passes the session's bytes as they are. Once +serving ends, it waits until the tty has sent what serving wrote, such as the +confirmation of a stop. + +`runtime-adapter-guest init TTY` runs `serve_terminal` on the tty TTY in place +of NVX's managed agent, as a VM's PID 1; see [VMs](#vms). It ends with 0 once +the host stopped it, and with 1, after a diagnostic on standard error, once +serving failed or the broker closed the tty. As PID 1, which must not exit, it +ends the VM with that status instead, through NVX's `/sbin/nvx-exit`, as the +managed agent does. + ## Host program `host::Client` is the host program's client of the guest agent. It runs over @@ -96,6 +112,92 @@ calls fail. A receive that times out doesn't: it keeps what arrived. A request ends the client on any failure after the framing accepted it, a timeout included, because an answer that arrived later would answer the wrong request. +## VMs + +The host program runs the guest agent in VMs of NVX's artifacts, through NVX's +supervisor, `nvx::host::supervisor`, each in a sandbox store of +`nvx::host::store`. + +### The guest agent in place of the managed agent + +NVX's initramfs hands a VM of the managed lifecycle to NVX's managed agent: its +`/init` runs +`exec /sbin/nvx-managed-agent /dev/hvc2 - nvx-managed UID GID NAME HOME` as +PID 1. `initramfs::write` copies that initramfs, and appends to the copy a +second archive, in the `newc` format, which Linux unpacks over the first: + +- the guest agent, at `/sbin/runtime-adapter-guest`; +- a script, `/sbin/runtime-adapter-init`, that runs + `runtime-adapter-guest init "$1"`, with the control tty, the first of the + arguments that `/init` passes the managed agent; +- a symbolic link from `/sbin/nvx-managed-agent` to the script. Linux removes a + file whose type an archive changes before it creates the new one, so the + link replaces the managed agent without writing to its file, which could + have other names. + +The second archive starts at a multiple of 4 bytes, as Linux requires of an +uncompressed archive, after zeros that Linux skips. NVX's own images don't +change. + +### The runtime identity + +The supervisor records the runtime identity of each launch, and its reattach +compares the recorded identity with the caller's. `runtime::Runtime::from_dir` +takes NVX's artifacts from a repository checkout or an extracted release, with +`nvx::host::artifacts::Artifacts::from_dir`, and their runtime identity from +`nvx::host::artifacts::ArtifactDir::runtime_identity`, which names each +artifact by the SHA-256 digest that the install metadata beside it records, +never from its contents: a release's `SHA256SUMS`, or the build provenance in a +checkout's `build/`, where `nvx.py download` also installs a release's. + +The VMs boot a copy of NVX's initramfs with the guest agent in it, so the +identity of their initramfs names both: +`sha256:DIGEST+runtime-adapter-guest/BUILD`. The guest agent has no install +metadata of its own, so the caller names its build, BUILD, which is the crate's +version unless the caller names another. A host program that ships another +build of the guest agent runs another runtime, and finds a VM of this one +mismatched. + +### Commands + +`runtime-adapter-host` takes NVX's artifacts from the directory that +`NVX_ARTIFACTS_DIR` names. Each command takes the guest agent's build from +`--guest-build BUILD`, the crate's version by default: + +- `launch STORE --guest-agent PATH [--hypervisor NAME]` launches a VM in the + sandbox store STORE, which it creates if needed, with the guest agent PATH, a + static `x86_64-unknown-linux-musl` build of `runtime-adapter-guest`, on the + hypervisor NAME, or else on the one that + `nvx::Hypervisor::from_env_or_default` selects. Once the launch has checked + that no VM of the sandbox runs, it writes the VM's initramfs into the store. + It connects to the VM's control endpoint with `nvx::host::session::connect`, + attaches with the launch's capability, exchanges a `Ping` and the longest + `Echo` with the guest agent, commits the launch, and exits, leaving the VM + running. A launch that fails ends its VM. +- `reattach STORE` takes the sandbox's lock, and reattaches to its VM, which + another process launched, and which it holds, although the VM is not its + child, through its verified process identity. It reads the capability from + the store, attaches a new session, in the session's next epoch, exchanges a + `Ping`, and exits, leaving the VM running. +- `stop STORE` reattaches and attaches as `reattach` does, then stops the VM: + the adapter's `Stop`, which the guest agent confirms with `Stopped` before it + ends the VM, is the stop's graceful step. It waits for the VM to exit, and + checks the VM's outcome report: a success, with status 0 and a complete + teardown. A VM whose guest agent does not confirm the stop is terminated, and + the stop fails. + +`reattach` and `stop` find a VM that another runtime identity launched +mismatched, and then signal nothing: the VM keeps running. The commands print +what they find as `KEY=VALUE` lines: `reattached` (`live`, `mismatched`, +`exited`, or `indeterminate`), `differences` (the parts of the runtime identity +in which a mismatched VM differs), `pid` (the VMM's process ID), `instance` and +`epoch` (the identity of the session that they attached to), `stopped` +(`graceful`, `forced`, or `timed-out`), `outcome` (such as `success 0`), and +`teardown` (`complete` or `incomplete`). They exit with 0 once they did what +they were asked, with 3 when they find the VM mismatched, and with 1, after a +diagnostic on standard error, when they fail. Without arguments, the program +reports the contract that it was built against. + ## Tests `tests/independent_build.rs` checks that every configuration builds as an @@ -157,6 +259,56 @@ Windows. They cover: - the answers that its requests check, and a request whose answer arrives too late. +Through a relay between its test broker and a pseudo-terminal, +`tests/guest_session.rs` also checks that `guest::serve_terminal` and +`runtime-adapter-guest init` serve a tty in raw mode, with an `Echo` of every +byte value, and that `init` fails at once on a path that is no tty. + +`tests/initramfs.rs` and `tests/runtime.rs` check, in the host-only +configuration, on Linux and Windows: + +- the archive that puts the guest agent in place of the managed agent: its + entries, their owners and modes, and the padding of its headers and data, + for guest agents of every length modulo 4; +- the copy of NVX's initramfs, which stays as it is, of every length modulo 4, + with the archive after it at a multiple of 4 bytes, and the files that a + failure names; +- the runtime identity of a checkout and of a release, which differs from + NVX's in the initramfs alone, another build of the guest agent, which differs + in the initramfs too, and missing install metadata, which fails as NVX + reports it. + +`tests/vm_restart.rs` checks the restart of an orchestrator, on a real +hypervisor, in the host-only configuration: + +- `runtime-adapter-host launch` launches a VM, and exits; +- another process's `reattach` holds the VM, which is not its child, and + attaches a new session, in a later epoch of the same broker instance; +- a third process's `stop` stops the VM: gracefully, with a successful outcome, + and with no runtime record left. + +In another test, a host program that ships another build of the guest agent +finds the VM mismatched in its initramfs, and signals nothing: the VM runs on, +under the same runtime record, until the host program of the right runtime +stops it. A test that fails ends the VM that it left running, and keeps the +sandbox's directory, whose path it prints, unless it can confirm that the VM +ended. The tests boot VMs, so they are ignored. They read the hypervisor from +`NVX_HYPERVISOR`, through `nvx::Hypervisor::from_env_or_default`, NVX's +artifacts, with their install metadata, from `NVX_ARTIFACTS_DIR`, and the guest +agent from `RUNTIME_ADAPTER_GUEST`. In a checkout where +`python3 scripts/nvx.py download` installed NVX's artifacts, run them in this +directory with: + +```bash +cargo build --locked --release --features guest \ + --target x86_64-unknown-linux-musl --bin runtime-adapter-guest +NVX_HYPERVISOR=kvm NVX_ARTIFACTS_DIR="$(cd ../.. && pwd)" \ + RUNTIME_ADAPTER_GUEST="$PWD/target/x86_64-unknown-linux-musl/release/runtime-adapter-guest" \ + cargo test --locked --features host -- --ignored +``` + +On Windows, take the guest agent from a Linux build. + Run the checks of each configuration in this directory. The guest-only configuration needs Linux and the musl target, which `rustup target add x86_64-unknown-linux-musl` installs: @@ -181,6 +333,10 @@ configuration. A change to the adapter is a test input of CI's [change classification](../../doc/ci.md#change-classification), so it runs the NVX microVM tests. Their [workflow](../../.github/workflows/run-nvx-microvm-tests.yml) builds the -host-only configuration on each backend and runs any ignored tests that it -declares, the adapter's hypervisor scenarios, with `NVX_HYPERVISOR` naming the -backend, which `nvx::Hypervisor::from_env_or_default` reads. +host-only configuration on each backend, in the jobs that run the guest and +sandbox steps, and runs its ignored tests, the adapter's VM scenarios, with +`NVX_HYPERVISOR` naming the backend, `NVX_ARTIFACTS_DIR` the checkout, into +which the job downloaded NVX's artifacts and their provenance, and +`RUNTIME_ADAPTER_GUEST` the guest agent. The Linux jobs build the guest agent +first, and the Windows job takes the one that CI's `distro-artifacts` job +builds on Linux. diff --git a/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs b/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs index 6b0395a14..01bf77985 100644 --- a/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs +++ b/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs @@ -6,6 +6,12 @@ //! the host and the guest, such as a control console in raw mode. It exits with 0 once the host //! stopped it or the broker closed the stream, and with 1, after a diagnostic on standard error, //! once serving failed. +//! - `runtime-adapter-guest init TTY` serves the host program on the tty TTY, which it puts in raw +//! mode itself, in place of NVX's managed agent, whose initramfs runs it on the VM's control tty +//! as PID 1; see `runtime_adapter::initramfs`. It then ends with 0 once the host stopped it, and +//! with 1, after a diagnostic on standard error, once serving failed or the broker closed the +//! tty. As PID 1, which must not exit, it ends the VM with that status instead, through NVX's +//! `/sbin/nvx-exit`, as NVX's managed agent does. //! //! Anything else is a usage error, with status 2. @@ -20,10 +26,14 @@ fn main() -> ExitCode { ExitCode::SUCCESS } [command] if command == runtime_adapter::GUEST_SERVE_COMMAND => serve(), + [command, tty] if command == runtime_adapter::GUEST_INIT_COMMAND => { + init(std::path::Path::new(tty)) + } _ => { eprintln!( - "usage: runtime-adapter-guest [{}]", - runtime_adapter::GUEST_SERVE_COMMAND + "usage: runtime-adapter-guest [{} | {} TTY]", + runtime_adapter::GUEST_SERVE_COMMAND, + runtime_adapter::GUEST_INIT_COMMAND, ); ExitCode::from(2) } @@ -46,6 +56,41 @@ fn serve() -> ExitCode { } } +/// NVX's program that ends the VM with an exit status, which its initramfs provides. +#[cfg(target_os = "linux")] +const NVX_EXIT: &str = "/sbin/nvx-exit"; + +/// Serves the session on the tty `tty` in place of NVX's managed agent, then ends with the status +/// of serving: as PID 1, by ending the VM through [`NVX_EXIT`]. +#[cfg(target_os = "linux")] +fn init(tty: &std::path::Path) -> ExitCode { + use runtime_adapter::guest::{self, Served}; + + let status: u8 = match guest::serve_terminal(tty) { + Ok(Served::Stopped) => 0, + Ok(Served::Closed) => { + eprintln!("runtime-adapter-guest: the broker closed {}", tty.display()); + 1 + } + Err(error) => { + eprintln!("runtime-adapter-guest: {error}"); + 1 + } + }; + if std::process::id() != 1 { + return ExitCode::from(status); + } + // PID 1 must not exit, or the kernel panics: `nvx-exit` replaces it and ends the VM. Should + // it fail to start, PID 1 waits, and the host stops the VM. + let error = std::os::unix::process::CommandExt::exec( + std::process::Command::new(NVX_EXIT).arg(status.to_string()), + ); + eprintln!("runtime-adapter-guest: cannot run {NVX_EXIT}: {error}"); + loop { + std::thread::sleep(std::time::Duration::from_secs(3600)); + } +} + /// Standard input and output as one stream, which carries nothing but the session's records. #[cfg(target_os = "linux")] struct Stdio { diff --git a/tests/runtime-adapter/src/bin/runtime-adapter-host.rs b/tests/runtime-adapter/src/bin/runtime-adapter-host.rs index 34471149b..84674ce9d 100644 --- a/tests/runtime-adapter/src/bin/runtime-adapter-host.rs +++ b/tests/runtime-adapter/src/bin/runtime-adapter-host.rs @@ -1,17 +1,455 @@ -//! The runtime adapter's host program. It reports the contract that it was built against and -//! whether `nvx` supports this host. +//! The runtime adapter's host program. +//! +//! - `runtime-adapter-host` reports the contract that it was built against and whether `nvx` +//! supports this host. +//! - `runtime-adapter-host launch STORE --guest-agent PATH [--hypervisor NAME]` launches a VM in +//! the sandbox store STORE, which it creates if needed, through NVX's supervisor, with the guest +//! agent PATH, a static `x86_64-unknown-linux-musl` build of `runtime-adapter-guest`, in place of +//! NVX's managed agent; see `runtime_adapter::initramfs`. It attaches to the VM's control +//! session, exchanges a `Ping` and the longest `Echo` with the guest agent, commits the launch, +//! and exits, leaving the VM running. The VM runs on the hypervisor NAME, `kvm`, `mshv`, or +//! `whp`, or else on the one that `nvx::Hypervisor::from_env_or_default` selects. +//! - `runtime-adapter-host reattach STORE` reattaches to the VM of STORE, which an earlier process +//! launched and which it holds although it is not its child, attaches a new session to it, in +//! the session's next epoch, and exchanges a `Ping`, leaving the VM running. +//! - `runtime-adapter-host stop STORE` reattaches and attaches as `reattach` does, then stops the +//! VM: it asks the guest agent to stop, waits for the VM to exit, and checks the VM's outcome +//! report. A VM whose guest agent does not confirm the stop is terminated. +//! +//! Each command takes NVX's artifacts from the directory that `NVX_ARTIFACTS_DIR` names, a +//! repository checkout or an extracted release, and the VM's runtime identity from NVX's identity +//! of them, with the guest agent's build that `--guest-build BUILD` names, this crate's version +//! by default; see `runtime_adapter::runtime`. Reattach finds a VM that another runtime identity +//! launched mismatched, and then signals nothing: the VM keeps running. +//! +//! The commands print what they find as `KEY=VALUE` lines: +//! +//! - `reattached`: what reattach found: `live`, `mismatched`, `exited`, or `indeterminate`; +//! - `differences`: the parts of the runtime identity in which a mismatched VM differs, separated +//! by commas; +//! - `pid`: the VMM's process ID; +//! - `instance` and `epoch`: the broker instance, in hexadecimal, and the epoch of the session +//! that the command attached to; +//! - `stopped`: how the stop ended: `graceful`, `forced`, or `timed-out`; +//! - `outcome`: the category and status code of the VM's outcome report, such as `success 0`; +//! - `teardown`: whether the VM's teardown completed: `complete` or `incomplete`. +//! +//! They exit with 0 once they did what they were asked, with 3 when reattach finds the VM +//! mismatched, and with 1, after a diagnostic on standard error, when they fail. Anything else is +//! a usage error, with status 2. +use std::env; +use std::error::Error; +use std::ffi::OsString; +use std::path::{Path, PathBuf}; use std::process::ExitCode; +use std::time::{Duration, Instant}; + +use nvx::Hypervisor; +use nvx::host::artifacts::Artifacts; +use nvx::host::instance::generate_capability; +use nvx::host::launch::LaunchSpec; +use nvx::host::report; +use nvx::host::session; +use nvx::host::store::{IdentityPart, SandboxStore}; +use nvx::host::supervisor::{self, Reattached, StopOutcome, Vmm, VmmCommand}; +use nvx::protocol::CAPABILITY_LEN; +use runtime_adapter::frame::MAX_BODY; +use runtime_adapter::host::Client; +use runtime_adapter::initramfs; +use runtime_adapter::runtime::{GUEST_BUILD, Runtime}; + +/// How long a command waits for the sandbox's lock. +const LOCK_TIMEOUT: Duration = Duration::from_secs(10); +/// How long a launch waits for the VM to boot and its guest agent to answer. +const START_TIMEOUT: Duration = Duration::from_secs(60); +/// How long reattach and stop wait to attach a session and for the guest agent's answer. +const SESSION_TIMEOUT: Duration = Duration::from_secs(30); +/// How long a stop waits for the guest agent's confirmation and for the VM to exit. +const STOP_TIMEOUT: Duration = Duration::from_secs(30); +/// How long a stop waits for a VM that it terminated to exit. +const FORCED_TIMEOUT: Duration = Duration::from_secs(10); +/// The name of the initramfs that boots the guest agent, which the launch writes into the +/// sandbox's store. +const INITRAMFS_NAME: &str = "runtime-adapter.cpio.gz"; +/// The exit status of a command that finds the VM mismatched. +const MISMATCHED: u8 = 3; +/// The command lines that the program accepts. +const USAGE: &str = concat!( + "usage: runtime-adapter-host\n", + " runtime-adapter-host launch STORE --guest-agent PATH [--hypervisor NAME]", + " [--guest-build BUILD]\n", + " runtime-adapter-host (reattach | stop) STORE [--guest-build BUILD]", +); + +/// What the program was asked to do. +enum Command { + /// Report the contract that the program was built against. + Report, + /// Launch a VM of the guest agent. + Launch { + /// The sandbox's store. + store: PathBuf, + /// The guest agent's executable. + guest_agent: PathBuf, + /// The hypervisor, unless `nvx::Hypervisor::from_env_or_default` selects it. + hypervisor: Option, + /// The guest agent's build. + guest_build: String, + }, + /// Reattach to the VM, and stop it if asked. + Reattach { + /// The sandbox's store. + store: PathBuf, + /// The guest agent's build. + guest_build: String, + /// Whether to stop the VM. + stop: bool, + }, +} fn main() -> ExitCode { - if std::env::args_os().len() > 1 { - eprintln!("usage: runtime-adapter-host"); - return ExitCode::from(2); + let command = match parse(env::args_os().skip(1).collect()) { + Ok(command) => command, + Err(problem) => { + eprintln!("runtime-adapter-host: {problem}\n{USAGE}"); + return ExitCode::from(2); + } + }; + let result = match command { + Command::Report => { + println!( + "runtime-adapter-host: {}; supported host: {}", + runtime_adapter::contract(), + nvx::host::platform::SUPPORTED, + ); + return ExitCode::SUCCESS; + } + Command::Launch { + store, + guest_agent, + hypervisor, + guest_build, + } => launch(&store, &guest_agent, hypervisor, &guest_build), + Command::Reattach { + store, + guest_build, + stop, + } => reattach(&store, &guest_build, stop), + }; + result.unwrap_or_else(|problem| { + eprintln!("runtime-adapter-host: {problem}"); + ExitCode::FAILURE + }) +} + +/// Parses the program's arguments, or says why they are not a command. +fn parse(arguments: Vec) -> Result { + let mut arguments = arguments.into_iter(); + let Some(name) = arguments.next() else { + return Ok(Command::Report); + }; + let name = match name.to_str() { + Some(name @ ("launch" | "reattach" | "stop")) => name.to_owned(), + _ => return Err(format!("unknown command {name:?}")), + }; + let store = PathBuf::from( + arguments + .next() + .ok_or_else(|| format!("{name} needs a sandbox store"))?, + ); + let (mut guest_agent, mut hypervisor, mut guest_build) = (None, None, None); + while let Some(option) = arguments.next() { + let value = arguments + .next() + .ok_or_else(|| format!("{option:?} needs a value"))?; + match (name.as_str(), option.to_str()) { + ("launch", Some("--guest-agent")) if guest_agent.is_none() => { + guest_agent = Some(PathBuf::from(value)); + } + ("launch", Some("--hypervisor")) if hypervisor.is_none() => { + let value = value + .to_str() + .ok_or_else(|| format!("unknown hypervisor {value:?}"))?; + hypervisor = Some(value.parse().map_err(|error| describe(&error))?); + } + (_, Some("--guest-build")) if guest_build.is_none() => { + let value = value + .into_string() + .map_err(|value| format!("the guest build {value:?} is not Unicode"))?; + guest_build = Some(value); + } + _ => return Err(format!("unexpected option {option:?} of {name}")), + } + } + let guest_build = guest_build.unwrap_or_else(|| GUEST_BUILD.to_owned()); + Ok(match name.as_str() { + "launch" => Command::Launch { + store, + guest_agent: guest_agent.ok_or("launch needs --guest-agent")?, + hypervisor, + guest_build, + }, + "reattach" => Command::Reattach { + store, + guest_build, + stop: false, + }, + _ => Command::Reattach { + store, + guest_build, + stop: true, + }, + }) +} + +/// Launches a VM of the guest agent `guest_agent` in the sandbox store `store`, on `hypervisor` +/// or else the one that `nvx::Hypervisor::from_env_or_default` selects, with the guest agent's +/// build `guest_build`. +fn launch( + store: &Path, + guest_agent: &Path, + hypervisor: Option, + guest_build: &str, +) -> Result { + let hypervisor = match hypervisor { + Some(hypervisor) => hypervisor, + None => Hypervisor::from_env_or_default().map_err(|error| describe(&error))?, + }; + let runtime = runtime(guest_build)?; + runtime + .artifacts + .probe() + .map_err(|error| describe(&error))?; + let store = SandboxStore::create(store).map_err(|error| describe(&error))?; + let mut locked = store + .lock(Some(Instant::now() + LOCK_TIMEOUT)) + .map_err(|error| describe(&error))?; + let capability = generate_capability().map_err(|error| describe(&error))?; + let initramfs = store.dir().join(INITRAMFS_NAME); + let spec = LaunchSpec::new( + Artifacts::new( + &runtime.artifacts.openvmm, + &runtime.artifacts.kernel, + &initramfs, + ), + hypervisor, + ); + let deadline = Instant::now() + START_TIMEOUT; + let starting = supervisor::launch(&mut locked, &runtime.identity, &capability, |endpoint| { + // The launch has checked that no VM of the sandbox runs, which could still read the + // initramfs of an earlier launch. + initramfs::write(&runtime.artifacts.initrd, guest_agent, &initramfs).map_err(|error| { + nvx::Error::io("cannot write the initramfs of the guest agent").with_source(error) + })?; + let arguments = spec.openvmm_arguments(endpoint, &store.outcome_path())?; + Ok(VmmCommand::new(&spec.openvmm, arguments, store.dir())) + }) + .map_err(|error| describe(&error))?; + let process = starting.process().reference().clone(); + println!("pid={}", process.pid); + // A launch that fails from here on ends its VM, once `starting` drops. + let failed = |problem: String| { + format!( + "{problem}; see the VMM's log {}", + store.log_path().display() + ) + }; + let transport = session::connect( + starting.endpoint(), + process.pid, + process.start_time, + deadline, + ) + .map_err(|error| failed(format!("cannot connect to the VM: {}", describe(&error))))?; + let mut client = attach(transport, &capability, deadline).map_err(failed)?; + client + .ping(deadline) + .and_then(|()| client.echo(&echo_body(), deadline)) + .map_err(|error| { + failed(format!( + "the guest agent did not answer: {}", + describe(&error) + )) + })?; + starting.commit().map_err(|error| describe(&error))?; + Ok(ExitCode::SUCCESS) +} + +/// Returns the body of the launch's `Echo`: the longest that the framing allows, of every byte +/// value, which spans several of the session's records, each way. +fn echo_body() -> Vec { + (0..=u8::MAX).cycle().take(MAX_BODY).collect() +} + +/// Reattaches to the VM of the sandbox store `store`, with the guest agent's build `guest_build`, +/// attaches a session to it, and exchanges a `Ping`, then stops the VM if `stop` asks. +fn reattach(store: &Path, guest_build: &str, stop: bool) -> Result { + let runtime = runtime(guest_build)?; + let store = SandboxStore::open(store).map_err(|error| describe(&error))?; + let mut locked = store + .lock(Some(Instant::now() + LOCK_TIMEOUT)) + .map_err(|error| describe(&error))?; + let reattached = + supervisor::reattach(&mut locked, &runtime.identity).map_err(|error| describe(&error))?; + let vmm = match reattached { + Reattached::Live(vmm) => { + println!("reattached=live"); + vmm + } + Reattached::Mismatched { vmm, differences } => { + println!("reattached=mismatched"); + println!("differences={}", names(&differences)); + println!("pid={}", vmm.record().process().pid); + return Ok(ExitCode::from(MISMATCHED)); + } + Reattached::Exited { .. } => { + println!("reattached=exited"); + return Err("no VM of the sandbox runs".to_owned()); + } + Reattached::Indeterminate(reason) => { + println!("reattached=indeterminate"); + return Err(format!( + "whether the VM of the sandbox runs cannot be determined: {reason}" + )); + } + }; + println!("pid={}", vmm.record().process().pid); + let deadline = Instant::now() + SESSION_TIMEOUT; + let attached = store + .read_capability() + .map_err(|error| describe(&error)) + .and_then(|capability| connect(&vmm, &capability, deadline)) + .and_then(|mut client| { + client + .ping(deadline) + .map_err(|error| format!("the guest agent did not answer: {}", describe(&error)))?; + Ok(client) + }); + if !stop { + return attached.map(|_| ExitCode::SUCCESS); } + // A VM whose guest agent cannot confirm the stop is terminated. + let stopped = vmm + .stop( + &mut locked, + Instant::now() + STOP_TIMEOUT, + Some(FORCED_TIMEOUT), + |_, deadline| { + attached?.stop(deadline).map_err(|error| { + format!( + "the guest agent did not confirm the stop: {}", + describe(&error) + ) + }) + }, + ) + .map_err(|error| describe(&error))?; println!( - "runtime-adapter-host: {}; supported host: {}", - runtime_adapter::contract(), - nvx::host::platform::SUPPORTED, + "stopped={}", + match stopped.outcome { + StopOutcome::Graceful => "graceful", + StopOutcome::Forced => "forced", + StopOutcome::TimedOut => "timed-out", + } ); - ExitCode::SUCCESS + if let Some(problem) = stopped.graceful_error { + return Err(problem); + } + if stopped.outcome != StopOutcome::Graceful { + return Err("the VM did not exit once the guest agent confirmed the stop".to_owned()); + } + let outcome = report::read(&store.outcome_path()).ok_or_else(|| { + format!( + "the VM left no outcome report {}", + store.outcome_path().display() + ) + })?; + let category = outcome.category.as_deref().unwrap_or("none"); + let status = outcome + .status_code + .map_or("none".to_owned(), |code| code.to_string()); + println!("outcome={category} {status}"); + println!( + "teardown={}", + match outcome.teardown_complete { + Some(true) => "complete", + Some(false) | None => "incomplete", + } + ); + if (category, outcome.status_code, outcome.teardown_complete) + != ("success", Some(0), Some(true)) + { + return Err("the VM's outcome is not a success with a complete teardown".to_owned()); + } + Ok(ExitCode::SUCCESS) +} + +/// Returns the runtime of NVX's artifacts in the directory that `NVX_ARTIFACTS_DIR` names, with +/// the guest agent's build `guest_build`. +fn runtime(guest_build: &str) -> Result { + let directory = env::var_os(Artifacts::ENV_DIR) + .filter(|directory| !directory.is_empty()) + .ok_or_else(|| { + format!( + "set {} to NVX's artifacts: a repository checkout or an extracted release", + Artifacts::ENV_DIR + ) + })?; + Runtime::from_dir(directory, guest_build).map_err(|error| describe(&error)) +} + +/// Connects to the control endpoint of the VM `vmm`, and attaches a session with `capability`, +/// by `deadline`. +fn connect( + vmm: &Vmm, + capability: &[u8; CAPABILITY_LEN], + deadline: Instant, +) -> Result { + let process = vmm.record().process(); + let transport = session::connect(vmm.endpoint(), process.pid, process.start_time, deadline) + .map_err(|error| format!("cannot connect to the VM: {}", describe(&error)))?; + attach(transport, capability, deadline) +} + +/// Attaches a session over `transport` with `capability` by `deadline`, and prints its identity. +fn attach( + transport: Box, + capability: &[u8; CAPABILITY_LEN], + deadline: Instant, +) -> Result { + let client = Client::attach(transport, capability, deadline) + .map_err(|error| format!("cannot attach to the VM: {}", describe(&error)))?; + let identity = client.identity(); + let instance: String = identity + .instance_id + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + println!("instance={instance}"); + println!("epoch={}", identity.epoch); + Ok(client) +} + +/// Returns the names of `parts`, separated by commas. +fn names(parts: &[IdentityPart]) -> String { + parts + .iter() + .map(|part| part.name()) + .collect::>() + .join(",") +} + +/// Describes `error` with the causes that its description does not end with already. +fn describe(error: &dyn Error) -> String { + let mut description = error.to_string(); + let mut source = error.source(); + while let Some(cause) = source { + let cause_description = cause.to_string(); + if !description.ends_with(&cause_description) { + description = format!("{description}: {cause_description}"); + } + source = cause.source(); + } + description } diff --git a/tests/runtime-adapter/src/guest.rs b/tests/runtime-adapter/src/guest.rs index c81470d21..6b31eb169 100644 --- a/tests/runtime-adapter/src/guest.rs +++ b/tests/runtime-adapter/src/guest.rs @@ -1,11 +1,13 @@ //! The adapter's guest agent: it serves the host program over the guest's end of the control //! session between the host and the guest, with the adapter's own [framing](crate::frame), through -//! `nvx::guest::nvxs` alone. +//! `nvx::guest::nvxs`, and on a tty through `nvx::guest::terminal`. use std::fmt; -use std::io::{Read, Write}; +use std::io::{self, Read, Write}; +use std::path::Path; use nvx::guest::nvxs::{GuestConnection, GuestConnectionError}; +use nvx::guest::terminal::Terminal; use nvx::protocol::OUTER_MIN_RECEIVE_WINDOW; use nvx::protocol::nvxs::GuestEvent; @@ -34,6 +36,8 @@ pub enum ServeError { Frame(FrameError), /// The host sent a message of a kind that only the guest sends. Unexpected(u8), + /// The tty of the session could not be opened or put in raw mode. + Terminal(io::Error), } impl fmt::Display for ServeError { @@ -47,6 +51,7 @@ impl fmt::Display for ServeError { "the host sent a message of kind {kind}, which only the guest sends" ) } + Self::Terminal(error) => error.fmt(formatter), } } } @@ -57,6 +62,7 @@ impl std::error::Error for ServeError { Self::Session(error) => Some(error), Self::Frame(error) => Some(error), Self::Unexpected(_) => None, + Self::Terminal(error) => Some(error), } } } @@ -122,3 +128,30 @@ pub fn serve(stream: S) -> Result { } } } + +/// Serves the host program on the tty at `path`, the guest's end of the control session, as +/// [`serve`] does, in place of NVX's managed agent on a VM's control tty. +/// +/// It opens the tty with NVX's [`Terminal`], as NVX's managed agent does: waiting while the kernel +/// sets the tty up during boot, and in raw mode, so that the tty passes the session's bytes as they +/// are. Once serving ends, it waits until the tty has sent what serving wrote, such as the +/// [`Message::Stopped`] that confirms a stop. +/// +/// It fails with [`ServeError::Terminal`] if the tty cannot be opened or put in raw mode, and +/// otherwise as [`serve`] does. +pub fn serve_terminal(path: &Path) -> Result { + let failed = |action: String| { + move |error: io::Error| { + ServeError::Terminal(io::Error::new(error.kind(), format!("{action}: {error}"))) + } + }; + let mut terminal = + Terminal::open(path).map_err(failed(format!("cannot open {}", path.display())))?; + terminal + .configure_raw() + .map_err(failed(format!("cannot put {} in raw mode", path.display())))?; + let served = serve(&mut terminal)?; + // As NVX's managed agent does, a drain that fails is no failure of serving, which ended. + let _ = terminal.drain(); + Ok(served) +} diff --git a/tests/runtime-adapter/src/initramfs.rs b/tests/runtime-adapter/src/initramfs.rs new file mode 100644 index 000000000..39920de60 --- /dev/null +++ b/tests/runtime-adapter/src/initramfs.rs @@ -0,0 +1,128 @@ +//! The initramfs of a VM that runs the adapter's guest agent in place of NVX's managed agent. +//! +//! NVX's initramfs boots through its `/init`, which hands a VM of the managed lifecycle to NVX's +//! managed agent, as PID 1, on the VM's control tty: +//! +//! ```text +//! exec /sbin/nvx-managed-agent /dev/hvc2 - nvx-managed UID GID NAME HOME +//! ``` +//! +//! [`write`](fn@write) copies that initramfs, and appends to the copy a second archive, in the +//! `newc` format, which Linux unpacks over the first: +//! +//! - the guest agent, at [`GUEST_AGENT`]; +//! - a script, at [`LAUNCHER`], that runs the guest agent as `runtime-adapter-guest init TTY`, +//! with the control tty, the first of the arguments that `/init` passes the managed agent; +//! - a symbolic link from [`MANAGED_AGENT`] to the script. Linux removes a file whose type an +//! archive changes before it creates the new one, so the link replaces the managed agent +//! without writing to its file, which could have other names. +//! +//! The second archive starts at a multiple of 4 bytes, as Linux requires of an uncompressed +//! archive, after the zeros that pad the copy, which Linux skips. NVX's initramfs itself stays as +//! it is. + +use std::fs::{self, File}; +use std::io::{self, Write}; +use std::path::Path; + +/// Path of the guest agent in the VM. +pub const GUEST_AGENT: &str = "/sbin/runtime-adapter-guest"; +/// Path of the script that runs the guest agent in place of NVX's managed agent. +pub const LAUNCHER: &str = "/sbin/runtime-adapter-init"; +/// Path of NVX's managed agent, which `/init` runs. +pub const MANAGED_AGENT: &str = "/sbin/nvx-managed-agent"; + +/// Returns the script at [`LAUNCHER`], which runs the guest agent's +/// [`GUEST_INIT_COMMAND`](crate::GUEST_INIT_COMMAND) on the control tty, the first of the +/// arguments that `/init` passes the managed agent. +pub fn launcher_script() -> String { + format!( + "#!/bin/sh\nexec {GUEST_AGENT} {} \"$1\"\n", + crate::GUEST_INIT_COMMAND + ) +} + +/// The magic number of a `newc` header. +const MAGIC: &[u8] = b"070701"; +/// The name of the entry that ends an archive. +const TRAILER: &str = "TRAILER!!!"; +/// The type bits of a regular file. +const REGULAR: u32 = 0o100_000; +/// The type bits of a symbolic link. +const SYMLINK: u32 = 0o120_000; + +/// Writes to `output` a copy of NVX's initramfs `base` with the guest agent `guest_agent`, a +/// static `x86_64-unknown-linux-musl` build of `runtime-adapter-guest`, in place of NVX's managed +/// agent, as the [module documentation](self) describes. +/// +/// Fails if a file cannot be read or written, or if the guest agent does not fit an archive +/// entry, whose size has 32 bits. The error names the file. +pub fn write(base: &Path, guest_agent: &Path, output: &Path) -> io::Result<()> { + let agent = fs::read(guest_agent).map_err(naming(guest_agent))?; + let archive = archive(&agent)?; + let mut file = File::create(output).map_err(naming(output))?; + let mut source = File::open(base).map_err(naming(base))?; + let length = io::copy(&mut source, &mut file).map_err(naming(output))?; + let padding = length.next_multiple_of(4) - length; + // `padding` is less than 4. + file.write_all(&[0; 3][..padding as usize]) + .and_then(|()| file.write_all(&archive)) + .map_err(naming(output)) +} + +/// Returns a function that names `path` in the message of an error. +fn naming(path: &Path) -> impl Fn(io::Error) -> io::Error + '_ { + move |error| io::Error::new(error.kind(), format!("{}: {error}", path.display())) +} + +/// Returns the archive that [`write`](fn@write) appends, with the guest agent `guest_agent`: +/// every entry that the [module documentation](self) lists, owned by root, then the trailer. +/// +/// Fails if the guest agent does not fit an archive entry, whose size has 32 bits. +pub fn archive(guest_agent: &[u8]) -> io::Result> { + let script = launcher_script(); + let entries: [(&str, u32, &[u8]); 3] = [ + (GUEST_AGENT, REGULAR | 0o755, guest_agent), + (LAUNCHER, REGULAR | 0o755, script.as_bytes()), + // A link's data is its target. + (MANAGED_AGENT, SYMLINK | 0o777, LAUNCHER.as_bytes()), + ]; + let mut archive = Vec::new(); + for (inode, (path, mode, data)) in (1..).zip(entries) { + // Archives name their entries relative to the root. + push(&mut archive, inode, &path[1..], mode, data)?; + } + push(&mut archive, 0, TRAILER, 0, &[])?; + Ok(archive) +} + +/// Appends to `archive` the entry `name`, with the inode number `inode`, the mode `mode`, and the +/// data `data`, each padded to a multiple of 4 bytes. +fn push(archive: &mut Vec, inode: u32, name: &str, mode: u32, data: &[u8]) -> io::Result<()> { + let size = u32::try_from(data.len()).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name} does not fit an archive entry, whose size has 32 bits"), + ) + })?; + // The name's size counts its terminating NUL. + let name_size = u32::try_from(name.len() + 1).expect("a constant name is short"); + // The inode, mode, owner and group, link count, modification time, size, device numbers of + // the file and of a device that it is, name size, and checksum, which `newc` leaves zero. + let fields = [inode, mode, 0, 0, 1, 0, size, 0, 0, 0, 0, name_size, 0]; + archive.extend_from_slice(MAGIC); + for field in fields { + archive.extend_from_slice(format!("{field:08X}").as_bytes()); + } + archive.extend_from_slice(name.as_bytes()); + archive.push(0); + pad(archive); + archive.extend_from_slice(data); + pad(archive); + Ok(()) +} + +/// Pads `archive` with zeros to a multiple of 4 bytes. +fn pad(archive: &mut Vec) { + archive.resize(archive.len().next_multiple_of(4), 0); +} diff --git a/tests/runtime-adapter/src/lib.rs b/tests/runtime-adapter/src/lib.rs index 50b0b7edb..71ddf1197 100644 --- a/tests/runtime-adapter/src/lib.rs +++ b/tests/runtime-adapter/src/lib.rs @@ -6,8 +6,8 @@ //! //! - protocol-only, without features: the control protocol, the contract versions, and the other //! items that `nvx` has without features, with the adapter's own [`frame`]s; -//! - host-only, with the `host` feature: the host API too, and the host program, in the `host` -//! module and the `runtime-adapter-host` binary; +//! - host-only, with the `host` feature: the host API too, and the host program, in the `host`, +//! `runtime`, and `initramfs` modules and the `runtime-adapter-host` binary; //! - guest-only, with the `guest` feature: the guest API too, and the guest agent, in the `guest` //! module and the `runtime-adapter-guest` binary, a static `x86_64-unknown-linux-musl` //! executable. @@ -15,18 +15,32 @@ //! The adapter brings its own framing to the control session between the host and the guest, in //! place of NVX's `NVXC` frames. The guest agent serves requests in that framing, and //! `runtime-adapter-guest serve` runs it on its standard input and output. The host program's -//! client attaches to the session with the launch capability and sends those requests. The -//! `runtime-adapter-host` binary reports the contract that it was built against. +//! client attaches to the session with the launch capability and sends those requests. +//! +//! The host program runs the guest agent in VMs of NVX's artifacts, in place of NVX's managed +//! agent, through NVX's supervisor: `runtime-adapter-host launch` starts a VM, and later +//! processes reattach to it and stop it with `runtime-adapter-host reattach` and `stop`, as an +//! orchestrator does after it restarts. The `runtime` module takes the VMs' runtime identity from +//! NVX's identity of its artifacts, with the guest agent's build in it, and the `initramfs` module +//! writes the initramfs that puts the guest agent in place of the managed agent, where +//! `runtime-adapter-guest init` serves the VM's control tty. pub mod frame; #[cfg(all(feature = "guest", target_os = "linux"))] pub mod guest; #[cfg(feature = "host")] pub mod host; +#[cfg(feature = "host")] +pub mod initramfs; +#[cfg(feature = "host")] +pub mod runtime; /// Command that starts the guest agent on the runtime adapter's standard streams. pub const GUEST_SERVE_COMMAND: &str = "serve"; +/// Command that starts the guest agent on the tty that follows it, in place of NVX's managed agent. +pub const GUEST_INIT_COMMAND: &str = "init"; + /// Describes the runtime contract that this build speaks: the control contract revision, and the /// versions of the microVM ABI, the NVXS records, and the NVXC frames. pub fn contract() -> String { diff --git a/tests/runtime-adapter/src/runtime.rs b/tests/runtime-adapter/src/runtime.rs new file mode 100644 index 000000000..a1383c84e --- /dev/null +++ b/tests/runtime-adapter/src/runtime.rs @@ -0,0 +1,64 @@ +//! The runtime of the adapter's VMs: NVX's artifacts, whose initramfs boots the guest agent in +//! place of NVX's managed agent, and the runtime identity that NVX's supervisor records for a +//! launch and checks when it reattaches. +//! +//! The identity starts from the one that NVX gives its artifacts, +//! [`ArtifactDir::runtime_identity`], which names each artifact by the SHA-256 digest that the +//! install metadata beside it records, never from its contents: a release's `SHA256SUMS`, or the +//! build provenance in a repository checkout's `build/`, where `nvx.py download` also installs a +//! release's. +//! +//! The VMs boot a copy of NVX's initramfs that holds the guest agent; see [`crate::initramfs`]. +//! Its identity names both: NVX's initramfs and the guest agent's build, +//! `sha256:DIGEST+runtime-adapter-guest/BUILD`. The guest agent has no install metadata of its +//! own, so the caller names its build, which is this crate's version, [`GUEST_BUILD`], unless the +//! caller ships another. A host program that ships another build of the guest agent runs another +//! runtime, so it finds the VMs of this one mismatched. + +use std::path::Path; + +use nvx::Result; +use nvx::host::artifacts::{ArtifactDir, Artifacts}; +use nvx::host::store::{ArtifactIdentity, RuntimeIdentity}; + +/// The guest agent's build that a host program ships unless it names another: this crate's +/// version. +pub const GUEST_BUILD: &str = env!("CARGO_PKG_VERSION"); + +/// NVX's artifacts, and the runtime identity of the adapter's VMs that boot them with the guest +/// agent. +#[derive(Debug, Clone)] +pub struct Runtime { + /// The OpenVMM executable, the guest kernel, and NVX's initramfs, which the VMs boot a copy + /// of, with the guest agent in it. + pub artifacts: Artifacts, + /// The runtime identity of the VMs. + pub identity: RuntimeIdentity, +} + +impl Runtime { + /// Returns the runtime of NVX's artifacts in `directory`, a repository checkout or an + /// extracted release, which [`Artifacts::from_dir`] finds there, with the guest agent's build + /// `guest_build`, as the [module documentation](self) describes. + /// + /// Fails as [`Artifacts::from_dir`] and [`ArtifactDir::runtime_identity`] do, and with + /// [`ErrorKind::Unsupported`](nvx::ErrorKind::Unsupported) if the identity of the initramfs + /// with `guest_build` is not an artifact identity. + pub fn from_dir(directory: impl AsRef, guest_build: &str) -> Result { + let directory = directory.as_ref(); + let artifacts = Artifacts::from_dir(directory)?; + let nvx = ArtifactDir::from_dir(directory)?.runtime_identity()?; + let identity = RuntimeIdentity::new( + nvx.openvmm, + nvx.kernel, + ArtifactIdentity::new(format!( + "{}+runtime-adapter-guest/{guest_build}", + nvx.initramfs + ))?, + ); + Ok(Self { + artifacts, + identity, + }) + } +} diff --git a/tests/runtime-adapter/tests/guest_session.rs b/tests/runtime-adapter/tests/guest_session.rs index f06dedd2c..7d591e31f 100644 --- a/tests/runtime-adapter/tests/guest_session.rs +++ b/tests/runtime-adapter/tests/guest_session.rs @@ -6,18 +6,24 @@ //! `runtime-adapter-guest` binary, whose standard input and output are the socket. The tests cover //! messages that span records, partial records, resets in the middle of a message, disconnects, //! record lengths, bootstrap identity, ERROR payloads, sequence and identity errors, and the -//! bounded buffer of the adapter's decoder. +//! bounded buffer of the adapter's decoder. Through a relay to a pseudo-terminal, they also cover +//! the guest agent on a tty, which it puts in raw mode, as it serves a VM's control tty. #![cfg(all(feature = "guest", target_os = "linux"))] +use std::fs::{File, OpenOptions}; use std::io::{self, Read, Write}; use std::net::Shutdown; use std::os::fd::OwnedFd; +use std::os::unix::fs::OpenOptionsExt; use std::os::unix::net::UnixStream; +use std::path::PathBuf; use std::process::{Child, Command, Output, Stdio}; use std::sync::mpsc::{self, Receiver, Sender}; use std::thread::{self, JoinHandle}; use std::time::{Duration, Instant}; +use nix::fcntl::OFlag; +use nix::pty; use nvx::guest::nvxs::GuestConnectionError; use nvx::protocol::OUTER_MAX_PAYLOAD; use nvx::protocol::nvxs::{ @@ -640,32 +646,53 @@ fn the_guest_agent_refuses_messages_that_its_framing_does_not_allow() { /// Starts the guest binary's `serve` with one end of a socket pair as its standard input and /// output, and returns the broker on the other end with the process. -fn start_binary() -> (Broker, Child) { +fn start_binary() -> (Broker, GuestProcess) { let (broker, guest) = UnixStream::pair().unwrap(); let output = guest.try_clone().unwrap(); - let child = Command::new(env!("CARGO_BIN_EXE_runtime-adapter-guest")) - .arg(runtime_adapter::GUEST_SERVE_COMMAND) - .stdin(Stdio::from(OwnedFd::from(guest))) - .stdout(Stdio::from(OwnedFd::from(output))) - .stderr(Stdio::piped()) - .spawn() - .unwrap(); + let child = GuestProcess::spawn( + Command::new(env!("CARGO_BIN_EXE_runtime-adapter-guest")) + .arg(runtime_adapter::GUEST_SERVE_COMMAND) + .stdin(Stdio::from(OwnedFd::from(guest))) + .stdout(Stdio::from(OwnedFd::from(output))) + .stderr(Stdio::piped()), + ); (Broker::new(broker), child) } -/// Returns the guest binary's output once it exits. If it does not exit in time, kills and reaps -/// it, and fails the test instead of hanging it. -fn finished(mut child: Child) -> Output { - let deadline = Instant::now() + TIMEOUT; - while child.try_wait().unwrap().is_none() { - if Instant::now() >= deadline { +/// A guest binary that a test started. Dropping it kills and reaps the process, so that a test +/// that fails while the process runs does not leave it running. +struct GuestProcess(Option); + +impl GuestProcess { + /// Starts the guest binary with `command`. + fn spawn(command: &mut Command) -> Self { + Self(Some(command.spawn().unwrap())) + } + + /// Returns the guest binary's output once it exits. If it does not exit in time, kills and + /// reaps it, and fails the test instead of hanging it. + fn finished(mut self) -> Output { + let deadline = Instant::now() + TIMEOUT; + let child = self.0.as_mut().expect("the process was not reaped"); + while child.try_wait().unwrap().is_none() { + if Instant::now() >= deadline { + let _ = child.kill(); + let output = self.0.take().unwrap().wait_with_output().unwrap(); + panic!("the guest binary did not exit in time: {output:?}"); + } + thread::sleep(Duration::from_millis(10)); + } + self.0.take().unwrap().wait_with_output().unwrap() + } +} + +impl Drop for GuestProcess { + fn drop(&mut self) { + if let Some(child) = &mut self.0 { let _ = child.kill(); - let output = child.wait_with_output().unwrap(); - panic!("the guest binary did not exit in time: {output:?}"); + let _ = child.wait(); } - thread::sleep(Duration::from_millis(10)); } - child.wait_with_output().unwrap() } #[test] @@ -677,7 +704,7 @@ fn the_guest_binary_serves_the_session_on_its_standard_streams() { let messages = [Message::Ping, echo.clone(), Message::Stop]; let replies = broker.exchange(&messages, OUTER_MAX_PAYLOAD, 3); assert_eq!(replies, [Message::Pong, echo, Message::Stopped]); - let output = finished(child); + let output = child.finished(); assert!(output.status.success(), "{output:?}"); assert!(output.stderr.is_empty(), "{output:?}"); @@ -685,7 +712,7 @@ fn the_guest_binary_serves_the_session_on_its_standard_streams() { let (mut broker, child) = start_binary(); broker.attach(FIRST); broker.close(); - let output = finished(child); + let output = child.finished(); assert!(output.status.success(), "{output:?}"); assert!(output.stderr.is_empty(), "{output:?}"); @@ -693,7 +720,7 @@ fn the_guest_binary_serves_the_session_on_its_standard_streams() { let (mut broker, child) = start_binary(); broker.attach(FIRST); broker.send_data(&[0, 0, 0, 0, 9], OUTER_MAX_PAYLOAD); - let output = finished(child); + let output = child.finished(); assert_eq!(output.status.code(), Some(1), "{output:?}"); let diagnostic = String::from_utf8_lossy(&output.stderr); assert_eq!( @@ -701,3 +728,104 @@ fn the_guest_binary_serves_the_session_on_its_standard_streams() { "runtime-adapter-guest: unknown message kind 9\n" ); } + +/// A pseudo-terminal whose tty the guest agent serves, opening it by its path, and whose master +/// end a relay joins to a socket pair, on whose other end the broker plays its end of the session, +/// as for the other tests. +struct Terminal { + /// The path of the tty. + path: PathBuf, + /// The test's own descriptor of the tty. Reads of the master end fail once no descriptor of + /// the tty is open, so it keeps the relay up until the guest agent opens the tty. + _tty: OwnedFd, +} + +/// Opens a pseudo-terminal, and returns it with the broker at the other end of its relay. +/// +/// Its descriptors are closed on exec, so that no process that a test starts holds its master +/// end, which would keep the tty from hanging up once the test ends. +fn terminal() -> (Broker, Terminal) { + let master = pty::posix_openpt(OFlag::O_RDWR | OFlag::O_NOCTTY | OFlag::O_CLOEXEC).unwrap(); + pty::grantpt(&master).unwrap(); + pty::unlockpt(&master).unwrap(); + let path = PathBuf::from(pty::ptsname_r(&master).unwrap()); + // The standard library opens files, and duplicates descriptors, close-on-exec. + let tty = OpenOptions::new() + .read(true) + .write(true) + .custom_flags(OFlag::O_NOCTTY.bits()) + .open(&path) + .unwrap(); + let (broker, relay) = UnixStream::pair().unwrap(); + let (mut from_broker, mut to_broker) = (relay.try_clone().unwrap(), relay); + let master = File::from(OwnedFd::from(master)); + let (mut to_guest, mut from_guest) = (master.try_clone().unwrap(), master); + thread::spawn(move || io::copy(&mut from_broker, &mut to_guest)); + thread::spawn(move || { + let _ = io::copy(&mut from_guest, &mut to_broker); + let _ = to_broker.shutdown(Shutdown::Write); + }); + let terminal = Terminal { + path, + _tty: OwnedFd::from(tty), + }; + (Broker::new(broker), terminal) +} + +#[test] +fn the_guest_agent_serves_a_tty_in_raw_mode() { + let (mut broker, terminal) = terminal(); + let path = terminal.path.clone(); + let serving = thread::spawn(move || guest::serve_terminal(&path)); + broker.attach(FIRST); + // Every byte value, which a tty that was not in raw mode would translate or act on: carriage + // returns and newlines, and the characters of signals, flow control, line editing, and the end + // of a file. The confirmation of the stop reaches the broker before serving returns. + let replies = broker.exchange(&[longest(), Message::Stop], OUTER_MAX_PAYLOAD, 2); + assert_eq!(replies, [longest(), Message::Stopped]); + assert_eq!(joined(serving).unwrap(), Served::Stopped); +} + +#[test] +fn the_guest_binary_serves_a_tty_in_place_of_the_managed_agent() { + // Unless it is PID 1, `init` exits with the status with which PID 1 would end the VM. + let (mut broker, terminal) = terminal(); + let child = GuestProcess::spawn( + Command::new(env!("CARGO_BIN_EXE_runtime-adapter-guest")) + .arg(runtime_adapter::GUEST_INIT_COMMAND) + .arg(&terminal.path) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()), + ); + broker.attach(FIRST); + let replies = broker.exchange(&[Message::Ping, Message::Stop], OUTER_MAX_PAYLOAD, 2); + assert_eq!(replies, [Message::Pong, Message::Stopped]); + let output = child.finished(); + assert!(output.status.success(), "{output:?}"); + assert!(output.stdout.is_empty(), "{output:?}"); + assert!(output.stderr.is_empty(), "{output:?}"); + + // A path that is no tty fails at once: one that cannot be opened, and one that has no + // terminal settings. + for (path, diagnostic, error) in [ + ("/", "cannot open /", 21), + ("/dev/null", "cannot put /dev/null in raw mode", 25), + ] { + let child = GuestProcess::spawn( + Command::new(env!("CARGO_BIN_EXE_runtime-adapter-guest")) + .args([runtime_adapter::GUEST_INIT_COMMAND, path]) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()), + ); + let output = child.finished(); + assert_eq!(output.status.code(), Some(1), "{path}: {output:?}"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.starts_with(&format!("runtime-adapter-guest: {diagnostic}: ")) + && stderr.ends_with(&format!(" (os error {error})\n")), + "{path}: {stderr}" + ); + } +} diff --git a/tests/runtime-adapter/tests/initramfs.rs b/tests/runtime-adapter/tests/initramfs.rs new file mode 100644 index 000000000..0dc2c550f --- /dev/null +++ b/tests/runtime-adapter/tests/initramfs.rs @@ -0,0 +1,158 @@ +//! The initramfs that boots the guest agent is a copy of NVX's initramfs, with an archive after it +//! that Linux unpacks over NVX's: the guest agent, the script that runs it, and the link that +//! replaces NVX's managed agent with the script. +#![cfg(feature = "host")] + +use std::fs; +use std::path::PathBuf; + +use runtime_adapter::initramfs::{self, GUEST_AGENT, LAUNCHER, MANAGED_AGENT}; + +/// An entry of a `newc` archive. +#[derive(Debug, PartialEq, Eq)] +struct Entry { + /// The entry's name, relative to the root. + name: String, + /// The inode number. + inode: u32, + /// The type and permission bits. + mode: u32, + /// The owner, the group, and the link count. + owner: (u32, u32, u32), + /// The data: a file's contents, or a link's target. + data: Vec, +} + +/// The length of a `newc` header. +const HEADER_LEN: usize = 110; + +/// Parses the `newc` archive `bytes`, which must end with its trailer and the trailer's padding, +/// and checks that its headers and data start at multiples of 4 bytes, and that its padding is +/// zeros. Returns its entries before the trailer. +fn parse(bytes: &[u8]) -> Vec { + let mut entries = Vec::new(); + let mut offset = 0; + loop { + assert_eq!(offset % 4, 0, "a header at {offset}"); + let header = &bytes[offset..offset + HEADER_LEN]; + assert_eq!(&header[..6], b"070701", "the magic number at {offset}"); + let field = |index: usize| { + let digits = &header[6 + 8 * index..14 + 8 * index]; + assert!( + digits + .iter() + .all(|digit| digit.is_ascii_digit() || (b'A'..=b'F').contains(digit)), + "{digits:?}" + ); + u32::from_str_radix(std::str::from_utf8(digits).unwrap(), 16).unwrap() + }; + // The device numbers and the checksum are zero. + for index in [7, 8, 9, 10, 12] { + assert_eq!(field(index), 0, "field {index} at {offset}"); + } + let name_start = offset + HEADER_LEN; + let name_end = name_start + field(11) as usize; + assert_eq!(bytes[name_end - 1], 0, "the name's NUL at {offset}"); + let name = String::from_utf8(bytes[name_start..name_end - 1].to_vec()).unwrap(); + let data_start = name_end.next_multiple_of(4); + let data_end = data_start + field(6) as usize; + let end = data_end.next_multiple_of(4); + assert!(bytes[name_end..data_start].iter().all(|&byte| byte == 0)); + assert!(bytes[data_end..end].iter().all(|&byte| byte == 0)); + if name == "TRAILER!!!" { + assert_eq!(end, bytes.len(), "the trailer ends the archive"); + return entries; + } + entries.push(Entry { + name, + inode: field(0), + mode: field(1), + owner: (field(2), field(3), field(4)), + data: bytes[data_start..data_end].to_vec(), + }); + offset = end; + } +} + +/// Returns the entries that the archive with the guest agent `agent` holds. +fn expected(agent: &[u8]) -> Vec { + let entry = |name: &str, inode, mode, data: &[u8]| Entry { + name: name.trim_start_matches('/').to_owned(), + inode, + mode, + owner: (0, 0, 1), + data: data.to_vec(), + }; + vec![ + entry(GUEST_AGENT, 1, 0o100_755, agent), + entry( + LAUNCHER, + 2, + 0o100_755, + initramfs::launcher_script().as_bytes(), + ), + entry(MANAGED_AGENT, 3, 0o120_777, LAUNCHER.as_bytes()), + ] +} + +#[test] +fn the_archive_replaces_the_managed_agent_with_a_link_to_the_guest_agent_launcher() { + // Guest agents of every length modulo 4 pad their data differently. + for length in 0..8 { + let agent = (1..=length).collect::>(); + let archive = initramfs::archive(&agent).unwrap(); + assert_eq!( + parse(&archive), + expected(&agent), + "a guest agent of {length} bytes" + ); + } +} + +#[test] +fn the_launcher_runs_the_guest_agent_on_the_control_tty() { + // `/init` passes the managed agent the control tty first. + let init = runtime_adapter::GUEST_INIT_COMMAND; + assert_eq!( + initramfs::launcher_script(), + format!("#!/bin/sh\nexec {GUEST_AGENT} {init} \"$1\"\n") + ); + assert_eq!(MANAGED_AGENT, "/sbin/nvx-managed-agent"); +} + +#[test] +fn the_initramfs_is_a_copy_of_nvx_s_with_the_archive_at_a_multiple_of_4_bytes() { + let directory = PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join("initramfs"); + fs::create_dir_all(&directory).unwrap(); + let (base, agent, output) = ( + directory.join("base.cpio.gz"), + directory.join("runtime-adapter-guest"), + directory.join("output.cpio.gz"), + ); + fs::write(&agent, b"guest agent").unwrap(); + let archive = initramfs::archive(b"guest agent").unwrap(); + // An earlier, longer output is replaced. + fs::write(&output, vec![0xee; 4096]).unwrap(); + for length in 0..8 { + let contents = vec![0xab; length]; + fs::write(&base, &contents).unwrap(); + initramfs::write(&base, &agent, &output).unwrap(); + let written = fs::read(&output).unwrap(); + let start = length.next_multiple_of(4); + assert_eq!(&written[..length], contents, "a base of {length} bytes"); + assert!(written[length..start].iter().all(|&byte| byte == 0)); + assert_eq!(&written[start..], archive, "a base of {length} bytes"); + } + // NVX's initramfs stays as it is. + assert_eq!(fs::read(&base).unwrap(), vec![0xab; 7]); + + // A failure names the file. + for (base, agent) in [ + (&directory.join("missing"), &agent), + (&base, &directory.join("missing")), + ] { + let error = initramfs::write(base, agent, &output).unwrap_err(); + let missing = directory.join("missing").display().to_string(); + assert!(error.to_string().contains(&missing), "{error}"); + } +} diff --git a/tests/runtime-adapter/tests/runtime.rs b/tests/runtime-adapter/tests/runtime.rs new file mode 100644 index 000000000..dbef63391 --- /dev/null +++ b/tests/runtime-adapter/tests/runtime.rs @@ -0,0 +1,136 @@ +//! The runtime of the adapter's VMs takes NVX's artifacts from a repository checkout or an +//! extracted release, and their runtime identity from NVX's, through the install metadata beside +//! them, with the guest agent's build in the identity of the initramfs. +#![cfg(feature = "host")] + +use std::fs; +use std::path::{Path, PathBuf}; + +use nvx::ErrorKind; +use nvx::host::artifacts::ArtifactDir; +use nvx::host::store::{ArtifactIdentity, IdentityPart, RuntimeIdentity}; +use runtime_adapter::runtime::{GUEST_BUILD, Runtime}; + +/// The digests that the install metadata of the test's artifacts records. +const OPENVMM: &str = "0a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f9"; +const KERNEL: &str = "1111111111111111111111111111111111111111111111111111111111111111"; +const INITRAMFS: &str = "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"; + +/// The build provenance of a checkout's artifacts: each record's name, field, and digest. +const PROVENANCE: [(&str, &str, &str); 3] = [ + ("openvmm.provenance.json", "executable_sha256", OPENVMM), + ("vmlinux.provenance.json", "kernel_sha256", KERNEL), + ("initramfs.provenance.json", "initramfs_sha256", INITRAMFS), +]; + +/// Creates an empty directory for the test `name`. +fn directory(name: &str) -> PathBuf { + let directory = PathBuf::from(env!("CARGO_TARGET_TMPDIR")) + .join("runtime") + .join(name); + let _ = fs::remove_dir_all(&directory); + fs::create_dir_all(&directory).unwrap(); + directory +} + +/// Lays out NVX's artifacts in `root`, with NVX's manifest and the install metadata that names +/// them: in the release layout, with its `SHA256SUMS`, if `release`, and otherwise in a +/// repository checkout's, with the build provenance in `build/`. Returns the path of the metadata +/// that names the kernel. +fn lay_out(root: &Path, release: bool) -> PathBuf { + let manifest = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../SOURCE-MANIFEST.json"); + fs::copy(manifest, root.join("SOURCE-MANIFEST.json")).unwrap(); + if release { + // A release is told apart by its built artifacts. + fs::create_dir_all(root.join("guest")).unwrap(); + fs::write(root.join("guest/vmlinux"), b"kernel").unwrap(); + let openvmm = if cfg!(windows) { + "bin/openvmm.exe" + } else { + "bin/openvmm" + }; + let sums = format!( + "{OPENVMM} {openvmm}\n{KERNEL} guest/vmlinux\n{INITRAMFS} guest/initramfs.cpio.gz\n" + ); + let path = root.join("SHA256SUMS"); + fs::write(&path, sums).unwrap(); + path + } else { + let build = root.join("build"); + fs::create_dir_all(&build).unwrap(); + for (name, field, digest) in PROVENANCE { + let record = format!("{{\"format\": 1, \"{field}\": \"{digest}\", \"other\": true}}"); + fs::write(build.join(name), record).unwrap(); + } + build.join("vmlinux.provenance.json") + } +} + +/// Returns the identity of the runtime of the test's artifacts with the guest agent's build +/// `guest_build`. +fn identity(guest_build: &str) -> RuntimeIdentity { + let sha256 = |digest| ArtifactIdentity::sha256(digest).unwrap(); + let initramfs = format!("sha256:{INITRAMFS}+runtime-adapter-guest/{guest_build}"); + RuntimeIdentity::new( + sha256(OPENVMM), + sha256(KERNEL), + ArtifactIdentity::new(initramfs).unwrap(), + ) +} + +#[test] +fn the_runtime_identity_is_nvx_s_with_the_guest_agent_s_build_in_either_layout() { + for (name, release, guest) in [("repository", false, "build"), ("release", true, "guest")] { + let root = directory(name); + lay_out(&root, release); + let runtime = Runtime::from_dir(&root, GUEST_BUILD).unwrap(); + assert_eq!(runtime.identity, identity(GUEST_BUILD), "{name}"); + // Only the initramfs, which holds the guest agent, differs from NVX's identity. + let nvx = ArtifactDir::from_dir(&root) + .unwrap() + .runtime_identity() + .unwrap(); + assert_eq!( + runtime.identity.differences(&nvx), + [IdentityPart::Initramfs], + "{name}" + ); + // The VMs boot a copy of NVX's Alpine initramfs. + let root = fs::canonicalize(&root).unwrap(); + let initrd = fs::canonicalize(runtime.artifacts.initrd.parent().unwrap()).unwrap(); + assert_eq!(initrd, root.join(guest), "{name}"); + assert_eq!( + runtime.artifacts.initrd.file_name().unwrap(), + "initramfs.cpio.gz" + ); + } +} + +#[test] +fn another_build_of_the_guest_agent_is_another_runtime() { + let root = directory("another-build"); + lay_out(&root, false); + let ours = Runtime::from_dir(&root, GUEST_BUILD).unwrap().identity; + let theirs = Runtime::from_dir(&root, "another-build").unwrap().identity; + assert_eq!(theirs, identity("another-build")); + assert_eq!(ours.differences(&theirs), [IdentityPart::Initramfs]); + + // A build that an artifact identity cannot hold fails. + for build in ["with space", &"x".repeat(64)] { + let error = Runtime::from_dir(&root, build).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Unsupported, "{build:?}: {error}"); + } +} + +#[test] +fn missing_install_metadata_fails_as_nvx_reports_it() { + for (name, release) in [("missing-provenance", false), ("missing-checksums", true)] { + let root = directory(name); + let metadata = lay_out(&root, release); + fs::remove_file(&metadata).unwrap(); + let error = Runtime::from_dir(&root, GUEST_BUILD).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Unavailable, "{name}: {error}"); + let missing = format!("{} is missing", metadata.display()); + assert!(error.message().contains(&missing), "{name}: {error}"); + } +} diff --git a/tests/runtime-adapter/tests/vm_restart.rs b/tests/runtime-adapter/tests/vm_restart.rs new file mode 100644 index 000000000..a3b0d5078 --- /dev/null +++ b/tests/runtime-adapter/tests/vm_restart.rs @@ -0,0 +1,316 @@ +//! The host program launches a VM of the guest agent through NVX's supervisor, and later processes +//! of it reattach to the VM and stop it, as an orchestrator does after it restarts, on a real +//! hypervisor, through public APIs only. +//! +//! The tests boot VMs, so they are ignored. They read: +//! +//! - `NVX_HYPERVISOR`: the hypervisor, through `nvx::Hypervisor::from_env_or_default`; +//! - `NVX_ARTIFACTS_DIR`: NVX's artifacts, a repository checkout or an extracted release, with +//! the install metadata from which NVX gives their runtime identity; +//! - `RUNTIME_ADAPTER_GUEST`: the guest agent, a static `x86_64-unknown-linux-musl` build of +//! `runtime-adapter-guest`. +//! +//! ```text +//! cargo test --locked --features host -- --ignored +//! ``` +#![cfg(feature = "host")] + +use std::collections::BTreeMap; +use std::convert::Infallible; +use std::env; +use std::error::Error; +use std::ffi::OsStr; +use std::fs::{self, File}; +use std::io; +use std::path::PathBuf; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::{Duration, Instant}; + +use nvx::Hypervisor; +use nvx::host::process::Process; +use nvx::host::store::{RuntimeRecord, SandboxStore}; +use nvx::host::supervisor::{self, Reattached, StopOutcome}; + +/// The variable that names the guest agent. +const GUEST_AGENT: &str = "RUNTIME_ADAPTER_GUEST"; +/// The longest that a command of the host program may take. +const COMMAND_TIMEOUT: Duration = Duration::from_secs(180); +/// How much of the end of the VMM's log a failure shows. +const LOG_TAIL: usize = 8192; + +/// A sandbox of a test: a directory of its own, which holds the sandbox's store and the output of +/// each command of the host program that the test runs. +/// +/// Dropping it ends a VM that a failed test left running, through its verified identity, and +/// removes the directory once no VM of the sandbox runs. Should that remain uncertain, it keeps +/// the directory, whose store a later reattach needs to end the VM, and prints its path. +struct Sandbox { + /// The test's directory. + root: PathBuf, + /// The sandbox's store, which the launch creates. + store: PathBuf, + /// The number of commands that the test ran. + commands: usize, +} + +impl Sandbox { + /// Creates the directory of the test `name`, in the system's temporary directory, so that the + /// path of a Linux control socket in it stays short. A directory that an earlier test process + /// kept stays as it is. + fn new(name: &str) -> Self { + let pid = std::process::id(); + let root = (0..) + .map(|attempt| env::temp_dir().join(format!("runtime-adapter-{name}-{pid}-{attempt}"))) + .find(|root| match fs::create_dir(root) { + Ok(()) => true, + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => false, + Err(error) => panic!("cannot create {}: {error}", root.display()), + }) + .expect("a directory name is free"); + Self { + store: root.join("store"), + root, + commands: 0, + } + } + + /// Launches the VM, and returns what the launch printed. + fn launch(&mut self) -> BTreeMap { + let hypervisor = Hypervisor::from_env_or_default().expect("NVX_HYPERVISOR is valid"); + let guest_agent = env::var_os(GUEST_AGENT).unwrap_or_else(|| { + panic!( + "{GUEST_AGENT} must name a static x86_64-unknown-linux-musl runtime-adapter-guest" + ) + }); + let options = [ + OsStr::new("--guest-agent"), + &guest_agent, + OsStr::new("--hypervisor"), + OsStr::new(hypervisor.as_str()), + ]; + let launched = self.run("launch", &options, 0); + let record = self.record().expect("the launch recorded its VM"); + assert_eq!(launched["pid"], record.process().pid.to_string()); + launched + } + + /// Runs the host program's `command` on the sandbox's store with `options`, checks that it + /// exits with `status`, and returns the `KEY=VALUE` lines that it printed. + fn run(&mut self, command: &str, options: &[&OsStr], status: i32) -> BTreeMap { + self.commands += 1; + let output = self.root.join(format!("{}-{command}", self.commands)); + let (stdout, stderr) = (output.with_extension("out"), output.with_extension("err")); + // Files, rather than pipes, take the output, so that nothing waits for a VMM that might + // hold them. + let mut child = Command::new(env!("CARGO_BIN_EXE_runtime-adapter-host")) + .arg(command) + .arg(&self.store) + .args(options) + .stdin(Stdio::null()) + .stdout(File::create(&stdout).unwrap()) + .stderr(File::create(&stderr).unwrap()) + .spawn() + .unwrap(); + let deadline = Instant::now() + COMMAND_TIMEOUT; + let exit = loop { + if let Some(exit) = child.try_wait().unwrap() { + break exit; + } + if Instant::now() >= deadline { + let _ = child.kill(); + let _ = child.wait(); + panic!("{command} did not end in time{}", self.diagnostics(&output)); + } + thread::sleep(Duration::from_millis(50)); + }; + assert_eq!( + exit.code(), + Some(status), + "{command} {options:?} ended with {exit}{}", + self.diagnostics(&output) + ); + let printed = fs::read_to_string(&stdout).unwrap(); + let mut lines = BTreeMap::new(); + for line in printed.lines() { + let (key, value) = line + .split_once('=') + .unwrap_or_else(|| panic!("{command} printed {line:?}")); + let earlier = lines.insert(key.to_owned(), value.to_owned()); + assert!(earlier.is_none(), "{command} printed {key} twice"); + } + lines + } + + /// Returns what the command whose output files start with `output` printed, and the end of the + /// VMM's log. + fn diagnostics(&self, output: &std::path::Path) -> String { + let read = |path: PathBuf| { + let bytes = fs::read(path).unwrap_or_default(); + let tail = bytes.len().saturating_sub(LOG_TAIL); + String::from_utf8_lossy(&bytes[tail..]).into_owned() + }; + format!( + "\nstdout:\n{}\nstderr:\n{}\nthe end of the VMM's log:\n{}", + read(output.with_extension("out")), + read(output.with_extension("err")), + read(self.store.join(nvx::host::store::LOG_NAME)), + ) + } + + /// Returns the sandbox's runtime record, if it has one. + fn record(&self) -> Option { + SandboxStore::open(&self.store).unwrap().runtime().unwrap() + } + + /// Returns the VMM of the sandbox's runtime record, if the record names one that runs. + fn vmm(&self) -> Option { + Process::open(self.record()?.process()).unwrap() + } + + /// Ends the VM that the sandbox's store records, whatever its runtime identity: reattach ends + /// an interrupted launch, and holds a VM that runs, which the stop then terminates through its + /// verified identity. Fails with the reason why a VM of the sandbox might still run. + fn end_vm(&self) -> Result<(), String> { + if !self.store.exists() { + return Ok(()); + } + let failed = |error: nvx::Error| describe(&error); + let store = SandboxStore::open(&self.store).map_err(failed)?; + let recorded = ( + store.runtime().map_err(failed)?, + store.journal().map_err(failed)?, + ); + let identity = match recorded { + (Some(record), _) => record.runtime().clone(), + (None, Some(journal)) => journal.runtime().clone(), + (None, None) => return Ok(()), + }; + let mut locked = store + .lock(Some(Instant::now() + Duration::from_secs(10))) + .map_err(failed)?; + let vmm = match supervisor::reattach(&mut locked, &identity).map_err(failed)? { + Reattached::Live(vmm) | Reattached::Mismatched { vmm, .. } => vmm, + Reattached::Exited { .. } => return Ok(()), + Reattached::Indeterminate(reason) => return Err(reason), + }; + let forced = Some(Duration::from_secs(10)); + let stopped = vmm + .stop(&mut locked, Instant::now(), forced, |_, _| { + Ok::<(), Infallible>(()) + }) + .map_err(failed)?; + match stopped.outcome { + StopOutcome::Graceful | StopOutcome::Forced => Ok(()), + StopOutcome::TimedOut => Err(format!( + "VMM process {} did not exit after it was terminated", + vmm.record().process().pid + )), + } + } +} + +impl Drop for Sandbox { + fn drop(&mut self) { + match self.end_vm() { + Ok(()) => { + let _ = fs::remove_dir_all(&self.root); + } + Err(reason) => eprintln!( + "kept {} because a VM of its sandbox might still run: {reason}", + self.root.display() + ), + } + } +} + +/// Describes `error`, with its causes. +fn describe(error: &dyn Error) -> String { + let mut description = error.to_string(); + let mut source = error.source(); + while let Some(cause) = source { + description = format!("{description}: {cause}"); + source = cause.source(); + } + description +} + +/// Returns the epoch that a command printed. +fn epoch(printed: &BTreeMap) -> u64 { + printed["epoch"].parse().unwrap() +} + +/// Checks that a command stopped the VM of `vmm` that `launched` printed, with the guest agent's +/// confirmation, after it attached a session in an epoch after `earlier`'s: the VM exited with +/// a successful outcome, and no runtime record names it. +fn assert_stopped( + sandbox: &Sandbox, + stopped: &BTreeMap, + launched: &BTreeMap, + earlier: &BTreeMap, + vmm: &Process, +) { + assert_eq!(stopped["reattached"], "live"); + assert_eq!(stopped["pid"], launched["pid"]); + assert_eq!(stopped["instance"], launched["instance"]); + assert!(epoch(stopped) > epoch(earlier), "{stopped:?} {earlier:?}"); + assert_eq!(stopped["stopped"], "graceful"); + assert_eq!(stopped["outcome"], "success 0"); + assert_eq!(stopped["teardown"], "complete"); + assert!(vmm.has_exited().unwrap()); + assert_eq!( + sandbox.record(), + None, + "the stop cleared the runtime record" + ); +} + +#[test] +#[ignore = "boots a VM; see the module documentation for what it needs"] +fn a_restarted_host_program_reattaches_to_the_vm_and_stops_it() { + let mut sandbox = Sandbox::new("restart"); + // The launching process has exited, and the VM runs on. + let launched = sandbox.launch(); + let vmm = sandbox + .vmm() + .expect("the VM runs after its launcher exited"); + + // Another process holds the VM, which is not its child, and attaches a new session, in the + // session's next epoch. + let reattached = sandbox.run("reattach", &[], 0); + assert_eq!(reattached["reattached"], "live"); + assert_eq!(reattached["pid"], launched["pid"]); + assert_eq!(reattached["instance"], launched["instance"]); + assert!(epoch(&reattached) > epoch(&launched), "{reattached:?}"); + assert!(!vmm.has_exited().unwrap()); + + // A third one stops it. + let stopped = sandbox.run("stop", &[], 0); + assert_stopped(&sandbox, &stopped, &launched, &reattached, &vmm); +} + +#[test] +#[ignore = "boots a VM; see the module documentation for what it needs"] +fn a_host_program_of_another_runtime_finds_the_vm_mismatched_and_signals_nothing() { + let mut sandbox = Sandbox::new("identity"); + let launched = sandbox.launch(); + let record = sandbox.record(); + let vmm = sandbox + .vmm() + .expect("the VM runs after its launcher exited"); + + // A host program that ships another build of the guest agent runs another runtime. + let options = [OsStr::new("--guest-build"), OsStr::new("another-build")]; + let mismatched = sandbox.run("reattach", &options, 3); + assert_eq!(mismatched["reattached"], "mismatched"); + assert_eq!(mismatched["differences"], "initramfs"); + assert_eq!(mismatched["pid"], launched["pid"]); + assert!(!mismatched.contains_key("epoch"), "{mismatched:?}"); + + // It signaled nothing: the VM runs on, under the same runtime record, and its guest agent + // serves the host program of the right runtime, which stops it. + assert!(!vmm.has_exited().unwrap()); + assert_eq!(sandbox.record(), record); + let stopped = sandbox.run("stop", &[], 0); + assert_stopped(&sandbox, &stopped, &launched, &launched, &vmm); +}