From 0a89f7b1c03545a6292961cada97dbbea979e899 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Wed, 21 Oct 2026 23:55:03 -0700 Subject: [PATCH 1/2] tests: launch the runtime adapter's VMs with a layered root The runtime adapter now launches VMs with a real layered root, which its own guest assembles through nvx::guest::root in a worker process, never PID 1. The guest library's root operations get real-kernel coverage before NVX has a Rust PID 1, and later root modes get a harness for their scenarios. - The host program plans the VM with NVX's typed SandboxPlan, launches it through NVX's supervisor with the OpenVMM arguments of NVX's sandbox start, and records the launch in its store. The boot tokens carry only what the boot contract defines; the rest of what the root needs goes in one root configuration message of the adapter's framing, after attach. - The appended initramfs archive also replaces /sbin/nvx-init-agent with a launcher that names the control tty itself, so that NVX's init agent never assembles the root, and keeps the blockless launcher. - PID 1 starts an unrelated child and the worker, collects every child's status, and ends the VM with the worker's. The worker attaches, checks the root configuration, reads the boot layout, bootstraps and assembles the root, answers RootReady or RootFailed with the stage that failed, serves pings, echoes and probes, and on Stop tears the root down and reports a teardown that failed. - A table of VM scenarios checks EROFS over an ext4 scratch through the worker's probes, and that a wrong image identity, an invalid root configuration, and a busy root's teardown each reach the host program with their stage while the VM still ends cleanly. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ddd9400a-9806-4d3d-b2cb-4320252cb090 --- .github/workflows/run-nvx-microvm-tests.yml | 15 +- doc/ci.md | 30 +- doc/project-structure.md | 8 +- tests/runtime-adapter/Cargo.lock | 1 + tests/runtime-adapter/Cargo.toml | 16 +- tests/runtime-adapter/README.md | 431 ++++++++-- .../src/bin/runtime-adapter-guest.rs | 101 ++- .../src/bin/runtime-adapter-host.rs | 476 +++++++++-- tests/runtime-adapter/src/frame.rs | 250 +++++- tests/runtime-adapter/src/guest.rs | 127 ++- tests/runtime-adapter/src/host.rs | 82 +- tests/runtime-adapter/src/initramfs.rs | 46 +- tests/runtime-adapter/src/layered.rs | 224 +++++ tests/runtime-adapter/src/lib.rs | 36 +- tests/runtime-adapter/src/probe.rs | 358 ++++++++ tests/runtime-adapter/src/root.rs | 738 +++++++++++++++++ tests/runtime-adapter/src/worker.rs | 415 ++++++++++ tests/runtime-adapter/src/worker/probes.rs | 475 +++++++++++ tests/runtime-adapter/tests/common/mod.rs | 246 ++++++ tests/runtime-adapter/tests/guest_session.rs | 773 +++++++++++++++++- tests/runtime-adapter/tests/host_session.rs | 250 +++++- tests/runtime-adapter/tests/initramfs.rs | 29 +- tests/runtime-adapter/tests/layered_launch.rs | 269 ++++++ tests/runtime-adapter/tests/vm_layered.rs | 490 +++++++++++ tests/runtime-adapter/tests/vm_restart.rs | 241 +----- 25 files changed, 5590 insertions(+), 537 deletions(-) create mode 100644 tests/runtime-adapter/src/layered.rs create mode 100644 tests/runtime-adapter/src/probe.rs create mode 100644 tests/runtime-adapter/src/root.rs create mode 100644 tests/runtime-adapter/src/worker.rs create mode 100644 tests/runtime-adapter/src/worker/probes.rs create mode 100644 tests/runtime-adapter/tests/common/mod.rs create mode 100644 tests/runtime-adapter/tests/layered_launch.rs create mode 100644 tests/runtime-adapter/tests/vm_layered.rs diff --git a/.github/workflows/run-nvx-microvm-tests.yml b/.github/workflows/run-nvx-microvm-tests.yml index 14faa9eef..689a7b173 100644 --- a/.github/workflows/run-nvx-microvm-tests.yml +++ b/.github/workflows/run-nvx-microvm-tests.yml @@ -1250,14 +1250,17 @@ jobs: # The runtime adapter's hypervisor scenarios on this backend: the ignored # tests of its host-only configuration, which read the backend through - # `nvx::Hypervisor::from_env_or_default`. Its host program launches a VM + # `nvx::Hypervisor::from_env_or_default`. Its host program launches VMs # of the artifacts above, with the runtime identity that their build # provenance gives, whose initramfs runs the adapter's guest agent in - # place of NVX's managed agent. Later processes of the host program - # reattach to the VM, as after a restart of the orchestrator, and stop - # it. The guest agent is a static musl executable: Linux builds it, and - # Windows takes the one that distro-artifacts builds on Linux, as it - # takes the Ubuntu and Azure Linux images. + # place of NVX's agents. Later processes of the host program reattach to + # a VM, as after a restart of the orchestrator, and stop it. The layered + # scenarios boot the Ubuntu EROFS image over a copy of the scratch + # template, from the guest artifacts, as the sandbox steps above do, and + # the guest agent's worker assembles the root outside PID 1. The guest + # agent is a static musl executable: Linux builds it, and Windows takes + # the one that distro-artifacts builds on Linux, as it takes the Ubuntu + # and Azure Linux images. - name: Build the runtime adapter's guest agent on Linux if: runner.os != 'Windows' && inputs.guests-and-sandboxes && !inputs.debug-kernel shell: bash diff --git a/doc/ci.md b/doc/ci.md index 61e1ce6a4..155189c86 100644 --- a/doc/ci.md +++ b/doc/ci.md @@ -1305,8 +1305,11 @@ guest-only configuration, that its guest agent serves the session, also on a tty that it puts in raw mode. Each runs against a test broker that sends it messages that span records, resets, disconnects, and malformed records. The host-only tests also check the initramfs that runs the guest agent in place of -NVX's managed agent, and the runtime identity that the host program takes from -NVX's identity of its artifacts. +NVX's agents, the runtime identity that the host program takes from NVX's +identity of its artifacts, and the plan, OpenVMM arguments, and record of a +launch with a layered root. The guest-only tests also run the guest agent's +worker of a layered root against a fake root: its root configuration, the +failure of each stage of the root, which it reports, and its teardown. The development release job depends on `aci-edge-sandboxes` and `nvx-rs` and requires their combined Linux/Windows results to be successful; failed, @@ -1330,15 +1333,20 @@ The same jobs build the runtime adapter's host-only configuration on their backend and run its ignored tests, its VM scenarios, with `NVX_HYPERVISOR` naming the backend. Through NVX's supervisor, the adapter's host program launches a VM of the job's artifacts, whose initramfs runs the adapter's guest -agent in place of NVX's managed agent, with the runtime identity that the -artifacts' build provenance gives. Later processes reattach to the VM, as after -a restart of the orchestrator, find it mismatched under another runtime -identity without signaling it, and stop it. The Linux jobs build the guest -agent, a static musl executable, first, and the Windows job takes the one that -`distro-artifacts` builds on Linux, as it takes the Ubuntu and Azure Linux -images. The adapter's other tests, including those of its guest agent, run in -the `nvx-rs` job. Changes under `tests/runtime-adapter/` therefore trigger the -NVX microVM test matrix too. +agent in place of NVX's agents, with the runtime identity that the artifacts' +build provenance gives. Later processes reattach to the VM, as after a restart +of the orchestrator, find it mismatched under another runtime identity without +signaling it, and stop it. The adapter's layered scenarios boot the Ubuntu +EROFS image over a copy of the scratch template, from the guest artifacts, +where the guest agent's worker, outside PID 1, assembles the root through +`nvx::guest::root`. They check the root, and that a failed assembly, an +invalid root configuration, and a failed teardown each reach the host program +with the stage that failed, and still end the VM cleanly. The Linux jobs build +the guest agent, a static musl executable, first, and the Windows job takes the +one that `distro-artifacts` builds on Linux, as it takes the Ubuntu and Azure +Linux images. The adapter's other tests, including those of its guest agent, +run in the `nvx-rs` job. Changes under `tests/runtime-adapter/` therefore +trigger the NVX microVM test matrix too. ## Adversarial campaigns diff --git a/doc/project-structure.md b/doc/project-structure.md index c69980da4..fc9363647 100644 --- a/doc/project-structure.md +++ b/doc/project-structure.md @@ -344,9 +344,11 @@ independent adapter's does. Its host program and its guest agent exchange the adapter's own framing over the control session between the host and the guest, and its tests check each of them against a test broker. Its host program also runs the guest agent in VMs of NVX's artifacts through NVX's -supervisor, and reattaches to them from other processes, which its ignored -tests check on each backend in CI. See its -[README](../tests/runtime-adapter/README.md). +supervisor, and reattaches to them from other processes. In VMs with a +layered root, the guest agent's worker assembles the root through +`nvx::guest::root`, outside PID 1, and the host program checks it with probes, +in a table of scenarios. Its ignored tests check the VMs on each backend in CI. +See its [README](../tests/runtime-adapter/README.md). ## Root files diff --git a/tests/runtime-adapter/Cargo.lock b/tests/runtime-adapter/Cargo.lock index 779ab6055..57f1e1771 100644 --- a/tests/runtime-adapter/Cargo.lock +++ b/tests/runtime-adapter/Cargo.lock @@ -167,6 +167,7 @@ version = "0.1.0" dependencies = [ "nix", "nvx", + "serde_json", ] [[package]] diff --git a/tests/runtime-adapter/Cargo.toml b/tests/runtime-adapter/Cargo.toml index d3af2a071..393e8f87c 100644 --- a/tests/runtime-adapter/Cargo.toml +++ b/tests/runtime-adapter/Cargo.toml @@ -15,13 +15,25 @@ publish = false # does. Without features, the adapter builds against the protocol-only `nvx`. default = [] # The host program: the host API of `nvx` and the `runtime-adapter-host` binary. -host = ["nvx/host"] +host = ["nvx/host", "dep:serde_json"] # The guest agent: the guest API of `nvx` and the `runtime-adapter-guest` binary, which builds as a # static `x86_64-unknown-linux-musl` executable. -guest = ["nvx/guest"] +guest = ["nvx/guest", "dep:nix"] [dependencies] nvx = { path = "../../nvx-rs", default-features = false } +# The host program records each launch in the sandbox's store as JSON, with the persisted form of +# its plan. The host side of `nvx` depends on the same release. +serde_json = { version = "1.0.120", optional = true } + +# The guest agent's PID 1 collects the status of every child and ends the unrelated one, whose +# system calls `nvx` leaves to the caller. The guest side of `nvx` depends on the same release, +# with these features among its own. +[target.'cfg(target_os = "linux")'.dependencies] +nix = { version = "0.31.3", default-features = false, features = [ + "process", + "signal", +], optional = true } # The guest agent's tests open pseudo-terminals. The guest side of `nvx` depends on the same # release, with this feature among its own. diff --git a/tests/runtime-adapter/README.md b/tests/runtime-adapter/README.md index c3c9a0deb..1ff2744ff 100644 --- a/tests/runtime-adapter/README.md +++ b/tests/runtime-adapter/README.md @@ -13,19 +13,25 @@ an adapter outside the repository does: The adapter brings its own application framing to the control session between the host and the guest, in place of NVX's `NVXC` frames. Its host program sends requests in that framing, and its guest agent serves them. The host program -also runs the guest agent in VMs of NVX's artifacts, in place of NVX's managed -agent, through NVX's supervisor, and finds a VM again after it restarts, as an +also runs the guest agent in VMs of NVX's artifacts, in place of NVX's agents, +through NVX's supervisor, and finds a VM again after it restarts, as an orchestrator does. +A VM may have a layered root: an image under an upper, which the guest agent's +worker assembles through `nvx::guest::root`, in a process of its own rather +than PID 1, as the host program's root configuration asks. The host program +then checks the root from inside the VM with the worker's probes, in a +[table of scenarios](#layered-scenarios), to which each new check adds a row. + ## Configurations Each configuration enables one side of `nvx`: | Configuration | Cargo options | `nvx` features | Modules | Binary | | --- | --- | --- | --- | --- | -| Protocol-only | none | none | `frame` | none | -| Host-only | `--features host` | `host` | `frame`, `host`, `runtime`, `initramfs` | `runtime-adapter-host` | -| Guest-only | `--features guest --target x86_64-unknown-linux-musl` | `guest` | `frame`, `guest` | `runtime-adapter-guest`, a static executable | +| Protocol-only | none | none | `frame`, `root`, `probe` | none | +| Host-only | `--features host` | `host` | `frame`, `root`, `probe`, `host`, `runtime`, `initramfs`, `layered` | `runtime-adapter-host` | +| Guest-only | `--features guest --target x86_64-unknown-linux-musl` | `guest` | `frame`, `root`, `probe`, `guest`, `worker` | `runtime-adapter-guest`, a static executable | ## Framing @@ -41,6 +47,12 @@ length, a little-endian `u32`, then the message's kind: | 3 | `Echo`: bytes that the guest sends back in an `Echo` of its own | at most 256 KiB, what four of the session's largest records carry | | 4 | `Stop`: asks the guest to stop serving | none | | 5 | `Stopped`: the guest's confirmation of `Stop`, its last message | none | +| 6 | `RootConfig`: the host's [root configuration](#root-configurations) of a layered root, which the guest answers with `RootReady` or `RootFailed` | the configuration, at most 4 KiB | +| 7 | `RootReady`: the guest assembled the root | none | +| 8 | `RootFailed`: a stage of the root failed: the guest's answer to `RootConfig`, or its report of a failed teardown before it confirms a `Stop` | the stage, then the problem, at most 4 KiB | +| 9 | `Probe`: asks the guest to run a [probe](#probes) on its root | the probe, at most 4 KiB | +| 10 | `Report`: the guest's answer to `Probe`: the facts that the probe found | at most 256 KiB | +| 11 | `ProbeFailed`: the guest's answer to a `Probe` that it does not know, or that failed | the problem, at most 4 KiB | The session keeps no message boundaries, so a message may span several of its records, and one record may end a message and start the next. `Decoder` @@ -48,6 +60,90 @@ reassembles the messages. It checks each header before it takes any byte of the body, and holds at most one partial message. A malformed message ends the stream, because the framing cannot find the next message after it. +The framing carries each body as bytes, within its kind's limit. The bodies of +kinds 6 and 8 to 11 have encodings of their own, which the `root` and `probe` +modules write and read. A body that does not decode leaves the stream intact, +at the start of the next message. The guest answers a root configuration that +does not decode with `RootFailed`, as a configuration that it refuses, and a +probe with `ProbeFailed`. The host program's client ends on an answer that does +not decode, since it cannot tell what the guest did. + +### Root configurations + +The boot tokens of `nvx::protocol::root` carry what NVX's boot contract +defines: the image's format, the identity that the guest expects of it, and +the upper. A root configuration, `root::RootConfig`, carries the rest that the +root needs, which only the adapter defines, once the host attaches. It is +UTF-8 text: one `NAME=VALUE` line per field, each ended by a newline, in any +order, and each at most once. Numbers are decimal, without a sign or leading +zeros: + +| Name | Value | Required | +| --- | --- | --- | +| `version` | 1 | yes | +| `runtime` | The runtime directory, below which the guest assembles the root | yes | +| `agent-cgroup` | The agent's cgroup, which holds the guest's own processes | yes | +| `workload-cgroup` | The workload cgroup, the agent's sibling | yes | +| `device-timeout-ms` | How long the guest waits for each block device, at most 600,000 | yes | +| `agent-memory-low` | The agent's `memory.low`, in bytes | no | +| `workload-memory-max` | The workload cgroup's `memory.max`, in bytes | no | +| `workload-pids-max` | The workload cgroup's `pids.max` | no | +| `distro-sectors` | The size of the image's block device, in 512-byte sectors | no | +| `scratch-sectors` | The size of the scratch block device, in 512-byte sectors | no | + +The paths are absolute and canonical guest paths of at most 4,095 bytes, +without a NUL or a newline: not `/`, and without an empty, `.`, or `..` +component, or a trailing `/`. The runtime directory holds no `,`, `:`, `\`, or +`=`, which the overlay's options reserve, and the two cgroups are siblings, +whose parent is not `/`. The optional numbers are positive. The host program +and the guest agent ship together, so the guest refuses every other +configuration, one with an unknown field included, rather than ignore what it +does not know. `RootConfig::default()` holds the values that the host program +sends unless its caller names others: the runtime directory +`/run/runtime-adapter`, the cgroups `/sys/fs/cgroup/agent` and +`/sys/fs/cgroup/workload`, a wait of 30 seconds, an agent's `memory.low` of +16 MiB, as NVX's own guest reserves, and a workload's `memory.max` of 128 MiB +and `pids.max` of 128. + +A stage of the root that fails is a `root::RootFailure`: the stage's code, one +byte, then the problem, as UTF-8 text. The guest's worker, and with it the VM, +then ends with the stage's status: + +| Code | Stage | What failed | Status | +| --- | --- | --- | --- | +| 1 | `configuration` | Receiving and checking the root configuration | 11 | +| 2 | `layout` | Reading the boot layout of the root from the kernel command line, and checking that the worker assembles it | 12 | +| 3 | `bootstrap` | Bootstrapping the guest for the root: its cgroups and their resource values | 13 | +| 4 | `assembly` | Assembling the root, such as from an image of another identity than the boot tokens expect | 14 | +| 5 | `teardown` | Tearing the root down, such as while a process holds it open | 15 | + +### Probes + +A probe, `probe::Probe`, is a named check that the host asks the guest's +worker to run on its layered root. It is UTF-8 text: the probe's name, which +starts with a lowercase ASCII letter, and holds lowercase ASCII letters, +digits, and `-`, then each of its arguments after one space, each without +whitespace or control characters. The worker answers with the facts that the +probe found, `probe::Facts`, also UTF-8 text: one `KEY=VALUE` line per fact, +each ended by a newline, in the order in which the probe found them, and each +key at most once. A key holds lowercase ASCII letters, digits, `.`, and `-`, +and a value escapes `\`, newlines, and carriage returns as `\\`, `\n`, and +`\r`. + +The worker runs the probes of `worker::probes`, none of which takes arguments: + +| Probe | Facts | +| --- | --- | +| `mounts` | For the image's mount, `distro`, the scratch's, `scratch`, and the overlay's, `root`: the number of mounts at its mount point, `NAME.count`, and of the topmost, its `NAME.type`, `NAME.source`, `NAME.options`, and `NAME.super`, the file system's own options | +| `contents` | The root's `os-release.id` and `root.mode`. It writes `.runtime-adapter-probe` at the top of the root, and reports what the root, `write.root`, the upper, `write.upper`, and the image, `write.lower`, hold at that name, and whether the image refuses a new file, `lower.write` | +| `cgroups` | The file system at the cgroups' parent, `parent.type`, and its `parent.subtree-control`; whether each cgroup exists, `agent.exists` and `workload.exists`; `agent.memory.low`, `workload.memory.max`, and `workload.pids.max`; the number of processes in each, `agent.processes` and `workload.processes`; and `user-namespaces`, the guest's limit of user namespaces, or `unavailable` without them | +| `processes` | For PID 1, `pid1`, the worker, `worker`, and PID 1's unrelated child, `sibling`: `NAME.pid`, and while the process exists, `NAME.state`, `NAME.ppid`, `NAME.root`, `NAME.cgroup`, `NAME.command`, and `NAME.namespaces`: `same` if it shares PID 1's, or the ones that differ | +| `hold` | It opens the root, and keeps it open until the worker ends, so that the teardown finds the root busy: `held`, the path that it holds | + +A value that a probe cannot read is `absent` if its file does not exist, and +otherwise `unreadable: PROBLEM`. A new check of a root is a new probe in this +table. + ## Guest agent `guest::serve` serves the host program on a stream to the guest's end of the @@ -58,11 +154,13 @@ control session, through `nvx::guest::nvxs`. It attaches, then: because a new epoch starts with a new message; - credits each payload back once it holds the payload, so that a message longer than the window still arrives whole; -- answers each `Ping` with `Pong`, sends each `Echo` back, and answers `Stop` - with `Stopped`, then returns. +- answers each `Ping` with `Pong`, sends each `Echo` back, answers each + `Probe` with `ProbeFailed`, since the VM has no layered root to probe, and + answers `Stop` with `Stopped`, then returns. -A failure of the session, a malformed message, and a message that only the -guest sends end serving with an error. +A failure of the session, a malformed message, a root configuration, which +only the worker of a VM with a layered root accepts, and a message that only +the guest sends end serving with an error. `runtime-adapter-guest serve` runs the guest agent on its standard input and output, which connect to the guest's end of the session, such as a control @@ -85,6 +183,51 @@ serving failed or the broker closed the tty. As PID 1, which must not exit, it ends the VM with that status instead, through NVX's `/sbin/nvx-exit`, as the managed agent does. +### PID 1 and the worker of a layered root + +`runtime-adapter-guest init TTY layered` runs PID 1 of a VM with a layered +root, in place of NVX's init agent; see [VMs](#vms). PID 1, +`worker::supervise`, stays minimal: + +1. It starts an unrelated child, `runtime-adapter-guest idle`, which sleeps + until a signal ends it, and stands for a daemon that a guest starts before + its agent. +2. It starts the worker, `runtime-adapter-guest worker TTY SIBLING`, with the + process ID of the unrelated child, which the worker's probes check, since + the guest's kernel does not list a process's children. +3. It collects the status of every child, adopted processes included, until + the worker ends, and then ends the VM with the worker's status, through + `/sbin/nvx-exit`. + +The worker, `worker::work`, is a process of its own, never PID 1. It opens the +tty with `nvx::guest::terminal`, as `serve_terminal` does, and then: + +1. attaches the session; +2. receives the host's root configuration, and checks it; +3. reads the boot layout of the root from the kernel command line, with + `nvx::protocol::root::RootLayout::parse`, then bootstraps the guest's + cgroups with `nvx::guest::root::bootstrap`, which moves the guest's + processes into the agent's cgroup, and assembles the root with + `nvx::guest::root::assemble`; +4. answers `RootReady`, or `RootFailed` with the stage that failed, and then + ends; +5. serves `Ping`, `Echo`, and `Probe`; +6. on `Stop`, tears the root down, reports a teardown that failed with + `RootFailed`, confirms the stop with `Stopped`, and ends. + +Before the root configuration, the worker accepts only the configuration, or a +`Stop`, which ends it without a root. It assembles an EROFS image over an ext4 +scratch, and refuses the other layouts that the boot tokens can name at the +layout stage. It ends with 0 once the host stopped it, after a teardown that +succeeded, with the stage's status, from 11 to 15, once a stage failed, and +with 1 once its session failed or the broker closed the tty, each but the +first after a diagnostic on standard error. Before it exits, it closes what its +probes hold, then unmounts a root that it still holds, as +`nvx::guest::root::Root` does when it drops, ignoring failures. A teardown that +failed leaves the root mounted until the VM ends. The worker runs the root's +operations through `worker::RootOperations`, so that tests run it against a +fake root. + ## Host program `host::Client` is the host program's client of the guest agent. It runs over @@ -99,7 +242,11 @@ transport to the session's host endpoint, such as the one that `receive` returns the guest agent's next message, whether it spans records or shares one with the next message; - `ping`, `echo` and `stop` each send a request and check its answer: `Pong`, - an `Echo` of the same bytes, and `Stopped`. + an `Echo` of the same bytes, and `Stopped`. `stop` also returns the failure + of a layered root's teardown, which the guest reports before it confirms; +- `configure_root` sends a root configuration, and returns how the guest + answered: `RootReady`, or the stage of the root that failed; +- `probe` sends a probe, and returns the facts that it found, or the problem. Besides the session's own buffers, the client holds at most the rest of one of the guest's records and one partial message. A reset of the session ends the @@ -107,10 +254,11 @@ attachment, and its partial message with it, because the next epoch starts with a new message, in a new attachment. A failure of the session, a send that fails or times out, a malformed message, -and a message that only the host program sends end the client, and its later -calls fail. A receive that times out doesn't: it keeps what arrived. A request -ends the client on any failure after the framing accepted it, a timeout -included, because an answer that arrived later would answer the wrong request. +an answer whose body does not decode, and a message that only the host program +sends end the client, and its later calls fail. A receive that times out +doesn't: it keeps what arrived. A request ends the client on any failure after +the framing accepted it, a timeout included, because an answer that arrived +later would answer the wrong request. ## VMs @@ -118,26 +266,33 @@ The host program runs the guest agent in VMs of NVX's artifacts, through NVX's supervisor, `nvx::host::supervisor`, each in a sandbox store of `nvx::host::store`. -### The guest agent in place of the managed agent +### The guest agent in place of NVX's agents NVX's initramfs hands a VM of the managed lifecycle to NVX's managed agent: its `/init` runs `exec /sbin/nvx-managed-agent /dev/hvc2 - nvx-managed UID GID NAME HOME` as -PID 1. `initramfs::write` copies that initramfs, and appends to the copy a -second archive, in the `newc` format, which Linux unpacks over the first: +PID 1. A VM with a layered root, whose kernel command line has +`nvx_sandbox=1`, goes to NVX's init agent instead, which assembles the root: +`/init` runs `exec /sbin/nvx-init-agent`, without arguments. +`initramfs::write` copies that initramfs, and appends to the copy a second +archive, in the `newc` format, which Linux unpacks over the first: - the guest agent, at `/sbin/runtime-adapter-guest`; - a script, `/sbin/runtime-adapter-init`, that runs `runtime-adapter-guest init "$1"`, with the control tty, the first of the arguments that `/init` passes the managed agent; -- a symbolic link from `/sbin/nvx-managed-agent` to the script. Linux removes a - file whose type an archive changes before it creates the new one, so the - link replaces the managed agent without writing to its file, which could - have other names. - -The second archive starts at a multiple of 4 bytes, as Linux requires of an -uncompressed archive, after zeros that Linux skips. NVX's own images don't -change. +- a symbolic link from `/sbin/nvx-managed-agent` to that script; +- a script, `/sbin/runtime-adapter-layered`, that runs + `runtime-adapter-guest init /dev/hvc2 layered`, which names the control tty + itself, since `/init` passes NVX's init agent none; +- a symbolic link from `/sbin/nvx-init-agent` to that script, so that NVX's + init agent never assembles the root in the adapter's VMs. + +Linux removes a file whose type an archive changes before it creates the new +one, so each link replaces NVX's agent without writing to its file, which could +have other names. The second archive starts at a multiple of 4 bytes, as Linux +requires of an uncompressed archive, after zeros that Linux skips. NVX's own +images don't change. ### The runtime identity @@ -158,22 +313,71 @@ version unless the caller names another. A host program that ships another build of the guest agent runs another runtime, and finds a VM of this one mismatched. +### Layered roots + +`layered::LayeredLaunch` plans a VM with a layered root with NVX's typed plan +of a cold layered launch, `nvx::host::sandbox::SandboxPlan`, with NVX's default +sandbox settings, and checks it with `SandboxPlan::validated`. The plan names +the images by absolute paths, since OpenVMM runs in the sandbox's store. The +VM's kernel command line is the plan's, without parameters of the caller: +`nvx_sandbox=1`, the image's token, such as +`nvx_layer=distro,0xd0003000,erofs:UUID`, the upper's, such as +`nvx_scratch=0xd0006000,ext4`, and those of the settings, which the worker +ignores. OpenVMM's arguments are those that NVX's own sandbox start passes: +the plan's, which attach the image and the scratch as block devices, then the +managed lifecycle in one process, NVX's default memory, the kernel, the +initramfs, that command line, no virtio console, the control console with the +launch capability on standard input, and the outcome report. + +The boot tokens carry only what NVX's boot contract defines. The launch sends +the rest in the [root configuration](#root-configurations), once it attaches: +the defaults, with the runtime directory that the caller names, as it is, so +that the guest checks it, and the size of each image in sectors, if it is a +whole number of them, which the guest checks against its block device. + +The launch records what it launched in the sandbox's store, beside NVX's +runtime records, in `runtime-adapter.json`, before it spawns the VMM: + +```json +{"format": 1, "mode": "layered", "kernelCommandLine": "nvx_sandbox=1 ...", + "plan": PLAN, "rootConfiguration": "version=1\nruntime=/run/runtime-adapter\n..."} +``` + +`mode` is `layered`, or `blockless` for a VM without a layered root, whose +`plan` and `rootConfiguration` are `null`. `plan` is the persisted form of the +VM's `SandboxPlan`, and `rootConfiguration` the root configuration that the +launch sends, whether or not the guest accepts it. + ### Commands `runtime-adapter-host` takes NVX's artifacts from the directory that `NVX_ARTIFACTS_DIR` names. Each command takes the guest agent's build from `--guest-build BUILD`, the crate's version by default: -- `launch STORE --guest-agent PATH [--hypervisor NAME]` launches a VM in the - sandbox store STORE, which it creates if needed, with the guest agent PATH, a - static `x86_64-unknown-linux-musl` build of `runtime-adapter-guest`, on the - hypervisor NAME, or else on the one that +- `launch STORE --guest-agent PATH [--hypervisor NAME] [LAYERED]` launches a VM + in the sandbox store STORE, which it creates if needed, with the guest agent + PATH, a static `x86_64-unknown-linux-musl` build of `runtime-adapter-guest`, + on the hypervisor NAME, or else on the one that `nvx::Hypervisor::from_env_or_default` selects. Once the launch has checked - that no VM of the sandbox runs, it writes the VM's initramfs into the store. - It connects to the VM's control endpoint with `nvx::host::session::connect`, - attaches with the launch's capability, exchanges a `Ping` and the longest - `Echo` with the guest agent, commits the launch, and exits, leaving the VM - running. A launch that fails ends its VM. + that no VM of the sandbox runs, it writes the VM's initramfs and its + [record](#layered-roots) into the store. It connects to the VM's control + endpoint with `nvx::host::session::connect`, attaches with the launch's + capability, exchanges a `Ping` and the longest `Echo` with the guest agent, + commits the launch, and exits, leaving the VM running. A launch that fails + ends its VM. + + LAYERED, + `--distro FORMAT:PATH [--distro-identity UUID] --upper (scratch:PATH | ram) [--runtime-dir DIR]`, + gives the VM a [layered root](#layered-roots). Its image is the file PATH, an + EROFS file system, `erofs`, or a GPT disk, `gpt`, of which the guest expects + the identity UUID, if given. Its upper is the ext4 scratch image PATH, or + RAM. The guest assembles the root below its runtime directory DIR, + `/run/runtime-adapter` by default. Once attached, the launch sends the root + configuration, and exchanges the `Ping` and the `Echo` once the guest + assembled the root. If the guest reports that a stage of the root failed + instead, the launch waits for the VM to end, abandons the launch, and checks + that the VM ended with the stage's status. The worker assembles only an + EROFS image over a scratch, and fails the other layouts at the layout stage. - `reattach STORE` takes the sandbox's lock, and reattaches to its VM, which another process launched, and which it holds, although the VM is not its child, through its verified process identity. It reads the capability from @@ -184,19 +388,34 @@ mismatched. ends the VM, is the stop's graceful step. It waits for the VM to exit, and checks the VM's outcome report: a success, with status 0 and a complete teardown. A VM whose guest agent does not confirm the stop is terminated, and - the stop fails. - -`reattach` and `stop` find a VM that another runtime identity launched -mismatched, and then signal nothing: the VM keeps running. The commands print -what they find as `KEY=VALUE` lines: `reattached` (`live`, `mismatched`, -`exited`, or `indeterminate`), `differences` (the parts of the runtime identity -in which a mismatched VM differs), `pid` (the VMM's process ID), `instance` and -`epoch` (the identity of the session that they attached to), `stopped` -(`graceful`, `forced`, or `timed-out`), `outcome` (such as `success 0`), and -`teardown` (`complete` or `incomplete`). They exit with 0 once they did what -they were asked, with 3 when they find the VM mismatched, and with 1, after a -diagnostic on standard error, when they fail. Without arguments, the program -reports the contract that it was built against. + the stop fails. The worker of a layered root tears the root down before it + confirms, and reports a teardown that failed first: the stop then checks + that the VM ended with the teardown's status instead. +- `probe STORE PROBE...` reattaches and attaches as `reattach` does, then runs + each PROBE in the VM's layered root, in order, and prints the facts that it + found. A PROBE is the text of a [probe](#probes): its name, then its + arguments, each after one space. A probe that fails fails the command. + +`reattach`, `stop`, and `probe` find a VM that another runtime identity +launched mismatched, and then signal nothing: the VM keeps running. The +commands print what they find as `KEY=VALUE` lines: `reattached` (`live`, +`mismatched`, `exited`, or `indeterminate`), `differences` (the parts of the +runtime identity in which a mismatched VM differs), `pid` (the VMM's process +ID), `instance` and `epoch` (the identity of the session that they attached +to), `root` (`ready` or `failed`: how the guest answered the root +configuration, or `failed` when a stop finds that the root's teardown failed), +`stage` and `problem` (the stage of the root that failed, such as `assembly`, +and the problem), `PROBE.KEY` (the fact KEY that the probe named PROBE found), +`stopped` (`graceful`, `forced`, or `timed-out`), `outcome` (such as +`success 0`, or `guest-exit 14` once the assembly failed), and `teardown` +(`complete` or `incomplete`: whether the VMM's teardown of the VM completed, +which the outcome report records apart from the root's). Values escape `\`, newlines, and +carriage returns as `\\`, `\n`, and `\r`. The commands exit with 0 once they +did what they were asked, with 3 when they find the VM mismatched, with 4 when +the guest reported that a stage of the root failed, and the VM ended with the +stage's status and a complete teardown, and with 1, after a diagnostic on +standard error, when they fail. Without arguments, the program reports the +contract that it was built against. ## Tests @@ -215,9 +434,14 @@ independent adapter's does: guest binary is a static executable: it has no program interpreter, needs no shared library, and keeps its stack non-executable. -The unit tests of `frame` run in every configuration. They cover each kind's -encoding, messages in pieces of any size and split at every byte, and the -headers that the framing refuses before their body arrives. +The unit tests of `frame`, `root`, and `probe` run in every configuration. They +cover each kind's encoding, messages in pieces of any size and split at every +byte, the headers that the framing refuses before their body arrives, and the +root configurations, failures, probes, and facts that decode, and those that +don't. In the guest-only configuration, the unit tests of `worker::probes` +cover what the probes read: the mount table with its escapes, the kernel's +records of a process, and `os-release`, and the probes that the worker +refuses. `tests/guest_session.rs` checks, in the guest-only configuration, that the guest agent serves the control session with the adapter's own framing. A test @@ -229,7 +453,9 @@ in the middle of a message, with the next epoch of the instance and with a new instance, streams that end at and inside a record, and the records and messages that the guest agent refuses: sequence, epoch, and instance errors, stale and identity-less resets, record lengths, bootstrap identity, ERROR -payloads, and malformed headers. +payloads, and malformed headers. It also checks that the guest agent of a VM +without a layered root answers probes with a failure, and refuses root +configurations. `tests/host_session.rs` checks, in the host-only configuration, that the host program's client attaches to the control session and exchanges the adapter's @@ -257,19 +483,42 @@ Windows. They cover: body arrives, and messages that only the host program sends; and a message beyond the framing's limit, which it refuses to send; - the answers that its requests check, and a request whose answer arrives too - late. + late; +- a layered root: the configuration that the client sends, the guest's + answers, ready or the stage that failed, probes and their facts, a stop + that learns that the teardown failed, and answers whose bodies do not + decode, which end the client. Through a relay between its test broker and a pseudo-terminal, `tests/guest_session.rs` also checks that `guest::serve_terminal` and `runtime-adapter-guest init` serve a tty in raw mode, with an `Echo` of every byte value, and that `init` fails at once on a path that is no tty. +`tests/guest_session.rs` also runs the worker against a fake root, behind its +test broker: a root that the worker assembles, probes, and tears down on a +stop, across a reset of the session; the failure of each stage, which the +worker reports before it ends with the stage's status, without the operations +of the later stages; a teardown that fails; before the root, the configuration +and the `Stop` that it accepts, and the messages that it refuses; and a second +configuration and a broker that closes the stream, which leave the root that +it assembled to its operations. Through the binary, it checks that +`runtime-adapter-guest init TTY layered`, which is not PID 1 there, starts the +worker, which fails at the layout stage on a kernel command line without a +layered root, and ends with its status, and checks `worker`, `idle`, and the +usage errors. Each guest binary that a test starts runs in a process group of +its own, with the processes that it starts, and keeps its process ID, and with +it the group's, until the test has killed the group. A binary that exits but +leaves processes of its group running fails its test. A test that fails, or +gives up on the binary, kills the whole group, even once the binary exited, so +that none of them outlives the test or holds its output open, which tests check +too. + `tests/initramfs.rs` and `tests/runtime.rs` check, in the host-only configuration, on Linux and Windows: -- the archive that puts the guest agent in place of the managed agent: its - entries, their owners and modes, and the padding of its headers and data, - for guest agents of every length modulo 4; +- the archive that puts the guest agent in place of NVX's agents: its entries, + their owners and modes, both launchers, and the padding of its headers and + data, for guest agents of every length modulo 4; - the copy of NVX's initramfs, which stays as it is, of every length modulo 4, with the archive after it at a multiple of 4 bytes, and the files that a failure names; @@ -278,6 +527,13 @@ configuration, on Linux and Windows: in the initramfs too, and missing install metadata, which fails as NVX reports it. +`tests/layered_launch.rs` checks, in the host-only configuration, on Linux and +Windows, the plan of a layered launch: its block devices and boot tokens, the +OpenVMM arguments of NVX's sandbox start, and the sizes of the images; the +options that it passes on as they are, for the guest to check; the record of +a launch, with and without a layered root; and the command lines that the host +program refuses, each with status 2. + `tests/vm_restart.rs` checks the restart of an orchestrator, on a real hypervisor, in the host-only configuration: @@ -290,13 +546,55 @@ hypervisor, in the host-only configuration: In another test, a host program that ships another build of the guest agent finds the VM mismatched in its initramfs, and signals nothing: the VM runs on, under the same runtime record, until the host program of the right runtime -stops it. A test that fails ends the VM that it left running, and keeps the -sandbox's directory, whose path it prints, unless it can confirm that the VM -ended. The tests boot VMs, so they are ignored. They read the hypervisor from -`NVX_HYPERVISOR`, through `nvx::Hypervisor::from_env_or_default`, NVX's -artifacts, with their install metadata, from `NVX_ARTIFACTS_DIR`, and the guest -agent from `RUNTIME_ADAPTER_GUEST`. In a checkout where -`python3 scripts/nvx.py download` installed NVX's artifacts, run them in this +stops it. + +### Layered scenarios + +`tests/vm_layered.rs` launches VMs with a layered root on a real hypervisor, in +the host-only configuration, through the host program's commands, and checks +that the guest's worker, outside PID 1, assembles the root, or reports the +stage that failed, and that the VM still ends cleanly. Each scenario is a row +of the table that its `scenarios!` macro declares, and an ignored test of its +own: + +| Scenario | Root | Expects | +| --- | --- | --- | +| `erofs_over_scratch` | NVX's Ubuntu EROFS image, with its own identity, over a copy of the scratch template | A root that the worker assembled: the image, read-only, under the overlay, whose upper and work directories are on the scratch, each mounted once below the runtime directory; the merged root's contents, with a file that reaches the upper and not the image; the agent's and the workload's cgroups, with their resource values; a worker that is PID 1's child, and PID 1's unrelated child, untouched; and a teardown that unmounts the scratch cleanly | +| `other_identity_fails_the_assembly` | The image, of which the launch expects another identity | The assembly's failure, which the launch reports, and a VM that ends with 14 before the guest mounts the scratch | +| `invalid_configuration_fails_its_stage` | A relative runtime directory | The configuration's failure, which the launch reports, and a VM that ends with 11 | +| `busy_root_fails_the_teardown` | The image over the scratch, which the `hold` probe keeps open | The teardown's failure, which the stop reports, and a VM that ends with 15, leaving the scratch mounted | + +A row names the root, the image, the identity that the launch expects of it, +the upper, and the runtime directory, and what the scenario expects: a root +that the guest assembles, with a check that probes it, a teardown that fails +after a check, or the stage that fails, with its problem. Each scenario checks +the launch's record, and whether the VM left the scratch's journal to recover, +which tells whether the guest unmounted the scratch. + +The scenarios use the fixtures of NVX's own layered scenarios, from NVX's +artifacts: the Ubuntu EROFS image, `ubuntu-distro.erofs`, its manifest, which +holds its UUID, and the scratch template, `ubuntu-smoke-scratch.ext4`, an +empty ext4 file system that the guest-artifact build makes, of which each +scenario boots a copy. Downloaded packages do not include the template, so make +it beside the image as the guest-artifact build does: + +```bash +truncate --size 64M ../../build/ubuntu-smoke-scratch.ext4 +mkfs.ext4 -q -F -m 0 -U 4f3c8bf7-1959-47b4-8811-62b17bbddaf4 \ + -E lazy_itable_init=0,lazy_journal_init=0 ../../build/ubuntu-smoke-scratch.ext4 +``` + +### Running the VM tests + +The VM tests, `tests/vm_restart.rs` and `tests/vm_layered.rs`, share their +sandboxes through `tests/common`. A test that fails ends the VM that it left +running, and keeps the sandbox's directory, whose path it prints, unless it can +confirm that the VM ended. The tests boot VMs, so they are ignored. They read +the hypervisor from `NVX_HYPERVISOR`, through +`nvx::Hypervisor::from_env_or_default`, NVX's artifacts, with their install +metadata and the fixtures, from `NVX_ARTIFACTS_DIR`, and the guest agent from +`RUNTIME_ADAPTER_GUEST`. In a checkout where `python3 scripts/nvx.py download` +installed NVX's artifacts, and with the scratch template, run them in this directory with: ```bash @@ -336,7 +634,8 @@ NVX microVM tests. Their host-only configuration on each backend, in the jobs that run the guest and sandbox steps, and runs its ignored tests, the adapter's VM scenarios, with `NVX_HYPERVISOR` naming the backend, `NVX_ARTIFACTS_DIR` the checkout, into -which the job downloaded NVX's artifacts and their provenance, and -`RUNTIME_ADAPTER_GUEST` the guest agent. The Linux jobs build the guest agent -first, and the Windows job takes the one that CI's `distro-artifacts` job -builds on Linux. +which the job downloaded NVX's artifacts, their provenance, and the fixtures +of the layered scenarios with the guest artifacts, as NVX's own layered +scenarios take them, and `RUNTIME_ADAPTER_GUEST` the guest agent. The Linux +jobs build the guest agent first, and the Windows job takes the one that CI's +`distro-artifacts` job builds on Linux. diff --git a/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs b/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs index 01bf77985..fc4e4d655 100644 --- a/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs +++ b/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs @@ -10,36 +10,72 @@ //! mode itself, in place of NVX's managed agent, whose initramfs runs it on the VM's control tty //! as PID 1; see `runtime_adapter::initramfs`. It then ends with 0 once the host stopped it, and //! with 1, after a diagnostic on standard error, once serving failed or the broker closed the -//! tty. As PID 1, which must not exit, it ends the VM with that status instead, through NVX's -//! `/sbin/nvx-exit`, as NVX's managed agent does. +//! tty. +//! - `runtime-adapter-guest init TTY layered` runs PID 1 of a VM with a layered root, in place of +//! NVX's init agent: it starts an unrelated child, `runtime-adapter-guest idle`, and the worker, +//! `runtime-adapter-guest worker TTY SIBLING`, collects the status of every child until the +//! worker ends, and ends with the worker's status; see `runtime_adapter::worker`. +//! - `runtime-adapter-guest worker TTY [SIBLING]` runs the worker on the tty TTY: it assembles the +//! layered root that the host configures, and serves the host program until the host stops it. +//! SIBLING is the process ID of PID 1's unrelated child, which its probes check. It exits with 0 +//! once the host stopped it, with 11 to 15 once a stage of the root failed, and with 1 once its +//! session failed or the broker closed the tty, each but the first after a diagnostic on +//! standard error. +//! - `runtime-adapter-guest idle` sleeps until a signal ends it. //! -//! Anything else is a usage error, with status 2. +//! As PID 1, which must not exit, `init` ends the VM with its status instead, through NVX's +//! `/sbin/nvx-exit`, as NVX's managed agent does. Anything else is a usage error, with status 2. use std::process::ExitCode; #[cfg(target_os = "linux")] fn main() -> ExitCode { + use runtime_adapter::{ + GUEST_IDLE_COMMAND, GUEST_INIT_COMMAND, GUEST_LAYERED_MODE, GUEST_SERVE_COMMAND, + GUEST_WORKER_COMMAND, + }; + use std::path::Path; + let arguments = std::env::args_os().skip(1).collect::>(); match arguments.as_slice() { [] => { println!("runtime-adapter-guest: {}", runtime_adapter::contract()); ExitCode::SUCCESS } - [command] if command == runtime_adapter::GUEST_SERVE_COMMAND => serve(), - [command, tty] if command == runtime_adapter::GUEST_INIT_COMMAND => { - init(std::path::Path::new(tty)) + [command] if command == GUEST_SERVE_COMMAND => serve(), + [command, tty] if command == GUEST_INIT_COMMAND => init(Path::new(tty)), + [command, tty, mode] if command == GUEST_INIT_COMMAND && mode == GUEST_LAYERED_MODE => { + end(runtime_adapter::worker::supervise(Path::new(tty))) } - _ => { - eprintln!( - "usage: runtime-adapter-guest [{} | {} TTY]", - runtime_adapter::GUEST_SERVE_COMMAND, - runtime_adapter::GUEST_INIT_COMMAND, - ); - ExitCode::from(2) + [command, tty, sibling @ ..] if command == GUEST_WORKER_COMMAND && sibling.len() < 2 => { + match sibling + .first() + .map(|id| id.to_str().and_then(|id| id.parse().ok())) + { + Some(None) => usage(), + Some(Some(sibling)) => worker(Path::new(tty), Some(sibling)), + None => worker(Path::new(tty), None), + } } + [command] if command == GUEST_IDLE_COMMAND => runtime_adapter::worker::idle(), + _ => usage(), } } +/// Reports a usage error. +#[cfg(target_os = "linux")] +fn usage() -> ExitCode { + eprintln!( + "usage: runtime-adapter-guest [{} | {} TTY [{}] | {} TTY [SIBLING] | {}]", + runtime_adapter::GUEST_SERVE_COMMAND, + runtime_adapter::GUEST_INIT_COMMAND, + runtime_adapter::GUEST_LAYERED_MODE, + runtime_adapter::GUEST_WORKER_COMMAND, + runtime_adapter::GUEST_IDLE_COMMAND, + ); + ExitCode::from(2) +} + /// Serves the session on standard input and output. #[cfg(target_os = "linux")] fn serve() -> ExitCode { @@ -61,7 +97,7 @@ fn serve() -> ExitCode { const NVX_EXIT: &str = "/sbin/nvx-exit"; /// Serves the session on the tty `tty` in place of NVX's managed agent, then ends with the status -/// of serving: as PID 1, by ending the VM through [`NVX_EXIT`]. +/// of serving. #[cfg(target_os = "linux")] fn init(tty: &std::path::Path) -> ExitCode { use runtime_adapter::guest::{self, Served}; @@ -77,6 +113,43 @@ fn init(tty: &std::path::Path) -> ExitCode { 1 } }; + end(status) +} + +/// Runs the worker on the tty `tty`, with the process ID of PID 1's unrelated child, if any, and +/// ends with its status. +#[cfg(target_os = "linux")] +fn worker(tty: &std::path::Path, sibling: Option) -> ExitCode { + use runtime_adapter::guest; + use runtime_adapter::worker::{self, LibraryRoot, Worked}; + + let mut operations = LibraryRoot::new(sibling); + let worked = guest::on_terminal(tty, |terminal| worker::work(terminal, &mut operations)); + let status = match worked { + Ok(worked) => { + match &worked { + Worked::Stopped => {} + Worked::Failed(failure) => eprintln!("runtime-adapter-guest: {failure}"), + Worked::Closed => { + eprintln!("runtime-adapter-guest: the broker closed {}", tty.display()); + } + } + worked.status() + } + Err(error) => { + eprintln!("runtime-adapter-guest: {error}"); + worker::SESSION_STATUS + } + }; + // Closes what the probes held, then unmounts a root that the worker still holds, ignoring + // failures, before the worker exits. + drop(operations); + ExitCode::from(status) +} + +/// Ends with `status`: as PID 1, by ending the VM through [`NVX_EXIT`]. +#[cfg(target_os = "linux")] +fn end(status: u8) -> ExitCode { if std::process::id() != 1 { return ExitCode::from(status); } diff --git a/tests/runtime-adapter/src/bin/runtime-adapter-host.rs b/tests/runtime-adapter/src/bin/runtime-adapter-host.rs index 84674ce9d..4ab596f3a 100644 --- a/tests/runtime-adapter/src/bin/runtime-adapter-host.rs +++ b/tests/runtime-adapter/src/bin/runtime-adapter-host.rs @@ -2,19 +2,35 @@ //! //! - `runtime-adapter-host` reports the contract that it was built against and whether `nvx` //! supports this host. -//! - `runtime-adapter-host launch STORE --guest-agent PATH [--hypervisor NAME]` launches a VM in -//! the sandbox store STORE, which it creates if needed, through NVX's supervisor, with the guest -//! agent PATH, a static `x86_64-unknown-linux-musl` build of `runtime-adapter-guest`, in place of -//! NVX's managed agent; see `runtime_adapter::initramfs`. It attaches to the VM's control +//! - `runtime-adapter-host launch STORE --guest-agent PATH [--hypervisor NAME] [LAYERED]` launches +//! a VM in the sandbox store STORE, which it creates if needed, through NVX's supervisor, with +//! the guest agent PATH, a static `x86_64-unknown-linux-musl` build of `runtime-adapter-guest`, +//! in place of NVX's agents; see `runtime_adapter::initramfs`. It records what it launches in +//! STORE; see `runtime_adapter::layered::LaunchRecord`. It attaches to the VM's control //! session, exchanges a `Ping` and the longest `Echo` with the guest agent, commits the launch, //! and exits, leaving the VM running. The VM runs on the hypervisor NAME, `kvm`, `mshv`, or //! `whp`, or else on the one that `nvx::Hypervisor::from_env_or_default` selects. +//! +//! LAYERED, `--distro FORMAT:PATH [--distro-identity UUID] --upper (scratch:PATH | ram) +//! [--runtime-dir DIR]`, gives the VM a layered root, which the guest agent's worker assembles; +//! see `runtime_adapter::layered` and `runtime_adapter::worker`. Its image is the raw file PATH, +//! an EROFS filesystem, `erofs`, or a GPT disk, `gpt`, of which the guest expects the identity +//! UUID, if given. Its upper is the ext4 scratch image PATH, or RAM. The guest assembles the +//! root below its runtime directory DIR, `/run/runtime-adapter` by default. Once attached, the +//! launch sends the root configuration, and exchanges the `Ping` and the `Echo` once the guest +//! assembled the root. If the guest reports that a stage of the root failed instead, the launch +//! waits for the VM to end, abandons it, and checks that the VM ended with the stage's status. //! - `runtime-adapter-host reattach STORE` reattaches to the VM of STORE, which an earlier process //! launched and which it holds although it is not its child, attaches a new session to it, in //! the session's next epoch, and exchanges a `Ping`, leaving the VM running. //! - `runtime-adapter-host stop STORE` reattaches and attaches as `reattach` does, then stops the -//! VM: it asks the guest agent to stop, waits for the VM to exit, and checks the VM's outcome -//! report. A VM whose guest agent does not confirm the stop is terminated. +//! VM: it asks the guest agent to stop, which first tears down a layered root, waits for the VM +//! to exit, and checks the VM's outcome report. A VM whose guest agent does not confirm the stop +//! is terminated. +//! - `runtime-adapter-host probe STORE PROBE...` reattaches and attaches as `reattach` does, then +//! runs each PROBE in the VM's layered root, and prints the facts that it found. A PROBE is the +//! text of a probe: its name, then its arguments, each after one space; see +//! `runtime_adapter::probe` and `runtime_adapter::worker::probes`. //! //! Each command takes NVX's artifacts from the directory that `NVX_ARTIFACTS_DIR` names, a //! repository checkout or an extracted release, and the VM's runtime identity from NVX's identity @@ -30,13 +46,21 @@ //! - `pid`: the VMM's process ID; //! - `instance` and `epoch`: the broker instance, in hexadecimal, and the epoch of the session //! that the command attached to; +//! - `root`: how the guest answered the root configuration: `ready` or `failed`, or `failed` +//! once a stop finds that the root's teardown failed; +//! - `stage` and `problem`: the stage of the root that failed, such as `assembly`, and the +//! problem; +//! - `PROBE.KEY`: the fact KEY that the probe named PROBE found; //! - `stopped`: how the stop ended: `graceful`, `forced`, or `timed-out`; //! - `outcome`: the category and status code of the VM's outcome report, such as `success 0`; //! - `teardown`: whether the VM's teardown completed: `complete` or `incomplete`. //! +//! Values escape `\`, newlines, and carriage returns as `\\`, `\n`, and `\r`. +//! //! They exit with 0 once they did what they were asked, with 3 when reattach finds the VM -//! mismatched, and with 1, after a diagnostic on standard error, when they fail. Anything else is -//! a usage error, with status 2. +//! mismatched, with 4 when the guest reported that a stage of the root failed, and the VM ended +//! with the stage's status and a complete teardown, and with 1, after a diagnostic on standard +//! error, when they fail. Anything else is a usage error, with status 2. use std::env; use std::error::Error; @@ -51,21 +75,28 @@ use nvx::host::instance::generate_capability; use nvx::host::launch::LaunchSpec; use nvx::host::report; use nvx::host::session; -use nvx::host::store::{IdentityPart, SandboxStore}; -use nvx::host::supervisor::{self, Reattached, StopOutcome, Vmm, VmmCommand}; +use nvx::host::store::{IdentityPart, LockedStore, SandboxStore}; +use nvx::host::supervisor::{self, Reattached, Starting, StopOutcome, Vmm, VmmCommand}; use nvx::protocol::CAPABILITY_LEN; +use nvx::protocol::root::{ImageFormat, RootImage}; +use nvx::uuid::Uuid; use runtime_adapter::frame::MAX_BODY; use runtime_adapter::host::Client; use runtime_adapter::initramfs; +use runtime_adapter::layered::{LaunchRecord, LayeredLaunch}; +use runtime_adapter::probe::{Probe, ProbeReply, escape}; +use runtime_adapter::root::{RootFailure, RootReply}; use runtime_adapter::runtime::{GUEST_BUILD, Runtime}; /// How long a command waits for the sandbox's lock. const LOCK_TIMEOUT: Duration = Duration::from_secs(10); -/// How long a launch waits for the VM to boot and its guest agent to answer. +/// How long a launch waits for the VM to boot, its guest agent to assemble a layered root, and to +/// answer. const START_TIMEOUT: Duration = Duration::from_secs(60); -/// How long reattach and stop wait to attach a session and for the guest agent's answer. +/// How long reattach, stop, and probe wait to attach a session and for the guest agent's answers. const SESSION_TIMEOUT: Duration = Duration::from_secs(30); -/// How long a stop waits for the guest agent's confirmation and for the VM to exit. +/// How long a stop waits for the guest agent's confirmation and for the VM to exit, and how long +/// a launch whose root failed waits for the VM to exit. const STOP_TIMEOUT: Duration = Duration::from_secs(30); /// How long a stop waits for a VM that it terminated to exit. const FORCED_TIMEOUT: Duration = Duration::from_secs(10); @@ -74,12 +105,18 @@ const FORCED_TIMEOUT: Duration = Duration::from_secs(10); const INITRAMFS_NAME: &str = "runtime-adapter.cpio.gz"; /// The exit status of a command that finds the VM mismatched. const MISMATCHED: u8 = 3; +/// The exit status of a command whose guest reported that a stage of the root failed, once the VM +/// ended with the stage's status. +const ROOT_FAILED: u8 = 4; /// The command lines that the program accepts. const USAGE: &str = concat!( "usage: runtime-adapter-host\n", " runtime-adapter-host launch STORE --guest-agent PATH [--hypervisor NAME]", " [--guest-build BUILD]\n", - " runtime-adapter-host (reattach | stop) STORE [--guest-build BUILD]", + " [--distro FORMAT:PATH [--distro-identity UUID]", + " --upper (scratch:PATH | ram) [--runtime-dir DIR]]\n", + " runtime-adapter-host (reattach | stop) STORE [--guest-build BUILD]\n", + " runtime-adapter-host probe STORE PROBE... [--guest-build BUILD]", ); /// What the program was asked to do. @@ -96,6 +133,8 @@ enum Command { hypervisor: Option, /// The guest agent's build. guest_build: String, + /// The VM's layered root, if it has one. + layered: Option, }, /// Reattach to the VM, and stop it if asked. Reattach { @@ -106,6 +145,46 @@ enum Command { /// Whether to stop the VM. stop: bool, }, + /// Reattach to the VM, and run probes in its layered root. + Probe { + /// The sandbox's store. + store: PathBuf, + /// The guest agent's build. + guest_build: String, + /// The probes, in order. + probes: Vec, + }, +} + +/// The layered root of a launch, as its options give it. +struct Layered { + /// The image. + distro: PathBuf, + /// The image's format, and the identity that the guest expects of it, if any. + image: RootImage, + /// The ext4 scratch image of the upper, or `None` for an upper in RAM. + scratch: Option, + /// The guest's runtime directory, unless the default. + runtime: Option, +} + +/// The options of a command. +#[derive(Default)] +struct Options { + /// `--guest-agent`. + guest_agent: Option, + /// `--hypervisor`. + hypervisor: Option, + /// `--guest-build`. + guest_build: Option, + /// `--distro`: the image's format and its file. + distro: Option<(ImageFormat, PathBuf)>, + /// `--distro-identity`. + identity: Option, + /// `--upper`: the scratch image, or `None` for RAM. + upper: Option>, + /// `--runtime-dir`. + runtime: Option, } fn main() -> ExitCode { @@ -130,12 +209,24 @@ fn main() -> ExitCode { guest_agent, hypervisor, guest_build, - } => launch(&store, &guest_agent, hypervisor, &guest_build), + layered, + } => launch( + &store, + &guest_agent, + hypervisor, + &guest_build, + layered.as_ref(), + ), Command::Reattach { store, guest_build, stop, } => reattach(&store, &guest_build, stop), + Command::Probe { + store, + guest_build, + probes, + } => probe(&store, &guest_build, &probes), }; result.unwrap_or_else(|problem| { eprintln!("runtime-adapter-host: {problem}"); @@ -150,7 +241,7 @@ fn parse(arguments: Vec) -> Result { return Ok(Command::Report); }; let name = match name.to_str() { - Some(name @ ("launch" | "reattach" | "stop")) => name.to_owned(), + Some(name @ ("launch" | "reattach" | "stop" | "probe")) => name.to_owned(), _ => return Err(format!("unknown command {name:?}")), }; let store = PathBuf::from( @@ -158,59 +249,139 @@ fn parse(arguments: Vec) -> Result { .next() .ok_or_else(|| format!("{name} needs a sandbox store"))?, ); - let (mut guest_agent, mut hypervisor, mut guest_build) = (None, None, None); - while let Some(option) = arguments.next() { + let mut options = Options::default(); + let mut probes = Vec::new(); + while let Some(argument) = arguments.next() { + let Some(option) = argument.to_str().filter(|option| option.starts_with("--")) else { + if name != "probe" { + return Err(format!("unexpected argument {argument:?} of {name}")); + } + let text = argument + .to_str() + .ok_or_else(|| format!("the probe {argument:?} is not Unicode"))?; + probes.push(Probe::parse(text).map_err(|error| error.problem().to_owned())?); + continue; + }; let value = arguments .next() .ok_or_else(|| format!("{option:?} needs a value"))?; - match (name.as_str(), option.to_str()) { - ("launch", Some("--guest-agent")) if guest_agent.is_none() => { - guest_agent = Some(PathBuf::from(value)); - } - ("launch", Some("--hypervisor")) if hypervisor.is_none() => { - let value = value - .to_str() - .ok_or_else(|| format!("unknown hypervisor {value:?}"))?; - hypervisor = Some(value.parse().map_err(|error| describe(&error))?); - } - (_, Some("--guest-build")) if guest_build.is_none() => { - let value = value - .into_string() - .map_err(|value| format!("the guest build {value:?} is not Unicode"))?; - guest_build = Some(value); - } - _ => return Err(format!("unexpected option {option:?} of {name}")), - } + parse_option(&mut options, &name, option, value)?; } - let guest_build = guest_build.unwrap_or_else(|| GUEST_BUILD.to_owned()); + let guest_build = options + .guest_build + .unwrap_or_else(|| GUEST_BUILD.to_owned()); Ok(match name.as_str() { - "launch" => Command::Launch { - store, - guest_agent: guest_agent.ok_or("launch needs --guest-agent")?, - hypervisor, - guest_build, - }, - "reattach" => Command::Reattach { + "launch" => { + let layered = match (options.distro, options.upper) { + (Some((format, distro)), Some(scratch)) => Some(Layered { + distro, + image: RootImage { + format, + identity: options.identity, + }, + scratch, + runtime: options.runtime, + }), + (Some(_), None) => return Err("--distro needs --upper".to_owned()), + (None, _) if options.identity.is_some() => { + return Err("--distro-identity needs --distro".to_owned()); + } + (None, Some(_)) => return Err("--upper needs --distro".to_owned()), + (None, None) if options.runtime.is_some() => { + return Err("--runtime-dir needs --distro".to_owned()); + } + (None, None) => None, + }; + Command::Launch { + store, + guest_agent: options.guest_agent.ok_or("launch needs --guest-agent")?, + hypervisor: options.hypervisor, + guest_build, + layered, + } + } + "probe" if probes.is_empty() => return Err("probe needs a probe".to_owned()), + "probe" => Command::Probe { store, guest_build, - stop: false, + probes, }, - _ => Command::Reattach { + stop => Command::Reattach { store, guest_build, - stop: true, + stop: stop == "stop", }, }) } +/// Parses `option` of the command `name`, with `value`, into `options`, or says why it is not one +/// of the command's options, or why its value is invalid. +fn parse_option( + options: &mut Options, + name: &str, + option: &str, + value: OsString, +) -> Result<(), String> { + let text = |value: OsString| { + value + .into_string() + .map_err(|value| format!("the value {value:?} of {option} is not Unicode")) + }; + match (name, option) { + ("launch", "--guest-agent") if options.guest_agent.is_none() => { + options.guest_agent = Some(PathBuf::from(value)); + } + ("launch", "--hypervisor") if options.hypervisor.is_none() => { + let hypervisor = text(value)?.parse().map_err(|error| describe(&error))?; + options.hypervisor = Some(hypervisor); + } + ("launch", "--distro") if options.distro.is_none() => { + let value = text(value)?; + let distro = value.split_once(':').and_then(|(format, path)| { + let format = ImageFormat::from_name(format)?; + (!path.is_empty()).then(|| (format, PathBuf::from(path))) + }); + options.distro = Some(distro.ok_or_else(|| { + format!("--distro takes FORMAT:PATH, with the format erofs or gpt, not {value:?}") + })?); + } + ("launch", "--distro-identity") if options.identity.is_none() => { + let value = text(value)?; + let identity = Uuid::parse(&value) + .ok_or_else(|| format!("--distro-identity takes a UUID, not {value:?}"))?; + options.identity = Some(identity); + } + ("launch", "--upper") if options.upper.is_none() => { + let value = text(value)?; + let upper = match value.split_once(':') { + _ if value == "ram" => Some(None), + Some(("scratch", path)) if !path.is_empty() => Some(Some(PathBuf::from(path))), + _ => None, + }; + options.upper = Some( + upper.ok_or_else(|| format!("--upper takes scratch:PATH or ram, not {value:?}"))?, + ); + } + ("launch", "--runtime-dir") if options.runtime.is_none() => { + options.runtime = Some(text(value)?); + } + (_, "--guest-build") if options.guest_build.is_none() => { + options.guest_build = Some(text(value)?); + } + _ => return Err(format!("unexpected option {option:?} of {name}")), + } + Ok(()) +} + /// Launches a VM of the guest agent `guest_agent` in the sandbox store `store`, on `hypervisor` /// or else the one that `nvx::Hypervisor::from_env_or_default` selects, with the guest agent's -/// build `guest_build`. +/// build `guest_build`, and with the layered root `layered`, if any. fn launch( store: &Path, guest_agent: &Path, hypervisor: Option, guest_build: &str, + layered: Option<&Layered>, ) -> Result { let hypervisor = match hypervisor { Some(hypervisor) => hypervisor, @@ -221,6 +392,17 @@ fn launch( .artifacts .probe() .map_err(|error| describe(&error))?; + let layered = layered + .map(|layered| { + LayeredLaunch::new( + &layered.distro, + layered.image, + layered.scratch.as_deref(), + layered.runtime.as_deref(), + ) + }) + .transpose() + .map_err(|error| describe(&error))?; let store = SandboxStore::create(store).map_err(|error| describe(&error))?; let mut locked = store .lock(Some(Instant::now() + LOCK_TIMEOUT)) @@ -242,7 +424,28 @@ fn launch( initramfs::write(&runtime.artifacts.initrd, guest_agent, &initramfs).map_err(|error| { nvx::Error::io("cannot write the initramfs of the guest agent").with_source(error) })?; - let arguments = spec.openvmm_arguments(endpoint, &store.outcome_path())?; + let outcome = store.outcome_path(); + let (arguments, kernel_command_line) = match &layered { + Some(layered) => ( + layered.openvmm_arguments( + hypervisor, + &spec.kernel, + &spec.initrd, + endpoint, + &outcome, + )?, + layered.kernel_command_line()?, + ), + None => ( + spec.openvmm_arguments(endpoint, &outcome)?, + spec.kernel_command_line()?, + ), + }; + let record = LaunchRecord { + kernel_command_line: &kernel_command_line, + layered: layered.as_ref(), + }; + record.write(store.dir())?; Ok(VmmCommand::new(&spec.openvmm, arguments, store.dir())) }) .map_err(|error| describe(&error))?; @@ -263,6 +466,24 @@ fn launch( ) .map_err(|error| failed(format!("cannot connect to the VM: {}", describe(&error))))?; let mut client = attach(transport, &capability, deadline).map_err(failed)?; + if let Some(layered) = &layered { + let reply = client + .configure_root(&layered.config, deadline) + .map_err(|error| { + failed(format!( + "the guest agent did not answer the root configuration: {}", + describe(&error) + )) + })?; + match reply { + RootReply::Ready => println!("root=ready"), + RootReply::Failed(failure) => { + print_failure(&failure); + drop(client); + return launch_failed(starting, &store, &failure).map_err(failed); + } + } + } client .ping(deadline) .and_then(|()| client.echo(&echo_body(), deadline)) @@ -276,6 +497,34 @@ fn launch( Ok(ExitCode::SUCCESS) } +/// Ends the launch `starting` of the sandbox `store`, whose guest reported `failure` and ends: it +/// waits for the VM to exit, abandons the launch, which terminates a VM that still runs, and +/// checks that the VM ended with the failed stage's status and a complete teardown. +fn launch_failed( + starting: Starting<'_>, + store: &SandboxStore, + failure: &RootFailure, +) -> Result { + let exited = starting + .process() + .wait(Instant::now() + STOP_TIMEOUT) + .map_err(|error| format!("cannot wait for the VM to exit: {error}"))?; + starting.abandon().map_err(|error| describe(&error))?; + println!("stopped={}", if exited { "graceful" } else { "forced" }); + if !exited { + return Err("the VM did not exit once its guest reported the failure".to_owned()); + } + check_outcome(store, "guest-exit", failure.stage().exit_status().into())?; + Ok(ExitCode::from(ROOT_FAILED)) +} + +/// Prints the root's `failure`. +fn print_failure(failure: &RootFailure) { + println!("root=failed"); + println!("stage={}", failure.stage()); + println!("problem={}", escape(failure.problem())); +} + /// Returns the body of the launch's `Echo`: the longest that the framing allows, of every byte /// value, which spans several of the session's records, each way. fn echo_body() -> Vec { @@ -290,31 +539,10 @@ fn reattach(store: &Path, guest_build: &str, stop: bool) -> Result { - println!("reattached=live"); - vmm - } - Reattached::Mismatched { vmm, differences } => { - println!("reattached=mismatched"); - println!("differences={}", names(&differences)); - println!("pid={}", vmm.record().process().pid); - return Ok(ExitCode::from(MISMATCHED)); - } - Reattached::Exited { .. } => { - println!("reattached=exited"); - return Err("no VM of the sandbox runs".to_owned()); - } - Reattached::Indeterminate(reason) => { - println!("reattached=indeterminate"); - return Err(format!( - "whether the VM of the sandbox runs cannot be determined: {reason}" - )); - } + let vmm = match hold(&mut locked, &runtime)? { + Ok(vmm) => vmm, + Err(status) => return Ok(status), }; - println!("pid={}", vmm.record().process().pid); let deadline = Instant::now() + SESSION_TIMEOUT; let attached = store .read_capability() @@ -330,18 +558,20 @@ fn reattach(store: &Path, guest_build: &str, stop: bool) -> Result Result { + check_outcome(&store, "success", 0)?; + Ok(ExitCode::SUCCESS) + } + Some(failure) => { + print_failure(&failure); + check_outcome(&store, "guest-exit", failure.stage().exit_status().into())?; + Ok(ExitCode::from(ROOT_FAILED)) + } + } +} + +/// Reattaches to the VM of the sandbox store `store`, with the guest agent's build `guest_build`, +/// attaches a session to it, and runs `probes` in the VM's layered root, printing the facts that +/// each found. +fn probe(store: &Path, guest_build: &str, probes: &[Probe]) -> Result { + let runtime = runtime(guest_build)?; + let store = SandboxStore::open(store).map_err(|error| describe(&error))?; + let mut locked = store + .lock(Some(Instant::now() + LOCK_TIMEOUT)) + .map_err(|error| describe(&error))?; + let vmm = match hold(&mut locked, &runtime)? { + Ok(vmm) => vmm, + Err(status) => return Ok(status), + }; + let deadline = Instant::now() + SESSION_TIMEOUT; + let capability = store.read_capability().map_err(|error| describe(&error))?; + let mut client = connect(&vmm, &capability, deadline)?; + for probe in probes { + let reply = client + .probe(probe, deadline) + .map_err(|error| format!("the guest agent did not answer: {}", describe(&error)))?; + match reply { + ProbeReply::Report(facts) => { + for (key, value) in facts.iter() { + println!("{}.{key}={}", probe.name(), escape(value)); + } + } + ProbeReply::Failed(problem) => return Err(problem), + } + } + Ok(ExitCode::SUCCESS) +} + +/// Reattaches to the VM of `locked`, with the runtime identity of `runtime`, and prints what it +/// found. Returns the VM, or the status with which the command ends once it finds the VM +/// mismatched, which it leaves running. +fn hold(locked: &mut LockedStore, runtime: &Runtime) -> Result, String> { + let reattached = + supervisor::reattach(locked, &runtime.identity).map_err(|error| describe(&error))?; + let vmm = match reattached { + Reattached::Live(vmm) => { + println!("reattached=live"); + vmm + } + Reattached::Mismatched { vmm, differences } => { + println!("reattached=mismatched"); + println!("differences={}", names(&differences)); + println!("pid={}", vmm.record().process().pid); + return Ok(Err(ExitCode::from(MISMATCHED))); + } + Reattached::Exited { .. } => { + println!("reattached=exited"); + return Err("no VM of the sandbox runs".to_owned()); + } + Reattached::Indeterminate(reason) => { + println!("reattached=indeterminate"); + return Err(format!( + "whether the VM of the sandbox runs cannot be determined: {reason}" + )); + } + }; + println!("pid={}", vmm.record().process().pid); + Ok(Ok(vmm)) +} + +/// Reads the outcome report of the VM of `store`, prints it, and checks that the VM ended with +/// the category `category`, the status code `status`, and a complete teardown. +fn check_outcome(store: &SandboxStore, category: &str, status: i64) -> Result<(), String> { let outcome = report::read(&store.outcome_path()).ok_or_else(|| { format!( "the VM left no outcome report {}", store.outcome_path().display() ) })?; - let category = outcome.category.as_deref().unwrap_or("none"); - let status = outcome + let found = outcome.category.as_deref().unwrap_or("none"); + let code = outcome .status_code .map_or("none".to_owned(), |code| code.to_string()); - println!("outcome={category} {status}"); + println!("outcome={found} {code}"); println!( "teardown={}", match outcome.teardown_complete { @@ -377,12 +687,14 @@ fn reattach(store: &Path, guest_build: &str, stop: bool) -> Result "incomplete", } ); - if (category, outcome.status_code, outcome.teardown_complete) - != ("success", Some(0), Some(true)) + if (found, outcome.status_code, outcome.teardown_complete) + != (category, Some(status), Some(true)) { - return Err("the VM's outcome is not a success with a complete teardown".to_owned()); + return Err(format!( + "the VM's outcome is not {category} {status} with a complete teardown" + )); } - Ok(ExitCode::SUCCESS) + Ok(()) } /// Returns the runtime of NVX's artifacts in the directory that `NVX_ARTIFACTS_DIR` names, with diff --git a/tests/runtime-adapter/src/frame.rs b/tests/runtime-adapter/src/frame.rs index 3a4656d4a..1b1147ef9 100644 --- a/tests/runtime-adapter/src/frame.rs +++ b/tests/runtime-adapter/src/frame.rs @@ -10,6 +10,11 @@ //! //! The framing cannot find the next message after a malformed one, so a [`FrameError`] ends the //! stream. +//! +//! The framing carries the body of each kind as bytes, within the kind's limit. The bodies of the +//! messages of a layered root and of its probes have encodings of their own, which the +//! [`root`](crate::root) and [`probe`](crate::probe) modules write and read: a body that does not +//! decode is a [`BodyError`], which the receiver answers or refuses, while the stream goes on. use std::fmt; @@ -22,6 +27,10 @@ pub const HEADER_LEN: usize = 5; /// the longest message, with its header, spans five records. pub const MAX_BODY: usize = 4 * OUTER_MAX_PAYLOAD; +/// Most bytes of the body of a root configuration, a root's failure, a probe, or a probe's failure: +/// short texts, unlike an echo or a probe's report. +pub const MAX_TEXT_BODY: usize = 4096; + /// A message of the adapter's framing. #[derive(Clone, Debug, PartialEq, Eq)] pub enum Message { @@ -35,6 +44,24 @@ pub enum Message { Stop, /// The guest's confirmation of [`Message::Stop`], its last message. Stopped, + /// The host's configuration of a layered root, what the root needs besides its boot tokens, + /// encoded as a [`RootConfig`](crate::root::RootConfig). The guest decodes and checks it + /// itself, so that it answers one that it refuses with [`Message::RootFailed`]. + RootConfig(Vec), + /// The guest's answer to [`Message::RootConfig`] once it has assembled the root. + RootReady, + /// A stage of the guest's layered root failed, encoded as a + /// [`RootFailure`](crate::root::RootFailure): the guest's answer to [`Message::RootConfig`], + /// or its report of a failed teardown before it confirms a [`Message::Stop`]. + RootFailed(Vec), + /// Asks the guest to run a probe, a named step of a scenario's checks, encoded as a + /// [`Probe`](crate::probe::Probe). + Probe(Vec), + /// The guest's answer to [`Message::Probe`]: what the probe found, encoded as + /// [`Facts`](crate::probe::Facts). + Report(Vec), + /// The guest's answer to a [`Message::Probe`] that failed: the problem, as UTF-8 text. + ProbeFailed(Vec), } impl Message { @@ -43,8 +70,9 @@ impl Message { self.parts().0.code() } - /// Returns the message's encoding: its header, then its body. An `Echo` body has at most - /// [`MAX_BODY`] bytes. + /// Returns the message's encoding: its header, then its body, which has at most the bytes that + /// its kind allows: [`MAX_BODY`] for an `Echo` or a `Report`, [`MAX_TEXT_BODY`] for a root + /// configuration, a root's failure, a probe, or a probe's failure, and none for the others. pub fn encode(&self) -> Result, FrameError> { let (kind, body) = self.parts(); if body.len() > kind.max_body() { @@ -70,6 +98,12 @@ impl Message { Self::Echo(body) => (Kind::Echo, body), Self::Stop => (Kind::Stop, &[]), Self::Stopped => (Kind::Stopped, &[]), + Self::RootConfig(body) => (Kind::RootConfig, body), + Self::RootReady => (Kind::RootReady, &[]), + Self::RootFailed(body) => (Kind::RootFailed, body), + Self::Probe(body) => (Kind::Probe, body), + Self::Report(body) => (Kind::Report, body), + Self::ProbeFailed(body) => (Kind::ProbeFailed, body), } } } @@ -87,19 +121,39 @@ enum Kind { Stop, /// [`Message::Stopped`]. Stopped, + /// [`Message::RootConfig`]. + RootConfig, + /// [`Message::RootReady`]. + RootReady, + /// [`Message::RootFailed`]. + RootFailed, + /// [`Message::Probe`]. + Probe, + /// [`Message::Report`]. + Report, + /// [`Message::ProbeFailed`]. + ProbeFailed, } impl Kind { + /// Every kind, in the order of its codes. + const ALL: [Self; 11] = [ + Self::Ping, + Self::Pong, + Self::Echo, + Self::Stop, + Self::Stopped, + Self::RootConfig, + Self::RootReady, + Self::RootFailed, + Self::Probe, + Self::Report, + Self::ProbeFailed, + ]; + /// Returns the kind with `code`, if the framing defines one. fn from_code(code: u8) -> Option { - match code { - 1 => Some(Self::Ping), - 2 => Some(Self::Pong), - 3 => Some(Self::Echo), - 4 => Some(Self::Stop), - 5 => Some(Self::Stopped), - _ => None, - } + Self::ALL.into_iter().find(|kind| kind.code() == code) } /// Returns the kind's code, the last byte of a header. @@ -110,14 +164,21 @@ impl Kind { Self::Echo => 3, Self::Stop => 4, Self::Stopped => 5, + Self::RootConfig => 6, + Self::RootReady => 7, + Self::RootFailed => 8, + Self::Probe => 9, + Self::Report => 10, + Self::ProbeFailed => 11, } } /// Returns the most bytes of a body of this kind. fn max_body(self) -> usize { match self { - Self::Echo => MAX_BODY, - Self::Ping | Self::Pong | Self::Stop | Self::Stopped => 0, + Self::Echo | Self::Report => MAX_BODY, + Self::RootConfig | Self::RootFailed | Self::Probe | Self::ProbeFailed => MAX_TEXT_BODY, + Self::Ping | Self::Pong | Self::Stop | Self::Stopped | Self::RootReady => 0, } } @@ -130,6 +191,12 @@ impl Kind { Self::Echo => Message::Echo(body), Self::Stop => Message::Stop, Self::Stopped => Message::Stopped, + Self::RootConfig => Message::RootConfig(body), + Self::RootReady => Message::RootReady, + Self::RootFailed => Message::RootFailed(body), + Self::Probe => Message::Probe(body), + Self::Report => Message::Report(body), + Self::ProbeFailed => Message::ProbeFailed(body), } } } @@ -140,7 +207,7 @@ impl Kind { pub enum FrameError { /// A kind that the framing does not define. Kind(u8), - /// A body longer than [`MAX_BODY`] bytes, or a body for a kind that has none. + /// A body longer than its kind allows: [`MAX_BODY`] bytes, [`MAX_TEXT_BODY`], or none. Length { /// Kind of the message. kind: u8, @@ -163,6 +230,41 @@ impl fmt::Display for FrameError { impl std::error::Error for FrameError {} +/// A body that does not decode as its kind's encoding, such as a root configuration that its guest +/// refuses. Unlike a [`FrameError`], it leaves the stream intact, at the start of the next message. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BodyError(String); + +impl BodyError { + /// Returns the error that `problem` describes. + pub fn new(problem: impl Into) -> Self { + Self(problem.into()) + } + + /// Returns the problem. + pub fn problem(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for BodyError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.0) + } +} + +impl std::error::Error for BodyError {} + +/// Returns the longest start of `text` of at most `limit` bytes that ends at a character's +/// boundary. +pub(crate) fn truncated(text: &str, limit: usize) -> &str { + let mut end = text.len().min(limit); + while !text.is_char_boundary(end) { + end -= 1; + } + &text[..end] +} + /// Reassembles messages from bytes that arrive in pieces of any size, such as the payloads of the /// control session's records. #[derive(Clone, Debug, Default)] @@ -247,7 +349,7 @@ impl Decoder { #[cfg(test)] mod tests { - use super::{Decoder, FrameError, HEADER_LEN, MAX_BODY, Message}; + use super::{Decoder, FrameError, HEADER_LEN, MAX_BODY, MAX_TEXT_BODY, Message}; /// Returns one message of each kind, with bodies of no, few, and the most bytes. fn messages() -> Vec { @@ -257,6 +359,17 @@ mod tests { Message::Echo(b"hello".to_vec()), Message::Pong, Message::Echo((0..=u8::MAX).cycle().take(MAX_BODY).collect()), + Message::RootConfig(b"version=1\n".to_vec()), + Message::RootConfig(vec![b'c'; MAX_TEXT_BODY]), + Message::RootReady, + Message::RootFailed(Vec::new()), + Message::RootFailed(vec![b'f'; MAX_TEXT_BODY]), + Message::Probe(b"mounts".to_vec()), + Message::Probe(vec![b'p'; MAX_TEXT_BODY]), + Message::Report(b"key=value\n".to_vec()), + Message::Report((0..=u8::MAX).rev().cycle().take(MAX_BODY).collect()), + Message::ProbeFailed(b"unknown probe".to_vec()), + Message::ProbeFailed(vec![b'e'; MAX_TEXT_BODY]), Message::Stop, Message::Stopped, ] @@ -298,6 +411,27 @@ mod tests { ); assert_eq!(Message::Stop.encode().unwrap(), [0, 0, 0, 0, 4]); assert_eq!(Message::Stopped.encode().unwrap(), [0, 0, 0, 0, 5]); + assert_eq!( + Message::RootConfig(b"c".to_vec()).encode().unwrap(), + [1, 0, 0, 0, 6, b'c'] + ); + assert_eq!(Message::RootReady.encode().unwrap(), [0, 0, 0, 0, 7]); + assert_eq!( + Message::RootFailed(b"f".to_vec()).encode().unwrap(), + [1, 0, 0, 0, 8, b'f'] + ); + assert_eq!( + Message::Probe(b"p".to_vec()).encode().unwrap(), + [1, 0, 0, 0, 9, b'p'] + ); + assert_eq!( + Message::Report(b"r".to_vec()).encode().unwrap(), + [1, 0, 0, 0, 10, b'r'] + ); + assert_eq!( + Message::ProbeFailed(b"e".to_vec()).encode().unwrap(), + [1, 0, 0, 0, 11, b'e'] + ); for message in messages() { assert_eq!(message.encode().unwrap()[HEADER_LEN - 1], message.kind()); } @@ -349,12 +483,23 @@ mod tests { #[test] fn malformed_headers_fail_before_their_body_is_taken() { let too_long = u32::try_from(MAX_BODY + 1).unwrap().to_le_bytes(); + let too_long_text = u32::try_from(MAX_TEXT_BODY + 1).unwrap().to_le_bytes(); + let text = |kind: u8| { + [ + too_long_text[0], + too_long_text[1], + too_long_text[2], + too_long_text[3], + kind, + ] + }; let cases = [ ([0, 0, 0, 0, 0], FrameError::Kind(0)), - ([0, 0, 0, 0, 6], FrameError::Kind(6)), + ([0, 0, 0, 0, 12], FrameError::Kind(12)), ([9, 0, 0, 0, 0xff], FrameError::Kind(0xff)), ([1, 0, 0, 0, 1], FrameError::Length { kind: 1, length: 1 }), ([1, 0, 0, 0, 4], FrameError::Length { kind: 4, length: 1 }), + ([1, 0, 0, 0, 7], FrameError::Length { kind: 7, length: 1 }), ( [too_long[0], too_long[1], too_long[2], too_long[3], 3], FrameError::Length { @@ -362,6 +507,13 @@ mod tests { length: MAX_BODY + 1, }, ), + ( + [too_long[0], too_long[1], too_long[2], too_long[3], 10], + FrameError::Length { + kind: 10, + length: MAX_BODY + 1, + }, + ), ( [0xff, 0xff, 0xff, 0xff, 3], FrameError::Length { @@ -369,6 +521,34 @@ mod tests { length: u32::MAX as usize, }, ), + ( + text(6), + FrameError::Length { + kind: 6, + length: MAX_TEXT_BODY + 1, + }, + ), + ( + text(8), + FrameError::Length { + kind: 8, + length: MAX_TEXT_BODY + 1, + }, + ), + ( + text(9), + FrameError::Length { + kind: 9, + length: MAX_TEXT_BODY + 1, + }, + ), + ( + text(11), + FrameError::Length { + kind: 11, + length: MAX_TEXT_BODY + 1, + }, + ), ]; for (header, error) in cases { let mut decoder = Decoder::new(); @@ -398,12 +578,38 @@ mod tests { fn encoding_refuses_a_body_beyond_the_limit() { let longest = Message::Echo(vec![0; MAX_BODY]).encode().unwrap(); assert_eq!(longest.len(), HEADER_LEN + MAX_BODY); - assert_eq!( - Message::Echo(vec![0; MAX_BODY + 1]).encode(), - Err(FrameError::Length { - kind: 3, - length: MAX_BODY + 1, - }) - ); + let report = Message::Report(vec![0; MAX_BODY]).encode().unwrap(); + assert_eq!(report.len(), HEADER_LEN + MAX_BODY); + let cases = [ + (Message::Echo(vec![0; MAX_BODY + 1]), 3, MAX_BODY + 1), + (Message::Report(vec![0; MAX_BODY + 1]), 10, MAX_BODY + 1), + ( + Message::RootConfig(vec![0; MAX_TEXT_BODY + 1]), + 6, + MAX_TEXT_BODY + 1, + ), + ( + Message::RootFailed(vec![0; MAX_TEXT_BODY + 1]), + 8, + MAX_TEXT_BODY + 1, + ), + ( + Message::Probe(vec![0; MAX_TEXT_BODY + 1]), + 9, + MAX_TEXT_BODY + 1, + ), + ( + Message::ProbeFailed(vec![0; MAX_TEXT_BODY + 1]), + 11, + MAX_TEXT_BODY + 1, + ), + ]; + for (message, kind, length) in cases { + assert_eq!( + message.encode(), + Err(FrameError::Length { kind, length }), + "{kind}" + ); + } } } diff --git a/tests/runtime-adapter/src/guest.rs b/tests/runtime-adapter/src/guest.rs index 6b31eb169..8667b7228 100644 --- a/tests/runtime-adapter/src/guest.rs +++ b/tests/runtime-adapter/src/guest.rs @@ -1,6 +1,10 @@ //! The adapter's guest agent: it serves the host program over the guest's end of the control //! session between the host and the guest, with the adapter's own [framing](crate::frame), through //! `nvx::guest::nvxs`, and on a tty through `nvx::guest::terminal`. +//! +//! [`session`] runs the session, and hands each of the host's messages to a handler, which +//! answers it and says whether serving goes on. [`serve`] runs it with the guest agent of a VM +//! without a layered root, and the [`worker`](crate::worker) with that of a VM with one. use std::fmt; use std::io::{self, Read, Write}; @@ -17,6 +21,9 @@ use crate::frame::{Decoder, FrameError, Message}; /// allows, so that the rest of a longer message arrives only as the guest credits what it took. pub const RECEIVE_WINDOW: u32 = OUTER_MIN_RECEIVE_WINDOW; +/// The problem with which the guest agent of a VM without a layered root answers a probe. +pub const NO_ROOT: &str = "the VM has no layered root to probe"; + /// How serving ended, when nothing failed. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Served { @@ -34,7 +41,8 @@ pub enum ServeError { Session(GuestConnectionError), /// The host sent a malformed message. Frame(FrameError), - /// The host sent a message of a kind that only the guest sends. + /// The host sent a message of a kind that the guest agent does not accept: one that only the + /// guest sends, or one that the guest agent does not take in its state. Unexpected(u8), /// The tty of the session could not be opened or put in raw mode. Terminal(io::Error), @@ -48,7 +56,7 @@ impl fmt::Display for ServeError { Self::Unexpected(kind) => { write!( formatter, - "the host sent a message of kind {kind}, which only the guest sends" + "the host sent a message of kind {kind}, which the guest agent does not accept" ) } Self::Terminal(error) => error.fmt(formatter), @@ -79,21 +87,45 @@ impl From for ServeError { } } -/// Serves the host program on `stream`, the guest's end of the control session, until the host -/// stops the guest or the broker closes the stream. +/// What the guest agent does once it handled a message from the host. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Step { + /// It sends these messages, in order, and serves on. + Continue(Vec), + /// It sends these messages, in order, then ends serving with the value, without reading + /// further. + End(Vec, T), +} + +/// How a [`session`] ended, when nothing failed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Ended { + /// The handler ended it, with this value. + Handled(T), + /// The broker closed the stream at a record boundary. + Closed, +} + +/// Runs the guest's end of the control session on `stream`, and hands each message of the host +/// to `handle`, until `handle` ends serving or the broker closes the stream. /// /// The guest attaches, then: /// /// - It acknowledges each reset of the session with [`RECEIVE_WINDOW`], once it dropped the -/// earlier epoch's partial message: a new epoch starts with a new message. +/// earlier epoch's partial message: a new epoch starts with a new message. The handler keeps +/// its state across epochs. /// - It credits each payload back once it holds the payload, so that a message longer than the /// window arrives whole. -/// - It answers [`Message::Ping`] with [`Message::Pong`], sends each [`Message::Echo`] back, and -/// answers [`Message::Stop`] with [`Message::Stopped`], then returns without reading further. +/// - It sends the messages with which `handle` answers each message, in order, and returns once +/// `handle` ends serving. /// -/// A failure of the session, a malformed message, and a message that only the guest sends end -/// serving with an error. -pub fn serve(stream: S) -> Result { +/// A failure of the session, a malformed message, and an error of `handle`, such as a message +/// that it does not accept, end serving with that error. A reply that the framing refuses fails +/// with [`ServeError::Frame`], before it is sent. +pub fn session( + stream: S, + mut handle: impl FnMut(Message) -> Result, ServeError>, +) -> Result, ServeError> { let mut connection = GuestConnection::new(stream); connection.attach()?; let mut decoder = Decoder::new(); @@ -105,7 +137,7 @@ pub fn serve(stream: S) -> Result { continue; } Ok(GuestEvent::Data(payload)) => payload, - Err(GuestConnectionError::Closed) => return Ok(Served::Closed), + Err(GuestConnectionError::Closed) => return Ok(Ended::Closed), Err(error) => return Err(error.into()), }; // A DATA payload has at most `OUTER_MAX_PAYLOAD` bytes, so its length fits a `u32`. @@ -114,32 +146,63 @@ pub fn serve(stream: S) -> Result { while !rest.is_empty() { let (consumed, message) = decoder.push(rest)?; rest = &rest[consumed..]; - let reply = match message { - None => continue, - Some(Message::Ping) => Message::Pong, - Some(Message::Echo(body)) => Message::Echo(body), - Some(Message::Stop) => { - connection.send(&Message::Stopped.encode()?)?; - return Ok(Served::Stopped); - } - Some(message) => return Err(ServeError::Unexpected(message.kind())), + let Some(message) = message else { + continue; }; - connection.send(&reply.encode()?)?; + let (replies, ended) = match handle(message)? { + Step::Continue(replies) => (replies, None), + Step::End(replies, value) => (replies, Some(value)), + }; + for reply in replies { + connection.send(&reply.encode()?)?; + } + if let Some(value) = ended { + return Ok(Ended::Handled(value)); + } } } } -/// Serves the host program on the tty at `path`, the guest's end of the control session, as -/// [`serve`] does, in place of NVX's managed agent on a VM's control tty. +/// Serves the host program on `stream`, the guest's end of the control session, in a VM without +/// a layered root, until the host stops the guest or the broker closes the stream. +/// +/// It runs the [`session`], in which it answers [`Message::Ping`] with [`Message::Pong`], sends +/// each [`Message::Echo`] back, answers each [`Message::Probe`] with a [`Message::ProbeFailed`] +/// that says [`NO_ROOT`], and answers [`Message::Stop`] with [`Message::Stopped`], then returns +/// without reading further. +/// +/// A failure of the session, a malformed message, and a message of another kind, which only the +/// guest sends or which configures a layered root, end serving with an error. +pub fn serve(stream: S) -> Result { + let ended = session(stream, |message| { + Ok(match message { + Message::Ping => Step::Continue(vec![Message::Pong]), + Message::Echo(body) => Step::Continue(vec![Message::Echo(body)]), + Message::Probe(_) => Step::Continue(vec![Message::ProbeFailed(NO_ROOT.into())]), + Message::Stop => Step::End(vec![Message::Stopped], ()), + message => return Err(ServeError::Unexpected(message.kind())), + }) + })?; + Ok(match ended { + Ended::Handled(()) => Served::Stopped, + Ended::Closed => Served::Closed, + }) +} + +/// Opens the tty at `path`, the guest's end of the control session, and returns what `run` +/// returns with it, as NVX's managed agent serves a VM's control tty. /// /// It opens the tty with NVX's [`Terminal`], as NVX's managed agent does: waiting while the kernel /// sets the tty up during boot, and in raw mode, so that the tty passes the session's bytes as they -/// are. Once serving ends, it waits until the tty has sent what serving wrote, such as the +/// are. Once `run` succeeds, it waits until the tty has sent what `run` wrote, such as the /// [`Message::Stopped`] that confirms a stop. /// /// It fails with [`ServeError::Terminal`] if the tty cannot be opened or put in raw mode, and -/// otherwise as [`serve`] does. -pub fn serve_terminal(path: &Path) -> Result { +/// otherwise as `run` does. +pub fn on_terminal( + path: &Path, + run: impl FnOnce(&mut Terminal) -> Result, +) -> Result { let failed = |action: String| { move |error: io::Error| { ServeError::Terminal(io::Error::new(error.kind(), format!("{action}: {error}"))) @@ -150,8 +213,16 @@ pub fn serve_terminal(path: &Path) -> Result { terminal .configure_raw() .map_err(failed(format!("cannot put {} in raw mode", path.display())))?; - let served = serve(&mut terminal)?; + let ran = run(&mut terminal)?; // As NVX's managed agent does, a drain that fails is no failure of serving, which ended. let _ = terminal.drain(); - Ok(served) + Ok(ran) +} + +/// Serves the host program on the tty at `path` as [`serve`] does, through [`on_terminal`], in +/// place of NVX's managed agent on a VM's control tty. +/// +/// It fails as [`on_terminal`] and [`serve`] do. +pub fn serve_terminal(path: &Path) -> Result { + on_terminal(path, |terminal| serve(terminal)) } diff --git a/tests/runtime-adapter/src/host.rs b/tests/runtime-adapter/src/host.rs index e901d8750..8bb2b078e 100644 --- a/tests/runtime-adapter/src/host.rs +++ b/tests/runtime-adapter/src/host.rs @@ -11,7 +11,9 @@ use nvx::host::platform::Transport; use nvx::protocol::CAPABILITY_LEN; use nvx::protocol::nvxs::Identity; -use crate::frame::{Decoder, FrameError, Message}; +use crate::frame::{BodyError, Decoder, FrameError, Message}; +use crate::probe::{Facts, Probe, ProbeReply}; +use crate::root::{RootConfig, RootFailure, RootReply}; /// A failure of a [`Client`]. #[derive(Debug)] @@ -22,6 +24,8 @@ pub enum ClientError { /// A malformed message: one that the guest agent sent, or one that the host program asked to /// send, which the client did not send. Frame(FrameError), + /// The guest agent answered a request with a body that does not decode as its kind's. + Body(BodyError), /// The guest agent sent a message of a kind that only the host program sends. Unexpected(u8), /// The guest agent answered a request with a message of another kind than the request's @@ -43,6 +47,7 @@ impl fmt::Display for ClientError { match self { Self::Session(error) => error.fmt(formatter), Self::Frame(error) => error.fmt(formatter), + Self::Body(error) => error.fmt(formatter), Self::Unexpected(kind) => write!( formatter, "the guest sent a message of kind {kind}, which only the host sends" @@ -62,6 +67,7 @@ impl std::error::Error for ClientError { match self { Self::Session(error) => Some(error), Self::Frame(error) => Some(error), + Self::Body(error) => Some(error), Self::Unexpected(_) | Self::Reply { .. } | Self::Echo | Self::Ended => None, } } @@ -84,8 +90,8 @@ impl From for ClientError { /// [`attach`](Self::attach) attaches with the launch capability. [`send`](Self::send) sends a /// message in as many of the session's records as it takes, and [`receive`](Self::receive) /// returns the guest agent's next message, whether it spans records or shares one with the next -/// message. [`ping`](Self::ping), [`echo`](Self::echo), and [`stop`](Self::stop) each send a -/// request and wait for its answer. +/// message. [`ping`](Self::ping), [`echo`](Self::echo), [`configure_root`](Self::configure_root), +/// [`probe`](Self::probe), and [`stop`](Self::stop) each send a request and wait for its answer. /// /// Besides the session's own buffers, the client holds at most the rest of one of the guest's /// records and one partial message, whose header [`Decoder`] checks before it takes any byte of @@ -100,8 +106,8 @@ impl From for ClientError { /// the session, a send that fails or times out, because the session cannot resume a message that /// it took in part, a malformed message, and a message that only the host program sends. A /// [`receive`](Self::receive) that times out doesn't: it keeps what arrived. A request ends the -/// client on any failure after the framing accepted it, a timeout included, because an answer that -/// arrived later would answer the wrong request. +/// client on any failure after the framing accepted it, a timeout and an answer whose body does +/// not decode included, because an answer that arrived later would answer the wrong request. pub struct Client { /// The host's end of the control session. connection: HostConnection, @@ -170,7 +176,12 @@ impl Client { self.taken += consumed; match message { None => {} - Some(message @ (Message::Ping | Message::Stop)) => { + Some( + message @ (Message::Ping + | Message::Stop + | Message::RootConfig(_) + | Message::Probe(_)), + ) => { return Err(self.fail(ClientError::Unexpected(message.kind()))); } Some(message) => return Ok(message), @@ -211,15 +222,68 @@ impl Client { } } + /// Sends the root configuration `config` of a VM with a layered root, and waits until + /// `deadline` for the guest agent's answer: `RootReady` once it assembled the root, or + /// `RootFailed` with the stage that failed, after which the guest ends. + /// + /// A configuration whose encoding is longer than + /// [`MAX_TEXT_BODY`](crate::frame::MAX_TEXT_BODY) bytes fails before anything is sent, and the + /// client stays usable. + pub fn configure_root( + &mut self, + config: &RootConfig, + deadline: Instant, + ) -> Result { + let request = Message::RootConfig(config.encode()); + match self.request(&request, deadline)? { + Message::RootReady => Ok(RootReply::Ready), + Message::RootFailed(body) => { + self.decoded(RootFailure::decode(&body).map(RootReply::Failed)) + } + reply => Err(self.mismatched(&request, &reply)), + } + } + + /// Sends `probe` to the guest agent of a VM with a layered root, and waits until `deadline` + /// for its answer: the facts that the probe found, or the problem. + pub fn probe(&mut self, probe: &Probe, deadline: Instant) -> Result { + let request = Message::Probe(probe.encode()); + match self.request(&request, deadline)? { + Message::Report(body) => self.decoded(Facts::decode(&body).map(ProbeReply::Report)), + Message::ProbeFailed(body) => { + let problem = String::from_utf8(body) + .map_err(|_| BodyError::new("the probe's failure is not UTF-8 text")); + self.decoded(problem.map(ProbeReply::Failed)) + } + reply => Err(self.mismatched(&request, &reply)), + } + } + /// Sends `Stop`, and waits until `deadline` for the guest agent's `Stopped`, its last message. - pub fn stop(&mut self, deadline: Instant) -> Result<(), ClientError> { + /// + /// The guest agent of a VM with a layered root tears the root down first, and reports a + /// teardown that failed with `RootFailed` before `Stopped`: the client returns that failure. + pub fn stop(&mut self, deadline: Instant) -> Result, ClientError> { let request = Message::Stop; - match self.request(&request, deadline)? { - Message::Stopped => Ok(()), + let failure = match self.request(&request, deadline)? { + Message::Stopped => return Ok(None), + Message::RootFailed(body) => self.decoded(RootFailure::decode(&body))?, + reply => return Err(self.mismatched(&request, &reply)), + }; + match self + .receive(Some(deadline)) + .map_err(|error| self.fail(error))? + { + Message::Stopped => Ok(Some(failure)), reply => Err(self.mismatched(&request, &reply)), } } + /// Returns `decoded`, the body of an answer, or ends the client because it does not decode. + fn decoded(&mut self, decoded: Result) -> Result { + decoded.map_err(|error| self.fail(ClientError::Body(error))) + } + /// Sends `request` by `deadline`, and returns the guest agent's next message. Once the framing /// accepted the request, any failure ends the client. fn request(&mut self, request: &Message, deadline: Instant) -> Result { diff --git a/tests/runtime-adapter/src/initramfs.rs b/tests/runtime-adapter/src/initramfs.rs index 39920de60..f4e866376 100644 --- a/tests/runtime-adapter/src/initramfs.rs +++ b/tests/runtime-adapter/src/initramfs.rs @@ -1,4 +1,4 @@ -//! The initramfs of a VM that runs the adapter's guest agent in place of NVX's managed agent. +//! The initramfs of a VM that runs the adapter's guest agent in place of NVX's agents. //! //! NVX's initramfs boots through its `/init`, which hands a VM of the managed lifecycle to NVX's //! managed agent, as PID 1, on the VM's control tty: @@ -7,15 +7,28 @@ //! exec /sbin/nvx-managed-agent /dev/hvc2 - nvx-managed UID GID NAME HOME //! ``` //! +//! and a VM with a layered root, whose kernel command line has `nvx_sandbox=1`, to NVX's init +//! agent, which assembles the root, without arguments: +//! +//! ```text +//! exec /sbin/nvx-init-agent +//! ``` +//! //! [`write`](fn@write) copies that initramfs, and appends to the copy a second archive, in the //! `newc` format, which Linux unpacks over the first: //! //! - the guest agent, at [`GUEST_AGENT`]; //! - a script, at [`LAUNCHER`], that runs the guest agent as `runtime-adapter-guest init TTY`, //! with the control tty, the first of the arguments that `/init` passes the managed agent; -//! - a symbolic link from [`MANAGED_AGENT`] to the script. Linux removes a file whose type an -//! archive changes before it creates the new one, so the link replaces the managed agent -//! without writing to its file, which could have other names. +//! - a symbolic link from [`MANAGED_AGENT`] to that script; +//! - a script, at [`LAYERED_LAUNCHER`], that runs the guest agent as +//! `runtime-adapter-guest init /dev/hvc2 layered`, PID 1 of a VM with a layered root, which +//! names the control tty, [`CONTROL_TTY`], itself, since `/init` passes NVX's init agent none; +//! - and a symbolic link from [`INIT_AGENT`] to that script, so that NVX's init agent never +//! assembles the root in the adapter's VMs. +//! +//! Linux removes a file whose type an archive changes before it creates the new one, so each link +//! replaces NVX's agent without writing to its file, which could have other names. //! //! The second archive starts at a multiple of 4 bytes, as Linux requires of an uncompressed //! archive, after the zeros that pad the copy, which Linux skips. NVX's initramfs itself stays as @@ -31,6 +44,13 @@ pub const GUEST_AGENT: &str = "/sbin/runtime-adapter-guest"; pub const LAUNCHER: &str = "/sbin/runtime-adapter-init"; /// Path of NVX's managed agent, which `/init` runs. pub const MANAGED_AGENT: &str = "/sbin/nvx-managed-agent"; +/// Path of the script that runs the guest agent's PID 1 of a VM with a layered root, in place of +/// NVX's init agent. +pub const LAYERED_LAUNCHER: &str = "/sbin/runtime-adapter-layered"; +/// Path of NVX's init agent, which `/init` runs in a VM with a layered root. +pub const INIT_AGENT: &str = "/sbin/nvx-init-agent"; +/// The VM's control tty, which `/init` requires of both lifecycles that NVX's agents serve. +pub const CONTROL_TTY: &str = "/dev/hvc2"; /// Returns the script at [`LAUNCHER`], which runs the guest agent's /// [`GUEST_INIT_COMMAND`](crate::GUEST_INIT_COMMAND) on the control tty, the first of the @@ -42,6 +62,17 @@ pub fn launcher_script() -> String { ) } +/// Returns the script at [`LAYERED_LAUNCHER`], which runs the guest agent's +/// [`GUEST_INIT_COMMAND`](crate::GUEST_INIT_COMMAND) on [`CONTROL_TTY`], in its +/// [`GUEST_LAYERED_MODE`](crate::GUEST_LAYERED_MODE). +pub fn layered_launcher_script() -> String { + format!( + "#!/bin/sh\nexec {GUEST_AGENT} {} {CONTROL_TTY} {}\n", + crate::GUEST_INIT_COMMAND, + crate::GUEST_LAYERED_MODE, + ) +} + /// The magic number of a `newc` header. const MAGIC: &[u8] = b"070701"; /// The name of the entry that ends an archive. @@ -53,7 +84,7 @@ const SYMLINK: u32 = 0o120_000; /// Writes to `output` a copy of NVX's initramfs `base` with the guest agent `guest_agent`, a /// static `x86_64-unknown-linux-musl` build of `runtime-adapter-guest`, in place of NVX's managed -/// agent, as the [module documentation](self) describes. +/// agent and init agent, as the [module documentation](self) describes. /// /// Fails if a file cannot be read or written, or if the guest agent does not fit an archive /// entry, whose size has 32 bits. The error names the file. @@ -81,11 +112,14 @@ fn naming(path: &Path) -> impl Fn(io::Error) -> io::Error + '_ { /// Fails if the guest agent does not fit an archive entry, whose size has 32 bits. pub fn archive(guest_agent: &[u8]) -> io::Result> { let script = launcher_script(); - let entries: [(&str, u32, &[u8]); 3] = [ + let layered_script = layered_launcher_script(); + let entries: [(&str, u32, &[u8]); 5] = [ (GUEST_AGENT, REGULAR | 0o755, guest_agent), (LAUNCHER, REGULAR | 0o755, script.as_bytes()), // A link's data is its target. (MANAGED_AGENT, SYMLINK | 0o777, LAUNCHER.as_bytes()), + (LAYERED_LAUNCHER, REGULAR | 0o755, layered_script.as_bytes()), + (INIT_AGENT, SYMLINK | 0o777, LAYERED_LAUNCHER.as_bytes()), ]; let mut archive = Vec::new(); for (inode, (path, mode, data)) in (1..).zip(entries) { diff --git a/tests/runtime-adapter/src/layered.rs b/tests/runtime-adapter/src/layered.rs new file mode 100644 index 000000000..83b21aa57 --- /dev/null +++ b/tests/runtime-adapter/src/layered.rs @@ -0,0 +1,224 @@ +//! The host program's VMs with a layered root: an image, on the `distro` block device, under an +//! upper, which the guest agent's worker assembles outside PID 1. +//! +//! The host plans the VM's block devices and the boot tokens of its root with NVX's typed plan of a +//! cold layered launch, a `nvx::host::sandbox::SandboxPlan`, and launches the VM through NVX's +//! supervisor, as it launches VMs without a layered root, with the OpenVMM arguments that NVX's +//! own sandbox start passes; see [`LayeredLaunch::openvmm_arguments`]. The boot tokens carry only +//! what NVX's boot contract defines: the image's format and the identity that the guest expects of +//! it, and the upper. Everything else that the root needs goes in one root configuration, a +//! [`RootConfig`], which the host sends once it attaches. +//! +//! The host records what it launched in the sandbox's store, beside NVX's runtime records, as a +//! [`LaunchRecord`]. + +use std::ffi::{OsStr, OsString}; +use std::fs; +use std::path::Path; + +use nvx::host::artifacts::absolute; +use nvx::host::launch::DEFAULT_MEMORY_MIB; +use nvx::host::sandbox::{SandboxDistro, SandboxPlan, SandboxSettings, SandboxUpper}; +use nvx::protocol::root::RootImage; +use nvx::{Error, Hypervisor, Result}; +use serde_json::{Value, json}; + +use crate::root::RootConfig; + +/// The name of the record of a launch in the sandbox's store. +pub const RECORD_NAME: &str = "runtime-adapter.json"; +/// The format of the record of a launch that this build writes. +pub const RECORD_FORMAT: u64 = 1; +/// The size of a sector of a block device, in bytes. +const SECTOR_SIZE: u64 = 512; + +/// A VM with a layered root: the plan of its image, its upper, and their boot tokens, and the +/// root configuration that the host sends its guest agent. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LayeredLaunch { + /// The plan of the VM's block devices and of the boot tokens of its root, with NVX's default + /// launch settings, whose tokens the worker ignores. + pub plan: SandboxPlan, + /// The root configuration that the host sends the guest agent once it attaches. + pub config: RootConfig, +} + +impl LayeredLaunch { + /// Returns the launch of the image at `distro`, in the format and with the expected identity of + /// `image`, over the ext4 scratch image at `scratch`, or over RAM without one. + /// + /// The root configuration takes its defaults, with the runtime directory `runtime` instead, if + /// any, as it is, so that the guest checks it, and with the size of each image, if it is a + /// whole number of sectors, which the guest checks against its block device. + /// + /// Fails with [`ErrorKind::Unsupported`](nvx::ErrorKind::Unsupported) if a path is not + /// Unicode, and otherwise as `SandboxPlan::new` and `SandboxPlan::validated` do, such as for an + /// image that is not a regular file. + pub fn new( + distro: &Path, + image: RootImage, + scratch: Option<&Path>, + runtime: Option<&str>, + ) -> Result { + // OpenVMM runs in the sandbox's store, so the plan names the images by absolute paths. + let distro = absolute(distro)?; + let scratch = scratch.map(absolute).transpose()?; + let upper = match &scratch { + Some(scratch) => SandboxUpper::scratch(unicode(scratch)?), + None => SandboxUpper::RAM, + }; + let plan = SandboxPlan::new( + SandboxDistro::new(unicode(&distro)?, image), + upper, + SandboxSettings::default(), + )?; + plan.validated()?; + let mut config = RootConfig { + distro_sectors: sectors(&distro)?, + scratch_sectors: scratch.as_deref().map(sectors).transpose()?.flatten(), + ..RootConfig::default() + }; + if let Some(runtime) = runtime { + runtime.clone_into(&mut config.runtime); + } + Ok(Self { plan, config }) + } + + /// Returns the VM's kernel command line: the plan's, `nvx_sandbox=1` and the boot tokens of + /// the root among them, without parameters of the caller. + /// + /// Fails as `SandboxPlan::kernel_command_line` does. + pub fn kernel_command_line(&self) -> Result { + self.plan.kernel_command_line("") + } + + /// Returns OpenVMM's arguments for the VM, as NVX's sandbox start composes them: the plan's, + /// which attach its block devices, then the managed lifecycle in one process on `hypervisor`, + /// NVX's default memory, the kernel at `kernel`, the initramfs at `initramfs`, the + /// [kernel command line](Self::kernel_command_line), no virtio console, the control console + /// listening on `endpoint` with the launch capability on standard input, and the outcome + /// report at `report`. + /// + /// Fails as `SandboxPlan::openvmm_arguments` and [`kernel_command_line`](Self::kernel_command_line) + /// do. + pub fn openvmm_arguments( + &self, + hypervisor: Hypervisor, + kernel: &Path, + initramfs: &Path, + endpoint: &str, + report: &Path, + ) -> Result> { + let mut arguments = self + .plan + .openvmm_arguments()? + .into_iter() + .map(OsString::from) + .collect::>(); + let memory = format!("{DEFAULT_MEMORY_MIB}M"); + let command_line = self.kernel_command_line()?; + let control = format!("listen={endpoint}"); + let options = [ + OsStr::new("--microvm-lifecycle"), + OsStr::new("managed"), + OsStr::new("--single-process"), + OsStr::new("--hypervisor"), + OsStr::new(hypervisor.as_str()), + OsStr::new("--memory"), + OsStr::new(&memory), + OsStr::new("--kernel"), + kernel.as_os_str(), + OsStr::new("--initrd"), + initramfs.as_os_str(), + OsStr::new("--cmdline"), + OsStr::new(&command_line), + OsStr::new("--virtio-console"), + OsStr::new("none"), + OsStr::new("--microvm-control-console"), + OsStr::new(&control), + OsStr::new("--microvm-control-auth-stdin"), + OsStr::new("--microvm-report"), + report.as_os_str(), + ]; + arguments.extend(options.map(OsStr::to_os_string)); + Ok(arguments) + } +} + +/// Returns `path` as text, which the plan takes. +fn unicode(path: &Path) -> Result<&str> { + path.to_str().ok_or_else(|| { + Error::unsupported(format!("the path {} is not Unicode text", path.display())) + }) +} + +/// Returns the size of the file at `path` in sectors, if it is a whole number of them. +fn sectors(path: &Path) -> Result> { + let length = fs::metadata(path) + .map_err(|error| { + Error::unavailable(format!("cannot read the size of {}", path.display())) + .with_source(error) + })? + .len(); + Ok(length + .is_multiple_of(SECTOR_SIZE) + .then_some(length / SECTOR_SIZE)) +} + +/// What the host program launched in a sandbox, which it records in the sandbox's store, in +/// [`RECORD_NAME`], before it spawns the VMM. The record is JSON in format [`RECORD_FORMAT`]: +/// +/// ```json +/// {"format": 1, "mode": "layered", "kernelCommandLine": "... nvx_sandbox=1 ...", +/// "plan": PLAN, "rootConfiguration": "version=1\nruntime=/run/runtime-adapter\n..."} +/// ``` +/// +/// The `mode` is `layered`, or `blockless` for a VM without a layered root, whose `plan` and +/// `rootConfiguration` are `null`. The `plan` is the persisted form of the VM's `SandboxPlan`, and +/// the `rootConfiguration` the encoding of the root configuration that the host sends, as it sends +/// it, whether or not the guest accepts it. +#[derive(Clone, Copy, Debug)] +pub struct LaunchRecord<'a> { + /// The VM's kernel command line. + pub kernel_command_line: &'a str, + /// The VM's layered root, if it has one. + pub layered: Option<&'a LayeredLaunch>, +} + +impl LaunchRecord<'_> { + /// Returns the record as JSON. + /// + /// Fails with [`ErrorKind::Other`](nvx::ErrorKind::Other) if the plan has no JSON form. + pub fn to_json(&self) -> Result { + let (mode, plan, config) = match self.layered { + Some(layered) => { + let plan = serde_json::to_value(&layered.plan).map_err(|error| { + Error::other("the plan of the layered root has no JSON form").with_source(error) + })?; + let config = String::from_utf8_lossy(&layered.config.encode()).into_owned(); + ("layered", plan, Value::String(config)) + } + None => ("blockless", Value::Null, Value::Null), + }; + Ok(json!({ + "format": RECORD_FORMAT, + "mode": mode, + "kernelCommandLine": self.kernel_command_line, + "plan": plan, + "rootConfiguration": config, + })) + } + + /// Writes the record in `directory`, the sandbox's store, in place of an earlier one. + /// + /// Fails as [`to_json`](Self::to_json) does, and with + /// [`ErrorKind::Io`](nvx::ErrorKind::Io) if the record cannot be written. + pub fn write(&self, directory: &Path) -> Result<()> { + let path = directory.join(RECORD_NAME); + let mut text = self.to_json()?.to_string(); + text.push('\n'); + fs::write(&path, text).map_err(|error| { + Error::io(format!("cannot write {}", path.display())).with_source(error) + }) + } +} diff --git a/tests/runtime-adapter/src/lib.rs b/tests/runtime-adapter/src/lib.rs index 71ddf1197..d948ef4ef 100644 --- a/tests/runtime-adapter/src/lib.rs +++ b/tests/runtime-adapter/src/lib.rs @@ -5,12 +5,13 @@ //! Each configuration enables one side of `nvx`: //! //! - protocol-only, without features: the control protocol, the contract versions, and the other -//! items that `nvx` has without features, with the adapter's own [`frame`]s; +//! items that `nvx` has without features, with the adapter's own [`frame`]s, and the bodies of +//! its messages about a layered [`root`] and its [`probe`]s; //! - host-only, with the `host` feature: the host API too, and the host program, in the `host`, -//! `runtime`, and `initramfs` modules and the `runtime-adapter-host` binary; +//! `runtime`, `initramfs`, and `layered` modules and the `runtime-adapter-host` binary; //! - guest-only, with the `guest` feature: the guest API too, and the guest agent, in the `guest` -//! module and the `runtime-adapter-guest` binary, a static `x86_64-unknown-linux-musl` -//! executable. +//! and `worker` modules and the `runtime-adapter-guest` binary, a static +//! `x86_64-unknown-linux-musl` executable. //! //! The adapter brings its own framing to the control session between the host and the guest, in //! place of NVX's `NVXC` frames. The guest agent serves requests in that framing, and @@ -24,6 +25,12 @@ //! NVX's identity of its artifacts, with the guest agent's build in it, and the `initramfs` module //! writes the initramfs that puts the guest agent in place of the managed agent, where //! `runtime-adapter-guest init` serves the VM's control tty. +//! +//! The host program also launches VMs with a layered root, of an image over an upper, which it +//! plans with NVX's typed plan, in the `layered` module. In those, the guest agent's PID 1 starts +//! a worker, in the `worker` module, which assembles the root through `nvx::guest::root` outside +//! PID 1, as the host's root configuration of the [`root`] module asks, and runs the [`probe`]s +//! with which the host checks the root. pub mod frame; #[cfg(all(feature = "guest", target_os = "linux"))] @@ -33,14 +40,33 @@ pub mod host; #[cfg(feature = "host")] pub mod initramfs; #[cfg(feature = "host")] +pub mod layered; +pub mod probe; +pub mod root; +#[cfg(feature = "host")] pub mod runtime; +#[cfg(all(feature = "guest", target_os = "linux"))] +pub mod worker; /// Command that starts the guest agent on the runtime adapter's standard streams. pub const GUEST_SERVE_COMMAND: &str = "serve"; -/// Command that starts the guest agent on the tty that follows it, in place of NVX's managed agent. +/// Command that starts the guest agent on the tty that follows it, in place of NVX's managed agent, +/// or, with [`GUEST_LAYERED_MODE`] after the tty, PID 1 of a VM with a layered root. pub const GUEST_INIT_COMMAND: &str = "init"; +/// The mode of [`GUEST_INIT_COMMAND`] that runs PID 1 of a VM with a layered root, which starts the +/// worker. +pub const GUEST_LAYERED_MODE: &str = "layered"; + +/// Command that starts the worker of a VM with a layered root on the tty that follows it, with the +/// process ID of PID 1's unrelated child after the tty. +pub const GUEST_WORKER_COMMAND: &str = "worker"; + +/// Command that starts PID 1's unrelated child in a VM with a layered root, which sleeps until a +/// signal ends it. +pub const GUEST_IDLE_COMMAND: &str = "idle"; + /// Describes the runtime contract that this build speaks: the control contract revision, and the /// versions of the microVM ABI, the NVXS records, and the NVXC frames. pub fn contract() -> String { diff --git a/tests/runtime-adapter/src/probe.rs b/tests/runtime-adapter/src/probe.rs new file mode 100644 index 000000000..37cab9906 --- /dev/null +++ b/tests/runtime-adapter/src/probe.rs @@ -0,0 +1,358 @@ +//! The adapter's probes: named checks that the host asks the guest's worker to run in a VM with a +//! layered root, and the facts with which the worker answers, which a scenario then checks. +//! +//! A [`Probe`] is UTF-8 text: the probe's name, then each of its arguments after one space. A name +//! starts with a lowercase ASCII letter, and holds lowercase ASCII letters, digits, and `-`, so +//! that it never reads as a command-line option, and an argument is text without whitespace or +//! control characters. The worker answers a probe that it ran with the [`Facts`] that it found, +//! and one that it does not know, or that failed, with the problem, as UTF-8 text. +//! +//! [`Facts`] are UTF-8 text too: one `KEY=VALUE` line per fact, each ended by a newline, in the +//! order in which the probe found them, and each key at most once. A key holds lowercase ASCII +//! letters, digits, `.`, and `-`, and a value [escapes](escape) `\`, newlines, and carriage +//! returns as `\\`, `\n`, and `\r`, so that each fact stays on one line. + +use std::fmt; + +use crate::frame::{BodyError, MAX_TEXT_BODY}; + +/// A probe: a named check that the guest's worker runs, with its arguments. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Probe { + /// The probe's name. + name: String, + /// The probe's arguments. + arguments: Vec, +} + +impl Probe { + /// Returns the probe `name` with `arguments`. + /// + /// Fails with a [`BodyError`] if the name or an argument breaks the rules of the + /// [module documentation](self), or if the probe's encoding is longer than + /// [`MAX_TEXT_BODY`] bytes. + pub fn new>( + name: impl Into, + arguments: impl IntoIterator, + ) -> Result { + let name = name.into(); + let valid_name = name.starts_with(|character: char| character.is_ascii_lowercase()) + && name + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-'); + if !valid_name { + return Err(BodyError::new(format!( + "the probe's name {name:?} is not a lowercase ASCII letter, then lowercase ASCII \ + letters, digits, and `-`" + ))); + } + let arguments = arguments.into_iter().map(Into::into).collect::>(); + let invalid = arguments.iter().find(|argument| { + argument.is_empty() + || argument + .chars() + .any(|character| character.is_whitespace() || character.is_control()) + }); + if let Some(argument) = invalid { + return Err(BodyError::new(format!( + "the {name} probe's argument {argument:?} is empty, or holds whitespace or a \ + control character" + ))); + } + let probe = Self { name, arguments }; + let length = probe.to_string().len(); + if length > MAX_TEXT_BODY { + return Err(BodyError::new(format!( + "the {} probe takes {length} bytes, more than {MAX_TEXT_BODY}", + probe.name + ))); + } + Ok(probe) + } + + /// Reads a probe from its text, its name and then each argument after one space, as a + /// scenario names it. + /// + /// Fails as [`new`](Self::new) does. + pub fn parse(text: &str) -> Result { + let mut words = text.split(' '); + let name = words.next().unwrap_or_default(); + Self::new(name, words) + } + + /// Returns the probe's name. + pub fn name(&self) -> &str { + &self.name + } + + /// Returns the probe's arguments. + pub fn arguments(&self) -> &[String] { + &self.arguments + } + + /// Returns the probe's encoding: its text. + pub fn encode(&self) -> Vec { + self.to_string().into_bytes() + } + + /// Reads a probe from its encoding. + /// + /// Fails with a [`BodyError`] if the body is not UTF-8 text, and otherwise as + /// [`parse`](Self::parse) does. + pub fn decode(body: &[u8]) -> Result { + let text = + std::str::from_utf8(body).map_err(|_| BodyError::new("the probe is not UTF-8 text"))?; + Self::parse(text) + } +} + +impl fmt::Display for Probe { + /// Writes the probe's text: its name, then each argument after one space. + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.name)?; + for argument in &self.arguments { + write!(formatter, " {argument}")?; + } + Ok(()) + } +} + +/// What a probe found: facts, each a key and a value, in the order in which the probe found them. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Facts(Vec<(String, String)>); + +impl Facts { + /// Returns no facts. + pub fn new() -> Self { + Self::default() + } + + /// Adds the fact that `key` has `value`. + /// + /// # Panics + /// + /// If `key` breaks the rules of the [module documentation](self), or if a fact has it + /// already: a probe names its facts itself. + pub fn push(&mut self, key: impl Into, value: impl Into) { + let key = key.into(); + assert!(valid_key(&key), "the fact's key {key:?} is not valid"); + assert!(self.get(&key).is_none(), "the fact {key} is repeated"); + self.0.push((key, value.into())); + } + + /// Returns the value of the fact `key`, if any. + pub fn get(&self, key: &str) -> Option<&str> { + self.0 + .iter() + .find(|(name, _)| name == key) + .map(|(_, value)| value.as_str()) + } + + /// Returns the facts, each as its key and its value, in order. + pub fn iter(&self) -> impl Iterator { + self.0 + .iter() + .map(|(key, value)| (key.as_str(), value.as_str())) + } + + /// Returns the number of facts. + pub fn len(&self) -> usize { + self.0.len() + } + + /// Returns whether there are no facts. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// Returns the facts' encoding: one `KEY=VALUE` line per fact, with the value + /// [escaped](escape). + pub fn encode(&self) -> Vec { + self.iter() + .flat_map(|(key, value)| format!("{key}={}\n", escape(value)).into_bytes()) + .collect() + } + + /// Reads facts from their encoding. + /// + /// Fails with a [`BodyError`] that names the first line that breaks the rules of the + /// [module documentation](self). + pub fn decode(body: &[u8]) -> Result { + let text = std::str::from_utf8(body) + .map_err(|_| BodyError::new("the facts are not UTF-8 text"))?; + let mut facts = Self::new(); + if text.is_empty() { + return Ok(facts); + } + let lines = text + .strip_suffix('\n') + .ok_or_else(|| BodyError::new("the facts do not end their last line with a newline"))?; + for line in lines.split('\n') { + let invalid = |problem: &str| BodyError::new(format!("the fact {line:?} {problem}")); + let (key, value) = line + .split_once('=') + .ok_or_else(|| invalid("has no value"))?; + if !valid_key(key) { + return Err(invalid("has an invalid key")); + } + if facts.get(key).is_some() { + return Err(invalid("repeats its key")); + } + let value = unescape(value).ok_or_else(|| invalid("has an invalid escape"))?; + facts.0.push((key.to_owned(), value)); + } + Ok(facts) + } +} + +/// Returns whether `key` is a valid key of a fact: lowercase ASCII letters, digits, `.`, and `-`. +fn valid_key(key: &str) -> bool { + !key.is_empty() + && key.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'.' || byte == b'-' + }) +} + +/// Returns `value` with `\`, newlines, and carriage returns escaped as `\\`, `\n`, and `\r`, as a +/// fact's encoding holds it, on one line. +pub fn escape(value: &str) -> String { + let mut escaped = String::with_capacity(value.len()); + for character in value.chars() { + match character { + '\\' => escaped.push_str("\\\\"), + '\n' => escaped.push_str("\\n"), + '\r' => escaped.push_str("\\r"), + character => escaped.push(character), + } + } + escaped +} + +/// Reverses [`escape`], or returns `None` for a `\` that starts no escape that it writes. +fn unescape(escaped: &str) -> Option { + let mut value = String::with_capacity(escaped.len()); + let mut characters = escaped.chars(); + while let Some(character) = characters.next() { + if character != '\\' { + value.push(character); + continue; + } + value.push(match characters.next()? { + '\\' => '\\', + 'n' => '\n', + 'r' => '\r', + _ => return None, + }); + } + Some(value) +} + +/// How the guest's worker answered a probe. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ProbeReply { + /// The probe ran, and found these facts. + Report(Facts), + /// The worker does not know the probe, or the probe failed, as the text says. + Failed(String), +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_probe_is_its_name_then_its_arguments() { + let probe = Probe::new("hold", ["/run/a", "ä"]).unwrap(); + assert_eq!(probe.name(), "hold"); + assert_eq!(probe.arguments(), ["/run/a", "ä"]); + assert_eq!(probe.encode(), "hold /run/a ä".as_bytes()); + assert_eq!(Probe::decode(&probe.encode()), Ok(probe.clone())); + assert_eq!(Probe::parse("hold /run/a ä"), Ok(probe)); + let bare = Probe::parse("mounts").unwrap(); + assert_eq!((bare.name(), bare.arguments()), ("mounts", &[][..])); + assert_eq!(bare.to_string(), "mounts"); + + let longest = format!("p {}", "a".repeat(MAX_TEXT_BODY - 2)); + assert_eq!( + Probe::parse(&longest).unwrap().encode().len(), + MAX_TEXT_BODY + ); + for (text, problem) in [ + ("", "name \"\" is not"), + ("Mounts", "name \"Mounts\" is not"), + ("mounts.all", "is not a lowercase ASCII letter, then"), + ("-mounts", "name \"-mounts\" is not"), + ("--mounts", "name \"--mounts\" is not"), + ("9p", "name \"9p\" is not"), + (" mounts", "name \"\" is not"), + ("mounts ", "argument \"\" is empty"), + ("mounts a", "argument \"\" is empty"), + ("mounts a\tb", "holds whitespace"), + ("mounts a\u{7f}", "a control character"), + ("mounts a\u{a0}b", "holds whitespace"), + (&format!("{longest}a"), "more than 4096"), + ] { + let error = Probe::parse(text).unwrap_err(); + assert!(error.problem().contains(problem), "{text:?}: {error}"); + } + let error = Probe::decode(b"mounts \xff").unwrap_err(); + assert_eq!(error.problem(), "the probe is not UTF-8 text"); + } + + #[test] + fn facts_are_one_line_each_with_escaped_values() { + let mut facts = Facts::new(); + assert!(facts.is_empty()); + facts.push("root.type", "overlay"); + facts.push("os-release.id", "a\\b\nc\rd=e"); + facts.push("empty", ""); + assert_eq!(facts.len(), 3); + assert_eq!(facts.get("root.type"), Some("overlay")); + assert_eq!(facts.get("missing"), None); + assert_eq!( + facts.iter().collect::>(), + [ + ("root.type", "overlay"), + ("os-release.id", "a\\b\nc\rd=e"), + ("empty", ""), + ] + ); + let encoded = facts.encode(); + assert_eq!( + encoded, + b"root.type=overlay\nos-release.id=a\\\\b\\nc\\rd=e\nempty=\n" + ); + assert_eq!(Facts::decode(&encoded), Ok(facts)); + assert_eq!(Facts::decode(b""), Ok(Facts::new())); + + for (body, problem) in [ + (&b"a=\xff\n"[..], "are not UTF-8 text"), + (b"a=b", "do not end their last line"), + (b"a=b\n\n", "\"\" has no value"), + (b"ab\n", "\"ab\" has no value"), + (b"A=b\n", "has an invalid key"), + (b"=b\n", "has an invalid key"), + (b"a b=c\n", "has an invalid key"), + (b"a=b\na=c\n", "repeats its key"), + (b"a=b\\\n", "has an invalid escape"), + (b"a=b\\t\n", "has an invalid escape"), + ] { + let error = Facts::decode(body).unwrap_err(); + assert!(error.problem().contains(problem), "{body:?}: {error}"); + } + } + + #[test] + #[should_panic(expected = "the fact a is repeated")] + fn a_probe_names_each_fact_once() { + let mut facts = Facts::new(); + facts.push("a", "1"); + facts.push("a", "2"); + } + + #[test] + #[should_panic(expected = "is not valid")] + fn a_probe_names_its_facts_with_valid_keys() { + Facts::new().push("Key", "value"); + } +} diff --git a/tests/runtime-adapter/src/root.rs b/tests/runtime-adapter/src/root.rs new file mode 100644 index 000000000..b40662d80 --- /dev/null +++ b/tests/runtime-adapter/src/root.rs @@ -0,0 +1,738 @@ +//! The adapter's messages about a guest's layered root: the host's root configuration, and the +//! stages of the root whose failures the guest reports. +//! +//! The boot tokens of `nvx::protocol::root` carry what NVX's boot contract defines: the image's +//! format and the identity that the guest expects of it, and the upper. A [`RootConfig`] carries +//! the rest that the root needs, which only the adapter defines, after the host attaches: the +//! runtime directory, the cgroups, their resource values, the sizes that the block devices must +//! have, and how long the guest waits for them. +//! +//! A root configuration is UTF-8 text: one `NAME=VALUE` line per field, each ended by a newline, +//! in any order, and each at most once. Numbers are decimal, without a sign or leading zeros: +//! +//! | Name | Value | Required | +//! | --- | --- | --- | +//! | `version` | [`CONFIG_VERSION`] | yes | +//! | `runtime` | The runtime directory, below which the guest assembles the root | yes | +//! | `agent-cgroup` | The agent's cgroup, which holds the guest's own processes | yes | +//! | `workload-cgroup` | The workload cgroup, the agent's sibling | yes | +//! | `device-timeout-ms` | How long the guest waits for each block device, at most 600,000 | yes | +//! | `agent-memory-low` | The agent's `memory.low`, in bytes | no | +//! | `workload-memory-max` | The workload cgroup's `memory.max`, in bytes | no | +//! | `workload-pids-max` | The workload cgroup's `pids.max` | no | +//! | `distro-sectors` | The size of the image's block device, in 512-byte sectors | no | +//! | `scratch-sectors` | The size of the scratch block device, in 512-byte sectors | no | +//! +//! The paths are absolute and canonical guest paths: not `/`, without an empty, `.`, or `..` +//! component, or a trailing `/`. The runtime directory holds no `,`, `:`, `\`, or `=`, which the +//! overlay's options reserve, and the two cgroups have the same parent. The optional numbers are +//! positive. [`RootConfig::decode`] refuses every other configuration, which the guest reports as +//! a failure of [`RootStage::Configuration`]. +//! +//! A [`RootFailure`] is a stage's code, one byte, then the problem as UTF-8 text. + +use std::collections::BTreeMap; +use std::fmt; +use std::time::Duration; + +use crate::frame::{BodyError, MAX_TEXT_BODY}; + +/// The version of the root configuration that this build writes and reads. +pub const CONFIG_VERSION: u32 = 1; + +/// The runtime directory unless the host names another. +pub const DEFAULT_RUNTIME: &str = "/run/runtime-adapter"; +/// The agent's cgroup unless the host names another. +pub const DEFAULT_AGENT_CGROUP: &str = "/sys/fs/cgroup/agent"; +/// The workload cgroup unless the host names another. +pub const DEFAULT_WORKLOAD_CGROUP: &str = "/sys/fs/cgroup/workload"; +/// The agent's `memory.low` unless the host names another: 16 MiB, as NVX's own guest reserves. +pub const DEFAULT_AGENT_MEMORY_LOW: u64 = 16 << 20; +/// The workload cgroup's `memory.max` unless the host names another: 128 MiB. +pub const DEFAULT_WORKLOAD_MEMORY_MAX: u64 = 128 << 20; +/// The workload cgroup's `pids.max` unless the host names another. +pub const DEFAULT_WORKLOAD_PIDS_MAX: u64 = 128; +/// How long the guest waits for each block device unless the host names another: 30 seconds, as +/// NVX's own guest waits. +pub const DEFAULT_DEVICE_TIMEOUT: Duration = Duration::from_secs(30); +/// The longest wait for a block device that a configuration may ask for: ten minutes. +pub const MAX_DEVICE_TIMEOUT: Duration = Duration::from_secs(600); + +/// The longest guest path of a configuration, in bytes: Linux's `PATH_MAX`, without the NUL. +const MAX_PATH_BYTES: usize = 4095; +/// The characters that a runtime directory may not hold, which the overlay's options reserve. +const RUNTIME_RESERVED: [char; 4] = [',', ':', '\\', '=']; + +/// The names of a configuration's fields. +const VERSION: &str = "version"; +/// See [`VERSION`]. +const RUNTIME: &str = "runtime"; +/// See [`VERSION`]. +const AGENT_CGROUP: &str = "agent-cgroup"; +/// See [`VERSION`]. +const WORKLOAD_CGROUP: &str = "workload-cgroup"; +/// See [`VERSION`]. +const DEVICE_TIMEOUT: &str = "device-timeout-ms"; +/// See [`VERSION`]. +const AGENT_MEMORY_LOW: &str = "agent-memory-low"; +/// See [`VERSION`]. +const WORKLOAD_MEMORY_MAX: &str = "workload-memory-max"; +/// See [`VERSION`]. +const WORKLOAD_PIDS_MAX: &str = "workload-pids-max"; +/// See [`VERSION`]. +const DISTRO_SECTORS: &str = "distro-sectors"; +/// See [`VERSION`]. +const SCRATCH_SECTORS: &str = "scratch-sectors"; +/// Every field, in the order in which [`RootConfig::encode`] writes them. +const FIELDS: [&str; 10] = [ + VERSION, + RUNTIME, + AGENT_CGROUP, + WORKLOAD_CGROUP, + DEVICE_TIMEOUT, + AGENT_MEMORY_LOW, + WORKLOAD_MEMORY_MAX, + WORKLOAD_PIDS_MAX, + DISTRO_SECTORS, + SCRATCH_SECTORS, +]; + +/// The host's configuration of a guest's layered root: everything that the root needs besides its +/// boot tokens, as the [module documentation](self) lists it. +/// +/// The guest paths are text, since the host may run on another operating system than the guest. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RootConfig { + /// The runtime directory, below which the guest assembles the root. + pub runtime: String, + /// The agent's cgroup, which holds the guest's own processes. + pub agent_cgroup: String, + /// The workload cgroup, the agent's sibling. + pub workload_cgroup: String, + /// How long the guest waits for each block device. + pub device_timeout: Duration, + /// The agent's `memory.low`, in bytes, if any. + pub agent_memory_low: Option, + /// The workload cgroup's `memory.max`, in bytes, if any. + pub workload_memory_max: Option, + /// The workload cgroup's `pids.max`, if any. + pub workload_pids_max: Option, + /// The size that the image's block device must have, in 512-byte sectors, if any. + pub distro_sectors: Option, + /// The size that the scratch block device must have, in 512-byte sectors, if any. + pub scratch_sectors: Option, +} + +impl Default for RootConfig { + /// Returns the configuration of the defaults: [`DEFAULT_RUNTIME`], [`DEFAULT_AGENT_CGROUP`], + /// [`DEFAULT_WORKLOAD_CGROUP`], [`DEFAULT_DEVICE_TIMEOUT`], [`DEFAULT_AGENT_MEMORY_LOW`], + /// [`DEFAULT_WORKLOAD_MEMORY_MAX`], and [`DEFAULT_WORKLOAD_PIDS_MAX`], with no sizes. + fn default() -> Self { + Self { + runtime: DEFAULT_RUNTIME.to_owned(), + agent_cgroup: DEFAULT_AGENT_CGROUP.to_owned(), + workload_cgroup: DEFAULT_WORKLOAD_CGROUP.to_owned(), + device_timeout: DEFAULT_DEVICE_TIMEOUT, + agent_memory_low: Some(DEFAULT_AGENT_MEMORY_LOW), + workload_memory_max: Some(DEFAULT_WORKLOAD_MEMORY_MAX), + workload_pids_max: Some(DEFAULT_WORKLOAD_PIDS_MAX), + distro_sectors: None, + scratch_sectors: None, + } + } +} + +impl RootConfig { + /// Returns the configuration's fields, as `NAME` and `VALUE`, in the order of the + /// [module documentation](self), without the optional fields that it lacks. + pub fn fields(&self) -> Vec<(&'static str, String)> { + let mut fields = vec![ + (VERSION, CONFIG_VERSION.to_string()), + (RUNTIME, self.runtime.clone()), + (AGENT_CGROUP, self.agent_cgroup.clone()), + (WORKLOAD_CGROUP, self.workload_cgroup.clone()), + (DEVICE_TIMEOUT, self.device_timeout.as_millis().to_string()), + ]; + let optional = [ + (AGENT_MEMORY_LOW, self.agent_memory_low), + (WORKLOAD_MEMORY_MAX, self.workload_memory_max), + (WORKLOAD_PIDS_MAX, self.workload_pids_max), + (DISTRO_SECTORS, self.distro_sectors), + (SCRATCH_SECTORS, self.scratch_sectors), + ]; + fields.extend( + optional + .into_iter() + .filter_map(|(name, value)| Some((name, value?.to_string()))), + ); + fields + } + + /// Returns the configuration's encoding, its [fields](Self::fields) as `NAME=VALUE` lines, + /// as the host sends it, whether or not the guest accepts it. + pub fn encode(&self) -> Vec { + self.fields() + .into_iter() + .flat_map(|(name, value)| format!("{name}={value}\n").into_bytes()) + .collect() + } + + /// Reads a configuration from its encoding, and checks it as the + /// [module documentation](self) describes. + /// + /// Fails with a [`BodyError`] that names the first field that breaks the rules. + pub fn decode(body: &[u8]) -> Result { + let refused = |problem: String| BodyError::new(format!("the root configuration {problem}")); + let text = std::str::from_utf8(body).map_err(|_| refused("is not UTF-8 text".into()))?; + let mut values = BTreeMap::new(); + if !text.is_empty() { + let lines = text + .strip_suffix('\n') + .ok_or_else(|| refused("does not end its last line with a newline".into()))?; + for line in lines.split('\n') { + let (name, value) = line + .split_once('=') + .ok_or_else(|| refused(format!("has a line without a value: {line:?}")))?; + if !FIELDS.contains(&name) { + return Err(refused(format!("names the unknown field {name:?}"))); + } + if values.insert(name, value).is_some() { + return Err(refused(format!("repeats the field {name}"))); + } + } + } + let required = |name: &str| { + values + .get(name) + .copied() + .ok_or_else(|| refused(format!("lacks the field {name}"))) + }; + let version = required(VERSION)?; + if version != CONFIG_VERSION.to_string() { + return Err(refused(format!( + "has version {version:?}, but this guest reads version {CONFIG_VERSION}" + ))); + } + let path = |name: &str| { + let path = required(name)?; + match path_problem(path) { + None => Ok(path.to_owned()), + Some(problem) => Err(refused(format!( + "names the {name} {path:?}, which {problem}" + ))), + } + }; + let runtime = path(RUNTIME)?; + if runtime.contains(RUNTIME_RESERVED) { + return Err(refused(format!( + "names a runtime directory with a `,`, `:`, `\\`, or `=`, which the overlay's \ + options reserve: {runtime:?}" + ))); + } + let agent_cgroup = path(AGENT_CGROUP)?; + let workload_cgroup = path(WORKLOAD_CGROUP)?; + if agent_cgroup == workload_cgroup || parent(&agent_cgroup) != parent(&workload_cgroup) { + return Err(refused(format!( + "names cgroups that are not siblings: {agent_cgroup:?} and {workload_cgroup:?}" + ))); + } + // The guest mounts the cgroup hierarchy at the cgroups' parent unless it is a cgroup. + if parent(&agent_cgroup).is_empty() { + return Err(refused(format!( + "names cgroups in the root directory: {agent_cgroup:?} and {workload_cgroup:?}" + ))); + } + let number = |name: &str| -> Result, BodyError> { + let Some(text) = values.get(name) else { + return Ok(None); + }; + match decimal(text) { + Some(value) if value > 0 => Ok(Some(value)), + _ => Err(refused(format!( + "has a {name} that is not a positive decimal number: {text:?}" + ))), + } + }; + let timeout = number(DEVICE_TIMEOUT)? + .ok_or_else(|| refused(format!("lacks the field {DEVICE_TIMEOUT}")))?; + let device_timeout = Duration::from_millis(timeout); + if device_timeout > MAX_DEVICE_TIMEOUT { + return Err(refused(format!( + "asks to wait {timeout} ms for each block device, longer than {} ms", + MAX_DEVICE_TIMEOUT.as_millis() + ))); + } + Ok(Self { + runtime, + agent_cgroup, + workload_cgroup, + device_timeout, + agent_memory_low: number(AGENT_MEMORY_LOW)?, + workload_memory_max: number(WORKLOAD_MEMORY_MAX)?, + workload_pids_max: number(WORKLOAD_PIDS_MAX)?, + distro_sectors: number(DISTRO_SECTORS)?, + scratch_sectors: number(SCRATCH_SECTORS)?, + }) + } +} + +/// Describes why `path` is not an absolute and canonical guest path, if it is not: the rest of a +/// sentence whose subject names the path. +fn path_problem(path: &str) -> Option<&'static str> { + if path.len() > MAX_PATH_BYTES { + return Some("is longer than 4095 bytes"); + } + if path.contains(['\0', '\n']) { + return Some("holds a NUL or a newline"); + } + let Some(relative) = path.strip_prefix('/') else { + return Some("is not absolute"); + }; + if relative + .split('/') + .any(|component| matches!(component, "" | "." | "..")) + { + return Some("is not canonical, or is the root"); + } + None +} + +/// Returns the parent of the canonical absolute path `path`. +fn parent(path: &str) -> &str { + path.rsplit_once('/').map_or("", |(parent, _)| parent) +} + +/// Parses decimal digits without a sign or leading zeros, as a `u64`. +fn decimal(text: &str) -> Option { + if text.is_empty() + || !text.bytes().all(|byte| byte.is_ascii_digit()) + || (text.len() > 1 && text.starts_with('0')) + { + return None; + } + text.parse().ok() +} + +/// The status with which the guest's worker, and with it the VM, ends once a stage of its root +/// failed: this base plus the stage's [code](RootStage::code). +pub const FAILURE_STATUS_BASE: u8 = 10; + +/// A stage of a guest's layered root, which a [`RootFailure`] names. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum RootStage { + /// Receiving and checking the root configuration. + Configuration, + /// Reading the boot layout of the root from the kernel command line, and checking that the + /// guest can assemble it. + Layout, + /// Bootstrapping the guest for the root: its cgroups, and user namespaces. + Bootstrap, + /// Assembling the root. + Assembly, + /// Tearing the root down. + Teardown, +} + +impl RootStage { + /// Every stage, in order. + pub const ALL: [Self; 5] = [ + Self::Configuration, + Self::Layout, + Self::Bootstrap, + Self::Assembly, + Self::Teardown, + ]; + + /// Returns the stage's code in a [`RootFailure`]'s body: its position, from 1. + pub const fn code(self) -> u8 { + match self { + Self::Configuration => 1, + Self::Layout => 2, + Self::Bootstrap => 3, + Self::Assembly => 4, + Self::Teardown => 5, + } + } + + /// Returns the stage that `code` numbers, if any. + pub fn from_code(code: u8) -> Option { + Self::ALL.into_iter().find(|stage| stage.code() == code) + } + + /// Returns the stage's name: `configuration`, `layout`, `bootstrap`, `assembly`, or + /// `teardown`. + pub const fn as_str(self) -> &'static str { + match self { + Self::Configuration => "configuration", + Self::Layout => "layout", + Self::Bootstrap => "bootstrap", + Self::Assembly => "assembly", + Self::Teardown => "teardown", + } + } + + /// Returns the stage named `name`, if any. + pub fn from_name(name: &str) -> Option { + Self::ALL.into_iter().find(|stage| stage.as_str() == name) + } + + /// Returns the status with which the guest's worker, and with it the VM, ends once this stage + /// failed: [`FAILURE_STATUS_BASE`] plus its code, from 11 for the configuration to 15 for + /// the teardown. + pub const fn exit_status(self) -> u8 { + FAILURE_STATUS_BASE + self.code() + } +} + +impl fmt::Display for RootStage { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.as_str()) + } +} + +/// The most bytes of a [`RootFailure`]'s problem: what its body holds after the stage's code. +pub const MAX_PROBLEM_LEN: usize = MAX_TEXT_BODY - 1; + +/// A failure of a stage of a guest's layered root, which the guest reports to the host. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RootFailure { + /// The stage that failed. + stage: RootStage, + /// What failed. + problem: String, +} + +impl RootFailure { + /// Returns the failure of `stage` that `problem` describes, which is cut to its first + /// [`MAX_PROBLEM_LEN`] bytes, at a character's boundary, if it is longer. + pub fn new(stage: RootStage, problem: impl Into) -> Self { + let mut problem = problem.into(); + let length = crate::frame::truncated(&problem, MAX_PROBLEM_LEN).len(); + problem.truncate(length); + Self { stage, problem } + } + + /// Returns the stage that failed. + pub fn stage(&self) -> RootStage { + self.stage + } + + /// Returns what failed. + pub fn problem(&self) -> &str { + &self.problem + } + + /// Returns the failure's encoding: the stage's code, then the problem. + pub fn encode(&self) -> Vec { + let mut body = vec![self.stage.code()]; + body.extend_from_slice(self.problem.as_bytes()); + body + } + + /// Reads a failure from its encoding. + /// + /// Fails with a [`BodyError`] for an empty body, an unknown stage, or a problem that is not + /// UTF-8 text. + pub fn decode(body: &[u8]) -> Result { + let Some((&code, problem)) = body.split_first() else { + return Err(BodyError::new("a root's failure names no stage")); + }; + let stage = RootStage::from_code(code).ok_or_else(|| { + BodyError::new(format!("a root's failure names the unknown stage {code}")) + })?; + let problem = std::str::from_utf8(problem) + .map_err(|_| BodyError::new("a root's failure is not UTF-8 text"))?; + Ok(Self::new(stage, problem)) + } +} + +impl fmt::Display for RootFailure { + /// Writes `the STAGE of the root failed: PROBLEM`. + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "the {} of the root failed: {}", + self.stage, self.problem + ) + } +} + +impl std::error::Error for RootFailure {} + +/// How the guest answered a root configuration. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RootReply { + /// The guest assembled the root. + Ready, + /// A stage of the root failed, and the guest ends. + Failed(RootFailure), +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Returns a configuration with every field. + fn full() -> RootConfig { + RootConfig { + distro_sectors: Some(116_960), + scratch_sectors: Some(131_072), + ..RootConfig::default() + } + } + + #[test] + fn a_configuration_is_one_line_per_field() { + assert_eq!( + String::from_utf8(full().encode()).unwrap(), + "version=1\n\ + runtime=/run/runtime-adapter\n\ + agent-cgroup=/sys/fs/cgroup/agent\n\ + workload-cgroup=/sys/fs/cgroup/workload\n\ + device-timeout-ms=30000\n\ + agent-memory-low=16777216\n\ + workload-memory-max=134217728\n\ + workload-pids-max=128\n\ + distro-sectors=116960\n\ + scratch-sectors=131072\n" + ); + let minimal = RootConfig { + agent_memory_low: None, + workload_memory_max: None, + workload_pids_max: None, + ..RootConfig::default() + }; + for config in [full(), minimal, RootConfig::default()] { + assert_eq!(RootConfig::decode(&config.encode()), Ok(config)); + } + // The fields may come in any order. + let mut lines = String::from_utf8(full().encode()) + .unwrap() + .lines() + .map(str::to_owned) + .collect::>(); + lines.reverse(); + let reversed = lines + .iter() + .map(|line| format!("{line}\n")) + .collect::(); + assert_eq!(RootConfig::decode(reversed.as_bytes()), Ok(full())); + } + + #[test] + fn a_configuration_that_breaks_the_rules_is_refused() { + let base = String::from_utf8(RootConfig::default().encode()).unwrap(); + let replaced = |name: &str, value: &str| { + base.lines() + .map(|line| match line.split_once('=') { + Some((field, _)) if field == name => format!("{name}={value}\n"), + _ => format!("{line}\n"), + }) + .collect::() + }; + let without = |name: &str| { + base.lines() + .filter(|line| !line.starts_with(&format!("{name}="))) + .map(|line| format!("{line}\n")) + .collect::() + }; + let cases = [ + (Vec::from(&b"version=1\xff\n"[..]), "is not UTF-8 text"), + ( + base.trim_end().as_bytes().to_vec(), + "does not end its last line", + ), + ( + format!("{base}\n").into_bytes(), + "has a line without a value", + ), + ( + format!("{base}color=blue\n").into_bytes(), + "names the unknown field \"color\"", + ), + ( + format!("{base}runtime=/run/other\n").into_bytes(), + "repeats the field runtime", + ), + (Vec::new(), "lacks the field version"), + (without(RUNTIME).into_bytes(), "lacks the field runtime"), + ( + without(DEVICE_TIMEOUT).into_bytes(), + "lacks the field device-timeout-ms", + ), + (replaced(VERSION, "2").into_bytes(), "has version \"2\""), + ( + replaced(RUNTIME, "run/relative").into_bytes(), + "is not absolute", + ), + ( + replaced(RUNTIME, "/").into_bytes(), + "is not canonical, or is the root", + ), + ( + replaced(RUNTIME, "/run//nvx").into_bytes(), + "is not canonical", + ), + ( + replaced(RUNTIME, "/run/./nvx").into_bytes(), + "is not canonical", + ), + ( + replaced(RUNTIME, "/run/../nvx").into_bytes(), + "is not canonical", + ), + ( + replaced(RUNTIME, "/run/nvx/").into_bytes(), + "is not canonical", + ), + ( + replaced(RUNTIME, "/run/a,b").into_bytes(), + "the overlay's options reserve", + ), + ( + replaced(RUNTIME, "/run/a:b").into_bytes(), + "the overlay's options reserve", + ), + ( + replaced(RUNTIME, "/run/a=b").into_bytes(), + "the overlay's options reserve", + ), + (replaced(RUNTIME, "/run/a\0b").into_bytes(), "holds a NUL"), + ( + replaced(RUNTIME, &format!("/{}", "a".repeat(4095))).into_bytes(), + "longer than 4095", + ), + ( + replaced(AGENT_CGROUP, "agent").into_bytes(), + "names the agent-cgroup \"agent\", which is not absolute", + ), + ( + replaced(WORKLOAD_CGROUP, "/sys/fs/cgroup/agent").into_bytes(), + "not siblings", + ), + ( + replaced(WORKLOAD_CGROUP, "/sys/fs/cgroup/agent/workload").into_bytes(), + "not siblings", + ), + ( + replaced(WORKLOAD_CGROUP, "/sys/fs/workload").into_bytes(), + "not siblings", + ), + ( + replaced(AGENT_CGROUP, "/agent") + .replace( + "workload-cgroup=/sys/fs/cgroup/workload", + "workload-cgroup=/workload", + ) + .into_bytes(), + "names cgroups in the root directory", + ), + ( + replaced(DEVICE_TIMEOUT, "0").into_bytes(), + "not a positive decimal number", + ), + ( + replaced(DEVICE_TIMEOUT, "600001").into_bytes(), + "longer than 600000 ms", + ), + ( + replaced(AGENT_MEMORY_LOW, "-1").into_bytes(), + "not a positive decimal number", + ), + ( + replaced(WORKLOAD_MEMORY_MAX, "0x10").into_bytes(), + "not a positive decimal number", + ), + ( + replaced(WORKLOAD_PIDS_MAX, "010").into_bytes(), + "not a positive decimal number", + ), + ( + replaced(WORKLOAD_PIDS_MAX, "").into_bytes(), + "not a positive decimal number", + ), + ( + format!("{base}distro-sectors=18446744073709551616\n").into_bytes(), + "not a positive decimal number", + ), + ( + format!("{base}scratch-sectors= 1\n").into_bytes(), + "not a positive decimal number", + ), + ]; + for (body, problem) in cases { + let error = RootConfig::decode(&body).unwrap_err(); + assert!( + error.problem().starts_with("the root configuration ") + && error.problem().contains(problem), + "{:?}: {error}", + String::from_utf8_lossy(&body) + ); + } + // The longest valid path, and the largest number. + let longest = format!("/{}", "a".repeat(4094)); + let large = format!("{base}distro-sectors={}\n", u64::MAX); + assert_eq!( + RootConfig::decode(replaced(RUNTIME, &longest).as_bytes()) + .unwrap() + .runtime, + longest + ); + assert_eq!( + RootConfig::decode(large.as_bytes()).unwrap().distro_sectors, + Some(u64::MAX) + ); + } + + #[test] + fn stages_have_codes_names_and_exit_statuses() { + let expected = [ + (RootStage::Configuration, 1, "configuration", 11), + (RootStage::Layout, 2, "layout", 12), + (RootStage::Bootstrap, 3, "bootstrap", 13), + (RootStage::Assembly, 4, "assembly", 14), + (RootStage::Teardown, 5, "teardown", 15), + ]; + for (stage, code, name, status) in expected { + assert_eq!(stage.code(), code); + assert_eq!(RootStage::from_code(code), Some(stage)); + assert_eq!(stage.as_str(), name); + assert_eq!(stage.to_string(), name); + assert_eq!(RootStage::from_name(name), Some(stage)); + assert_eq!(stage.exit_status(), status); + } + assert_eq!(RootStage::from_code(0), None); + assert_eq!(RootStage::from_code(6), None); + assert_eq!(RootStage::from_name("mapping"), None); + } + + #[test] + fn a_failure_is_its_stage_then_its_problem() { + let failure = RootFailure::new(RootStage::Assembly, "the UUID differs"); + assert_eq!(failure.encode(), b"\x04the UUID differs"); + assert_eq!(RootFailure::decode(&failure.encode()), Ok(failure.clone())); + assert_eq!( + failure.to_string(), + "the assembly of the root failed: the UUID differs" + ); + let empty = RootFailure::new(RootStage::Teardown, ""); + assert_eq!(RootFailure::decode(&[5]), Ok(empty)); + + // A long problem is cut at a character's boundary, so that the body fits its limit. + let long = "é".repeat(MAX_TEXT_BODY); + let failure = RootFailure::new(RootStage::Configuration, long); + assert_eq!(failure.problem().len(), MAX_PROBLEM_LEN - 1); + assert!(failure.encode().len() <= MAX_TEXT_BODY); + assert_eq!(RootFailure::decode(&failure.encode()), Ok(failure)); + + for (body, problem) in [ + (&b""[..], "names no stage"), + (b"\x00problem", "unknown stage 0"), + (b"\x06problem", "unknown stage 6"), + (b"\x01\xffproblem", "not UTF-8 text"), + ] { + let error = RootFailure::decode(body).unwrap_err(); + assert!(error.problem().contains(problem), "{body:?}: {error}"); + } + } +} diff --git a/tests/runtime-adapter/src/worker.rs b/tests/runtime-adapter/src/worker.rs new file mode 100644 index 000000000..5430b0416 --- /dev/null +++ b/tests/runtime-adapter/src/worker.rs @@ -0,0 +1,415 @@ +//! The adapter's worker: the guest agent of a VM with a layered root. It assembles the root through +//! `nvx::guest::root` in a process of its own, never PID 1, and serves the host program until the +//! host stops it. +//! +//! # PID 1 and the worker +//! +//! In a VM with a layered root, NVX's `/init` runs `/sbin/nvx-init-agent`, which the adapter's +//! initramfs replaces with `runtime-adapter-guest init /dev/hvc2 layered`; see the host side's +//! `initramfs` module. That PID 1, [`supervise`], stays minimal: +//! +//! 1. It starts an unrelated child, `runtime-adapter-guest idle`, which stands for a daemon that a +//! guest starts before its agent, and which the worker leaves as it is. +//! 2. It starts the worker, `runtime-adapter-guest worker /dev/hvc2 SIBLING`, with the process ID +//! of the unrelated child, which the worker's probes check. +//! 3. It collects the status of every child, adopted processes included, until the worker ends, +//! and then ends the VM with the worker's status, through NVX's `/sbin/nvx-exit`. +//! +//! The worker, [`work`], runs the session in this order: +//! +//! 1. It opens the tty and attaches the session. +//! 2. It receives the host's root configuration, a [`RootConfig`], and checks it. +//! 3. It reads the boot layout of the root from the kernel command line with +//! `nvx::protocol::root::RootLayout::parse`, then bootstraps the guest and assembles the root. +//! 4. It answers `RootReady`, or `RootFailed` with the stage that failed, and then ends. +//! 5. It serves `Ping`, `Echo`, and `Probe`, whose probes the [`probes`] module runs. +//! 6. On `Stop`, it tears the root down, reports a teardown that failed with `RootFailed`, +//! confirms the stop with `Stopped`, and ends. +//! +//! Before the root configuration, it accepts only the configuration, or a `Stop`, which ends it +//! without a root. It assembles an EROFS image over an ext4 scratch, and refuses the other layouts +//! that the boot tokens can name at the layout stage. +//! +//! The worker, and with it the VM, ends with 0 once the host stopped it, after a teardown that +//! succeeded, with the stage's [exit status](RootStage::exit_status) once a stage failed, from 11 +//! for the configuration to 15 for the teardown, and with [`SESSION_STATUS`] once its session +//! failed or the broker closed it. + +pub mod probes; + +use std::fs::{self, File}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::Duration; + +use nix::errno::Errno; +use nix::sys::signal::{self, Signal}; +use nix::sys::wait::{self, WaitPidFlag, WaitStatus}; +use nix::unistd::Pid; +use nvx::guest::cgroup::Layout; +use nvx::guest::root::{self, Assembly, Bootstrap, Device, Resources, Root}; +use nvx::protocol::root::{ + DISTRO_MMIO_BASE, ImageFormat, RootLayout, RootUpper, SCRATCH_MMIO_BASE, +}; + +use crate::frame::{MAX_BODY, MAX_TEXT_BODY, Message, truncated}; +use crate::guest::{self, Ended, ServeError, Step}; +use crate::probe::{Facts, Probe}; +use crate::root::{RootConfig, RootFailure, RootStage}; + +/// The status with which the worker ends once its session failed or the broker closed it, and +/// with which PID 1 ends the VM if a child cannot start or the worker's status cannot be collected. +pub const SESSION_STATUS: u8 = 1; + +/// The kernel command line, from which the worker reads the boot layout of the root. +const COMMAND_LINE: &str = "/proc/cmdline"; + +/// The operations of a layered root that the worker runs: those of `nvx::guest::root` in a VM, in +/// [`LibraryRoot`], and a fake's in tests. Each fails with the problem, as text. +pub trait RootOperations { + /// Returns the kernel command line, from which the worker reads the boot layout of the root. + fn command_line(&mut self) -> Result, String>; + + /// Bootstraps the guest for the root that `config` configures: its cgroups, and their + /// resource values. + fn bootstrap(&mut self, config: &RootConfig) -> Result<(), String>; + + /// Assembles the root of `layout`, which the boot tokens name, as `config` configures it. + fn assemble(&mut self, config: &RootConfig, layout: &RootLayout) -> Result<(), String>; + + /// Runs `probe` on the assembled root, and returns what it found. + fn probe(&mut self, probe: &Probe) -> Result; + + /// Tears the assembled root down. + fn teardown(&mut self) -> Result<(), String>; +} + +/// How the worker's session ended, when it did not fail. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Worked { + /// The host stopped the worker, which tore its root down, if it had assembled one. + Stopped, + /// A stage of the root failed, which the worker reported to the host. + Failed(RootFailure), + /// The broker closed the stream at a record boundary. + Closed, +} + +impl Worked { + /// Returns the status with which the worker ends: 0 once the host stopped it, the stage's + /// [exit status](RootStage::exit_status) once a stage failed, and [`SESSION_STATUS`] once the + /// broker closed the stream. + pub fn status(&self) -> u8 { + match self { + Self::Stopped => 0, + Self::Failed(failure) => failure.stage().exit_status(), + Self::Closed => SESSION_STATUS, + } + } +} + +/// Runs the worker's session on `stream`, the guest's end of the control session, with the root +/// operations `operations`, as the [module documentation](self) describes, until the host stops +/// the worker, a stage of the root fails, or the broker closes the stream. +/// +/// A failure of the session, a malformed message, and a message that the worker does not accept +/// in its state end the session with an error, and leave a root that it assembled to +/// `operations`. +pub fn work( + stream: S, + operations: &mut dyn RootOperations, +) -> Result { + let mut ready = false; + let ended = guest::session(stream, |message| { + if ready { + return serve_root(operations, message); + } + let step = configure_root(operations, message)?; + ready = matches!(step, Step::Continue(_)); + Ok(step) + })?; + Ok(match ended { + Ended::Handled(worked) => worked, + Ended::Closed => Worked::Closed, + }) +} + +/// Handles `message` before the root is ready: a root configuration, whose root it assembles +/// through `operations`, or a `Stop`. +fn configure_root( + operations: &mut dyn RootOperations, + message: Message, +) -> Result, ServeError> { + match message { + Message::RootConfig(body) => Ok(match assemble(operations, &body) { + Ok(()) => Step::Continue(vec![Message::RootReady]), + Err(failure) => Step::End( + vec![Message::RootFailed(failure.encode())], + Worked::Failed(failure), + ), + }), + Message::Stop => Ok(Step::End(vec![Message::Stopped], Worked::Stopped)), + message => Err(ServeError::Unexpected(message.kind())), + } +} + +/// Handles `message` once the root is ready, through `operations`. +fn serve_root( + operations: &mut dyn RootOperations, + message: Message, +) -> Result, ServeError> { + Ok(match message { + Message::Ping => Step::Continue(vec![Message::Pong]), + Message::Echo(body) => Step::Continue(vec![Message::Echo(body)]), + Message::Probe(body) => Step::Continue(vec![run_probe(operations, &body)]), + Message::Stop => match operations.teardown() { + Ok(()) => Step::End(vec![Message::Stopped], Worked::Stopped), + Err(problem) => { + let failure = RootFailure::new(RootStage::Teardown, problem); + Step::End( + vec![Message::RootFailed(failure.encode()), Message::Stopped], + Worked::Failed(failure), + ) + } + }, + message => return Err(ServeError::Unexpected(message.kind())), + }) +} + +/// Checks the root configuration `body`, reads the boot layout of the root, bootstraps the guest, +/// and assembles the root, through `operations`, or returns the failure of the stage that failed. +fn assemble(operations: &mut dyn RootOperations, body: &[u8]) -> Result<(), RootFailure> { + let config = RootConfig::decode(body) + .map_err(|error| RootFailure::new(RootStage::Configuration, error.problem()))?; + let layout = + layout(operations).map_err(|problem| RootFailure::new(RootStage::Layout, problem))?; + operations + .bootstrap(&config) + .map_err(|problem| RootFailure::new(RootStage::Bootstrap, problem))?; + operations + .assemble(&config, &layout) + .map_err(|problem| RootFailure::new(RootStage::Assembly, problem)) +} + +/// Reads the boot layout of the root from the kernel command line, through `operations`, and +/// checks that the worker assembles it: an EROFS image over an ext4 scratch. +fn layout(operations: &mut dyn RootOperations) -> Result { + let command_line = operations.command_line()?; + let layout = RootLayout::parse(&command_line) + .map_err(|error| format!("the kernel command line has invalid boot tokens: {error}"))? + .ok_or("the kernel command line names no layered root")?; + if (layout.image.format, layout.upper) != (ImageFormat::Erofs, RootUpper::Scratch) { + return Err(format!( + "the worker assembles only an EROFS image over an ext4 scratch, not {} over {}", + layout.image.format, layout.upper + )); + } + Ok(layout) +} + +/// Runs the probe `body` through `operations`, and returns the worker's answer: a report of what +/// the probe found, or the problem. +fn run_probe(operations: &mut dyn RootOperations, body: &[u8]) -> Message { + let found = Probe::decode(body) + .map_err(|error| error.problem().to_owned()) + .and_then(|probe| operations.probe(&probe)); + let problem = match found { + Ok(facts) => { + let report = facts.encode(); + if report.len() <= MAX_BODY { + return Message::Report(report); + } + format!( + "the probe's report takes {} bytes, more than {MAX_BODY}", + report.len() + ) + } + Err(problem) => problem, + }; + Message::ProbeFailed(truncated(&problem, MAX_TEXT_BODY).as_bytes().to_vec()) +} + +/// The root operations of `nvx::guest::root` on the guest's kernel, whose probes the [`probes`] +/// module runs. +/// +/// Dropping it closes the descriptors that probes keep open, then unmounts a root that it still +/// holds, as `nvx::guest::root::Root` does when it drops, ignoring failures. +pub struct LibraryRoot { + /// The process ID of PID 1's unrelated child, which the probes check, if PID 1 named one. + sibling: Option, + /// What the bootstrap prepared, once it ran. + bootstrap: Option, + /// The descriptors that probes keep open until the worker ends, such as those of `hold`. + /// They close before the root drops. + held: Vec, + /// The root, from its assembly until its teardown. + root: Option, +} + +impl LibraryRoot { + /// Returns the operations, with the process ID of PID 1's unrelated child, if any. + pub fn new(sibling: Option) -> Self { + Self { + sibling, + bootstrap: None, + held: Vec::new(), + root: None, + } + } +} + +impl RootOperations for LibraryRoot { + /// Reads `/proc/cmdline`. + fn command_line(&mut self) -> Result, String> { + fs::read(COMMAND_LINE).map_err(|error| format!("cannot read {COMMAND_LINE}: {error}")) + } + + /// Runs `nvx::guest::root::bootstrap` with the configuration's cgroups and resource values. + fn bootstrap(&mut self, config: &RootConfig) -> Result<(), String> { + let layout = Layout::new(&config.agent_cgroup, &config.workload_cgroup) + .map_err(|error| error.to_string())?; + let resources = Resources { + agent_memory_low: config.agent_memory_low, + workload_memory_max: config.workload_memory_max, + workload_pids_max: config.workload_pids_max, + }; + let bootstrap = root::bootstrap(&layout, &resources).map_err(|error| error.to_string())?; + self.bootstrap = Some(bootstrap); + Ok(()) + } + + /// Runs `nvx::guest::root::assemble` on the block devices at the MMIO bases of their roles, + /// with the identity that the boot tokens expect of the image, and the configuration's + /// runtime directory, device sizes, and timeout. + fn assemble(&mut self, config: &RootConfig, layout: &RootLayout) -> Result<(), String> { + let assembly = Assembly { + runtime: PathBuf::from(&config.runtime), + distro: Device { + mmio: DISTRO_MMIO_BASE, + sectors: config.distro_sectors, + }, + distro_uuid: layout.image.identity, + scratch: Device { + mmio: SCRATCH_MMIO_BASE, + sectors: config.scratch_sectors, + }, + timeout: config.device_timeout, + }; + let root = root::assemble(&assembly).map_err(|error| error.to_string())?; + self.root = Some(root); + Ok(()) + } + + /// Runs one of the [`probes`]. + fn probe(&mut self, probe: &Probe) -> Result { + let (Some(bootstrap), Some(root)) = (&self.bootstrap, &self.root) else { + return Err("the worker holds no root to probe".to_owned()); + }; + let context = probes::Context { + runtime: root.runtime(), + root: root.path(), + agent: bootstrap.layout().agent(), + workload: bootstrap.layout().workload(), + sibling: self.sibling, + }; + probes::run(probe, &context, &mut self.held) + } + + /// Runs `nvx::guest::root::Root::teardown`. A root whose teardown failed keeps its mounts. + fn teardown(&mut self) -> Result<(), String> { + match self.root.take() { + Some(root) => root.teardown().map_err(|error| error.to_string()), + None => Ok(()), + } + } +} + +/// Runs PID 1 of a VM with a layered root, as the [module documentation](self) describes, with +/// the worker on the tty `tty`, and returns the worker's status, or [`SESSION_STATUS`] if a child +/// cannot start or the worker's status cannot be collected. +/// +/// As PID 1, it leaves the unrelated child to the end of the VM. Elsewhere, as in tests, it then +/// kills the unrelated child and collects its status. +pub fn supervise(tty: &Path) -> u8 { + let program = match std::env::current_exe() { + Ok(program) => program, + Err(error) => { + eprintln!("runtime-adapter-guest: cannot find its own executable: {error}"); + return SESSION_STATUS; + } + }; + let sibling = match start(Command::new(&program).arg(crate::GUEST_IDLE_COMMAND)) { + Ok(sibling) => sibling, + Err(error) => { + eprintln!("runtime-adapter-guest: cannot start PID 1's unrelated child: {error}"); + return SESSION_STATUS; + } + }; + let worker = start( + Command::new(&program) + .arg(crate::GUEST_WORKER_COMMAND) + .arg(tty) + .arg(sibling.to_string()), + ); + let status = match worker { + Ok(worker) => collect(worker), + Err(error) => { + eprintln!("runtime-adapter-guest: cannot start the worker: {error}"); + SESSION_STATUS + } + }; + if std::process::id() != 1 { + // A child keeps its process ID until its status is collected, so the signal reaches the + // unrelated child unless `collect` collected its status already. + let sibling = pid(sibling); + if wait::waitpid(sibling, Some(WaitPidFlag::WNOHANG)) == Ok(WaitStatus::StillAlive) { + let _ = signal::kill(sibling, Signal::SIGKILL); + let _ = wait::waitpid(sibling, None); + } + } + status +} + +/// Starts `command` with no standard input, and returns its process ID, whose status the caller +/// collects. +fn start(command: &mut Command) -> std::io::Result { + // PID 1 collects the status of every child through `wait`, rather than through the handle. + #[allow(clippy::zombie_processes)] + let child = command.stdin(Stdio::null()).spawn()?; + Ok(child.id()) +} + +/// Collects the status of every child until the child `worker` ends, and returns the worker's +/// status: its exit status, or 128 plus the signal that ended it, as a shell reports it. +fn collect(worker: u32) -> u8 { + let worker = pid(worker); + loop { + match wait::wait() { + Ok(WaitStatus::Exited(child, code)) if child == worker => { + return u8::try_from(code).unwrap_or(SESSION_STATUS); + } + Ok(WaitStatus::Signaled(child, signal, _)) if child == worker => { + return u8::try_from(128 + signal as i32).unwrap_or(SESSION_STATUS); + } + Ok(_) | Err(Errno::EINTR) => {} + Err(error) => { + eprintln!("runtime-adapter-guest: cannot collect the worker's status: {error}"); + return SESSION_STATUS; + } + } + } +} + +/// Returns the process ID `id` as `nix` takes it. +fn pid(id: u32) -> Pid { + // Linux process IDs are at most 2^22. + Pid::from_raw(i32::try_from(id).unwrap_or(i32::MAX)) +} + +/// Runs PID 1's unrelated child, which sleeps until a signal ends it. +pub fn idle() -> ! { + loop { + std::thread::sleep(Duration::from_secs(3600)); + } +} diff --git a/tests/runtime-adapter/src/worker/probes.rs b/tests/runtime-adapter/src/worker/probes.rs new file mode 100644 index 000000000..80f24317e --- /dev/null +++ b/tests/runtime-adapter/src/worker/probes.rs @@ -0,0 +1,475 @@ +//! The probes that the worker runs on its assembled root: named checks of a scenario, each of which +//! reports what it finds as [`Facts`]. +//! +//! | Probe | Facts | +//! | --- | --- | +//! | `mounts` | For the image's mount, `distro`, the scratch's, `scratch`, and the overlay's, `root`: the number of mounts at its mount point, `NAME.count`, and of the topmost, its `NAME.type`, `NAME.source`, `NAME.options`, and `NAME.super`, the file system's own options | +//! | `contents` | The root's `os-release.id` and `root.mode`. It writes [`PROBE_FILE`] in the root, and reports what the root, `write.root`, the upper, `write.upper`, and the image, `write.lower`, hold at that name, and whether the image refuses a new file, `lower.write` | +//! | `cgroups` | The file system at the cgroups' parent, `parent.type`, and its `parent.subtree-control`; whether each cgroup exists, `agent.exists` and `workload.exists`; `agent.memory.low`, `workload.memory.max`, and `workload.pids.max`; the number of processes in each, `agent.processes` and `workload.processes`; and `user-namespaces`, the guest's limit of user namespaces, or `unavailable` without them | +//! | `processes` | For PID 1, `pid1`, the worker, `worker`, and PID 1's unrelated child, `sibling`: `NAME.pid`, and while the process exists, `NAME.state`, `NAME.ppid`, `NAME.root`, `NAME.cgroup`, `NAME.command`, and `NAME.namespaces`: `same` if it shares PID 1's, or the ones that differ | +//! | `hold` | It opens the root, and keeps it open until the worker ends, so that the teardown finds the root busy: `held`, the path that it holds | +//! +//! None takes arguments. A value that a probe cannot read is `absent` if its file does not exist, +//! and otherwise `unreadable: PROBLEM`. + +use std::ffi::OsString; +use std::fs::{self, File}; +use std::io; +use std::os::unix::ffi::OsStringExt; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; + +use nix::errno::Errno; + +use crate::probe::{Facts, Probe}; + +/// The file that the `contents` probe writes at the top of the root. +pub const PROBE_FILE: &str = ".runtime-adapter-probe"; +/// What the `contents` probe writes in [`PROBE_FILE`]. +pub const PROBE_CONTENTS: &str = "runtime-adapter"; +/// The file that the `contents` probe tries to create at the top of the image. +const LOWER_FILE: &str = ".runtime-adapter-lower"; +/// The mount table of the worker's mount namespace. +const MOUNTINFO: &str = "/proc/self/mountinfo"; +/// The guest's limit of user namespaces, which only a kernel with user namespaces has. +const USER_NAMESPACES: &str = "/proc/sys/user/max_user_namespaces"; +/// The namespaces that the `processes` probe compares with PID 1's, of those that the guest's +/// kernel has. +const NAMESPACES: [&str; 7] = ["cgroup", "ipc", "mnt", "net", "pid", "user", "uts"]; + +/// What the probes check: the worker's root and cgroups, and PID 1's unrelated child. +#[derive(Clone, Copy, Debug)] +pub struct Context<'a> { + /// The runtime directory, below which `nvx::guest::root` mounts the root: the image at + /// `layers/distro`, the scratch, with the overlay's `upper` directory, at `scratch`, and the + /// overlay, the root, at `rootfs`. + pub runtime: &'a Path, + /// The root, the overlay's mount point. + pub root: &'a Path, + /// The agent's cgroup. + pub agent: &'a Path, + /// The workload cgroup. + pub workload: &'a Path, + /// The process ID of PID 1's unrelated child, if PID 1 named one. + pub sibling: Option, +} + +impl Context<'_> { + /// Returns the image's mount point. + fn distro(&self) -> PathBuf { + self.runtime.join("layers/distro") + } + + /// Returns the scratch's mount point. + fn scratch(&self) -> PathBuf { + self.runtime.join("scratch") + } +} + +/// Runs `probe` with `context`, and returns its facts. `held` keeps the descriptors that a probe +/// keeps open until the worker ends. +/// +/// Fails with the problem for a probe that the [module documentation](self) does not list, one +/// with arguments, and one that fails. +pub fn run(probe: &Probe, context: &Context<'_>, held: &mut Vec) -> Result { + let name = probe.name(); + if !["mounts", "contents", "cgroups", "processes", "hold"].contains(&name) { + return Err(format!("the worker runs no probe named {name}")); + } + if !probe.arguments().is_empty() { + return Err(format!("the {name} probe takes no arguments")); + } + let facts = match name { + "mounts" => mounts(context), + "contents" => contents(context), + "cgroups" => cgroups(context), + "processes" => processes(context), + _ => hold(context, held), + }; + facts.map_err(|error| format!("the {name} probe failed: {error}")) +} + +/// Runs the `mounts` probe. +fn mounts(context: &Context<'_>) -> io::Result { + let table = mount_table(&fs::read_to_string(MOUNTINFO)?)?; + let mut facts = Facts::new(); + for (name, point) in [ + ("distro", context.distro()), + ("scratch", context.scratch()), + ("root", context.root.to_owned()), + ] { + let mounts = table + .iter() + .filter(|mount| mount.point == point) + .collect::>(); + facts.push(format!("{name}.count"), mounts.len().to_string()); + // Of the mounts at a point, the table lists the topmost last. + if let Some(mount) = mounts.last() { + facts.push(format!("{name}.type"), mount.kind.as_str()); + facts.push(format!("{name}.source"), mount.source.as_str()); + facts.push(format!("{name}.options"), mount.options.as_str()); + facts.push(format!("{name}.super"), mount.super_options.as_str()); + } + } + Ok(facts) +} + +/// Runs the `contents` probe. +fn contents(context: &Context<'_>) -> io::Result { + let root = context.root; + let mut facts = Facts::new(); + // The image's own file first: `/etc/os-release` may be a link that resolves outside the root. + let identifier = ["usr/lib/os-release", "etc/os-release"] + .iter() + .find_map(|path| fs::read_to_string(root.join(path)).ok()) + .and_then(|text| os_release_id(&text).map(str::to_owned)); + facts.push("os-release.id", identifier.as_deref().unwrap_or("absent")); + let mode = fs::metadata(root)?.permissions().mode() & 0o7777; + facts.push("root.mode", format!("{mode:o}")); + fs::write(root.join(PROBE_FILE), PROBE_CONTENTS)?; + facts.push("write.root", read(&root.join(PROBE_FILE))); + let upper = context.scratch().join("upper").join(PROBE_FILE); + facts.push("write.upper", read(&upper)); + facts.push("write.lower", read(&context.distro().join(PROBE_FILE))); + let lower = match File::create_new(context.distro().join(LOWER_FILE)) { + Ok(_) => "allowed".to_owned(), + Err(error) if error.raw_os_error() == Some(Errno::EROFS as i32) => "refused".to_owned(), + Err(error) => format!("failed: {error}"), + }; + facts.push("lower.write", lower); + Ok(facts) +} + +/// Runs the `cgroups` probe. +fn cgroups(context: &Context<'_>) -> io::Result { + let parent = context.agent.parent().unwrap_or(Path::new("/")); + let table = mount_table(&fs::read_to_string(MOUNTINFO)?)?; + let parent_type = table + .iter() + .rev() + .find(|mount| mount.point == parent) + .map_or("none", |mount| mount.kind.as_str()); + let mut facts = Facts::new(); + facts.push("parent.type", parent_type); + facts.push( + "parent.subtree-control", + read(&parent.join("cgroup.subtree_control")), + ); + for (name, cgroup) in [("agent", context.agent), ("workload", context.workload)] { + let exists = if cgroup.is_dir() { "yes" } else { "no" }; + facts.push(format!("{name}.exists"), exists); + } + facts.push("agent.memory.low", read(&context.agent.join("memory.low"))); + for value in ["memory.max", "pids.max"] { + facts.push( + format!("workload.{value}"), + read(&context.workload.join(value)), + ); + } + for (name, cgroup) in [("agent", context.agent), ("workload", context.workload)] { + let processes = fs::read_to_string(cgroup.join("cgroup.procs"))?; + facts.push( + format!("{name}.processes"), + processes.lines().count().to_string(), + ); + } + let user_namespaces = Path::new(USER_NAMESPACES); + let limit = if user_namespaces.exists() { + read(user_namespaces) + } else { + "unavailable".to_owned() + }; + facts.push("user-namespaces", limit); + Ok(facts) +} + +/// Runs the `processes` probe. +fn processes(context: &Context<'_>) -> io::Result { + let init = namespaces(1)?; + let mut facts = Facts::new(); + let processes = [ + ("pid1", Some(1)), + ("worker", Some(std::process::id())), + ("sibling", context.sibling), + ]; + for (name, id) in processes { + let Some(id) = id else { + facts.push(format!("{name}.pid"), "none"); + continue; + }; + facts.push(format!("{name}.pid"), id.to_string()); + let directory = PathBuf::from(format!("/proc/{id}")); + let stat = match fs::read_to_string(directory.join("stat")) { + Err(error) if error.kind() == io::ErrorKind::NotFound => { + facts.push(format!("{name}.state"), "gone"); + continue; + } + stat => stat?, + }; + let (state, parent) = parse_stat(&stat).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("the status of process {id} is malformed"), + ) + })?; + facts.push(format!("{name}.state"), state); + facts.push(format!("{name}.ppid"), parent); + let root = fs::read_link(directory.join("root"))?; + facts.push(format!("{name}.root"), root.display().to_string()); + let cgroup = fs::read_to_string(directory.join("cgroup"))?; + facts.push( + format!("{name}.cgroup"), + unified_cgroup(&cgroup).unwrap_or("none"), + ); + let command = fs::read(directory.join("cmdline"))?; + facts.push(format!("{name}.command"), command_text(&command)); + let differ = namespaces(id)? + .into_iter() + .zip(&init) + .filter(|((_, own), (_, first))| own != first) + .map(|((namespace, _), _)| namespace) + .collect::>(); + let shared = if differ.is_empty() { + "same".to_owned() + } else { + format!("differ:{}", differ.join(",")) + }; + facts.push(format!("{name}.namespaces"), shared); + } + Ok(facts) +} + +/// Runs the `hold` probe, which keeps its descriptor in `held`. +fn hold(context: &Context<'_>, held: &mut Vec) -> io::Result { + held.push(File::open(context.root)?); + let mut facts = Facts::new(); + facts.push("held", context.root.display().to_string()); + Ok(facts) +} + +/// Returns the namespaces of the process `id`, each as its name and its identity, in the order +/// of [`NAMESPACES`], with `None` for one that the kernel does not have. +fn namespaces(id: u32) -> io::Result)>> { + NAMESPACES + .into_iter() + .map( + |namespace| match fs::read_link(format!("/proc/{id}/ns/{namespace}")) { + Ok(identity) => Ok((namespace, Some(identity))), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok((namespace, None)), + Err(error) => Err(error), + }, + ) + .collect() +} + +/// Returns the trimmed contents of the file at `path`, `absent` if it does not exist, or +/// `unreadable: PROBLEM`. +fn read(path: &Path) -> String { + match fs::read(path) { + Ok(bytes) => String::from_utf8_lossy(&bytes).trim().to_owned(), + Err(error) if error.kind() == io::ErrorKind::NotFound => "absent".to_owned(), + Err(error) => format!("unreadable: {error}"), + } +} + +/// A mount of a mount table. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Mount { + /// The mount point. + point: PathBuf, + /// The mount's options. + options: String, + /// The file system's type. + kind: String, + /// The mount's source. + source: String, + /// The file system's own options. + super_options: String, +} + +/// Parses a mount table in the format of `/proc/PID/mountinfo`, in its order: +/// `ID PARENT MAJOR:MINOR ROOT POINT OPTIONS [OPTIONAL...] - TYPE SOURCE SUPER`, with `\NNN` octal +/// escapes in its paths and source. +fn mount_table(table: &str) -> io::Result> { + table + .lines() + .map(|line| { + let fields = line.split(' ').collect::>(); + let separator = fields.iter().position(|field| *field == "-"); + let mount = match (separator, fields.as_slice()) { + (Some(separator), [_, _, _, _, point, options, ..]) if separator >= 6 => { + match fields[separator + 1..] { + [kind, source, super_options] => Some(Mount { + point: PathBuf::from(OsString::from_vec(unescape(point))), + options: (*options).to_owned(), + kind: kind.to_owned(), + source: String::from_utf8_lossy(&unescape(source)).into_owned(), + super_options: super_options.to_owned(), + }), + _ => None, + } + } + _ => None, + }; + mount.ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("the mount table has a malformed line: {line:?}"), + ) + }) + }) + .collect() +} + +/// Reverses the `\NNN` octal escapes of a field of a mount table. +fn unescape(field: &str) -> Vec { + let bytes = field.as_bytes(); + let mut unescaped = Vec::with_capacity(bytes.len()); + let mut index = 0; + while index < bytes.len() { + let escape = bytes.get(index + 1..index + 4).and_then(|digits| { + let octal = digits.iter().all(|digit| (b'0'..=b'7').contains(digit)); + let text = std::str::from_utf8(digits).ok()?; + octal.then(|| u8::from_str_radix(text, 8).ok()).flatten() + }); + match escape { + Some(byte) if bytes[index] == b'\\' => { + unescaped.push(byte); + index += 4; + } + _ => { + unescaped.push(bytes[index]); + index += 1; + } + } + } + unescaped +} + +/// Returns the state and the parent's process ID from a process's `/proc/PID/stat`, whose command +/// is parenthesized and may hold any character, `)` and spaces included. +fn parse_stat(stat: &str) -> Option<(&str, &str)> { + let (_, rest) = stat.rsplit_once(')')?; + let mut fields = rest.split_whitespace(); + Some((fields.next()?, fields.next()?)) +} + +/// Returns a process's cgroup of the unified hierarchy from its `/proc/PID/cgroup`: the path of +/// the line `0::PATH`. +fn unified_cgroup(cgroup: &str) -> Option<&str> { + cgroup.lines().find_map(|line| line.strip_prefix("0::")) +} + +/// Returns a process's command line from its `/proc/PID/cmdline`, whose arguments end with NULs, +/// with the arguments separated by spaces. +fn command_text(command: &[u8]) -> String { + command + .split(|byte| *byte == 0) + .filter(|argument| !argument.is_empty()) + .map(String::from_utf8_lossy) + .collect::>() + .join(" ") +} + +/// Returns the `ID` of an `os-release` file, without its quotes. +fn os_release_id(text: &str) -> Option<&str> { + text.lines() + .find_map(|line| line.strip_prefix("ID=")) + .map(|value| value.trim().trim_matches(['"', '\''])) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_mount_table_is_read_with_its_escapes() { + let table = "\ +22 1 0:21 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw +41 30 254:0 / /run/a\\040b/layers/distro ro,relatime - erofs /dev/vda ro,user_xattr +42 30 254:16 / /run/a\\040b/scratch rw,nosuid,nodev,relatime - ext4 /dev/vdb rw +43 30 0:40 / /run/a\\040b/rootfs rw,nosuid,nodev,relatime - overlay overlay \ +rw,lowerdir=/run/a\\040b/layers/distro,metacopy=on,xino=on +"; + let table = mount_table(table).unwrap(); + assert_eq!(table.len(), 4); + assert_eq!( + table[1], + Mount { + point: PathBuf::from("/run/a b/layers/distro"), + options: "ro,relatime".to_owned(), + kind: "erofs".to_owned(), + source: "/dev/vda".to_owned(), + super_options: "ro,user_xattr".to_owned(), + } + ); + assert_eq!(table[0].point, Path::new("/proc")); + assert_eq!(table[3].kind, "overlay"); + assert_eq!(mount_table("").unwrap(), []); + for line in [ + "1 2 3 4 5 6 - a b", + "1 2 3 4 - a b c", + "1 2 3 4 5 6 7 a b c", + ] { + let error = mount_table(line).unwrap_err(); + assert!(error.to_string().contains("malformed line"), "{line:?}"); + } + } + + #[test] + fn escapes_are_three_octal_digits() { + assert_eq!(unescape("a\\040b\\134c"), b"a b\\c"); + assert_eq!(unescape("\\012\\011"), b"\n\t"); + // What starts no escape stays as it is. + assert_eq!(unescape("a\\08b\\04"), b"a\\08b\\04"); + assert_eq!(unescape("\\"), b"\\"); + } + + #[test] + fn process_records_are_read_as_the_kernel_writes_them() { + assert_eq!(parse_stat("17 (a) b (c)) S 1 17 17 0 -1"), Some(("S", "1"))); + assert_eq!(parse_stat("17 (idle) R 1"), Some(("R", "1"))); + assert_eq!(parse_stat("17 (idle) R"), None); + assert_eq!(parse_stat("17 idle R 1"), None); + assert_eq!(unified_cgroup("0::/agent\n"), Some("/agent")); + assert_eq!(unified_cgroup("1:name=x:/a\n0::/\n"), Some("/")); + assert_eq!(unified_cgroup("1:name=x:/a\n"), None); + assert_eq!( + command_text(b"/sbin/runtime-adapter-guest\0idle\0"), + "/sbin/runtime-adapter-guest idle" + ); + assert_eq!(command_text(b""), ""); + } + + #[test] + fn the_os_release_id_loses_its_quotes() { + let text = "NAME=\"Ubuntu\"\nID=ubuntu\nID_LIKE=debian\n"; + assert_eq!(os_release_id(text), Some("ubuntu")); + assert_eq!(os_release_id("ID=\"alpine\"\n"), Some("alpine")); + assert_eq!(os_release_id("ID='x'"), Some("x")); + assert_eq!(os_release_id("NAME=x\n"), None); + } + + #[test] + fn unknown_probes_and_arguments_are_refused() { + let context = Context { + runtime: Path::new("/run/runtime-adapter"), + root: Path::new("/run/runtime-adapter/rootfs"), + agent: Path::new("/sys/fs/cgroup/agent"), + workload: Path::new("/sys/fs/cgroup/workload"), + sibling: None, + }; + let mut held = Vec::new(); + let unknown = Probe::parse("mapping").unwrap(); + assert_eq!( + run(&unknown, &context, &mut held), + Err("the worker runs no probe named mapping".to_owned()) + ); + let with_argument = Probe::parse("hold /").unwrap(); + assert_eq!( + run(&with_argument, &context, &mut held), + Err("the hold probe takes no arguments".to_owned()) + ); + assert!(held.is_empty()); + } +} diff --git a/tests/runtime-adapter/tests/common/mod.rs b/tests/runtime-adapter/tests/common/mod.rs new file mode 100644 index 000000000..50f9ae16c --- /dev/null +++ b/tests/runtime-adapter/tests/common/mod.rs @@ -0,0 +1,246 @@ +//! What the VM tests share: the sandbox of a test, in which they run the host program's commands, +//! and the environment that they read. +//! +//! The tests read: +//! +//! - `NVX_HYPERVISOR`: the hypervisor, through `nvx::Hypervisor::from_env_or_default`; +//! - `NVX_ARTIFACTS_DIR`: NVX's artifacts, a repository checkout or an extracted release, with +//! the install metadata from which NVX gives their runtime identity; +//! - `RUNTIME_ADAPTER_GUEST`: the guest agent, a static `x86_64-unknown-linux-musl` build of +//! `runtime-adapter-guest`. +// Each test crate uses its own part of this module. +#![allow(dead_code)] + +use std::collections::BTreeMap; +use std::convert::Infallible; +use std::env; +use std::error::Error; +use std::ffi::OsStr; +use std::fs::{self, File}; +use std::io; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::{Duration, Instant}; + +use nvx::Hypervisor; +use nvx::host::process::Process; +use nvx::host::store::{RuntimeRecord, SandboxStore}; +use nvx::host::supervisor::{self, Reattached, StopOutcome}; + +/// The variable that names the guest agent. +pub(crate) const GUEST_AGENT: &str = "RUNTIME_ADAPTER_GUEST"; +/// The longest that a command of the host program may take. +const COMMAND_TIMEOUT: Duration = Duration::from_secs(180); +/// How much of the end of the VMM's log a failure shows. +const LOG_TAIL: usize = 8192; + +/// The `KEY=VALUE` lines that a command of the host program printed. +pub(crate) type Printed = BTreeMap; + +/// A sandbox of a test: a directory of its own, which holds the sandbox's store, the output of +/// each command of the host program that the test runs, and any file that the test makes for its +/// VM. +/// +/// Dropping it ends a VM that a failed test left running, through its verified identity, and +/// removes the directory once no VM of the sandbox runs. Should that remain uncertain, it keeps +/// the directory, whose store a later reattach needs to end the VM, and prints its path. +pub(crate) struct Sandbox { + /// The test's directory. + pub(crate) root: PathBuf, + /// The sandbox's store, which the launch creates. + pub(crate) store: PathBuf, + /// The number of commands that the test ran. + commands: usize, +} + +impl Sandbox { + /// Creates the directory of the test `name`, in the system's temporary directory, so that the + /// path of a Linux control socket in it stays short. A directory that an earlier test process + /// kept stays as it is. + pub(crate) fn new(name: &str) -> Self { + let pid = std::process::id(); + let root = (0..) + .map(|attempt| env::temp_dir().join(format!("runtime-adapter-{name}-{pid}-{attempt}"))) + .find(|root| match fs::create_dir(root) { + Ok(()) => true, + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => false, + Err(error) => panic!("cannot create {}: {error}", root.display()), + }) + .expect("a directory name is free"); + Self { + store: root.join("store"), + root, + commands: 0, + } + } + + /// Launches the VM with the guest agent, on the hypervisor that `NVX_HYPERVISOR` selects, and + /// with `options` too, checks that the launch exits with `status`, and returns what it + /// printed. A launch that succeeded recorded its VM, which runs. + pub(crate) fn launch(&mut self, options: &[&OsStr], status: i32) -> Printed { + let hypervisor = Hypervisor::from_env_or_default().expect("NVX_HYPERVISOR is valid"); + let guest_agent = env::var_os(GUEST_AGENT).unwrap_or_else(|| { + panic!( + "{GUEST_AGENT} must name a static x86_64-unknown-linux-musl runtime-adapter-guest" + ) + }); + let mut arguments = vec![ + OsStr::new("--guest-agent"), + &guest_agent, + OsStr::new("--hypervisor"), + OsStr::new(hypervisor.as_str()), + ]; + arguments.extend(options); + let launched = self.run("launch", &arguments, status); + if status == 0 { + let record = self.record().expect("the launch recorded its VM"); + assert_eq!(launched["pid"], record.process().pid.to_string()); + } + launched + } + + /// Runs the host program's `command` on the sandbox's store with `options`, checks that it + /// exits with `status`, and returns the `KEY=VALUE` lines that it printed. + pub(crate) fn run(&mut self, command: &str, options: &[&OsStr], status: i32) -> Printed { + self.commands += 1; + let output = self.root.join(format!("{}-{command}", self.commands)); + let (stdout, stderr) = (output.with_extension("out"), output.with_extension("err")); + // Files, rather than pipes, take the output, so that nothing waits for a VMM that might + // hold them. + let mut child = Command::new(env!("CARGO_BIN_EXE_runtime-adapter-host")) + .arg(command) + .arg(&self.store) + .args(options) + .stdin(Stdio::null()) + .stdout(File::create(&stdout).unwrap()) + .stderr(File::create(&stderr).unwrap()) + .spawn() + .unwrap(); + let deadline = Instant::now() + COMMAND_TIMEOUT; + let exit = loop { + if let Some(exit) = child.try_wait().unwrap() { + break exit; + } + if Instant::now() >= deadline { + let _ = child.kill(); + let _ = child.wait(); + panic!("{command} did not end in time{}", self.diagnostics(&output)); + } + thread::sleep(Duration::from_millis(50)); + }; + assert_eq!( + exit.code(), + Some(status), + "{command} {options:?} ended with {exit}{}", + self.diagnostics(&output) + ); + let printed = fs::read_to_string(&stdout).unwrap(); + let mut lines = BTreeMap::new(); + for line in printed.lines() { + let (key, value) = line + .split_once('=') + .unwrap_or_else(|| panic!("{command} printed {line:?}")); + let earlier = lines.insert(key.to_owned(), value.to_owned()); + assert!(earlier.is_none(), "{command} printed {key} twice"); + } + lines + } + + /// Returns what the command whose output files start with `output` printed, and the end of the + /// VMM's log. + fn diagnostics(&self, output: &Path) -> String { + let read = |path: PathBuf| { + let bytes = fs::read(path).unwrap_or_default(); + let tail = bytes.len().saturating_sub(LOG_TAIL); + String::from_utf8_lossy(&bytes[tail..]).into_owned() + }; + format!( + "\nstdout:\n{}\nstderr:\n{}\nthe end of the VMM's log:\n{}", + read(output.with_extension("out")), + read(output.with_extension("err")), + read(self.store.join(nvx::host::store::LOG_NAME)), + ) + } + + /// Returns the sandbox's runtime record, if it has one. + pub(crate) fn record(&self) -> Option { + SandboxStore::open(&self.store).unwrap().runtime().unwrap() + } + + /// Returns the VMM of the sandbox's runtime record, if the record names one that runs. + pub(crate) fn vmm(&self) -> Option { + Process::open(self.record()?.process()).unwrap() + } + + /// Ends the VM that the sandbox's store records, whatever its runtime identity: reattach ends + /// an interrupted launch, and holds a VM that runs, which the stop then terminates through its + /// verified identity. Fails with the reason why a VM of the sandbox might still run. + fn end_vm(&self) -> Result<(), String> { + if !self.store.exists() { + return Ok(()); + } + let failed = |error: nvx::Error| describe(&error); + let store = SandboxStore::open(&self.store).map_err(failed)?; + let recorded = ( + store.runtime().map_err(failed)?, + store.journal().map_err(failed)?, + ); + let identity = match recorded { + (Some(record), _) => record.runtime().clone(), + (None, Some(journal)) => journal.runtime().clone(), + (None, None) => return Ok(()), + }; + let mut locked = store + .lock(Some(Instant::now() + Duration::from_secs(10))) + .map_err(failed)?; + let vmm = match supervisor::reattach(&mut locked, &identity).map_err(failed)? { + Reattached::Live(vmm) | Reattached::Mismatched { vmm, .. } => vmm, + Reattached::Exited { .. } => return Ok(()), + Reattached::Indeterminate(reason) => return Err(reason), + }; + let forced = Some(Duration::from_secs(10)); + let stopped = vmm + .stop(&mut locked, Instant::now(), forced, |_, _| { + Ok::<(), Infallible>(()) + }) + .map_err(failed)?; + match stopped.outcome { + StopOutcome::Graceful | StopOutcome::Forced => Ok(()), + StopOutcome::TimedOut => Err(format!( + "VMM process {} did not exit after it was terminated", + vmm.record().process().pid + )), + } + } +} + +impl Drop for Sandbox { + fn drop(&mut self) { + match self.end_vm() { + Ok(()) => { + let _ = fs::remove_dir_all(&self.root); + } + Err(reason) => eprintln!( + "kept {} because a VM of its sandbox might still run: {reason}", + self.root.display() + ), + } + } +} + +/// Describes `error`, with its causes. +pub(crate) fn describe(error: &dyn Error) -> String { + let mut description = error.to_string(); + let mut source = error.source(); + while let Some(cause) = source { + description = format!("{description}: {cause}"); + source = cause.source(); + } + description +} + +/// Returns the epoch that a command printed. +pub(crate) fn epoch(printed: &Printed) -> u64 { + printed["epoch"].parse().unwrap() +} diff --git a/tests/runtime-adapter/tests/guest_session.rs b/tests/runtime-adapter/tests/guest_session.rs index 7d591e31f..0428f3332 100644 --- a/tests/runtime-adapter/tests/guest_session.rs +++ b/tests/runtime-adapter/tests/guest_session.rs @@ -8,14 +8,21 @@ //! record lengths, bootstrap identity, ERROR payloads, sequence and identity errors, and the //! bounded buffer of the adapter's decoder. Through a relay to a pseudo-terminal, they also cover //! the guest agent on a tty, which it puts in raw mode, as it serves a VM's control tty. +//! +//! The worker of a layered root runs its session with fake root operations, which record the +//! worker's calls and fail as a test asks: the tests cover the order of its stages, each stage +//! that fails, the requests that it serves once the root is ready, its stop, and the messages that +//! it refuses. The binary's PID 1 of a layered root, and its worker, run on a pseudo-terminal too, +//! on the test's host, whose kernel command line names no layered root. #![cfg(all(feature = "guest", target_os = "linux"))] -use std::fs::{File, OpenOptions}; +use std::fs::{self, File, OpenOptions}; use std::io::{self, Read, Write}; use std::net::Shutdown; use std::os::fd::OwnedFd; use std::os::unix::fs::OpenOptionsExt; use std::os::unix::net::UnixStream; +use std::os::unix::process::{CommandExt, ExitStatusExt}; use std::path::PathBuf; use std::process::{Child, Command, Output, Stdio}; use std::sync::mpsc::{self, Receiver, Sender}; @@ -24,13 +31,21 @@ use std::time::{Duration, Instant}; use nix::fcntl::OFlag; use nix::pty; +use nix::sys::signal::{self, Signal}; +use nix::sys::wait::{self, Id, WaitPidFlag, WaitStatus}; +use nix::unistd::Pid; use nvx::guest::nvxs::GuestConnectionError; use nvx::protocol::OUTER_MAX_PAYLOAD; use nvx::protocol::nvxs::{ ErrorCode, Identity, Record, RecordError, RecordParser, RecordType, SessionError, encode_record, }; +use nvx::protocol::root::{RootImage, RootLayout, RootUpper}; +use nvx::uuid::Uuid; use runtime_adapter::frame::{Decoder, FrameError, HEADER_LEN, MAX_BODY, Message}; -use runtime_adapter::guest::{self, RECEIVE_WINDOW, ServeError, Served}; +use runtime_adapter::guest::{self, NO_ROOT, RECEIVE_WINDOW, ServeError, Served}; +use runtime_adapter::probe::{Facts, Probe}; +use runtime_adapter::root::{RootConfig, RootFailure, RootStage}; +use runtime_adapter::worker::{self, RootOperations, Worked}; /// Identity of the broker's first epoch. const FIRST: Identity = Identity { @@ -306,15 +321,15 @@ fn start() -> (Broker, JoinHandle>) { (Broker::new(broker), serving) } -/// Returns how the guest agent's `serve` ended, and fails the test if it does not end in time, so -/// that a guest that keeps serving fails the test instead of hanging it. -fn joined(serving: JoinHandle>) -> Result { +/// Returns what `thread` returned, and fails the test if it does not end in time, so that a guest +/// that keeps serving fails the test instead of hanging it. +fn joined(thread: JoinHandle) -> T { let deadline = Instant::now() + TIMEOUT; - while !serving.is_finished() { + while !thread.is_finished() { assert!(Instant::now() < deadline, "the guest agent ends in time"); thread::sleep(Duration::from_millis(10)); } - serving.join().unwrap() + thread.join().unwrap() } /// Starts the guest agent, attaches it in the first epoch, sends it `bytes`, and returns the @@ -616,7 +631,7 @@ fn the_guest_agent_refuses_messages_that_its_framing_does_not_allow() { }; let cases = [ (header([0; 4], 0), FrameError::Kind(0)), - (header([0; 4], 9), FrameError::Kind(9)), + (header([0; 4], 12), FrameError::Kind(12)), ( header(too_long, 3), FrameError::Length { @@ -635,8 +650,18 @@ fn the_guest_agent_refuses_messages_that_its_framing_does_not_allow() { assert!(failed, "{message:?}: {error:?}"); } - // The guest agent answers requests, and refuses the messages that only it sends. - for message in [Message::Pong, Message::Stopped] { + // The guest agent answers requests, and refuses the messages that only it sends, and the + // configuration of a layered root, which its VM does not have. + let refused_messages = [ + Message::Pong, + Message::Stopped, + Message::RootReady, + Message::RootFailed(vec![1]), + Message::Report(Vec::new()), + Message::ProbeFailed(Vec::new()), + Message::RootConfig(RootConfig::default().encode()), + ]; + for message in refused_messages { let bytes = record(RecordType::Data, FIRST, 1, &message.encode().unwrap()); let error = refused(&bytes); let failed = matches!(&error, ServeError::Unexpected(kind) if *kind == message.kind()); @@ -644,6 +669,25 @@ fn the_guest_agent_refuses_messages_that_its_framing_does_not_allow() { } } +#[test] +fn the_guest_agent_of_a_vm_without_a_layered_root_answers_probes_with_a_failure() { + let (mut broker, serving) = start(); + broker.attach(FIRST); + let messages = [ + Message::Probe(b"mounts".to_vec()), + Message::Ping, + Message::Stop, + ]; + let replies = broker.exchange(&messages, OUTER_MAX_PAYLOAD, 3); + let expected = [ + Message::ProbeFailed(NO_ROOT.as_bytes().to_vec()), + Message::Pong, + Message::Stopped, + ]; + assert_eq!(replies, expected); + assert_eq!(joined(serving).unwrap(), Served::Stopped); +} + /// Starts the guest binary's `serve` with one end of a socket pair as its standard input and /// output, and returns the broker on the other end with the process. fn start_binary() -> (Broker, GuestProcess) { @@ -659,40 +703,126 @@ fn start_binary() -> (Broker, GuestProcess) { (Broker::new(broker), child) } -/// A guest binary that a test started. Dropping it kills and reaps the process, so that a test -/// that fails while the process runs does not leave it running. +/// A guest binary that a test started, in a process group of its own, which also holds the +/// processes that the binary starts, such as those of PID 1 of a layered root. The binary's status +/// is collected only once its group has been killed: until then, the binary keeps its process ID, +/// even after it exits, and the group keeps that ID, which no other process can take. Dropping it +/// kills the group and reaps the binary, so that a test that fails while they run leaves none of +/// them running, even if the binary exited first. struct GuestProcess(Option); +/// How a guest binary that a test started failed to finish, with what it wrote. Either way, its +/// process group was killed, and its status collected. +#[derive(Debug)] +enum Unfinished { + /// The binary did not exit in time. + Late(Output), + /// The binary exited, but left these processes of its group running. + Orphans(Output, Vec), +} + impl GuestProcess { - /// Starts the guest binary with `command`. + /// Starts the guest binary with `command`, in a process group of its own. fn spawn(command: &mut Command) -> Self { - Self(Some(command.spawn().unwrap())) + Self(Some(command.process_group(0).spawn().unwrap())) } - /// Returns the guest binary's output once it exits. If it does not exit in time, kills and - /// reaps it, and fails the test instead of hanging it. - fn finished(mut self) -> Output { - let deadline = Instant::now() + TIMEOUT; - let child = self.0.as_mut().expect("the process was not reaped"); - while child.try_wait().unwrap().is_none() { + /// Returns the guest binary's process ID, which is the ID of its process group. + fn id(&self) -> u32 { + self.0.as_ref().expect("the process was not reaped").id() + } + + /// Returns whether the guest binary has exited, without collecting its status. + fn exited(&self) -> bool { + let pid = Pid::from_raw(i32::try_from(self.id()).unwrap()); + let flags = WaitPidFlag::WEXITED | WaitPidFlag::WNOHANG | WaitPidFlag::WNOWAIT; + !matches!( + wait::waitid(Id::Pid(pid), flags).unwrap(), + WaitStatus::StillAlive + ) + } + + /// Returns the guest binary's output once it exits, and every process that it started with + /// it. Otherwise kills its process group, and fails the test instead of hanging it. + fn finished(self) -> Output { + self.finished_within(TIMEOUT) + .unwrap_or_else(|unfinished| panic!("the guest binary did not finish: {unfinished:?}")) + } + + /// Returns the guest binary's output once it exits within `timeout`, and no other process of + /// its group outlives it. Otherwise kills its process group, whose processes may all hold the + /// binary's output open, and fails with what the binary wrote. + fn finished_within(mut self, timeout: Duration) -> Result { + let deadline = Instant::now() + timeout; + while !self.exited() { if Instant::now() >= deadline { - let _ = child.kill(); - let output = self.0.take().unwrap().wait_with_output().unwrap(); - panic!("the guest binary did not exit in time: {output:?}"); + self.kill(); + return Err(Unfinished::Late(self.reaped())); } thread::sleep(Duration::from_millis(10)); } - self.0.take().unwrap().wait_with_output().unwrap() + let orphans = running_in_group(self.id()); + self.kill(); + let output = self.reaped(); + if orphans.is_empty() { + Ok(output) + } else { + Err(Unfinished::Orphans(output, orphans)) + } + } + + /// Kills the guest binary's process group, which is still the binary's, because its status + /// has not been collected. + fn kill(&self) { + let group = Pid::from_raw(i32::try_from(self.id()).unwrap()); + let _ = signal::killpg(group, Signal::SIGKILL); + } + + /// Collects the guest binary's status, and its output, which ends once every process that + /// holds it has ended. + fn reaped(&mut self) -> Output { + let child = self.0.take().expect("the process was not reaped"); + child.wait_with_output().unwrap() } } impl Drop for GuestProcess { fn drop(&mut self) { - if let Some(child) = &mut self.0 { - let _ = child.kill(); - let _ = child.wait(); + if self.0.is_some() { + self.kill(); + let _ = self.0.take().unwrap().wait(); + } + } +} + +/// Returns the processes of the process group `group` that have not exited: all but zombies. +fn running_in_group(group: u32) -> Vec { + let group = group.to_string(); + let mut running = Vec::new(); + for entry in fs::read_dir("/proc").unwrap().flatten() { + let Some(pid) = entry + .file_name() + .to_str() + .and_then(|name| name.parse().ok()) + else { + continue; + }; + // A process may end while the scan reads it. + let Ok(stat) = fs::read_to_string(entry.path().join("stat")) else { + continue; + }; + // The fields after the command's name, which may hold spaces and parentheses, start + // after its last `)`: the state, the parent's process ID, then the process group. + let Some((_, fields)) = stat.rsplit_once(')') else { + continue; + }; + let mut fields = fields.split_whitespace(); + let state = fields.next(); + if fields.nth(1) == Some(group.as_str()) && !matches!(state, Some("Z" | "X")) { + running.push(pid); } } + running } #[test] @@ -719,13 +849,13 @@ fn the_guest_binary_serves_the_session_on_its_standard_streams() { // The host sends a malformed message. let (mut broker, child) = start_binary(); broker.attach(FIRST); - broker.send_data(&[0, 0, 0, 0, 9], OUTER_MAX_PAYLOAD); + broker.send_data(&[0, 0, 0, 0, 12], OUTER_MAX_PAYLOAD); let output = child.finished(); assert_eq!(output.status.code(), Some(1), "{output:?}"); let diagnostic = String::from_utf8_lossy(&output.stderr); assert_eq!( diagnostic, - "runtime-adapter-guest: unknown message kind 9\n" + "runtime-adapter-guest: unknown message kind 12\n" ); } @@ -829,3 +959,588 @@ fn the_guest_binary_serves_a_tty_in_place_of_the_managed_agent() { ); } } + +/// An operation of a layered root, which a [`FakeRoot`] fails when a test asks. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Operation { + /// Reading the kernel command line. + CommandLine, + /// The bootstrap. + Bootstrap, + /// The assembly. + Assemble, + /// A probe. + Probe, + /// The teardown. + Teardown, +} + +/// A call of a [`FakeRoot`]'s operations, with what the worker passed. +#[derive(Clone, Debug, PartialEq, Eq)] +enum Call { + /// The kernel command line was read. + CommandLine, + /// The bootstrap ran with the configuration. + Bootstrap(RootConfig), + /// The assembly ran with the configuration and the layout. + Assemble(RootConfig, RootLayout), + /// The probe ran. + Probe(Probe), + /// The teardown ran. + Teardown, +} + +/// Root operations that record the worker's calls, and fail as a test asks. +#[derive(Debug)] +struct FakeRoot { + /// The kernel command line that the operations read. + command_line: Vec, + /// The operation that fails, and its problem, if any. + failing: Option<(Operation, &'static str)>, + /// The worker's calls, in order. + calls: Vec, +} + +impl FakeRoot { + /// Returns operations whose kernel command line names `layout`. + fn booting(layout: &RootLayout) -> Self { + let command_line = format!("console=hvc0 nvx_sandbox=1 {}", layout.tokens().join(" ")); + Self::reading(command_line.as_bytes()) + } + + /// Returns operations whose kernel command line is `command_line`. + fn reading(command_line: &[u8]) -> Self { + Self { + command_line: command_line.to_vec(), + failing: None, + calls: Vec::new(), + } + } + + /// Returns operations of the layout that the worker assembles, whose `operation` fails with + /// `problem`. + fn failing(operation: Operation, problem: &'static str) -> Self { + Self { + failing: Some((operation, problem)), + ..Self::booting(&layout()) + } + } + + /// Fails if `operation` fails. + fn result(&self, operation: Operation) -> Result<(), String> { + match self.failing { + Some((failing, problem)) if failing == operation => Err(problem.to_owned()), + _ => Ok(()), + } + } +} + +impl RootOperations for FakeRoot { + fn command_line(&mut self) -> Result, String> { + self.calls.push(Call::CommandLine); + self.result(Operation::CommandLine)?; + Ok(self.command_line.clone()) + } + + fn bootstrap(&mut self, config: &RootConfig) -> Result<(), String> { + self.calls.push(Call::Bootstrap(config.clone())); + self.result(Operation::Bootstrap) + } + + fn assemble(&mut self, config: &RootConfig, layout: &RootLayout) -> Result<(), String> { + self.calls.push(Call::Assemble(config.clone(), *layout)); + self.result(Operation::Assemble) + } + + fn probe(&mut self, probe: &Probe) -> Result { + self.calls.push(Call::Probe(probe.clone())); + self.result(Operation::Probe)?; + Ok(found(probe)) + } + + fn teardown(&mut self) -> Result<(), String> { + self.calls.push(Call::Teardown); + self.result(Operation::Teardown) + } +} + +/// Returns the facts that a [`FakeRoot`] finds with `probe`. +fn found(probe: &Probe) -> Facts { + let mut facts = Facts::new(); + facts.push("name", probe.name()); + facts.push("arguments", probe.arguments().join(" ")); + facts +} + +/// Returns the layout that the worker assembles: an EROFS image, with the identity that the guest +/// expects of it, over an ext4 scratch. +fn layout() -> RootLayout { + RootLayout { + image: RootImage::erofs(Uuid::parse("0a490905-41be-ed5f-984e-63eb7e5e20b1")), + upper: RootUpper::Scratch, + } +} + +/// The thread of a worker's session, which returns its root operations, with how the session +/// ended. +type Working = JoinHandle<(FakeRoot, Result)>; + +/// Starts the worker's session in a thread, with the root operations `root`, on one end of a +/// socket pair, and returns the broker on the other end with the thread. +fn start_worker(root: FakeRoot) -> (Broker, Working) { + let (broker, guest) = UnixStream::pair().unwrap(); + let working = thread::spawn(move || { + let mut root = root; + let worked = worker::work(guest, &mut root); + (root, worked) + }); + (Broker::new(broker), working) +} + +/// Returns the default root configuration's message. +fn configuration() -> Message { + Message::RootConfig(RootConfig::default().encode()) +} + +/// Returns the probe that `text` names. +fn probe(text: &str) -> Probe { + Probe::parse(text).unwrap() +} + +#[test] +fn the_worker_assembles_the_root_serves_it_and_tears_it_down_on_stop() { + let (mut broker, working) = start_worker(FakeRoot::booting(&layout())); + broker.attach(FIRST); + let replies = broker.exchange(&[configuration()], OUTER_MAX_PAYLOAD, 1); + assert_eq!(replies, [Message::RootReady]); + + // It serves pings, echoes, and probes, in records of several sizes. + let messages = [Message::Ping, longest(), Message::Probe(b"mounts".to_vec())]; + let expected = [ + Message::Pong, + longest(), + Message::Report(found(&probe("mounts")).encode()), + ]; + for size in [7, OUTER_MAX_PAYLOAD] { + let replies = broker.exchange(&messages, size, expected.len()); + assert_eq!(replies, expected, "records of {size} bytes"); + } + + // Its root outlives a reset of the session, as when the host reattaches. A probe that the + // framing of probes refuses, or that fails, is answered with its problem. + broker.settle(); + broker.reset(Identity { epoch: 2, ..FIRST }); + let messages = [ + Message::Probe(b"hold /run".to_vec()), + Message::Probe(b"Mounts".to_vec()), + ]; + let replies = broker.exchange(&messages, OUTER_MAX_PAYLOAD, 2); + assert_eq!( + replies[0], + Message::Report(found(&probe("hold /run")).encode()) + ); + let Message::ProbeFailed(problem) = &replies[1] else { + panic!("{replies:?}"); + }; + let problem = String::from_utf8_lossy(problem); + assert!(problem.contains("the probe's name \"Mounts\""), "{problem}"); + + let replies = broker.exchange(&[Message::Stop], OUTER_MAX_PAYLOAD, 1); + assert_eq!(replies, [Message::Stopped]); + let (root, worked) = joined(working); + let worked = worked.unwrap(); + assert_eq!((&worked, worked.status()), (&Worked::Stopped, 0)); + assert!(broker.read().is_none(), "the worker ended its stream"); + let config = RootConfig::default(); + let mut expected = vec![ + Call::CommandLine, + Call::Bootstrap(config.clone()), + Call::Assemble(config, layout()), + ]; + expected.extend([probe("mounts"), probe("mounts"), probe("hold /run")].map(Call::Probe)); + expected.push(Call::Teardown); + assert_eq!(root.calls, expected); +} + +#[test] +fn the_worker_reports_the_stage_that_failed_and_ends_with_its_status() { + let config = RootConfig::default(); + let read = || vec![Call::CommandLine]; + let bootstrapped = || vec![Call::CommandLine, Call::Bootstrap(config.clone())]; + let gpt = RootLayout { + image: RootImage::gpt(None), + upper: RootUpper::Scratch, + }; + let ram = RootLayout { + upper: RootUpper::Ram, + ..layout() + }; + let cases = [ + ( + "a configuration of another version", + FakeRoot::booting(&layout()), + b"version=2\n".to_vec(), + RootStage::Configuration, + "the root configuration has version \"2\"", + Vec::new(), + ), + ( + "a relative runtime directory", + FakeRoot::booting(&layout()), + RootConfig { + runtime: "run/relative".to_owned(), + ..RootConfig::default() + } + .encode(), + RootStage::Configuration, + "names the runtime \"run/relative\", which is not absolute", + Vec::new(), + ), + ( + "a kernel command line that cannot be read", + FakeRoot::failing(Operation::CommandLine, "cannot read /proc/cmdline"), + config.encode(), + RootStage::Layout, + "cannot read /proc/cmdline", + read(), + ), + ( + "no layered root", + FakeRoot::reading(b"console=hvc0 quiet"), + config.encode(), + RootStage::Layout, + "the kernel command line names no layered root", + read(), + ), + ( + "invalid boot tokens", + FakeRoot::reading(b"nvx_layer=distro,0xd0003000,erofs"), + config.encode(), + RootStage::Layout, + "the kernel command line has invalid boot tokens: ", + read(), + ), + ( + "a GPT image", + FakeRoot::booting(&gpt), + config.encode(), + RootStage::Layout, + "assembles only an EROFS image over an ext4 scratch, not gpt over scratch", + read(), + ), + ( + "a RAM upper", + FakeRoot::booting(&ram), + config.encode(), + RootStage::Layout, + "not erofs over ram", + read(), + ), + ( + "a bootstrap that failed", + FakeRoot::failing(Operation::Bootstrap, "failed to create the agent's cgroup"), + config.encode(), + RootStage::Bootstrap, + "failed to create the agent's cgroup", + bootstrapped(), + ), + ( + "an assembly that failed", + FakeRoot::failing(Operation::Assemble, "the distro layer's UUID is wrong"), + config.encode(), + RootStage::Assembly, + "the distro layer's UUID is wrong", + [ + bootstrapped(), + vec![Call::Assemble(config.clone(), layout())], + ] + .concat(), + ), + ]; + for (name, root, body, stage, problem, calls) in cases { + let (mut broker, working) = start_worker(root); + broker.attach(FIRST); + let replies = broker.exchange(&[Message::RootConfig(body)], OUTER_MAX_PAYLOAD, 1); + let [Message::RootFailed(reported)] = replies.as_slice() else { + panic!("{name}: {replies:?}"); + }; + let reported = RootFailure::decode(reported).unwrap(); + assert_eq!(reported.stage(), stage, "{name}: {reported}"); + assert!(reported.problem().contains(problem), "{name}: {reported}"); + // It ends its stream after the failure. + assert!(broker.read().is_none(), "{name}"); + let (root, worked) = joined(working); + let worked = worked.unwrap(); + assert_eq!(worked, Worked::Failed(reported), "{name}"); + assert_eq!(worked.status(), stage.exit_status(), "{name}"); + assert_eq!(root.calls, calls, "{name}"); + } +} + +#[test] +fn the_worker_reports_a_teardown_that_failed_before_it_confirms_the_stop() { + let problem = "failed to unmount the overlay: Device or resource busy (os error 16)"; + let (mut broker, working) = start_worker(FakeRoot::failing(Operation::Teardown, problem)); + broker.attach(FIRST); + let replies = broker.exchange(&[configuration(), Message::Stop], OUTER_MAX_PAYLOAD, 3); + let failure = RootFailure::new(RootStage::Teardown, problem); + let expected = [ + Message::RootReady, + Message::RootFailed(failure.encode()), + Message::Stopped, + ]; + assert_eq!(replies, expected); + let (root, worked) = joined(working); + let worked = worked.unwrap(); + assert_eq!(worked, Worked::Failed(failure)); + assert_eq!(worked.status(), 15); + assert_eq!(root.calls.last(), Some(&Call::Teardown)); +} + +#[test] +fn the_worker_accepts_only_its_configuration_or_a_stop_before_its_root() { + // A stop ends the worker without a root. + let (mut broker, working) = start_worker(FakeRoot::booting(&layout())); + broker.attach(FIRST); + let replies = broker.exchange(&[Message::Stop], OUTER_MAX_PAYLOAD, 1); + assert_eq!(replies, [Message::Stopped]); + let (root, worked) = joined(working); + assert_eq!(worked.unwrap(), Worked::Stopped); + assert_eq!(root.calls, []); + + // Any other message ends it with an error, as do the messages that only the guest sends. + let refused = [ + Message::Ping, + Message::Echo(b"early".to_vec()), + Message::Probe(b"mounts".to_vec()), + Message::Pong, + Message::RootReady, + Message::Report(Vec::new()), + ]; + for message in refused { + let (mut broker, working) = start_worker(FakeRoot::booting(&layout())); + broker.attach(FIRST); + broker.send_data(&message.encode().unwrap(), OUTER_MAX_PAYLOAD); + let (root, worked) = joined(working); + let failed = + matches!(&worked, Err(ServeError::Unexpected(kind)) if *kind == message.kind()); + assert!(failed, "{message:?}: {worked:?}"); + assert_eq!(root.calls, [], "{message:?}"); + } + + // Once the root is ready, a second configuration ends it with an error too, and the worker + // leaves the root that it assembled to its operations. + let (mut broker, working) = start_worker(FakeRoot::booting(&layout())); + broker.attach(FIRST); + let replies = broker.exchange(&[configuration()], OUTER_MAX_PAYLOAD, 1); + assert_eq!(replies, [Message::RootReady]); + broker.send_data(&configuration().encode().unwrap(), OUTER_MAX_PAYLOAD); + let (root, worked) = joined(working); + let failed = matches!(&worked, Err(ServeError::Unexpected(6))); + assert!(failed, "{worked:?}"); + assert!(!root.calls.contains(&Call::Teardown), "{:?}", root.calls); + + // So does a broker that closes the stream. + let (mut broker, working) = start_worker(FakeRoot::booting(&layout())); + broker.attach(FIRST); + let replies = broker.exchange(&[configuration()], OUTER_MAX_PAYLOAD, 1); + assert_eq!(replies, [Message::RootReady]); + broker.close(); + let (root, worked) = joined(working); + let worked = worked.unwrap(); + assert_eq!((&worked, worked.status()), (&Worked::Closed, 1)); + assert!(!root.calls.contains(&Call::Teardown), "{:?}", root.calls); +} + +/// Starts the guest binary with `arguments`, its standard input closed and its output piped. +fn spawn_binary(arguments: &[&std::ffi::OsStr]) -> GuestProcess { + GuestProcess::spawn( + Command::new(env!("CARGO_BIN_EXE_runtime-adapter-guest")) + .args(arguments) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()), + ) +} + +#[test] +fn the_guest_binary_runs_pid_1_and_the_worker_of_a_layered_root() { + use std::ffi::OsStr; + + let init = OsStr::new(runtime_adapter::GUEST_INIT_COMMAND); + let layered = OsStr::new(runtime_adapter::GUEST_LAYERED_MODE); + let worker = OsStr::new(runtime_adapter::GUEST_WORKER_COMMAND); + + // PID 1 starts the worker on the tty, which reports that the test's host boots no layered + // root. Unless it is PID 1, PID 1 then ends its unrelated child, and exits with the worker's + // status, with which PID 1 would end the VM. + let (mut broker, pty) = terminal(); + let child = spawn_binary(&[init, pty.path.as_os_str(), layered]); + broker.attach(FIRST); + let replies = broker.exchange(&[configuration()], OUTER_MAX_PAYLOAD, 1); + let [Message::RootFailed(reported)] = replies.as_slice() else { + panic!("{replies:?}"); + }; + let reported = RootFailure::decode(reported).unwrap(); + assert_eq!(reported.stage(), RootStage::Layout, "{reported}"); + let output = child.finished(); + assert_eq!(output.status.code(), Some(12), "{output:?}"); + assert!(output.stdout.is_empty(), "{output:?}"); + assert_eq!( + String::from_utf8_lossy(&output.stderr), + format!("runtime-adapter-guest: {reported}\n") + ); + + // The worker alone, with and without the unrelated child's process ID, stops before it + // configures a root. + for sibling in [None, Some(OsStr::new("1"))] { + let (mut broker, pty) = terminal(); + let mut arguments = vec![worker, pty.path.as_os_str()]; + arguments.extend(sibling); + let child = spawn_binary(&arguments); + broker.attach(FIRST); + let replies = broker.exchange(&[Message::Stop], OUTER_MAX_PAYLOAD, 1); + assert_eq!(replies, [Message::Stopped]); + let output = child.finished(); + assert!(output.status.success(), "{output:?}"); + assert!(output.stderr.is_empty(), "{output:?}"); + } + + // A worker that cannot open its tty fails at once, and PID 1 exits with its status. + for arguments in [ + [init, OsStr::new("/"), layered], + [worker, OsStr::new("/"), OsStr::new("7")], + ] { + let output = spawn_binary(&arguments).finished(); + assert_eq!(output.status.code(), Some(1), "{arguments:?}: {output:?}"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.starts_with("runtime-adapter-guest: cannot open /: "), + "{arguments:?}: {stderr}" + ); + } + + // The unrelated child sleeps until a signal ends it. + let idle = spawn_binary(&[OsStr::new(runtime_adapter::GUEST_IDLE_COMMAND)]); + thread::sleep(Duration::from_millis(200)); + assert!(!idle.exited()); + drop(idle); + + // Anything else is a usage error. + let usage: [&[&str]; 5] = [ + &["init", "/dev/null", "blockless"], + &["worker"], + &["worker", "/dev/null", "sibling"], + &["worker", "/dev/null", "1", "2"], + &["idle", "now"], + ]; + for arguments in usage { + let arguments = arguments.iter().map(OsStr::new).collect::>(); + let output = spawn_binary(&arguments).finished(); + assert_eq!(output.status.code(), Some(2), "{arguments:?}: {output:?}"); + } +} + +/// Starts PID 1 of a layered root on the tty of `pty`, and waits until its unrelated child and +/// its worker, which waits for the broker, run too, all of them with the binary's output. Returns +/// the binary, and the processes of its group. +fn layered_pid_1(pty: &Terminal) -> (GuestProcess, Vec) { + use std::ffi::OsStr; + + let child = spawn_binary(&[ + OsStr::new(runtime_adapter::GUEST_INIT_COMMAND), + pty.path.as_os_str(), + OsStr::new(runtime_adapter::GUEST_LAYERED_MODE), + ]); + let deadline = Instant::now() + TIMEOUT; + loop { + let running = running_in_group(child.id()); + if running.len() == 3 { + return (child, running); + } + assert!( + Instant::now() < deadline, + "PID 1 did not start its children: {running:?}" + ); + thread::sleep(Duration::from_millis(10)); + } +} + +/// Ends PID 1 of a layered root, which `layered_pid_1` started, before its children, as a crash +/// would, and waits until it exited. +fn crash(pid_1: &GuestProcess) { + signal::kill( + Pid::from_raw(i32::try_from(pid_1.id()).unwrap()), + Signal::SIGKILL, + ) + .unwrap(); + let deadline = Instant::now() + TIMEOUT; + while !pid_1.exited() { + assert!(Instant::now() < deadline, "PID 1 did not end"); + thread::sleep(Duration::from_millis(10)); + } +} + +/// Waits until no process of the process group `group` runs. +fn ended(group: u32) { + let deadline = Instant::now() + TIMEOUT; + loop { + let running = running_in_group(group); + if running.is_empty() { + return; + } + assert!(Instant::now() < deadline, "{running:?} still run"); + thread::sleep(Duration::from_millis(10)); + } +} + +#[test] +fn a_test_that_gives_up_on_pid_1_ends_every_process_that_it_started() { + // Giving up kills the whole group, so that the output, which every process of the group + // holds, ends, and none of the processes runs on. + let (_broker, pty) = terminal(); + let (child, _) = layered_pid_1(&pty); + let group = child.id(); + let unfinished = child.finished_within(Duration::from_millis(100)); + let Err(Unfinished::Late(output)) = unfinished else { + panic!("{unfinished:?}"); + }; + assert_eq!( + output.status.signal(), + Some(Signal::SIGKILL as i32), + "{output:?}" + ); + ended(group); +} + +#[test] +fn a_test_ends_every_process_that_outlives_pid_1() { + // PID 1 ends before its children, which hold its output. The test fails, and kills them, + // rather than wait for the output to end. + let (_broker, pty) = terminal(); + let (child, mut children) = layered_pid_1(&pty); + let group = child.id(); + crash(&child); + let unfinished = child.finished_within(TIMEOUT); + let Err(Unfinished::Orphans(output, mut orphans)) = unfinished else { + panic!("{unfinished:?}"); + }; + assert_eq!( + output.status.signal(), + Some(Signal::SIGKILL as i32), + "{output:?}" + ); + children.retain(|&pid| pid != group); + children.sort_unstable(); + orphans.sort_unstable(); + assert_eq!(orphans, children); + ended(group); + + // A test that fails once PID 1 ended kills them too. + let (_broker, pty) = terminal(); + let (child, _) = layered_pid_1(&pty); + let group = child.id(); + crash(&child); + drop(child); + ended(group); +} diff --git a/tests/runtime-adapter/tests/host_session.rs b/tests/runtime-adapter/tests/host_session.rs index a1cfe1716..629268114 100644 --- a/tests/runtime-adapter/tests/host_session.rs +++ b/tests/runtime-adapter/tests/host_session.rs @@ -6,8 +6,9 @@ //! and on Windows. The tests cover the attachment and its bootstrap identity, waits and refusals, //! messages that span records and records that share messages, partial records, record lengths, //! ERROR payloads, sequence and identity errors, resets and disconnects in the middle of a message, -//! timeouts, sends that fail, the answers that requests check, and the bounded buffer of the -//! adapter's decoder. +//! timeouts, sends that fail, the answers that requests check, the bounded buffer of the adapter's +//! decoder, and the requests of a layered root: its configuration, its probes, and a stop whose +//! teardown failed. #![cfg(feature = "host")] use std::collections::VecDeque; @@ -24,8 +25,10 @@ use nvx::protocol::nvxs::{ use nvx::protocol::{ CAPABILITY_LEN, OUTER_HEADER_LEN, OUTER_MAX_PAYLOAD, OUTER_MIN_RECEIVE_WINDOW, }; -use runtime_adapter::frame::{Decoder, FrameError, HEADER_LEN, MAX_BODY, Message}; +use runtime_adapter::frame::{Decoder, FrameError, HEADER_LEN, MAX_BODY, MAX_TEXT_BODY, Message}; use runtime_adapter::host::{Client, ClientError}; +use runtime_adapter::probe::{Facts, Probe, ProbeReply}; +use runtime_adapter::root::{RootConfig, RootFailure, RootReply, RootStage}; /// Launch capability that the test broker accepts. const CAPABILITY: [u8; CAPABILITY_LEN] = [0xa5; CAPABILITY_LEN]; @@ -512,7 +515,7 @@ fn the_client_attaches_and_exchanges_messages_that_span_records() { for body in &bodies { client.echo(body, deadline()).unwrap(); } - client.stop(deadline()).unwrap(); + assert_eq!(client.stop(deadline()).unwrap(), None); drop(client); let mut expected = vec![Message::Ping]; @@ -1036,9 +1039,10 @@ fn the_client_refuses_messages_that_its_framing_does_not_allow() { // body beyond the limit. let too_long = u32::try_from(MAX_BODY + 1).unwrap().to_le_bytes(); let header = |length: [u8; 4], kind: u8| [&length[..], &[kind]].concat(); + let too_long_text = u32::try_from(MAX_TEXT_BODY + 1).unwrap().to_le_bytes(); let cases = [ (header([0; 4], 0), FrameError::Kind(0)), - (header([0; 4], 9), FrameError::Kind(9)), + (header([0; 4], 12), FrameError::Kind(12)), ( header(too_long, 3), FrameError::Length { @@ -1057,6 +1061,17 @@ fn the_client_refuses_messages_that_its_framing_does_not_allow() { header(1u32.to_le_bytes(), 2), FrameError::Length { kind: 2, length: 1 }, ), + ( + header(too_long_text, 8), + FrameError::Length { + kind: 8, + length: MAX_TEXT_BODY + 1, + }, + ), + ( + header(1u32.to_le_bytes(), 7), + FrameError::Length { kind: 7, length: 1 }, + ), ]; for (bytes, expected) in cases { let (mut broker, mut client) = attached(FIRST, usize::MAX); @@ -1069,7 +1084,13 @@ fn the_client_refuses_messages_that_its_framing_does_not_allow() { // The client takes the guest agent's messages, and refuses those that only the host program // sends. - for message in [Message::Ping, Message::Stop] { + let host_messages = [ + Message::Ping, + Message::Stop, + Message::RootConfig(RootConfig::default().encode()), + Message::Probe(b"mounts".to_vec()), + ]; + for message in host_messages { let (mut broker, mut client) = attached(FIRST, usize::MAX); broker.send(RecordType::Data, &message.encode().unwrap()); let refused = client.receive(Some(deadline())); @@ -1111,44 +1132,94 @@ fn requests_check_their_answers_and_end_the_client_when_they_fail() { /// The kinds of a request and of its answer that a failure reports, or `None` for an echo of /// other bytes. type Kinds = Option<(u8, u8)>; - // Each request, and an answer that doesn't answer it. - let cases: [(&str, Request, Message, Kinds); 5] = [ + let configure: Request = |client| { + client + .configure_root(&RootConfig::default(), deadline()) + .map(drop) + }; + let probe: Request = |client| { + let probe = Probe::parse("mounts").unwrap(); + client.probe(&probe, deadline()).map(drop) + }; + let stop: Request = |client| client.stop(deadline()).map(drop); + let teardown = RootFailure::new(RootStage::Teardown, "busy").encode(); + // Each request, and answers that don't answer it. + let cases: [(&str, Request, Vec, Kinds); 11] = [ ( "Ping answered with Stopped", |client| client.ping(deadline()), - Message::Stopped, + vec![Message::Stopped], Some((1, 5)), ), ( "Echo answered with Pong", |client| client.echo(b"abc", deadline()), - Message::Pong, + vec![Message::Pong], Some((3, 2)), ), ( "Echo answered with other bytes", |client| client.echo(b"abc", deadline()), - Message::Echo(b"abd".to_vec()), + vec![Message::Echo(b"abd".to_vec())], None, ), ( "Stop answered with Pong", - |client| client.stop(deadline()), - Message::Pong, + stop, + vec![Message::Pong], Some((4, 2)), ), ( "Stop answered with an Echo", - |client| client.stop(deadline()), - Message::Echo(Vec::new()), + stop, + vec![Message::Echo(Vec::new())], Some((4, 3)), ), + ( + "Stop answered with RootReady", + stop, + vec![Message::RootReady], + Some((4, 7)), + ), + ( + "Stop answered with a teardown that failed, then Pong", + stop, + vec![Message::RootFailed(teardown.clone()), Message::Pong], + Some((4, 2)), + ), + ( + "Stop answered with a teardown that failed, then another", + stop, + vec![ + Message::RootFailed(teardown.clone()), + Message::RootFailed(teardown), + ], + Some((4, 8)), + ), + ( + "a root configuration answered with Pong", + configure, + vec![Message::Pong], + Some((6, 2)), + ), + ( + "a root configuration answered with a Report", + configure, + vec![Message::Report(Vec::new())], + Some((6, 10)), + ), + ( + "a probe answered with RootReady", + probe, + vec![Message::RootReady], + Some((9, 7)), + ), ]; - for (name, request, reply, kinds) in cases { + for (name, request, replies, kinds) in cases { let (mut broker, client) = attached(FIRST, usize::MAX); let requesting = spawn(client, request); broker.messages(1); - broker.send_data(&reply.encode().unwrap(), OUTER_MAX_PAYLOAD); + broker.send_data(&encode(&replies), OUTER_MAX_PAYLOAD); let (mut client, result) = joined(requesting); let error = result.unwrap_err(); let failed = match (&error, kinds) { @@ -1174,3 +1245,148 @@ fn requests_check_their_answers_and_end_the_client_when_they_fail() { broker.send_data(&Message::Pong.encode().unwrap(), OUTER_MAX_PAYLOAD); assert_ended(&mut client); } + +#[test] +fn the_client_configures_a_layered_root_probes_it_and_learns_how_its_teardown_ended() { + let (mut broker, client) = attached(FIRST, usize::MAX); + + // The guest assembles the root. + let configuring = spawn(client, |client| { + client.configure_root(&RootConfig::default(), deadline()) + }); + let expected = Message::RootConfig(RootConfig::default().encode()); + assert_eq!(broker.messages(1), [expected]); + broker.send_data(&Message::RootReady.encode().unwrap(), OUTER_MAX_PAYLOAD); + let (client, configured) = joined(configuring); + assert_eq!(configured.unwrap(), RootReply::Ready); + + // A probe's facts, in records of 7 bytes, and a probe that failed. + let mut facts = Facts::new(); + facts.push("root.type", "overlay"); + facts.push("root.options", "rw,nosuid,nodev"); + let probing = spawn(client, |client| { + client.probe(&Probe::parse("mounts").unwrap(), deadline()) + }); + assert_eq!(broker.messages(1), [Message::Probe(b"mounts".to_vec())]); + broker.send_data(&Message::Report(facts.encode()).encode().unwrap(), 7); + let (client, probed) = joined(probing); + assert_eq!(probed.unwrap(), ProbeReply::Report(facts)); + let probing = spawn(client, |client| { + client.probe(&Probe::parse("mapping /a").unwrap(), deadline()) + }); + assert_eq!(broker.messages(1), [Message::Probe(b"mapping /a".to_vec())]); + let problem = "the worker runs no probe named mapping"; + let failed = Message::ProbeFailed(problem.as_bytes().to_vec()); + broker.send_data(&failed.encode().unwrap(), OUTER_MAX_PAYLOAD); + let (client, probed) = joined(probing); + assert_eq!(probed.unwrap(), ProbeReply::Failed(problem.to_owned())); + + // The teardown failed, which the guest reports before it confirms the stop, in one record. + let failure = RootFailure::new(RootStage::Teardown, "failed to unmount the overlay: busy"); + let stopping = spawn(client, |client| client.stop(deadline())); + assert_eq!(broker.messages(1), [Message::Stop]); + let replies = [Message::RootFailed(failure.encode()), Message::Stopped]; + broker.send_data(&encode(&replies), OUTER_MAX_PAYLOAD); + let (_, stopped) = joined(stopping); + assert_eq!(stopped.unwrap(), Some(failure)); +} + +#[test] +fn the_client_returns_the_stage_of_the_root_that_failed() { + for stage in RootStage::ALL { + let failure = RootFailure::new(stage, format!("the {stage} failed")); + let (mut broker, client) = attached(FIRST, usize::MAX); + let configuring = spawn(client, |client| { + client.configure_root(&RootConfig::default(), deadline()) + }); + broker.messages(1); + let reply = Message::RootFailed(failure.encode()); + broker.send_data(&reply.encode().unwrap(), OUTER_MAX_PAYLOAD); + let (_, configured) = joined(configuring); + assert_eq!(configured.unwrap(), RootReply::Failed(failure)); + } + + // A root configuration that the framing refuses fails before it is sent, and the client + // stays usable. + let (mut broker, mut client) = attached(FIRST, usize::MAX); + let config = RootConfig { + runtime: format!("/{}", "a".repeat(MAX_TEXT_BODY)), + ..RootConfig::default() + }; + let refused = client.configure_root(&config, deadline()); + let too_long = matches!( + refused, + Err(ClientError::Frame(FrameError::Length { kind: 6, .. })) + ); + assert!(too_long, "{refused:?}"); + let pinging = spawn(client, |client| client.ping(deadline())); + assert_eq!(broker.answer(OUTER_MAX_PAYLOAD), Message::Ping); + joined(pinging).1.unwrap(); +} + +#[test] +fn answers_whose_bodies_do_not_decode_end_the_client() { + /// A request of the client. + type Request = fn(&mut Client) -> Result<(), ClientError>; + let configure: Request = |client| { + client + .configure_root(&RootConfig::default(), deadline()) + .map(drop) + }; + let probe: Request = |client| { + let probe = Probe::parse("mounts").unwrap(); + client.probe(&probe, deadline()).map(drop) + }; + let stop: Request = |client| client.stop(deadline()).map(drop); + let cases: [(&str, Request, Message, &str); 6] = [ + ( + "a root's failure without a stage", + configure, + Message::RootFailed(Vec::new()), + "names no stage", + ), + ( + "a root's failure of an unknown stage", + configure, + Message::RootFailed(vec![9]), + "the unknown stage 9", + ), + ( + "facts without a value", + probe, + Message::Report(b"root.type\n".to_vec()), + "has no value", + ), + ( + "facts with an invalid escape", + probe, + Message::Report(b"root.type=a\\b\n".to_vec()), + "has an invalid escape", + ), + ( + "a probe's failure that is not UTF-8 text", + probe, + Message::ProbeFailed(vec![0xff]), + "the probe's failure is not UTF-8 text", + ), + ( + "a teardown's failure that is not UTF-8 text", + stop, + Message::RootFailed(vec![5, 0xff]), + "is not UTF-8 text", + ), + ]; + for (name, request, reply, problem) in cases { + let (mut broker, client) = attached(FIRST, usize::MAX); + let requesting = spawn(client, request); + broker.messages(1); + broker.send_data(&reply.encode().unwrap(), OUTER_MAX_PAYLOAD); + let (mut client, result) = joined(requesting); + let failed = matches!( + &result, + Err(ClientError::Body(error)) if error.problem().contains(problem) + ); + assert!(failed, "{name}: {result:?}"); + assert_ended(&mut client); + } +} diff --git a/tests/runtime-adapter/tests/initramfs.rs b/tests/runtime-adapter/tests/initramfs.rs index 0dc2c550f..da118a9db 100644 --- a/tests/runtime-adapter/tests/initramfs.rs +++ b/tests/runtime-adapter/tests/initramfs.rs @@ -1,12 +1,14 @@ //! The initramfs that boots the guest agent is a copy of NVX's initramfs, with an archive after it -//! that Linux unpacks over NVX's: the guest agent, the script that runs it, and the link that -//! replaces NVX's managed agent with the script. +//! that Linux unpacks over NVX's: the guest agent, the scripts that run it, and the links that +//! replace NVX's managed agent and init agent with the scripts. #![cfg(feature = "host")] use std::fs; use std::path::PathBuf; -use runtime_adapter::initramfs::{self, GUEST_AGENT, LAUNCHER, MANAGED_AGENT}; +use runtime_adapter::initramfs::{ + self, CONTROL_TTY, GUEST_AGENT, INIT_AGENT, LAUNCHER, LAYERED_LAUNCHER, MANAGED_AGENT, +}; /// An entry of a `newc` archive. #[derive(Debug, PartialEq, Eq)] @@ -92,11 +94,18 @@ fn expected(agent: &[u8]) -> Vec { initramfs::launcher_script().as_bytes(), ), entry(MANAGED_AGENT, 3, 0o120_777, LAUNCHER.as_bytes()), + entry( + LAYERED_LAUNCHER, + 4, + 0o100_755, + initramfs::layered_launcher_script().as_bytes(), + ), + entry(INIT_AGENT, 5, 0o120_777, LAYERED_LAUNCHER.as_bytes()), ] } #[test] -fn the_archive_replaces_the_managed_agent_with_a_link_to_the_guest_agent_launcher() { +fn the_archive_replaces_nvx_s_agents_with_links_to_the_guest_agent_launchers() { // Guest agents of every length modulo 4 pad their data differently. for length in 0..8 { let agent = (1..=length).collect::>(); @@ -110,7 +119,7 @@ fn the_archive_replaces_the_managed_agent_with_a_link_to_the_guest_agent_launche } #[test] -fn the_launcher_runs_the_guest_agent_on_the_control_tty() { +fn the_launchers_run_the_guest_agent_on_the_control_tty() { // `/init` passes the managed agent the control tty first. let init = runtime_adapter::GUEST_INIT_COMMAND; assert_eq!( @@ -118,6 +127,16 @@ fn the_launcher_runs_the_guest_agent_on_the_control_tty() { format!("#!/bin/sh\nexec {GUEST_AGENT} {init} \"$1\"\n") ); assert_eq!(MANAGED_AGENT, "/sbin/nvx-managed-agent"); + + // It passes NVX's init agent nothing, so the layered launcher names the control tty, which + // `/init` requires of both lifecycles. + let layered = runtime_adapter::GUEST_LAYERED_MODE; + assert_eq!( + initramfs::layered_launcher_script(), + format!("#!/bin/sh\nexec {GUEST_AGENT} {init} /dev/hvc2 {layered}\n") + ); + assert_eq!(CONTROL_TTY, "/dev/hvc2"); + assert_eq!(INIT_AGENT, "/sbin/nvx-init-agent"); } #[test] diff --git a/tests/runtime-adapter/tests/layered_launch.rs b/tests/runtime-adapter/tests/layered_launch.rs new file mode 100644 index 000000000..284f00249 --- /dev/null +++ b/tests/runtime-adapter/tests/layered_launch.rs @@ -0,0 +1,269 @@ +//! The host program plans a VM with a layered root with NVX's typed plan, composes OpenVMM's +//! arguments as NVX's sandbox start does, records what it launches, and refuses layered options +//! that do not make a root. The tests need no hypervisor. +#![cfg(feature = "host")] + +use std::ffi::OsString; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use nvx::Hypervisor; +use nvx::host::sandbox::SandboxPlan; +use nvx::protocol::root::{RootImage, RootLayout, RootUpper}; +use nvx::uuid::Uuid; +use runtime_adapter::layered::{LaunchRecord, LayeredLaunch, RECORD_NAME}; +use runtime_adapter::root::RootConfig; +use serde_json::{Value, json}; + +/// Returns a directory of the test `name`, empty, below Cargo's temporary directory. +fn directory(name: &str) -> PathBuf { + let directory = Path::new(env!("CARGO_TARGET_TMPDIR")).join(name); + let _ = fs::remove_dir_all(&directory); + fs::create_dir_all(&directory).unwrap(); + directory +} + +/// Writes a file of `length` bytes at `path`, and returns the path. +fn image(path: PathBuf, length: usize) -> PathBuf { + fs::write(&path, vec![0; length]).unwrap(); + path +} + +/// Returns the identity that the tests expect of their images. +fn identity() -> Option { + Uuid::parse("0a490905-41be-ed5f-984e-63eb7e5e20b1") +} + +#[test] +fn a_layered_launch_plans_its_blocks_and_boot_tokens_and_measures_its_images() { + let directory = directory("layered-plan"); + let distro = image(directory.join("distro.erofs"), 3 * 512); + let scratch = image(directory.join("scratch.ext4"), 8 * 512); + let launch = + LayeredLaunch::new(&distro, RootImage::erofs(identity()), Some(&scratch), None).unwrap(); + let config = RootConfig { + distro_sectors: Some(3), + scratch_sectors: Some(8), + ..RootConfig::default() + }; + assert_eq!(launch.config, config); + let layout = RootLayout { + image: RootImage::erofs(identity()), + upper: RootUpper::Scratch, + }; + assert_eq!(launch.plan.layout(), layout); + + // The kernel command line carries the boot tokens of the root, which the guest reads back. + let command_line = launch.kernel_command_line().unwrap(); + let tokens = command_line.split(' ').collect::>(); + assert!(tokens.contains(&"nvx_sandbox=1"), "{command_line}"); + for token in layout.tokens() { + assert!( + tokens.contains(&token.as_str()), + "{token} in {command_line}" + ); + } + assert_eq!( + RootLayout::parse(command_line.as_bytes()).unwrap(), + Some(layout) + ); + + // OpenVMM's arguments: the plan's, which attach the image read-only and the scratch, then + // those of NVX's sandbox start. + let (kernel, initramfs) = (directory.join("vmlinux"), directory.join("initramfs")); + let report = directory.join("outcome.json"); + let arguments = launch + .openvmm_arguments( + Hypervisor::Kvm, + &kernel, + &initramfs, + "/run/control.sock", + &report, + ) + .unwrap(); + let plan = launch.plan.openvmm_arguments().unwrap(); + let blocks = [ + format!("distro:file:{},ro", distro.display()), + format!("scratch:file:{}", scratch.display()), + ]; + for block in &blocks { + assert!(plan.contains(block), "{block} in {plan:?}"); + } + let (planned, rest) = arguments.split_at(plan.len()); + assert_eq!(planned, plan.iter().map(OsString::from).collect::>()); + let expected: [OsString; 20] = [ + "--microvm-lifecycle".into(), + "managed".into(), + "--single-process".into(), + "--hypervisor".into(), + "kvm".into(), + "--memory".into(), + "256M".into(), + "--kernel".into(), + kernel.into(), + "--initrd".into(), + initramfs.into(), + "--cmdline".into(), + command_line.into(), + "--virtio-console".into(), + "none".into(), + "--microvm-control-console".into(), + "listen=/run/control.sock".into(), + "--microvm-control-auth-stdin".into(), + "--microvm-report".into(), + report.into(), + ]; + assert_eq!(rest, expected); +} + +#[test] +fn a_layered_launch_passes_what_the_guest_checks_as_it_is() { + let directory = directory("layered-options"); + // An image that is not a whole number of sectors has no size, an upper in RAM no scratch, and + // a runtime directory replaces the default as it is, so that the guest checks it. + let distro = image(directory.join("distro.img"), 1000); + let launch = + LayeredLaunch::new(&distro, RootImage::gpt(None), None, Some("run/relative")).unwrap(); + let config = RootConfig { + runtime: "run/relative".to_owned(), + ..RootConfig::default() + }; + assert_eq!(launch.config, config); + assert_eq!(launch.plan.layout().upper, RootUpper::Ram); + let arguments = launch.plan.openvmm_arguments().unwrap(); + assert!( + !arguments + .iter() + .any(|argument| argument.starts_with("scratch:")), + "{arguments:?}" + ); + + // A relative image is planned by its absolute path, since OpenVMM runs in the sandbox's store. + let relative = distro.strip_prefix(std::env::current_dir().unwrap()); + if let Ok(relative) = relative { + let launch = LayeredLaunch::new(relative, RootImage::gpt(None), None, None).unwrap(); + assert_eq!(launch.plan.distro().path(), distro); + } + + // An image that does not exist is refused. + let missing = directory.join("missing.erofs"); + let error = LayeredLaunch::new(&missing, RootImage::erofs(None), None, None).unwrap_err(); + assert!( + error.to_string().contains("distro EROFS layer not found"), + "{error}" + ); +} + +#[test] +fn a_launch_records_what_it_launched() { + let directory = directory("layered-record"); + let distro = image(directory.join("distro.erofs"), 512); + let scratch = image(directory.join("scratch.ext4"), 512); + let launch = + LayeredLaunch::new(&distro, RootImage::erofs(identity()), Some(&scratch), None).unwrap(); + let record = LaunchRecord { + kernel_command_line: "nvx_sandbox=1", + layered: Some(&launch), + }; + let json = record.to_json().unwrap(); + assert_eq!(json["format"], 1); + assert_eq!(json["mode"], "layered"); + assert_eq!(json["kernelCommandLine"], "nvx_sandbox=1"); + let config = String::from_utf8(launch.config.encode()).unwrap(); + assert_eq!(json["rootConfiguration"], config.as_str()); + let plan: SandboxPlan = serde_json::from_value(json["plan"].clone()).unwrap(); + assert_eq!(plan, launch.plan); + + // The record replaces an earlier one in the store. + fs::write(directory.join(RECORD_NAME), "earlier").unwrap(); + record.write(&directory).unwrap(); + let written = fs::read_to_string(directory.join(RECORD_NAME)).unwrap(); + assert_eq!(serde_json::from_str::(&written).unwrap(), json); + + let blockless = LaunchRecord { + kernel_command_line: "hostname=nvx-sandbox", + layered: None, + }; + let expected = json!({ + "format": 1, + "mode": "blockless", + "kernelCommandLine": "hostname=nvx-sandbox", + "plan": null, + "rootConfiguration": null, + }); + assert_eq!(blockless.to_json().unwrap(), expected); +} + +#[test] +fn the_host_program_refuses_options_that_do_not_make_a_command() { + let launch = ["launch", "store", "--guest-agent", "agent"]; + let uuid = "0a490905-41be-ed5f-984e-63eb7e5e20b1"; + let cases: [(&[&str], &str); 14] = [ + (&["--distro", "erofs:image"], "--distro needs --upper"), + (&["--upper", "ram"], "--upper needs --distro"), + ( + &["--distro-identity", uuid], + "--distro-identity needs --distro", + ), + (&["--runtime-dir", "/run/a"], "--runtime-dir needs --distro"), + ( + &["--distro", "squashfs:image"], + "--distro takes FORMAT:PATH", + ), + (&["--distro", "erofs:"], "--distro takes FORMAT:PATH"), + (&["--distro", "image"], "--distro takes FORMAT:PATH"), + (&["--upper", "tmpfs"], "--upper takes scratch:PATH or ram"), + ( + &["--upper", "scratch:"], + "--upper takes scratch:PATH or ram", + ), + ( + &["--distro-identity", "identity"], + "--distro-identity takes a UUID", + ), + ( + &["--upper", "ram", "--upper", "ram"], + "unexpected option \"--upper\"", + ), + (&["mounts"], "unexpected argument \"mounts\" of launch"), + (&["--hypervisor", "qemu"], "qemu"), + (&["--distro"], "\"--distro\" needs a value"), + ]; + let mut commands = cases + .iter() + .map(|(options, problem)| ([&launch[..], options].concat(), *problem)) + .collect::>(); + commands.extend([ + (vec!["probe", "store"], "probe needs a probe"), + ( + vec!["probe", "store", "Mounts"], + "the probe's name \"Mounts\"", + ), + (vec!["probe", "store", "mounts "], "is empty"), + ( + vec!["probe", "store", "mounts", "--guest-agent", "agent"], + "unexpected option \"--guest-agent\" of probe", + ), + ( + vec!["stop", "store", "mounts"], + "unexpected argument \"mounts\" of stop", + ), + ( + vec!["reattach", "store", "--distro", "erofs:image"], + "unexpected option \"--distro\" of reattach", + ), + ]); + for (arguments, problem) in commands { + let output = Command::new(env!("CARGO_BIN_EXE_runtime-adapter-host")) + .args(&arguments) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(2), "{arguments:?}: {output:?}"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains(problem) && stderr.contains("usage: runtime-adapter-host"), + "{arguments:?}: {stderr}" + ); + } +} diff --git a/tests/runtime-adapter/tests/vm_layered.rs b/tests/runtime-adapter/tests/vm_layered.rs new file mode 100644 index 000000000..5da2e832f --- /dev/null +++ b/tests/runtime-adapter/tests/vm_layered.rs @@ -0,0 +1,490 @@ +//! The host program launches VMs with a layered root, which the guest agent's worker assembles +//! through `nvx::guest::root` in a process of its own, never PID 1, on a real hypervisor, through +//! public APIs only. +//! +//! Each scenario of the table that [`scenarios!`] declares is an ignored test, which launches a VM +//! with a [`Root`] and checks what it [`Expect`]s: a root that the guest assembles, which the +//! scenario probes and the stop tears down, or the stage of the root that fails. A new scenario is +//! a new row, with its own checks, through the probes of the worker. +//! +//! The tests boot VMs, so they are ignored. Besides what the `common` module lists, they read the +//! fixtures that NVX's own layered scenarios use, from NVX's artifacts in `NVX_ARTIFACTS_DIR`: +//! +//! - the Ubuntu EROFS image, `ubuntu-distro.erofs`, and its manifest, which holds its UUID; +//! - the scratch template, `ubuntu-smoke-scratch.ext4`, an empty ext4 file system that NVX's CI +//! makes with the guest artifacts, of which each scenario boots a copy. +//! +//! ```text +//! cargo test --locked --features host --test vm_layered -- --ignored +//! ``` +#![cfg(feature = "host")] + +mod common; + +use std::collections::BTreeSet; +use std::env; +use std::ffi::{OsStr, OsString}; +use std::fs::{self, File}; +use std::io::{Read, Seek, SeekFrom}; +use std::path::{Path, PathBuf}; + +use nvx::host::artifacts::{ArtifactDir, Artifacts, Guest}; +use runtime_adapter::layered::RECORD_NAME; +use runtime_adapter::root::{DEFAULT_RUNTIME, RootConfig, RootStage}; +use serde_json::Value; + +use common::{Printed, Sandbox}; + +/// The scratch template among NVX's artifacts. +const SCRATCH_TEMPLATE: &str = "ubuntu-smoke-scratch.ext4"; +/// The exit status of the host program's command whose guest reported that a stage of the root +/// failed. +const ROOT_FAILED: i32 = 4; + +/// The fixtures of the scenarios, among NVX's artifacts. +struct Fixtures { + /// NVX's Ubuntu EROFS image. + distro: PathBuf, + /// The image's UUID, from its manifest. + identity: String, + /// The scratch template. + scratch: PathBuf, +} + +impl Fixtures { + /// Finds the fixtures among NVX's artifacts in `NVX_ARTIFACTS_DIR`. + fn find() -> Self { + let directory = env::var_os(Artifacts::ENV_DIR) + .unwrap_or_else(|| panic!("{} must name NVX's artifacts", Artifacts::ENV_DIR)); + let artifacts = ArtifactDir::from_dir(directory).unwrap(); + let distro = artifacts.distro_layer(Guest::Ubuntu).unwrap(); + let manifest = artifacts.distro_manifest(Guest::Ubuntu).unwrap(); + let scratch = artifacts.artifact(SCRATCH_TEMPLATE); + for (path, fixture) in [ + (&distro, "the Ubuntu EROFS image"), + (&manifest, "the manifest of the Ubuntu EROFS image"), + ( + &scratch, + "the scratch template, an empty ext4 file system, which `mkfs.ext4` makes", + ), + ] { + assert!(path.is_file(), "{fixture} is missing: {}", path.display()); + } + let manifest: Value = serde_json::from_str(&fs::read_to_string(&manifest).unwrap()) + .expect("the manifest is JSON"); + let identity = manifest["uuid"] + .as_str() + .expect("the manifest holds the image's UUID") + .to_owned(); + Self { + distro, + identity, + scratch, + } + } +} + +/// The image of a scenario's root. +#[derive(Clone, Copy, Debug)] +enum Image { + /// NVX's Ubuntu EROFS image. + Erofs, +} + +/// The identity that a scenario's launch expects of the image. +#[derive(Clone, Copy, Debug)] +enum Identity { + /// The image's own, from its manifest. + Own, + /// Another, which the image does not have. + Other, +} + +/// The upper of a scenario's root. +#[derive(Clone, Copy, Debug)] +enum Upper { + /// A copy of the scratch template. + Scratch, +} + +/// The layered root of a scenario's launch. +#[derive(Clone, Copy, Debug)] +struct Root { + /// The image. + image: Image, + /// The identity that the launch expects of the image. + identity: Identity, + /// The upper. + upper: Upper, + /// The guest's runtime directory, as the launch passes it, unless the default. + runtime: Option<&'static str>, +} + +/// The root of the scenarios unless they say otherwise: NVX's EROFS image, with its own identity, +/// over a copy of the scratch template. +const ROOT: Root = Root { + image: Image::Erofs, + identity: Identity::Own, + upper: Upper::Scratch, + runtime: None, +}; + +impl Root { + /// Returns the launch's options of the root, with the `fixtures` and the `scratch` image. + fn options(&self, fixtures: &Fixtures, scratch: &Path) -> Vec { + let Image::Erofs = self.image; + let mut distro = OsString::from("erofs:"); + distro.push(&fixtures.distro); + let identity = match self.identity { + Identity::Own => fixtures.identity.clone(), + Identity::Other => other_identity(&fixtures.identity), + }; + let Upper::Scratch = self.upper; + let mut upper = OsString::from("scratch:"); + upper.push(scratch); + let mut options = [ + "--distro".into(), + distro, + "--distro-identity".into(), + identity.into(), + "--upper".into(), + upper, + ] + .to_vec(); + if let Some(runtime) = self.runtime { + options.extend(["--runtime-dir".into(), runtime.into()]); + } + options + } +} + +/// Returns a UUID other than `identity`, which differs in its last digit. +fn other_identity(identity: &str) -> String { + let (start, last) = identity.split_at(identity.len() - 1); + format!("{start}{}", if last == "0" { "1" } else { "0" }) +} + +/// What a scenario expects of its VM. +enum Expect { + /// The guest assembles the root, the check probes the VM, and the stop tears the root down, + /// which leaves the scratch unmounted. + Ready(fn(&mut Vm<'_>)), + /// The guest assembles the root, the check keeps it busy, and the stop reports the teardown + /// that failed with this problem, which leaves the scratch mounted when the VM ends. + TeardownFails(fn(&mut Vm<'_>), &'static str), + /// The guest reports that the stage failed with this problem, and the VM ends with its + /// status, before the guest mounted the scratch. + Fails(RootStage, &'static str), +} + +/// A scenario: a VM with a layered root, and what the scenario expects of it. +struct Scenario { + /// The root of the VM. + root: Root, + /// What the scenario expects. + expect: Expect, +} + +/// A VM of a scenario, whose guest assembled its root. +struct Vm<'a> { + /// The scenario's sandbox. + sandbox: &'a mut Sandbox, + /// What the launch printed. + launched: Printed, + /// The guest's runtime directory, below which it assembled the root. + runtime: &'static str, +} + +impl Vm<'_> { + /// Runs `probes` in the VM's root, and returns what they found, each fact as `PROBE.KEY`. + fn probe(&mut self, probes: &[&str]) -> Printed { + let probes = probes.iter().map(OsStr::new).collect::>(); + let printed = self.sandbox.run("probe", &probes, 0); + assert_eq!(printed["reattached"], "live"); + assert_eq!(printed["pid"], self.launched["pid"]); + printed + } +} + +/// Declares the table of scenarios: each row, `NAME: SCENARIO;`, is an ignored test, `NAME`, which +/// runs `SCENARIO`. +macro_rules! scenarios { + ($($(#[doc = $doc:literal])* $name:ident: $scenario:expr;)*) => { + $( + $(#[doc = $doc])* + #[test] + #[ignore = "boots a VM; see the module documentation for what it needs"] + fn $name() { + run(stringify!($name), $scenario); + } + )* + }; +} + +scenarios! { + /// An EROFS image over an ext4 scratch, of the identity that the launch expects: the merged + /// root's contents and mounts, the agent's and the workload's cgroups, PID 1's unrelated + /// child, and a clean teardown. + erofs_over_scratch: Scenario { + root: ROOT, + expect: Expect::Ready(check_erofs_over_scratch), + }; + /// An image of another identity than the launch expects, which the assembly refuses. + other_identity_fails_the_assembly: Scenario { + root: Root { identity: Identity::Other, ..ROOT }, + expect: Expect::Fails(RootStage::Assembly, "the distro layer's UUID is "), + }; + /// A relative runtime directory, which the guest refuses with the configuration. + invalid_configuration_fails_its_stage: Scenario { + root: Root { runtime: Some("run/runtime-adapter"), ..ROOT }, + expect: Expect::Fails(RootStage::Configuration, "which is not absolute"), + }; + /// A root that a probe holds open, whose teardown fails. + busy_root_fails_the_teardown: Scenario { + root: ROOT, + expect: Expect::TeardownFails(hold_root, "failed to unmount the overlay: "), + }; +} + +/// Runs the scenario `name`, `scenario`. +fn run(name: &str, scenario: Scenario) { + let fixtures = Fixtures::find(); + let mut sandbox = Sandbox::new(name); + let scratch = sandbox.root.join("scratch.ext4"); + fs::copy(&fixtures.scratch, &scratch).unwrap(); + let root = scenario.root; + let options = root.options(&fixtures, &scratch); + let options = options.iter().map(OsString::as_os_str).collect::>(); + let (check, failure) = match scenario.expect { + Expect::Fails(stage, problem) => { + let launched = sandbox.launch(&options, ROOT_FAILED); + assert_failed(&launched, stage, problem); + assert_eq!(launched["stopped"], "graceful"); + // The launch ended: no runtime record names the VM. + assert_eq!(sandbox.record(), None); + assert!(!needs_recovery(&scratch), "the guest mounted the scratch"); + return; + } + Expect::Ready(check) => (check, None), + Expect::TeardownFails(check, problem) => (check, Some(problem)), + }; + let launched = sandbox.launch(&options, 0); + assert_eq!(launched["root"], "ready"); + check_record(&sandbox, &fixtures, &root, &scratch); + let vmm = sandbox + .vmm() + .expect("the VM runs after its launcher exited"); + let mut vm = Vm { + sandbox: &mut sandbox, + launched: launched.clone(), + runtime: root.runtime.unwrap_or(DEFAULT_RUNTIME), + }; + check(&mut vm); + + let status = if failure.is_some() { ROOT_FAILED } else { 0 }; + let stopped = sandbox.run("stop", &[], status); + assert_eq!(stopped["reattached"], "live"); + assert_eq!(stopped["pid"], launched["pid"]); + assert_eq!(stopped["stopped"], "graceful"); + assert!(vmm.has_exited().unwrap()); + assert_eq!( + sandbox.record(), + None, + "the stop cleared the runtime record" + ); + match failure { + None => { + assert!(!stopped.contains_key("root"), "{stopped:?}"); + assert_eq!(stopped["outcome"], "success 0"); + assert_eq!(stopped["teardown"], "complete"); + assert!( + !needs_recovery(&scratch), + "the stop left the scratch mounted" + ); + } + Some(problem) => { + assert_failed(&stopped, RootStage::Teardown, problem); + assert!( + needs_recovery(&scratch), + "the failed teardown unmounted the scratch" + ); + } + } +} + +/// Checks that a command printed that the guest reported that `stage` failed with `problem`, and +/// that the VM then ended with the stage's status and a complete teardown. +fn assert_failed(printed: &Printed, stage: RootStage, problem: &str) { + assert_eq!(printed["root"], "failed", "{printed:?}"); + assert_eq!(printed["stage"], stage.as_str(), "{printed:?}"); + assert!(printed["problem"].contains(problem), "{printed:?}"); + let outcome = format!("guest-exit {}", stage.exit_status()); + assert_eq!(printed["outcome"], outcome, "{printed:?}"); + assert_eq!(printed["teardown"], "complete", "{printed:?}"); +} + +/// Checks what the launch recorded in the sandbox's store: a VM with the layered `root` of +/// `fixtures` over `scratch`, the boot tokens of its plan, and the root configuration that the +/// host sent. +fn check_record(sandbox: &Sandbox, fixtures: &Fixtures, root: &Root, scratch: &Path) { + let record = fs::read_to_string(sandbox.store.join(RECORD_NAME)).unwrap(); + let record: Value = serde_json::from_str(&record).unwrap(); + assert_eq!(record["format"], 1); + assert_eq!(record["mode"], "layered"); + let plan = &record["plan"]; + assert_eq!(plan["distro"]["format"], "erofs"); + assert_eq!(plan["distro"]["identity"], fixtures.identity.as_str()); + let path = |value: &Value| PathBuf::from(value.as_str().unwrap()); + assert_eq!(path(&plan["distro"]["path"]), fixtures.distro); + assert_eq!(plan["upper"]["kind"], "scratch"); + assert_eq!(path(&plan["upper"]["path"]), scratch); + let tokens = record["kernelCommandLine"] + .as_str() + .unwrap() + .split(' ') + .collect::>(); + let layer = format!("nvx_layer=distro,0xd0003000,erofs:{}", fixtures.identity); + for token in ["nvx_sandbox=1", &layer, "nvx_scratch=0xd0006000,ext4"] { + assert!(tokens.contains(token), "{token} in {tokens:?}"); + } + let sectors = |path: &Path| Some(fs::metadata(path).unwrap().len() / 512); + let config = RootConfig { + runtime: root.runtime.unwrap_or(DEFAULT_RUNTIME).to_owned(), + distro_sectors: sectors(&fixtures.distro), + scratch_sectors: sectors(scratch), + ..RootConfig::default() + }; + let sent = String::from_utf8(config.encode()).unwrap(); + assert_eq!(record["rootConfiguration"], sent.as_str()); +} + +/// Returns whether the ext4 file system at `path` needs its journal recovered: whether a mount +/// left its superblock's `needs_recovery` feature set, which a clean unmount clears. +fn needs_recovery(path: &Path) -> bool { + let mut superblock = [0; 1024]; + let mut file = File::open(path).unwrap(); + file.seek(SeekFrom::Start(1024)).unwrap(); + file.read_exact(&mut superblock).unwrap(); + let magic = u16::from_le_bytes([superblock[0x38], superblock[0x39]]); + assert_eq!( + magic, + 0xef53, + "{} holds an ext4 file system", + path.display() + ); + let incompatible = u32::from_le_bytes(superblock[0x60..0x64].try_into().unwrap()); + incompatible & 0x4 != 0 +} + +/// Returns the options of a list of mount options, separated by commas. +fn options(list: &str) -> BTreeSet<&str> { + list.split(',').collect() +} + +/// Checks the VM of the `erofs_over_scratch` scenario. +fn check_erofs_over_scratch(vm: &mut Vm<'_>) { + let facts = vm.probe(&["mounts", "contents", "cgroups", "processes"]); + let runtime = vm.runtime; + + // The image, read-only, under the overlay, whose upper and work directories are on the + // scratch, all once, below the runtime directory. + for (mount, kind, flags) in [ + ("distro", "erofs", &["ro"][..]), + ("scratch", "ext4", &["rw", "nosuid", "nodev"]), + ("root", "overlay", &["rw", "nosuid", "nodev"]), + ] { + assert_eq!(facts[&format!("mounts.{mount}.count")], "1", "{mount}"); + assert_eq!(facts[&format!("mounts.{mount}.type")], kind, "{mount}"); + let found = options(&facts[&format!("mounts.{mount}.options")]); + assert!( + found.is_superset(&flags.iter().copied().collect()), + "{mount}: {found:?}" + ); + } + let layers = [ + format!("lowerdir={runtime}/layers/distro"), + format!("upperdir={runtime}/scratch/upper"), + format!("workdir={runtime}/scratch/work"), + "metacopy=on".to_owned(), + "xino=on".to_owned(), + ]; + let overlay = options(&facts["mounts.root.super"]); + for option in &layers { + assert!(overlay.contains(option.as_str()), "{option} in {overlay:?}"); + } + + // The merged root: the image's files, and a file that the worker wrote, which reaches the + // upper and not the image, which refuses writes. + assert_eq!(facts["contents.os-release.id"], "ubuntu"); + assert_eq!(facts["contents.root.mode"], "755"); + assert_eq!(facts["contents.write.root"], "runtime-adapter"); + assert_eq!(facts["contents.write.upper"], "runtime-adapter"); + assert_eq!(facts["contents.write.lower"], "absent"); + assert_eq!(facts["contents.lower.write"], "refused"); + + // The agent's and the workload's cgroups, siblings in the cgroup v2 hierarchy, with the + // configuration's resource values, and no user namespaces. + let config = RootConfig::default(); + assert_eq!(facts["cgroups.parent.type"], "cgroup2"); + assert_eq!(facts["cgroups.agent.exists"], "yes"); + assert_eq!(facts["cgroups.workload.exists"], "yes"); + let values = [ + ("cgroups.agent.memory.low", config.agent_memory_low), + ("cgroups.workload.memory.max", config.workload_memory_max), + ("cgroups.workload.pids.max", config.workload_pids_max), + ]; + for (fact, value) in values { + assert_eq!(facts[fact], value.unwrap().to_string(), "{fact}"); + } + assert_eq!(facts["cgroups.workload.processes"], "0"); + let user_namespaces = facts["cgroups.user-namespaces"].as_str(); + assert!( + ["0", "unavailable"].contains(&user_namespaces), + "{user_namespaces}" + ); + + // The worker is not PID 1, but its child. PID 1's unrelated child runs on, untouched: in PID + // 1's root directory and namespaces, like the worker, and in the agent's cgroup, into which + // the bootstrap moved every process of the guest. + let agent = Path::new(&config.agent_cgroup); + let parent = agent.parent().unwrap(); + let agent = format!("/{}", agent.strip_prefix(parent).unwrap().display()); + assert_ne!(facts["processes.worker.pid"], "1"); + assert_eq!(facts["processes.worker.ppid"], "1"); + assert_eq!(facts["processes.sibling.ppid"], "1"); + let state = facts["processes.sibling.state"].as_str(); + assert!(["S", "R"].contains(&state), "{state}"); + let commands = [ + ("pid1", "init /dev/hvc2 layered"), + ("worker", "worker /dev/hvc2"), + ("sibling", "idle"), + ]; + for (process, command) in commands { + let found = &facts[&format!("processes.{process}.command")]; + let expected = format!("/sbin/runtime-adapter-guest {command}"); + assert!(found.starts_with(&expected), "{process}: {found}"); + assert_eq!( + facts[&format!("processes.{process}.root")], + "/", + "{process}" + ); + let namespaces = &facts[&format!("processes.{process}.namespaces")]; + assert_eq!(namespaces, "same", "{process}"); + assert_eq!( + facts[&format!("processes.{process}.cgroup")], + agent, + "{process}" + ); + } + let sibling = &facts["processes.sibling.pid"]; + assert!( + facts["processes.worker.command"].ends_with(&format!(" {sibling}")), + "{facts:?}" + ); +} + +/// Keeps the root of the `busy_root_fails_the_teardown` scenario busy. +fn hold_root(vm: &mut Vm<'_>) { + let facts = vm.probe(&["hold"]); + assert_eq!(facts["hold.held"], format!("{}/rootfs", vm.runtime)); +} diff --git a/tests/runtime-adapter/tests/vm_restart.rs b/tests/runtime-adapter/tests/vm_restart.rs index a3b0d5078..220c261ae 100644 --- a/tests/runtime-adapter/tests/vm_restart.rs +++ b/tests/runtime-adapter/tests/vm_restart.rs @@ -2,252 +2,29 @@ //! of it reattach to the VM and stop it, as an orchestrator does after it restarts, on a real //! hypervisor, through public APIs only. //! -//! The tests boot VMs, so they are ignored. They read: -//! -//! - `NVX_HYPERVISOR`: the hypervisor, through `nvx::Hypervisor::from_env_or_default`; -//! - `NVX_ARTIFACTS_DIR`: NVX's artifacts, a repository checkout or an extracted release, with -//! the install metadata from which NVX gives their runtime identity; -//! - `RUNTIME_ADAPTER_GUEST`: the guest agent, a static `x86_64-unknown-linux-musl` build of -//! `runtime-adapter-guest`. +//! The tests boot VMs, so they are ignored. They read what the `common` module lists. //! //! ```text //! cargo test --locked --features host -- --ignored //! ``` #![cfg(feature = "host")] -use std::collections::BTreeMap; -use std::convert::Infallible; -use std::env; -use std::error::Error; +mod common; + use std::ffi::OsStr; -use std::fs::{self, File}; -use std::io; -use std::path::PathBuf; -use std::process::{Command, Stdio}; -use std::thread; -use std::time::{Duration, Instant}; -use nvx::Hypervisor; use nvx::host::process::Process; -use nvx::host::store::{RuntimeRecord, SandboxStore}; -use nvx::host::supervisor::{self, Reattached, StopOutcome}; - -/// The variable that names the guest agent. -const GUEST_AGENT: &str = "RUNTIME_ADAPTER_GUEST"; -/// The longest that a command of the host program may take. -const COMMAND_TIMEOUT: Duration = Duration::from_secs(180); -/// How much of the end of the VMM's log a failure shows. -const LOG_TAIL: usize = 8192; - -/// A sandbox of a test: a directory of its own, which holds the sandbox's store and the output of -/// each command of the host program that the test runs. -/// -/// Dropping it ends a VM that a failed test left running, through its verified identity, and -/// removes the directory once no VM of the sandbox runs. Should that remain uncertain, it keeps -/// the directory, whose store a later reattach needs to end the VM, and prints its path. -struct Sandbox { - /// The test's directory. - root: PathBuf, - /// The sandbox's store, which the launch creates. - store: PathBuf, - /// The number of commands that the test ran. - commands: usize, -} - -impl Sandbox { - /// Creates the directory of the test `name`, in the system's temporary directory, so that the - /// path of a Linux control socket in it stays short. A directory that an earlier test process - /// kept stays as it is. - fn new(name: &str) -> Self { - let pid = std::process::id(); - let root = (0..) - .map(|attempt| env::temp_dir().join(format!("runtime-adapter-{name}-{pid}-{attempt}"))) - .find(|root| match fs::create_dir(root) { - Ok(()) => true, - Err(error) if error.kind() == io::ErrorKind::AlreadyExists => false, - Err(error) => panic!("cannot create {}: {error}", root.display()), - }) - .expect("a directory name is free"); - Self { - store: root.join("store"), - root, - commands: 0, - } - } - - /// Launches the VM, and returns what the launch printed. - fn launch(&mut self) -> BTreeMap { - let hypervisor = Hypervisor::from_env_or_default().expect("NVX_HYPERVISOR is valid"); - let guest_agent = env::var_os(GUEST_AGENT).unwrap_or_else(|| { - panic!( - "{GUEST_AGENT} must name a static x86_64-unknown-linux-musl runtime-adapter-guest" - ) - }); - let options = [ - OsStr::new("--guest-agent"), - &guest_agent, - OsStr::new("--hypervisor"), - OsStr::new(hypervisor.as_str()), - ]; - let launched = self.run("launch", &options, 0); - let record = self.record().expect("the launch recorded its VM"); - assert_eq!(launched["pid"], record.process().pid.to_string()); - launched - } - - /// Runs the host program's `command` on the sandbox's store with `options`, checks that it - /// exits with `status`, and returns the `KEY=VALUE` lines that it printed. - fn run(&mut self, command: &str, options: &[&OsStr], status: i32) -> BTreeMap { - self.commands += 1; - let output = self.root.join(format!("{}-{command}", self.commands)); - let (stdout, stderr) = (output.with_extension("out"), output.with_extension("err")); - // Files, rather than pipes, take the output, so that nothing waits for a VMM that might - // hold them. - let mut child = Command::new(env!("CARGO_BIN_EXE_runtime-adapter-host")) - .arg(command) - .arg(&self.store) - .args(options) - .stdin(Stdio::null()) - .stdout(File::create(&stdout).unwrap()) - .stderr(File::create(&stderr).unwrap()) - .spawn() - .unwrap(); - let deadline = Instant::now() + COMMAND_TIMEOUT; - let exit = loop { - if let Some(exit) = child.try_wait().unwrap() { - break exit; - } - if Instant::now() >= deadline { - let _ = child.kill(); - let _ = child.wait(); - panic!("{command} did not end in time{}", self.diagnostics(&output)); - } - thread::sleep(Duration::from_millis(50)); - }; - assert_eq!( - exit.code(), - Some(status), - "{command} {options:?} ended with {exit}{}", - self.diagnostics(&output) - ); - let printed = fs::read_to_string(&stdout).unwrap(); - let mut lines = BTreeMap::new(); - for line in printed.lines() { - let (key, value) = line - .split_once('=') - .unwrap_or_else(|| panic!("{command} printed {line:?}")); - let earlier = lines.insert(key.to_owned(), value.to_owned()); - assert!(earlier.is_none(), "{command} printed {key} twice"); - } - lines - } - - /// Returns what the command whose output files start with `output` printed, and the end of the - /// VMM's log. - fn diagnostics(&self, output: &std::path::Path) -> String { - let read = |path: PathBuf| { - let bytes = fs::read(path).unwrap_or_default(); - let tail = bytes.len().saturating_sub(LOG_TAIL); - String::from_utf8_lossy(&bytes[tail..]).into_owned() - }; - format!( - "\nstdout:\n{}\nstderr:\n{}\nthe end of the VMM's log:\n{}", - read(output.with_extension("out")), - read(output.with_extension("err")), - read(self.store.join(nvx::host::store::LOG_NAME)), - ) - } - /// Returns the sandbox's runtime record, if it has one. - fn record(&self) -> Option { - SandboxStore::open(&self.store).unwrap().runtime().unwrap() - } - - /// Returns the VMM of the sandbox's runtime record, if the record names one that runs. - fn vmm(&self) -> Option { - Process::open(self.record()?.process()).unwrap() - } - - /// Ends the VM that the sandbox's store records, whatever its runtime identity: reattach ends - /// an interrupted launch, and holds a VM that runs, which the stop then terminates through its - /// verified identity. Fails with the reason why a VM of the sandbox might still run. - fn end_vm(&self) -> Result<(), String> { - if !self.store.exists() { - return Ok(()); - } - let failed = |error: nvx::Error| describe(&error); - let store = SandboxStore::open(&self.store).map_err(failed)?; - let recorded = ( - store.runtime().map_err(failed)?, - store.journal().map_err(failed)?, - ); - let identity = match recorded { - (Some(record), _) => record.runtime().clone(), - (None, Some(journal)) => journal.runtime().clone(), - (None, None) => return Ok(()), - }; - let mut locked = store - .lock(Some(Instant::now() + Duration::from_secs(10))) - .map_err(failed)?; - let vmm = match supervisor::reattach(&mut locked, &identity).map_err(failed)? { - Reattached::Live(vmm) | Reattached::Mismatched { vmm, .. } => vmm, - Reattached::Exited { .. } => return Ok(()), - Reattached::Indeterminate(reason) => return Err(reason), - }; - let forced = Some(Duration::from_secs(10)); - let stopped = vmm - .stop(&mut locked, Instant::now(), forced, |_, _| { - Ok::<(), Infallible>(()) - }) - .map_err(failed)?; - match stopped.outcome { - StopOutcome::Graceful | StopOutcome::Forced => Ok(()), - StopOutcome::TimedOut => Err(format!( - "VMM process {} did not exit after it was terminated", - vmm.record().process().pid - )), - } - } -} - -impl Drop for Sandbox { - fn drop(&mut self) { - match self.end_vm() { - Ok(()) => { - let _ = fs::remove_dir_all(&self.root); - } - Err(reason) => eprintln!( - "kept {} because a VM of its sandbox might still run: {reason}", - self.root.display() - ), - } - } -} - -/// Describes `error`, with its causes. -fn describe(error: &dyn Error) -> String { - let mut description = error.to_string(); - let mut source = error.source(); - while let Some(cause) = source { - description = format!("{description}: {cause}"); - source = cause.source(); - } - description -} - -/// Returns the epoch that a command printed. -fn epoch(printed: &BTreeMap) -> u64 { - printed["epoch"].parse().unwrap() -} +use common::{Printed, Sandbox, epoch}; /// Checks that a command stopped the VM of `vmm` that `launched` printed, with the guest agent's /// confirmation, after it attached a session in an epoch after `earlier`'s: the VM exited with /// a successful outcome, and no runtime record names it. fn assert_stopped( sandbox: &Sandbox, - stopped: &BTreeMap, - launched: &BTreeMap, - earlier: &BTreeMap, + stopped: &Printed, + launched: &Printed, + earlier: &Printed, vmm: &Process, ) { assert_eq!(stopped["reattached"], "live"); @@ -270,7 +47,7 @@ fn assert_stopped( fn a_restarted_host_program_reattaches_to_the_vm_and_stops_it() { let mut sandbox = Sandbox::new("restart"); // The launching process has exited, and the VM runs on. - let launched = sandbox.launch(); + let launched = sandbox.launch(&[], 0); let vmm = sandbox .vmm() .expect("the VM runs after its launcher exited"); @@ -293,7 +70,7 @@ fn a_restarted_host_program_reattaches_to_the_vm_and_stops_it() { #[ignore = "boots a VM; see the module documentation for what it needs"] fn a_host_program_of_another_runtime_finds_the_vm_mismatched_and_signals_nothing() { let mut sandbox = Sandbox::new("identity"); - let launched = sandbox.launch(); + let launched = sandbox.launch(&[], 0); let record = sandbox.record(); let vmm = sandbox .vmm() From adcb739d56bebaed26690952cecaadfc983ce98c Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Thu, 22 Oct 2026 17:14:16 -0700 Subject: [PATCH 2/2] tests: check the launch record of failed layered launches The layered scenarios whose launch fails returned before they checked the launch's record, although the adapter's README says that each scenario checks it. They now check it too: the record holds the plan, with the image identity that the launch expected, and the root configuration that the host sent, whether or not the guest accepted it. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ddd9400a-9806-4d3d-b2cb-4320252cb090 --- tests/runtime-adapter/tests/vm_layered.rs | 25 +++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tests/runtime-adapter/tests/vm_layered.rs b/tests/runtime-adapter/tests/vm_layered.rs index 5da2e832f..3d508764d 100644 --- a/tests/runtime-adapter/tests/vm_layered.rs +++ b/tests/runtime-adapter/tests/vm_layered.rs @@ -130,15 +130,20 @@ const ROOT: Root = Root { }; impl Root { + /// Returns the identity that the launch expects of the image of `fixtures`. + fn identity(&self, fixtures: &Fixtures) -> String { + match self.identity { + Identity::Own => fixtures.identity.clone(), + Identity::Other => other_identity(&fixtures.identity), + } + } + /// Returns the launch's options of the root, with the `fixtures` and the `scratch` image. fn options(&self, fixtures: &Fixtures, scratch: &Path) -> Vec { let Image::Erofs = self.image; let mut distro = OsString::from("erofs:"); distro.push(&fixtures.distro); - let identity = match self.identity { - Identity::Own => fixtures.identity.clone(), - Identity::Other => other_identity(&fixtures.identity), - }; + let identity = self.identity(fixtures); let Upper::Scratch = self.upper; let mut upper = OsString::from("scratch:"); upper.push(scratch); @@ -260,6 +265,9 @@ fn run(name: &str, scenario: Scenario) { let launched = sandbox.launch(&options, ROOT_FAILED); assert_failed(&launched, stage, problem); assert_eq!(launched["stopped"], "graceful"); + // The launch recorded what it launched, with the root configuration that it sent, which + // the guest refused, or accepted before a later stage failed. + check_record(&sandbox, &fixtures, &root, &scratch); // The launch ended: no runtime record names the VM. assert_eq!(sandbox.record(), None); assert!(!needs_recovery(&scratch), "the guest mounted the scratch"); @@ -324,16 +332,17 @@ fn assert_failed(printed: &Printed, stage: RootStage, problem: &str) { } /// Checks what the launch recorded in the sandbox's store: a VM with the layered `root` of -/// `fixtures` over `scratch`, the boot tokens of its plan, and the root configuration that the -/// host sent. +/// `fixtures` over `scratch`, the image of the identity that the launch expects, the boot tokens of +/// its plan, and the root configuration that the host sent, whether or not the guest accepted it. fn check_record(sandbox: &Sandbox, fixtures: &Fixtures, root: &Root, scratch: &Path) { let record = fs::read_to_string(sandbox.store.join(RECORD_NAME)).unwrap(); let record: Value = serde_json::from_str(&record).unwrap(); assert_eq!(record["format"], 1); assert_eq!(record["mode"], "layered"); let plan = &record["plan"]; + let identity = root.identity(fixtures); assert_eq!(plan["distro"]["format"], "erofs"); - assert_eq!(plan["distro"]["identity"], fixtures.identity.as_str()); + assert_eq!(plan["distro"]["identity"], identity.as_str()); let path = |value: &Value| PathBuf::from(value.as_str().unwrap()); assert_eq!(path(&plan["distro"]["path"]), fixtures.distro); assert_eq!(plan["upper"]["kind"], "scratch"); @@ -343,7 +352,7 @@ fn check_record(sandbox: &Sandbox, fixtures: &Fixtures, root: &Root, scratch: &P .unwrap() .split(' ') .collect::>(); - let layer = format!("nvx_layer=distro,0xd0003000,erofs:{}", fixtures.identity); + let layer = format!("nvx_layer=distro,0xd0003000,erofs:{identity}"); for token in ["nvx_sandbox=1", &layer, "nvx_scratch=0xd0006000,ext4"] { assert!(tokens.contains(token), "{token} in {tokens:?}"); }