From 7304b4c379a5b8d8db77b5df38dddee7e0d394d9 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Thu, 22 Oct 2026 15:45:09 -0700 Subject: [PATCH 1/4] nvx-rs: assemble a guest root from a GPT disk's layers A guest root's image can now be a GPT disk, in an Image::Gpt with the caller's expected identities, its disk GUID and its partitions' unique GUIDs and sizes, each if it knows them, and a bound on the size of its first partition. The assembly reads and checks the disk's table with nvx::guest::gpt::read_disk, at the device's logical block size and size, and uses the partitions that the kernel created when it found the disk, without comparing them with the table or asking the kernel to reread it. The first partition holds opaque bytes, which the assembly reads up to the caller's bound, without their trailing padding, and returns through Root::first_partition. It never probes or mounts the first partition. Every later partition holds a layer, an ext4 file system, which the assembly mounts read-only, nosuid and nodev, at layers/pN, for the partition's number N, from the node that devtmpfs created for its partition under the kernel's name, such as vda2, and the overlay stacks the layers with partition 2's at its base and the last partition's at its top. The layers mount without noload, and the assembly refuses a layer whose superblock isn't ext4's, or has the needs_recovery feature: its read-only device can't take the recovery, which noload or norecovery among the superblock's own default mount options would make the kernel skip. The disk, its first partition, the partitions' nodes, and the layers' superblocks are checked before anything is mounted, as are the overlay's options, whose length limits a disk's layers: below /run/nvx, a root mounts up to 195 layers, those of partitions 2 through 196, whatever the overlay mode and the upper. Each Lower reports its partition, and the teardown unmounts the layers from the top down. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 30bab0d7-7c3e-4120-9add-6c9cc1dc9e7e --- nvx-rs/README.md | 2 +- nvx-rs/src/guest/root.rs | 323 +++++++++++-- nvx-rs/src/guest/root/block.rs | 117 ++++- nvx-rs/src/guest/root/kernel.rs | 64 ++- nvx-rs/src/guest/root/tests.rs | 834 +++++++++++++++++++++++++++++--- 5 files changed, 1217 insertions(+), 123 deletions(-) diff --git a/nvx-rs/README.md b/nvx-rs/README.md index 8ad93cd21..4e4f5eac9 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -99,7 +99,7 @@ nvx-rs/ | `nvx::guest::exec::binding` | `Binding`: an `nvx::exec::Runner` bound to an `Execution`, the glue that a guest adapter would otherwise write. The runner's read allowances become grants, its discards discards, the input that it accepted writes and then a close, its terminations a cancellation, a disconnection, or a termination, and its settlement a kill of the execution's cgroup and checks until no process occupies it; without a cgroup, nothing proves that the program's descendants are gone, so the execution is fenced. It feeds the execution's events back to the runner, statuses that a caller who reaps hands over with `observe` included, and never blocks: the caller polls its descriptors and its input writer until its deadline, and each call takes a bounded number of steps, so that no execution holds the caller's loop | | `nvx::guest::time` | x86_64: `TimeAbi`, the steps of NVX time ABI v1 as typed calls in the caller's process, for any guest init or agent: the cold-boot time step and, separately, the start of the time daemon in a child process that the caller supervises, the VM generation ID, a `CaptureRequest` that repairs a `Restore`, and a `RestoreFinish`. A call takes the permission for OpenVMM's ports on the calling thread for each port transaction alone, and fails with `TimeError::PortRelease` if the thread may still hold some of it, which a process that it started would inherit, and from which `nvx::guest::spawn` starts none. No call waits for a child that it did not start. `nvx-time` runs its steps through them | | `nvx::guest::reseed` | `Seed::read` and `reseed`: a restore's fresh entropy, which starts with its `GenerationId`, and the reseed of the kernel's random number generator with it, in the caller's process. `nvx-reseed` runs them | -| `nvx::guest::root` | The operations of a layered root, for a guest init or agent, whether or not it is PID 1. `bootstrap`: the agent's and the workload's sibling cgroups of a `Layout`, with every process of their parent moved into the agent's, the agent's `memory.low`, and the workload cgroup prepared through `ExecutionCgroup::prepare`, with its `memory.max` and `pids.max`, and user namespaces kept unavailable. `assemble`: the `Assembly`'s block devices, found by their MMIO windows, checked against their roles' access modes and their expected sizes, and flushed, and the `Root` that it mounts, at a runtime directory, a canonical path that no symbolic link leads through, that it holds exclusively until its teardown, where nothing is mounted already, once it has checked everything: the read-only layer of the `Image` that the caller names, which it never probes, an EROFS image whose expected `nvx::uuid::Uuid` it checks in its superblock, under an overlay mounted `nosuid,nodev` with the one option set of the caller's `OverlayMode`, `metacopy=on,xino=on` or, for lowers that may carry untrusted overlay xattrs, `metacopy=off,redirect_dir=nofollow`, with no fallback, whose upper and work directories are on the `Upper`: an ext4 scratch mounted `nosuid,nodev`, whose superblock it checks for a journal that the caller's `Journal` policy forbids, or a tmpfs mounted `nosuid,nodev` at the kernel's default size. It refuses overlay options longer than `MOUNT_OPTIONS_LIMIT`, the most that a mount passes, and an overlay that the kernel's mount table shows read-only or without the mode's options in effect, since the kernel mounts one so, rather than fail, when its upper can't hold its work directory or its extended attributes. The `Root` reports its `Lower` layers, its `UpperKind`, and its `OverlayMode`. `Root::mount_share` and `Root::mount_aggregate`: virtio-fs live shares at targets in the root that it checks, against the targets that `protocol::share` reserves among others, with one aggregate per root, which it attaches through its `mappings::Mapper` beneath the overlay, with NVX's container root's reserved targets: a share's device mounted out of the root, then moved onto its target, and a copy of each child's entry of the aggregate. `Root::map`: a caller's host mappings, through the same mapper. A share, a child, or a mapping that fails latches the root. `Root::teardown`: the shares and mappings in reverse order, then an aggregate's own mount, the overlay, the layers from the top down, and the upper, with every failed unmount reported. Each failure is a typed `Error`. The operations collect no child's status, change neither the caller's root nor its credentials, start no async runtime, and never power the VM off | +| `nvx::guest::root` | The operations of a layered root, for a guest init or agent, whether or not it is PID 1. `bootstrap`: the agent's and the workload's sibling cgroups of a `Layout`, with every process of their parent moved into the agent's, the agent's `memory.low`, and the workload cgroup prepared through `ExecutionCgroup::prepare`, with its `memory.max` and `pids.max`, and user namespaces kept unavailable. `assemble`: the `Assembly`'s block devices, found by their MMIO windows, checked against their roles' access modes and their expected sizes, and flushed, and the `Root` that it mounts, at a runtime directory, a canonical path that no symbolic link leads through, that it holds exclusively until its teardown, where nothing is mounted already, once it has checked everything: the read-only layers of the `Image` that the caller names, which it never probes: an EROFS image, whose expected `nvx::uuid::Uuid` it checks in its superblock, or a GPT disk whose partition table it reads and checks with `nvx::guest::gpt::read_disk`, against the caller's `nvx::gpt::Expected` values, whose first partition it reads, up to the caller's bound, as opaque bytes that it never mounts, and whose later partitions, at least one, hold ext4 layers that it mounts `ro,nosuid,nodev`, from the block devices that the kernel created when it found the disk, once their superblocks show that no journal needs a recovery, which their read-only device can't take, partition 2's at the overlay's base and the last partition's at its top, under an overlay mounted `nosuid,nodev` with the one option set of the caller's `OverlayMode`, `metacopy=on,xino=on` or, for lowers that may carry untrusted overlay xattrs, `metacopy=off,redirect_dir=nofollow`, with no fallback, whose upper and work directories are on the `Upper`: an ext4 scratch mounted `nosuid,nodev`, whose superblock it checks for a journal that the caller's `Journal` policy forbids, or a tmpfs mounted `nosuid,nodev` at the kernel's default size. It refuses overlay options longer than `MOUNT_OPTIONS_LIMIT`, the most that a mount passes, which, below `/run/nvx`, fit up to 195 layers of a GPT disk, and an overlay that the kernel's mount table shows read-only or without the mode's options in effect, since the kernel mounts one so, rather than fail, when its upper can't hold its work directory or its extended attributes. The `Root` reports the first partition's bytes, its `Lower` layers, its `UpperKind`, and its `OverlayMode`. `Root::mount_share` and `Root::mount_aggregate`: virtio-fs live shares at targets in the root that it checks, against the targets that `protocol::share` reserves among others, with one aggregate per root, which it attaches through its `mappings::Mapper` beneath the overlay, with NVX's container root's reserved targets: a share's device mounted out of the root, then moved onto its target, and a copy of each child's entry of the aggregate. `Root::map`: a caller's host mappings, through the same mapper. A share, a child, or a mapping that fails latches the root. `Root::teardown`: the shares and mappings in reverse order, then an aggregate's own mount, the overlay, the layers from the top down, and the upper, with every failed unmount reported. Each failure is a typed `Error`. The operations collect no child's status, change neither the caller's root nor its credentials, start no async runtime, and never power the VM off | | `nvx::guest::gpt` | `read_disk`: the partition table of a GPT disk, read and checked with `nvx::gpt`'s `Disk::read` through the disk's block device, which the caller opened, at the logical block size and size that `BLKSSZGET` and `BLKGETSIZE64` report, with the caller's `Expected` values, such as the disk GUID that a boot parameter names. It only reads the device, at explicit offsets, so it never moves the file's position. It neither compares the partitions that the kernel created with the table nor asks the kernel to reread the table. Each failure is a typed `DeviceError` | Lowering functions take the names by which messages refer to their inputs, such diff --git a/nvx-rs/src/guest/root.rs b/nvx-rs/src/guest/root.rs index 52e396269..e81653295 100644 --- a/nvx-rs/src/guest/root.rs +++ b/nvx-rs/src/guest/root.rs @@ -8,10 +8,13 @@ //! - [`assemble`] finds the block devices of an [`Assembly`] by their MMIO windows, checks and //! flushes them, and mounts the root: an overlay, mounted `nosuid,nodev` with the option set of //! the caller's [`OverlayMode`], which it checks the kernel applied, of the read-only layers of -//! the `distro` image, in the [`Image`] format that the caller names, which the assembly never -//! probes, over an upper, an ext4 scratch or a tmpfs. It returns the [`Root`] that owns the -//! mounts, holds the [runtime directory](#runtime-directory), and reports what it mounted: the -//! [`Lower`] layers, the [`UpperKind`], and the [`OverlayMode`]. +//! the `distro` image over an upper. The caller names the image's format, which the assembly +//! never probes: an EROFS file system, the overlay's one lower layer, or a GPT disk, whose first +//! partition holds opaque bytes that the root returns, and whose later partitions hold the +//! layers, ext4 file systems. The upper is an ext4 scratch or a tmpfs. It returns the [`Root`] +//! that owns the mounts, holds the [runtime directory](#runtime-directory), and reports what it +//! mounted: the first partition's bytes, the [`Lower`] layers, the [`UpperKind`], and the +//! [`OverlayMode`]. //! - [`Root::mount_share`] and [`Root::mount_aggregate`] mount the VMM's live shares at targets in //! the root, and [`Root::map`] applies a caller's host mappings there, through the root's //! [`Mapper`], as [live shares and mappings](#live-shares-and-mappings) describes. @@ -30,7 +33,8 @@ //! The root's mounts are below a runtime directory that the caller chooses, such as `/run/nvx`, //! which holds: //! -//! - `layers/distro`, the EROFS image; +//! - `layers/distro`, an EROFS image, or `layers/pN`, the layer in partition N of a GPT disk, for +//! each partition after the first; //! - `scratch`, an ext4 scratch, or `tmpfs`, a tmpfs, with the overlay's `upper` and `work` //! directories; //! - `rootfs`, the overlay, which is the root; @@ -53,8 +57,10 @@ //! //! The overlay's options name its layers' and its upper's directories by these paths, and //! `mount(2)` passes a file system at most [`MOUNT_OPTIONS_LIMIT`] bytes of options. An assembly -//! refuses longer options, which a long runtime directory makes, before it mounts anything, -//! rather than let the kernel cut them short. +//! refuses longer options before it mounts anything, rather than let the kernel cut them short, +//! so the length of the runtime directory's path limits the layers of a GPT disk: below +//! `/run/nvx`, a root mounts up to 195 layers, those of partitions 2 through 196, whichever the +//! overlay mode and the upper. //! //! # Live shares and mappings //! @@ -96,6 +102,7 @@ //! use std::path::Path; //! use std::time::Duration; //! +//! use nvx::gpt::Expected; //! use nvx::guest::cgroup::Layout; //! use nvx::guest::mappings::Access; //! use nvx::guest::root::{self, Assembly, Device, Image, OverlayMode, Resources, Upper}; @@ -111,12 +118,17 @@ //! let mut root = root::assemble(&Assembly { //! runtime: "/run/nvx".into(), //! distro: Device { mmio: DISTRO_MMIO_BASE, sectors: None }, -//! image: Image::Erofs { uuid: Some("8d2f3a4e-1c5b-4e7a-9f60-2b3c4d5e6f70".parse()?) }, +//! image: Image::Gpt { +//! expected: Expected::new().disk_guid("8d2f3a4e-1c5b-4e7a-9f60-2b3c4d5e6f70".parse()?), +//! first_partition_bound: 64 * 1024, +//! }, //! upper: Upper::Tmpfs, //! overlay: OverlayMode::UntrustedLowers, //! timeout: Duration::from_secs(30), //! })?; -//! println!("{} layer over a {:?} upper", root.lowers().len(), root.upper()); +//! let configuration = root.first_partition().unwrap_or_default(); +//! let layers = root.lowers().len(); +//! println!("the first partition holds {} bytes over {layers} layers", configuration.len()); //! root.mount_share("workspace", Path::new("/workspace"), Access::ReadWrite)?; //! // Run the workload in `bootstrap.workload()`, in the root at `root.path()`; then: //! root.teardown()?; @@ -132,7 +144,7 @@ mod share; mod tests; use std::fmt; -use std::fs; +use std::fs::{self, File}; use std::io; use std::os::unix::ffi::OsStrExt; use std::os::unix::fs::PermissionsExt; @@ -140,8 +152,10 @@ use std::path::{Path, PathBuf}; use std::time::Duration; use self::kernel::{Kernel, Paths, SystemKernel}; +use super::gpt::DeviceError; use super::mappings::{Mapper, MappingError, below_its_root}; use super::sys::{self, Errno, MountFlags}; +use crate::gpt::{Expected, GptError, Partition}; use crate::uuid::Uuid; pub use self::bootstrap::{Bootstrap, BootstrapStep, Resources, bootstrap}; @@ -149,6 +163,8 @@ pub use self::bootstrap::{Bootstrap, BootstrapStep, Resources, bootstrap}; pub(crate) use self::kernel::{Call, FakeKernel, Operation, attach_calls}; pub use self::share::{ShareChild, ShareKind, ShareRefusal}; +/// The directory of the layers' mount points, below the runtime directory. +const LAYERS_DIRECTORY: &str = "layers"; /// The EROFS image's mount point, below the runtime directory. const DISTRO_DIRECTORY: &str = "layers/distro"; /// The ext4 scratch's mount point, below the runtime directory. @@ -190,7 +206,8 @@ pub const MOUNT_OPTIONS_LIMIT: usize = 4095; #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] #[non_exhaustive] pub enum Role { - /// The read-only EROFS image, the overlay's lower layer. + /// The read-only image: an EROFS file system, the overlay's lower layer, or a GPT disk, whose + /// partitions after the first hold the overlay's lower layers. Distro, /// The writable ext4 scratch, which holds the overlay's upper and work directories. Scratch, @@ -236,6 +253,10 @@ pub enum DeviceStep { Superblock, /// Reading its access mode: read-only or writable. Access, + /// Opening it, to read a GPT disk. + Open, + /// Reading the status of the node of a GPT disk's partition. + Status, } impl fmt::Display for DeviceStep { @@ -246,6 +267,8 @@ impl fmt::Display for DeviceStep { Self::Flush => "flush the buffers of", Self::Superblock => "read the superblock of", Self::Access => "read the access mode of", + Self::Open => "open", + Self::Status => "read the status of", }) } } @@ -281,6 +304,26 @@ pub enum Image { /// The UUID that the file system's superblock must hold, if the caller expects one. uuid: Option, }, + /// A GPT disk, whose partition table an assembly reads and checks through + /// [`guest::gpt::read_disk`](super::gpt::read_disk), and whose partitions it uses as the + /// kernel created them when it found the disk: it neither compares them with the table nor + /// asks the kernel to reread it. Its first partition holds opaque bytes, which the assembly + /// returns through [`Root::first_partition`], and never probes or mounts. Each later + /// partition, from partition 2 on, holds a layer, an ext4 file system that the assembly + /// mounts at `layers/pN`, for the partition's number N, `ro,nosuid,nodev`, without `noload`. + /// It refuses, before it mounts anything, a layer whose superblock has the `needs_recovery` + /// feature: the layer's read-only device can't take the recovery, which `noload` or + /// `norecovery` among the superblock's own default mount options would skip. The overlay + /// stacks the layers with partition 2's at its base and the last partition's at its top. A + /// disk has at least one layer, as [`nvx::gpt`](crate::gpt) requires. + Gpt { + /// What the caller expects of the disk beyond its format: its GUID, and its partitions' + /// unique GUIDs and sizes, each if the caller knows it. + expected: Expected, + /// The most bytes that the first partition may hold: a larger first partition is refused + /// before any of it is read. + first_partition_bound: usize, + }, } /// The file system that holds the overlay's upper and work directories, two separate @@ -365,7 +408,7 @@ impl OverlayMode { /// 1. checks the runtime directory's path; /// 2. creates the runtime directory, as `mkdir -p` does, and refuses it if a symbolic link leads /// through it; -/// 3. creates the mount points of the EROFS image, `layers/distro`, of the upper, `scratch` or +/// 3. creates the mount points of an EROFS image, `layers/distro`, of the upper, `scratch` or /// `tmpfs`, and of the root, `rootfs`, a component at a time, and refuses a component that /// exists as anything but a directory, a symbolic link included; takes the runtime directory's /// exclusive lock, which the root holds from then on, and refuses the runtime directory if @@ -374,12 +417,19 @@ impl OverlayMode { /// bind mount of a directory; /// 4. finds the distro device, polling for it every 50 ms until `timeout` passes, checks that it /// is read-only and, if the caller expects a size, of that size, and flushes its buffers; -/// 5. checks the expected UUID, if any, in the EROFS image's superblock, and reads nothing else; +/// 5. for an EROFS image, checks the expected UUID, if any, in its superblock, and reads nothing +/// else; for a GPT disk, reads and checks its partition table as +/// [`guest::gpt::read_disk`](super::gpt::read_disk) does, reads the first partition, checks +/// that the node of each layer's partition, which devtmpfs created when the kernel found the +/// disk, exists, checks in each layer's superblock that it is ext4 and lacks the +/// `needs_recovery` feature, and creates the layers' mount points, `layers/pN`, as step 3 +/// does; /// 6. for an ext4 scratch, finds, checks, and flushes the scratch device as step 4 does, except /// that it must be writable, and, if the caller forbids a journal, checks that its superblock /// lacks the `has_journal` feature; /// 7. checks that the overlay's options fit in [`MOUNT_OPTIONS_LIMIT`] bytes; -/// 8. mounts the EROFS image read-only at `layers/distro`; +/// 8. mounts the layers: an EROFS image read-only, or a GPT disk's ext4 layers read-only with +/// `nosuid` and `nodev`; /// 9. mounts the upper with `nosuid` and `nodev`: the scratch's ext4 file system, or a tmpfs /// whose root directory has mode 0755; /// 10. creates the upper's `upper` and `work` directories unless they exist, with mode 0755 @@ -423,6 +473,7 @@ fn assemble_with( upper: None, upper_kind: assembly.upper.kind(), overlay_mode: assembly.overlay, + first_partition: None, shares: Vec::new(), mapper: None, mapped: Vec::new(), @@ -447,15 +498,19 @@ pub struct Lower { device: PathBuf, /// The layer's mount point. path: PathBuf, + /// The GPT partition that holds the layer, if a GPT disk does. + partition: Option, } impl Lower { - /// Returns the layer's mount: [`Mount::Distro`] for an EROFS image. + /// Returns the layer's mount: [`Mount::Distro`] for an EROFS image, or [`Mount::Layer`] for a + /// layer of a GPT disk. pub fn mount(&self) -> &Mount { &self.mount } - /// Returns the node of the layer's block device: the distro device that holds an EROFS image. + /// Returns the node of the layer's block device: the distro device that holds an EROFS image, + /// or the block device of a GPT disk's partition. pub fn device(&self) -> &Path { &self.device } @@ -464,6 +519,12 @@ impl Lower { pub fn path(&self) -> &Path { &self.path } + + /// Returns the GPT partition that holds the layer, with its number, GUIDs, and extent, or + /// `None` for an EROFS image. + pub fn partition(&self) -> Option<&Partition> { + self.partition.as_ref() + } } /// The handle of an assembled layered root, which owns its mounts, those of the assembly and @@ -489,6 +550,8 @@ pub struct Root { upper_kind: UpperKind, /// The overlay's mode. overlay_mode: OverlayMode, + /// The bytes of a GPT disk's first partition, without their trailing padding. + first_partition: Option>, /// The mounts of live shares that the root unmounts by path, and their mount points, in mount /// order: an aggregate's, and a share's that stays at `staging` because its unmount failed. shares: Vec<(Mount, PathBuf)>, @@ -521,7 +584,16 @@ impl Root { self.latched } - /// Returns the overlay's layers, the base first: the EROFS image's one layer. + /// Returns the bytes of a GPT disk's first partition: all of its bytes, less its trailing NUL + /// bytes and ASCII whitespace, as opaque bytes that need not be UTF-8 and that the root never + /// interprets. Returns `None` for an EROFS image. + pub fn first_partition(&self) -> Option<&[u8]> { + self.first_partition.as_deref() + } + + /// Returns the overlay's layers, the base first: an EROFS image, or the layers of a GPT disk + /// in the order of their partitions, from partition 2's, the base, to the last partition's, + /// the top. pub fn lowers(&self) -> &[Lower] { &self.lowers } @@ -567,7 +639,13 @@ impl Root { }); } } - let distro_point = create_below(&runtime, DISTRO_DIRECTORY)?; + let mut points = Vec::new(); + match assembly.image { + Image::Erofs { .. } => points.push(create_below(&runtime, DISTRO_DIRECTORY)?), + Image::Gpt { .. } => { + create_below(&runtime, LAYERS_DIRECTORY)?; + } + } let upper_point = create_below( &runtime, match assembly.upper.kind() { @@ -575,9 +653,10 @@ impl Root { UpperKind::Tmpfs => TMPFS_DIRECTORY, }, )?; - let root_point = create_below(&runtime, ROOT_DIRECTORY)?; + points.push(upper_point.clone()); + points.push(create_below(&runtime, ROOT_DIRECTORY)?); self.reservation = Some(reserve(&runtime)?); - for point in [&distro_point, &upper_point, &root_point] { + for point in &points { self.check_unoccupied(point, false)?; } // A root whose teardown failed may have left mounted an upper of the other kind, or an @@ -596,13 +675,21 @@ impl Root { assembly.distro, assembly.timeout, )?; - let Image::Erofs { uuid } = &assembly.image; - check_erofs_uuid(&device, *uuid)?; - let lowers = vec![Lower { - mount: Mount::Distro, - device, - path: distro_point, - }]; + let lowers = match &assembly.image { + Image::Erofs { uuid } => { + check_erofs_uuid(&device, *uuid)?; + vec![Lower { + mount: Mount::Distro, + device, + path: runtime.join(DISTRO_DIRECTORY), + partition: None, + }] + } + Image::Gpt { + expected, + first_partition_bound, + } => self.gpt_layers(device, expected, *first_partition_bound)?, + }; let scratch = match assembly.upper { Upper::Scratch { device, journal } => { let device = block::prepare( @@ -623,12 +710,19 @@ impl Root { let work = upper_point.join(WORK_DIRECTORY); let options = overlay_options(&lowers, &upper, &work, assembly.overlay)?; for lower in lowers { + let (filesystem, flags) = match lower.partition { + None => (b"erofs".as_slice(), MountFlags::MS_RDONLY), + Some(_) => ( + b"ext4".as_slice(), + MountFlags::MS_RDONLY | MountFlags::MS_NOSUID | MountFlags::MS_NODEV, + ), + }; self.mount_at( &lower.mount, Some(lower.device.as_os_str().as_bytes()), &lower.path, - Some(b"erofs"), - MountFlags::MS_RDONLY, + Some(filesystem), + flags, None, )?; self.lowers.push(lower); @@ -727,6 +821,64 @@ impl Root { Ok(()) } + /// Reads and checks the partition table of the GPT disk on the distro device `device`, against + /// `expected`, keeps the bytes of its first partition, at most `first_partition_bound`, and + /// returns its layers, in the order of their partitions, on the block devices that the kernel + /// created for them, with their mount points, which it creates below the runtime directory. + fn gpt_layers( + &mut self, + device: PathBuf, + expected: &Expected, + first_partition_bound: usize, + ) -> Result, Error> { + let (disk, first_partition) = { + let file = File::open(&device).map_err(|error| Error::Device { + role: Role::Distro, + step: DeviceStep::Open, + device: device.clone(), + error, + })?; + let disk = self + .kernel + .read_disk(&device, &file, expected) + .map_err(|error| Error::Disk { + device: device.clone(), + error, + })?; + let first_partition = disk + .read_first_partition(&file, first_partition_bound) + .map_err(|error| Error::FirstPartition { + device: device.clone(), + error, + })?; + (disk, first_partition) + }; + // The first partition is the one numbered 1, and every other partition holds a layer. + let layers: Vec = disk + .partitions() + .iter() + .filter(|partition| partition.number() > 1) + .copied() + .collect(); + let numbers: Vec = layers.iter().map(Partition::number).collect(); + let nodes = block::partition_nodes(&device, &numbers)?; + let mut lowers = Vec::with_capacity(layers.len()); + for (partition, node) in layers.into_iter().zip(nodes) { + let number = partition.number(); + check_layer(number, &node)?; + let path = create_below(&self.runtime, &format!("{LAYERS_DIRECTORY}/p{number}"))?; + self.check_unoccupied(&path, false)?; + lowers.push(Lower { + mount: Mount::Layer { number }, + device: node, + path, + partition: Some(partition), + }); + } + self.first_partition = Some(first_partition); + Ok(lowers) + } + /// Refuses the directory `path` where a file system, or a bind mount of a directory, is /// mounted already, and one that can't be checked, unless `may_be_missing` and it doesn't /// exist. @@ -842,6 +994,10 @@ impl fmt::Debug for Root { .field("lowers", &self.lowers) .field("upper", &self.upper) .field("upper_kind", &self.upper_kind) + .field( + "first_partition_bytes", + &self.first_partition.as_ref().map(Vec::len), + ) .field("shares", &self.shares) .field("mapped", &self.mapped) .field("latched", &self.latched) @@ -873,9 +1029,9 @@ fn check_erofs_uuid(device: &Path, expected: Option) -> Result<(), Error> /// Checks that the ext4 file system on the scratch device `device` has no journal, in its /// superblock's feature flags, and reads nothing else. fn check_no_journal(device: &Path) -> Result<(), Error> { - match block::ext4_has_journal(device) { - Ok(Some(false)) => Ok(()), - Ok(Some(true)) => Err(Error::Journal { + match block::ext4_features(device) { + Ok(Some(features)) if !features.journal => Ok(()), + Ok(Some(_)) => Err(Error::Journal { device: device.to_owned(), }), Ok(None) => Err(Error::NotExt4 { @@ -890,6 +1046,33 @@ fn check_no_journal(device: &Path) -> Result<(), Error> { } } +/// Checks that the layer in partition `number` of a GPT disk, on the block device `device`, holds +/// an ext4 file system whose journal needs no recovery, in its superblock's feature flags, and +/// reads nothing else. +/// +/// The layer's device is read-only, so the kernel can't recover its journal. Without `noload`, +/// the kernel refuses to mount such a layer, but with `noload` or `norecovery` among the +/// superblock's own default mount options, it would mount the layer without the recovery. +fn check_layer(number: u8, device: &Path) -> Result<(), Error> { + match block::ext4_features(device) { + Ok(Some(features)) if !features.recovery => Ok(()), + Ok(Some(_)) => Err(Error::Recovery { + number, + device: device.to_owned(), + }), + Ok(None) => Err(Error::NotExt4Layer { + number, + device: device.to_owned(), + }), + Err(error) => Err(Error::Device { + role: Role::Distro, + step: DeviceStep::Superblock, + device: device.to_owned(), + error, + }), + } +} + /// Returns the overlay's options: its lower directories, the mount points of `lowers` from the /// top down, its `upper` and `work` directories, and the option set of `mode`, or fails if they /// are longer than [`MOUNT_OPTIONS_LIMIT`]. @@ -1010,6 +1193,12 @@ fn create_scratch_directory(path: &Path) -> io::Result<()> { pub enum Mount { /// The EROFS image, at `layers/distro` in the runtime directory. Distro, + /// The layer in a partition of a GPT disk, at `layers/pN` in the runtime directory, for its + /// partition's number N. + Layer { + /// The number of the layer's partition: 2 for the base, and more for each layer above. + number: u8, + }, /// The ext4 scratch, at `scratch` in the runtime directory. Scratch, /// The tmpfs upper, at `tmpfs` in the runtime directory. @@ -1046,6 +1235,7 @@ impl fmt::Display for Mount { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Distro => formatter.write_str("the distro layer"), + Self::Layer { number } => write!(formatter, "the layer in partition {number}"), Self::Scratch => formatter.write_str("the scratch"), Self::Tmpfs => formatter.write_str("the tmpfs upper"), Self::Overlay => formatter.write_str("the overlay"), @@ -1162,6 +1352,44 @@ pub enum Error { /// The image's UUID. actual: Uuid, }, + /// The GPT disk on the distro device failed its validation: its partition table could not be + /// read, is invalid, or differs from the caller's expectations. + Disk { + /// The distro device's node. + device: PathBuf, + /// Why the disk failed. + error: DeviceError, + }, + /// The first partition of the GPT disk on the distro device is larger than the caller's + /// bound, or could not be read. + FirstPartition { + /// The distro device's node. + device: PathBuf, + /// Why the first partition failed. + error: GptError, + }, + /// The kernel created no block device for a layer's partition: no node has its name. + Partition { + /// The partition's number. + number: u8, + /// The node where the partition's block device should be. + node: PathBuf, + }, + /// A layer's partition of the GPT disk holds no ext4 file system. + NotExt4Layer { + /// The partition's number. + number: u8, + /// The node of the partition's block device. + device: PathBuf, + }, + /// The ext4 file system of a layer's partition needs a journal recovery, which its read-only + /// device can't take. + Recovery { + /// The partition's number. + number: u8, + /// The node of the partition's block device. + device: PathBuf, + }, /// The scratch device holds no ext4 file system, so it has no journal feature to check. NotExt4 { /// The device's node. @@ -1172,8 +1400,8 @@ pub enum Error { /// The device's node. device: PathBuf, }, - /// The overlay's options are longer than [`MOUNT_OPTIONS_LIMIT`], as the paths of a long - /// runtime directory make them. + /// The overlay's options are longer than [`MOUNT_OPTIONS_LIMIT`]: its lower directories, with + /// the runtime directory's path, are too many or too long for one mount. OverlayOptions { /// The options' length, in bytes. length: usize, @@ -1294,6 +1522,33 @@ impl fmt::Display for Error { formatter, "the distro layer's UUID is {actual}, expected {expected}" ), + Self::Disk { device, error } => write!( + formatter, + "the GPT disk on the distro device {} failed its validation: {error}", + device.display() + ), + Self::FirstPartition { device, error } => write!( + formatter, + "failed to read the first partition of the GPT disk on the distro device {}: \ + {error}", + device.display() + ), + Self::Partition { number, node } => write!( + formatter, + "the distro's partition {number} has no block device at {}", + node.display() + ), + Self::NotExt4Layer { number, device } => write!( + formatter, + "the distro's partition {number}, {}, holds no ext4 file system", + device.display() + ), + Self::Recovery { number, device } => write!( + formatter, + "the layer in partition {number}, {}, needs a journal recovery, which its \ + read-only device can't take", + device.display() + ), Self::NotExt4 { device } => write!( formatter, "the scratch device {} holds no ext4 file system", diff --git a/nvx-rs/src/guest/root/block.rs b/nvx-rs/src/guest/root/block.rs index ace489a45..07ca596b9 100644 --- a/nvx-rs/src/guest/root/block.rs +++ b/nvx-rs/src/guest/root/block.rs @@ -1,5 +1,5 @@ //! The block devices that the VMM attaches in the root's roles, which the base addresses of their -//! MMIO windows name. +//! MMIO windows name, and the block devices of a GPT disk's partitions. //! //! The VMM declares each device on the kernel command line, so the kernel names its MMIO window //! `virtio-mmio.N` in `/proc/iomem`, and creates its block device below the window's device in @@ -7,6 +7,7 @@ use std::fs::{self, File}; use std::io; +use std::os::unix::ffi::OsStrExt; use std::os::unix::fs::FileExt; use std::path::{Path, PathBuf}; use std::time::Duration; @@ -39,6 +40,22 @@ const EXT4_MAGIC: u16 = 0xEF53; const EXT4_COMPAT_RANGE: std::ops::Range = 0x5C..0x60; /// The compatible feature flag of an ext4 file system that has a journal, `has_journal`. const EXT4_HAS_JOURNAL: u32 = 0x0004; +/// The range of the ext4 superblock's incompatible feature flags, a little-endian `u32`. +const EXT4_INCOMPAT_RANGE: std::ops::Range = 0x60..0x64; +/// The incompatible feature flag of an ext4 file system whose journal needs a recovery, +/// `needs_recovery`, which the kernel sets while it has the file system mounted writable, and +/// clears once it unmounts it cleanly. +const EXT4_NEEDS_RECOVERY: u32 = 0x0004; + +/// The features of an ext4 file system that the root checks, from its superblock's feature +/// flags. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Ext4Features { + /// Whether the file system has a journal. + pub(super) journal: bool, + /// Whether its journal needs a recovery. + pub(super) recovery: bool, +} /// Finds the block device of `role` that the VMM attached at `device`'s MMIO window, polling /// until it appears for as long as `timeout` allows, checks that its access mode is the role's, @@ -127,6 +144,47 @@ fn resolve( Err(Error::Missing { role, mmio }) } +/// Returns the node of the block device of each of the partitions `numbers` of the GPT disk whose +/// node is `disk`, in order. +/// +/// The kernel created the block devices of the disk's partitions from the table that the disk +/// held when the kernel found it, and devtmpfs created their nodes beside the disk's, under the +/// names that the kernel gives them: the disk's name and the partition's number, with a `p` +/// between them if the disk's name ends with a digit, as `vda2` and `nvme0n1p2` are. It waits for +/// none: a partition whose node is missing fails at once. +pub(super) fn partition_nodes(disk: &Path, numbers: &[u8]) -> Result, Error> { + numbers + .iter() + .map(|&number| { + let node = partition_node(disk, number); + match fs::symlink_metadata(&node) { + Ok(_) => Ok(node), + Err(error) if error.kind() == io::ErrorKind::NotFound => { + Err(Error::Partition { number, node }) + } + Err(error) => Err(Error::Device { + role: Role::Distro, + step: DeviceStep::Status, + device: node, + error, + }), + } + }) + .collect() +} + +/// Returns the node of partition `number` of the disk whose node is `disk`, under the name that +/// the kernel gives the partition's block device. +fn partition_node(disk: &Path, number: u8) -> PathBuf { + let disk_name = disk.file_name().unwrap_or_default(); + let mut name = disk_name.to_owned(); + if disk_name.as_bytes().last().is_some_and(u8::is_ascii_digit) { + name.push("p"); + } + name.push(number.to_string()); + disk.with_file_name(name) +} + /// Returns the name of the block device below the virtio MMIO window at `mmio`, or `None` while /// the kernel lists no such window, or has not probed it yet. fn find(paths: &Paths, mmio: u64) -> io::Result> { @@ -217,11 +275,11 @@ pub(super) fn erofs_uuid(device: &Path) -> io::Result> { ))) } -/// Reads whether the ext4 file system on the block device `device` has a journal, from its -/// superblock's compatible feature flags, and reads nothing else, or returns `None` if the device +/// Reads the features of the ext4 file system on the block device `device` that the root checks, +/// from its superblock's feature flags, and reads nothing else, or returns `None` if the device /// holds no ext4 superblock. -pub(super) fn ext4_has_journal(device: &Path) -> io::Result> { - let mut superblock = [0; EXT4_COMPAT_RANGE.end]; +pub(super) fn ext4_features(device: &Path) -> io::Result> { + let mut superblock = [0; EXT4_INCOMPAT_RANGE.end]; match File::open(device)?.read_exact_at(&mut superblock, EXT4_SUPERBLOCK) { Ok(()) => {} Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => return Ok(None), @@ -235,12 +293,17 @@ pub(super) fn ext4_has_journal(device: &Path) -> io::Result> { if magic != EXT4_MAGIC { return Ok(None); } - let compatible = u32::from_le_bytes( - superblock[EXT4_COMPAT_RANGE] - .try_into() - .expect("the feature flags are four bytes"), - ); - Ok(Some(compatible & EXT4_HAS_JOURNAL != 0)) + let flags = |range: std::ops::Range| { + u32::from_le_bytes( + superblock[range] + .try_into() + .expect("the feature flags are four bytes"), + ) + }; + Ok(Some(Ext4Features { + journal: flags(EXT4_COMPAT_RANGE) & EXT4_HAS_JOURNAL != 0, + recovery: flags(EXT4_INCOMPAT_RANGE) & EXT4_NEEDS_RECOVERY != 0, + })) } #[cfg(test)] @@ -256,6 +319,16 @@ mod tests { } } + #[test] + fn a_partition_s_node_has_the_name_that_the_kernel_gives_it() { + let node = |disk: &str, number| partition_node(Path::new(disk), number); + assert_eq!(node("/dev/vda", 2), Path::new("/dev/vda2")); + assert_eq!(node("/dev/vdaa", 196), Path::new("/dev/vdaa196")); + // A disk whose name ends with a digit has a `p` before the partition's number. + assert_eq!(node("/dev/nvme0n1", 2), Path::new("/dev/nvme0n1p2")); + assert_eq!(node("/dev/loop7", 12), Path::new("/dev/loop7p12")); + } + #[test] fn the_superblock_names_an_erofs_uuid_or_none() { let temporary = tempfile::tempdir().unwrap(); @@ -281,26 +354,32 @@ mod tests { } #[test] - fn the_superblock_tells_whether_an_ext4_file_system_has_a_journal() { + fn the_superblock_tells_whether_an_ext4_journal_exists_and_needs_a_recovery() { let temporary = tempfile::tempdir().unwrap(); let device = temporary.path().join("device"); + let features = |journal, recovery| Some(Ext4Features { journal, recovery }); let mut image = vec![0u8; 2048]; image[1024 + 0x38..1024 + 0x3A].copy_from_slice(&EXT4_MAGIC.to_le_bytes()); - // Other compatible features, such as `dir_index`, are not a journal. + // Other features, such as `dir_index` and `extent`, are neither. image[1024 + 0x5C..1024 + 0x60].copy_from_slice(&0x0020_u32.to_le_bytes()); + image[1024 + 0x60..1024 + 0x64].copy_from_slice(&0x0040_u32.to_le_bytes()); fs::write(&device, &image).unwrap(); - assert_eq!(ext4_has_journal(&device).unwrap(), Some(false)); + assert_eq!(ext4_features(&device).unwrap(), features(false, false)); image[1024 + 0x5C] |= 0x04; fs::write(&device, &image).unwrap(); - assert_eq!(ext4_has_journal(&device).unwrap(), Some(true)); + assert_eq!(ext4_features(&device).unwrap(), features(true, false)); + image[1024 + 0x60] |= 0x04; + fs::write(&device, &image).unwrap(); + assert_eq!(ext4_features(&device).unwrap(), features(true, true)); // Another magic number, or a device too short for the feature flags, holds no ext4. image[1024 + 0x38] ^= 1; fs::write(&device, &image).unwrap(); - assert_eq!(ext4_has_journal(&device).unwrap(), None); - fs::write(&device, &image[..1024 + 0x5E]).unwrap(); - assert_eq!(ext4_has_journal(&device).unwrap(), None); + assert_eq!(ext4_features(&device).unwrap(), None); + image[1024 + 0x38] ^= 1; + fs::write(&device, &image[..1024 + 0x62]).unwrap(); + assert_eq!(ext4_features(&device).unwrap(), None); assert_eq!( - ext4_has_journal(&temporary.path().join("missing")) + ext4_features(&temporary.path().join("missing")) .unwrap_err() .kind(), io::ErrorKind::NotFound diff --git a/nvx-rs/src/guest/root/kernel.rs b/nvx-rs/src/guest/root/kernel.rs index 54a0b8698..1d70c2e65 100644 --- a/nvx-rs/src/guest/root/kernel.rs +++ b/nvx-rs/src/guest/root/kernel.rs @@ -2,18 +2,21 @@ //! //! The operations read the kernel's text files and walk directories through the standard //! library, at the paths of a [`Paths`], so that a test relocates them to a temporary directory. -//! What a test cannot do in a temporary directory, such as a mount, an unmount, or a flush of a -//! block device's buffers, and the writes to the kernel's control files, the check for a mount +//! What a test cannot do in a temporary directory, such as a mount, an unmount, a flush of a +//! block device's buffers, or a read of a GPT disk's partition table at the geometry that its +//! block device reports, and the writes to the kernel's control files, the check for a mount //! point, and the options of a mount, whose effects and answers a test simulates, go through a //! [`Kernel`], which also performs the mapper's operations for the root's live shares and //! mappings. use std::ffi::CStr; -use std::fs; +use std::fs::{self, File}; use std::io; use std::path::{Path, PathBuf}; use std::time::Duration; +use crate::gpt::{Disk, Expected}; +use crate::guest::gpt::{self, DeviceError}; use crate::guest::mappings::Kernel as MappingKernel; use crate::guest::parse::parse_decimal; use crate::guest::sys::{self, Errno, Mode, MountFlags, OpenFlags}; @@ -25,13 +28,14 @@ const MOUNT_TABLE: &str = "/proc/self/mountinfo"; /// The kernel's operations that the root operations perform, other than reads: mounts and /// unmounts, flushes of a block device's buffers, writes of a control file, and the sleeps -/// between two polls for a block device; the check for a mount point and the options of a -/// mount, which depend on them; and the mapper's operations, through which the root mounts -/// its live shares and mappings. +/// between two polls for a block device; the check for a mount point and the options of a mount, +/// which depend on them; since only a block device reports its geometry, the read of a GPT disk's +/// partition table; and the mapper's operations, through which the root mounts its live shares +/// and mappings. /// /// [`SystemKernel`] performs them. The tests' fake records the operations, simulates the effect -/// of a cgroup's membership file and its own mounts, and fails the operations that a test -/// chooses. +/// of a cgroup's membership file and its own mounts, reads a GPT disk's table from a regular +/// file, and fails the operations that a test chooses. pub(super) trait Kernel: MappingKernel + Send + Sync { /// Mounts `source` at `target`, as `mount(2)` does, with the file system's own options /// `data`. @@ -50,6 +54,16 @@ pub(super) trait Kernel: MappingKernel + Send + Sync { /// Writes the dirty buffers of the block device `device`, then drops its cached ones. fn flush(&mut self, device: &CStr) -> Result<(), Errno>; + /// Reads and checks the partition table of the GPT disk that `file` opens, the whole-disk + /// block device whose node is `device`, with the caller's `expected` values, at the device's + /// logical block size and size, as [`gpt::read_disk`] does. + fn read_disk( + &mut self, + device: &Path, + file: &File, + expected: &Expected, + ) -> Result; + /// Writes `value` whole to the existing control file `path`, such as a cgroup's or a /// sysctl's, and reports a failure to close it. fn write(&mut self, path: &CStr, value: &[u8]) -> Result<(), Errno>; @@ -94,6 +108,15 @@ impl Kernel for SystemKernel { descriptor.close() } + fn read_disk( + &mut self, + _device: &Path, + file: &File, + expected: &Expected, + ) -> Result { + gpt::read_disk(file, expected) + } + fn write(&mut self, path: &CStr, value: &[u8]) -> Result<(), Errno> { let descriptor = sys::open( path, @@ -166,12 +189,15 @@ pub(crate) use fake::{Call, FakeKernel, Operation, attach_calls}; #[cfg(test)] mod fake { use std::ffi::{CStr, OsStr}; + use std::fs::File; use std::os::unix::ffi::OsStrExt; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, PoisonError}; use std::time::Duration; use super::Kernel; + use crate::gpt::{Disk, Expected}; + use crate::guest::gpt::DeviceError; use crate::guest::mappings::{Access, Kernel as MappingKernel}; use crate::guest::sys::{ self, Errno, FileDescriptor, FileKind, FileStatus, Mode, MountAttributes, MountFlags, @@ -185,6 +211,9 @@ mod fake { /// `open_tree`, before the tree's number. const TREE_PREFIX: &[u8] = b"/memfd:nvx-fake-tree-"; + /// The logical block size of the disks whose tables the fake kernel reads, in bytes. + const LOGICAL_BLOCK_SIZE: u32 = 512; + /// An operation of the kernel, which a test fails. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum Operation { @@ -241,6 +270,8 @@ mod fake { Unmount(String), /// A flush of a block device's buffers. Flush(String), + /// A read of the partition table of the GPT disk on a block device, which its node names. + ReadDisk(String), /// A write of a value to a control file. Write(String, String), /// A copy, or an open, of the mount of a file by `open_tree`. @@ -372,6 +403,8 @@ mod fake { /// It shows a mount's options as `ro` or `rw`, as its flags have `MS_RDONLY` or not, then the /// options that the mount passed, unless a test chose others, as for a kernel that applied /// other options than those passed, or that shows options at their defaults. + /// + /// It reads the table of a GPT disk, a regular file of 512-byte blocks, with [`Disk::read`]. #[derive(Clone, Default)] pub(crate) struct FakeKernel(Arc>); @@ -706,6 +739,21 @@ mod fake { self.record(Call::Flush(text(device)), Operation::Flush, &path(device)) } + fn read_disk( + &mut self, + device: &Path, + file: &File, + expected: &Expected, + ) -> Result { + self.with(|state| { + state + .calls + .push(Call::ReadDisk(device.display().to_string())); + }); + let size = file.metadata().map_err(DeviceError::Size)?.len(); + Disk::read(file, LOGICAL_BLOCK_SIZE, size, expected).map_err(DeviceError::Table) + } + fn write(&mut self, file: &CStr, value: &[u8]) -> Result<(), Errno> { let call = Call::Write(text(file), String::from_utf8_lossy(value).into_owned()); let file = path(file); diff --git a/nvx-rs/src/guest/root/tests.rs b/nvx-rs/src/guest/root/tests.rs index 442b83d03..0bf28d3dd 100644 --- a/nvx-rs/src/guest/root/tests.rs +++ b/nvx-rs/src/guest/root/tests.rs @@ -11,6 +11,7 @@ use std::time::Duration; use super::kernel::{Call, FakeKernel, Operation, Paths, super_options}; use super::*; +use crate::gpt::ExpectedPartition; use crate::guest::cgroup::Layout; use crate::guest::mappings::{Access, SourceRoot}; use crate::guest::sys::MountAttributes; @@ -21,6 +22,21 @@ const DISTRO: u64 = 0xd000_0000; const SCRATCH: u64 = 0x1_d000_1000; /// The UUID of the test's EROFS image. const UUID: Uuid = Uuid::from_bytes([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]); +/// The GUID of the tests' GPT disks. +const DISK_GUID: Uuid = Uuid::from_bytes([0xD1; 16]); +/// The partition type GUID of the tests' GPT partitions: Linux file system data, +/// `0fc63daf-8483-4772-8e79-3d69d8477de4`. +const LINUX_DATA: Uuid = Uuid::from_bytes([ + 0x0f, 0xc6, 0x3d, 0xaf, 0x84, 0x83, 0x47, 0x72, 0x8e, 0x79, 0x3d, 0x69, 0xd8, 0x47, 0x7d, 0xe4, +]); +/// The size of the tests' GPT disks' logical blocks, in bytes. +const BLOCK: usize = 512; +/// The size of each layer of the tests' GPT disks, in blocks: the first 2 KiB of an ext4 file +/// system, which hold its superblock. +const LAYER_BLOCKS: usize = 4; +/// The tests' bound for the first partition, in bytes. +const FIRST_PARTITION_BOUND: usize = 64 * 1024; + /// A guest in a temporary directory, and the fake kernel that the operations get clones of. struct Guest { /// The temporary directory, which the guest holds only so that dropping it removes the @@ -108,6 +124,31 @@ impl Guest { } } + /// Returns an assembly of a GPT disk on the guest's distro device, with the tests' bound for + /// the first partition and no other expectation, over `upper`, with the overlay mode + /// `overlay`. + fn gpt_assembly(&self, upper: Upper, overlay: OverlayMode) -> Assembly { + Assembly { + image: Image::Gpt { + expected: Expected::new(), + first_partition_bound: FIRST_PARTITION_BOUND, + }, + upper, + overlay, + ..self.assembly() + } + } + + /// Writes `disk` to the distro device, and creates the device nodes of its layers' + /// partitions, with their layers' bytes, as devtmpfs does once the kernel has read the disk's + /// table. + fn insert(&self, disk: &GptDisk) { + fs::write(self.path("dev/vda"), disk.image()).unwrap(); + for number in 2..=disk.partitions() { + write(&self.path(&format!("dev/vda{number}")), disk.layer(number)); + } + } + /// Assembles `assembly` with a clone of the guest's kernel. fn assemble(&self, assembly: &Assembly) -> Result { assemble_with(Box::new(self.kernel.clone()), &self.paths, assembly) @@ -202,6 +243,141 @@ fn set_read_only(root: &Path, name: &str, read_only: bool) { ); } +/// A GPT disk of 512-byte blocks that a test writes, with the tests' disk GUID: its first +/// partition, which holds the test's bytes in as many blocks as they take, and at least one, then +/// the layers, of [`LAYER_BLOCKS`] blocks each, the start of an ext4 file system without a +/// journal, whose first byte is its partition's number. Partition N's unique GUID is +/// [`GptDisk::guid`]`(N)`. +struct GptDisk { + /// The first partition's bytes. + first_partition: Vec, + /// The layers' bytes, partition 2's first. + layers: Vec>, +} + +impl GptDisk { + /// Returns a disk whose first partition holds `first_partition`, with `layers` layers. + fn new(first_partition: &[u8], layers: u8) -> Self { + let layers = (2..=layers.saturating_add(1)) + .map(|number| { + let mut layer = ext4_image(false); + layer[0] = number; + layer + }) + .collect(); + Self { + first_partition: first_partition.to_vec(), + layers, + } + } + + /// Returns the number of partitions, the first included. + fn partitions(&self) -> u8 { + u8::try_from(self.layers.len() + 1).unwrap() + } + + /// Returns the bytes of the layer in partition `number`. + fn layer(&self, number: u8) -> &[u8] { + &self.layers[usize::from(number) - 2] + } + + /// Returns the bytes of the layer in partition `number`, for a test to change. + fn layer_mut(&mut self, number: u8) -> &mut [u8] { + &mut self.layers[usize::from(number) - 2] + } + + /// Returns the unique GUID of partition `number`. + fn guid(number: u8) -> Uuid { + let mut bytes = [0x11; 16]; + bytes[15] = number; + Uuid::from_bytes(bytes) + } + + /// Returns the number of the first partition's blocks. + fn first_partition_blocks(&self) -> usize { + self.first_partition.len().div_ceil(BLOCK).max(1) + } + + /// Returns the size of partition `number`, in bytes. + fn size(&self, number: u8) -> u64 { + let blocks = if number == 1 { + self.first_partition_blocks() + } else { + LAYER_BLOCKS + }; + (blocks * BLOCK) as u64 + } + + /// Writes the disk: its protective MBR, its primary header and partition entry array, and its + /// partitions, without the backup table, which the reader never reads. + fn image(&self) -> Vec { + // At least UEFI's 128 entries, four to a block. + let entries = usize::from(self.partitions()).max(128).next_multiple_of(4); + let array_blocks = entries * 128 / BLOCK; + let first_usable = 2 + array_blocks; + let last_usable = + first_usable + self.first_partition_blocks() + self.layers.len() * LAYER_BLOCKS - 1; + // The alternate header, after the room for its array. + let alternate = last_usable + array_blocks + 1; + let mut image = vec![0; (alternate + 1) * BLOCK]; + image[446 + 4] = 0xEE; + image[446 + 8..446 + 12].copy_from_slice(&1u32.to_le_bytes()); + let size = u32::try_from(alternate).unwrap(); + image[446 + 12..446 + 16].copy_from_slice(&size.to_le_bytes()); + image[510..512].copy_from_slice(&[0x55, 0xAA]); + let mut array = vec![0; entries * 128]; + let mut lba = first_usable; + for number in 1..=self.partitions() { + let blocks = self.size(number) as usize / BLOCK; + let entry = &mut array[(usize::from(number) - 1) * 128..][..128]; + entry[..16].copy_from_slice(&LINUX_DATA.to_bytes_le()); + entry[16..32].copy_from_slice(&Self::guid(number).to_bytes_le()); + entry[32..40].copy_from_slice(&(lba as u64).to_le_bytes()); + entry[40..48].copy_from_slice(&((lba + blocks - 1) as u64).to_le_bytes()); + let bytes = if number == 1 { + &self.first_partition + } else { + self.layer(number) + }; + image[lba * BLOCK..][..bytes.len()].copy_from_slice(bytes); + lba += blocks; + } + image[2 * BLOCK..][..array.len()].copy_from_slice(&array); + let header = &mut image[BLOCK..2 * BLOCK]; + header[..8].copy_from_slice(b"EFI PART"); + header[8..12].copy_from_slice(&0x0001_0000_u32.to_le_bytes()); + header[12..16].copy_from_slice(&92_u32.to_le_bytes()); + header[24..32].copy_from_slice(&1_u64.to_le_bytes()); + header[32..40].copy_from_slice(&(alternate as u64).to_le_bytes()); + header[40..48].copy_from_slice(&(first_usable as u64).to_le_bytes()); + header[48..56].copy_from_slice(&(last_usable as u64).to_le_bytes()); + header[56..72].copy_from_slice(&DISK_GUID.to_bytes_le()); + header[72..80].copy_from_slice(&2_u64.to_le_bytes()); + header[80..84].copy_from_slice(&u32::try_from(entries).unwrap().to_le_bytes()); + header[84..88].copy_from_slice(&128_u32.to_le_bytes()); + header[88..92].copy_from_slice(&crc32(&array).to_le_bytes()); + let crc = crc32(&header[..92]); + header[16..20].copy_from_slice(&crc.to_le_bytes()); + image + } +} + +/// Returns the CRC-32 of `bytes` that GPT records: IEEE 802.3's, reflected. +fn crc32(bytes: &[u8]) -> u32 { + let mut crc = u32::MAX; + for &byte in bytes { + crc ^= u32::from(byte); + for _ in 0..8 { + crc = if crc & 1 == 1 { + (crc >> 1) ^ 0xEDB8_8320 + } else { + crc >> 1 + }; + } + } + !crc +} + /// Returns the first 2 KiB of an ext4 file system, whose superblock has the `has_journal` /// feature if `journal`. fn ext4_image(journal: bool) -> Vec { @@ -290,12 +466,14 @@ fn assembly_mounts_the_image_and_the_scratch_under_the_overlay() { assert_eq!(root.path(), guest.path("run/nvx/rootfs")); assert_eq!(root.runtime(), guest.path("run/nvx")); // The root reports what it mounted: the image's one layer, the scratch, and the mode. + assert_eq!(root.first_partition(), None); let [lower] = root.lowers() else { panic!("{:?}", root.lowers()); }; assert_eq!(lower.mount(), &Mount::Distro); assert_eq!(lower.device(), guest.path("dev/vda")); assert_eq!(lower.path(), guest.path("run/nvx/layers/distro")); + assert_eq!(lower.partition(), None); assert_eq!(root.upper(), UpperKind::Scratch); assert_eq!(root.overlay(), OverlayMode::TrustedLowers); root.teardown().unwrap(); @@ -946,71 +1124,201 @@ fn below(runtime: &str, names: &[&str]) -> Vec { } #[test] -fn both_overlay_modes_assemble_on_both_uppers_with_their_own_option_sets() { - for upper in [scratch_upper(Journal::Allowed), Upper::Tmpfs] { - for mode in [OverlayMode::TrustedLowers, OverlayMode::UntrustedLowers] { - let case = format!("{upper:?}, {mode:?}"); - let guest = Guest::new(); - let assembly = Assembly { - upper, - overlay: mode, - ..guest.assembly() - }; - let root = guest.assemble(&assembly).unwrap(); - let runtime = guest.text("run/nvx"); - let flags = MountFlags::MS_NOSUID | MountFlags::MS_NODEV; - let (point, upper_mount) = match upper.kind() { - UpperKind::Scratch => ( - "scratch", - mount_call( - Some(&guest.text("dev/vdb")), - &format!("{runtime}/scratch"), - Some("ext4"), - flags, - None, - ), - ), - UpperKind::Tmpfs => ( - "tmpfs", - mount_call( - Some("tmpfs"), - &format!("{runtime}/tmpfs"), - Some("tmpfs"), - flags, - Some("mode=0755"), - ), - ), - }; - // The upper, then the overlay, with the mode's one option set, and the flags - // `nosuid,nodev`. - let overlay = mount_call( +fn a_gpt_disk_s_layers_are_mounted_under_the_overlay_top_first() { + let guest = Guest::new(); + guest.insert(&GptDisk::new(b"{\"configuration\": true}\n", 3)); + let assembly = guest.gpt_assembly(scratch_upper(Journal::Allowed), OverlayMode::TrustedLowers); + let root = guest.assemble(&assembly).unwrap(); + let runtime = guest.text("run/nvx"); + let layer = |number: u8| { + mount_call( + Some(&guest.text(&format!("dev/vda{number}"))), + &format!("{runtime}/layers/p{number}"), + Some("ext4"), + MountFlags::MS_RDONLY | MountFlags::MS_NOSUID | MountFlags::MS_NODEV, + None, + ) + }; + // The disk is flushed, then its table read and checked, and the scratch checked, before any + // mount; the overlay stacks the layers with partition 2's at its base and partition 4's at + // its top. + let options = overlay_data( + &runtime, + &["layers/p4", "layers/p3", "layers/p2"], + "scratch", + OverlayMode::TrustedLowers, + ); + assert_eq!( + guest.kernel.calls(), + [ + Call::Flush(guest.text("dev/vda")), + Call::ReadDisk(guest.text("dev/vda")), + Call::Flush(guest.text("dev/vdb")), + layer(2), + layer(3), + layer(4), + mount_call( + Some(&guest.text("dev/vdb")), + &format!("{runtime}/scratch"), + Some("ext4"), + MountFlags::MS_NOSUID | MountFlags::MS_NODEV, + None, + ), + mount_call( Some("overlay"), &format!("{runtime}/rootfs"), Some("overlay"), - flags, - Some(&overlay_data(&runtime, &["layers/distro"], point, mode)), - ); - let calls = guest.kernel.calls(); - assert_eq!(calls[calls.len() - 2..], [upper_mount, overlay], "{case}"); - // A tmpfs upper looks for no scratch device. - let scratch_flushed = calls.contains(&Call::Flush(guest.text("dev/vdb"))); - assert_eq!( - scratch_flushed, - upper.kind() == UpperKind::Scratch, - "{case}" - ); - assert_eq!(root.upper(), upper.kind(), "{case}"); - assert_eq!(root.overlay(), mode, "{case}"); - for directory in ["upper", "work"] { - let path = guest.path(&format!("run/nvx/{point}/{directory}")); - assert!(path.is_dir(), "{case}: {directory}"); + MountFlags::MS_NOSUID | MountFlags::MS_NODEV, + Some(&options), + ), + ] + ); + // The first partition comes back as its bytes, without the padding after them, and is never + // mounted. + assert_eq!( + root.first_partition(), + Some(&b"{\"configuration\": true}"[..]) + ); + let numbers: Vec = root + .lowers() + .iter() + .map(|lower| lower.partition().unwrap().number()) + .collect(); + assert_eq!(numbers, [2, 3, 4]); + for lower in root.lowers() { + let partition = lower.partition().unwrap(); + let number = partition.number(); + assert_eq!(lower.mount(), &Mount::Layer { number }); + assert_eq!(lower.device(), guest.path(&format!("dev/vda{number}"))); + assert_eq!( + lower.path(), + guest.path(&format!("run/nvx/layers/p{number}")) + ); + assert_eq!(partition.unique_guid(), GptDisk::guid(number)); + } + assert_eq!(root.upper(), UpperKind::Scratch); + assert_eq!(root.overlay(), OverlayMode::TrustedLowers); + root.teardown().unwrap(); + assert_eq!( + guest.kernel.unmounts(), + below( + &runtime, + &["rootfs", "layers/p4", "layers/p3", "layers/p2", "scratch"] + ) + ); + + // A layer's mount point where a file system is mounted already is refused before any + // mount. + let guest = Guest::new(); + guest.insert(&GptDisk::new(b"configuration", 3)); + let third = guest.path("run/nvx/layers/p3"); + fs::create_dir_all(&third).unwrap(); + guest + .kernel + .clone() + .mount( + Some(c"tmpfs"), + &sys::path_c_string(&third).unwrap(), + Some(c"tmpfs"), + MountFlags::empty(), + None, + ) + .unwrap(); + let mounts = guest.kernel.mounts(); + let assembly = guest.gpt_assembly(scratch_upper(Journal::Allowed), OverlayMode::TrustedLowers); + let error = guest.assemble(&assembly).unwrap_err(); + assert!( + matches!(error.error(), Error::Occupied { path } if *path == third), + "{error:?}" + ); + assert_eq!(guest.kernel.mounts(), mounts); +} + +#[test] +fn both_overlay_modes_assemble_on_both_uppers_with_their_own_option_sets() { + let modes = [OverlayMode::TrustedLowers, OverlayMode::UntrustedLowers]; + for gpt in [false, true] { + for upper in [scratch_upper(Journal::Allowed), Upper::Tmpfs] { + for mode in modes { + let case = format!("GPT {gpt}, {upper:?}, {mode:?}"); + let guest = Guest::new(); + let mut assembly = Assembly { + upper, + overlay: mode, + ..guest.assembly() + }; + let lowers: &[&str] = if gpt { + guest.insert(&GptDisk::new(b"configuration", 2)); + assembly.image = Image::Gpt { + expected: Expected::new(), + first_partition_bound: FIRST_PARTITION_BOUND, + }; + &["layers/p3", "layers/p2"] + } else { + &["layers/distro"] + }; + let root = guest.assemble(&assembly).unwrap(); + let runtime = guest.text("run/nvx"); + let flags = MountFlags::MS_NOSUID | MountFlags::MS_NODEV; + let (point, upper_mount) = match upper.kind() { + UpperKind::Scratch => ( + "scratch", + mount_call( + Some(&guest.text("dev/vdb")), + &format!("{runtime}/scratch"), + Some("ext4"), + flags, + None, + ), + ), + UpperKind::Tmpfs => ( + "tmpfs", + mount_call( + Some("tmpfs"), + &format!("{runtime}/tmpfs"), + Some("tmpfs"), + flags, + Some("mode=0755"), + ), + ), + }; + // The upper, then the overlay, with the mode's one option set, and the flags + // `nosuid,nodev`. + let overlay = mount_call( + Some("overlay"), + &format!("{runtime}/rootfs"), + Some("overlay"), + flags, + Some(&overlay_data(&runtime, lowers, point, mode)), + ); + let calls = guest.kernel.calls(); + assert_eq!(calls[calls.len() - 2..], [upper_mount, overlay], "{case}"); + // A tmpfs upper looks for no scratch device. + let scratch_flushed = calls.contains(&Call::Flush(guest.text("dev/vdb"))); + assert_eq!( + scratch_flushed, + upper.kind() == UpperKind::Scratch, + "{case}" + ); + assert_eq!(root.upper(), upper.kind(), "{case}"); + assert_eq!(root.overlay(), mode, "{case}"); + assert_eq!(root.first_partition().is_some(), gpt, "{case}"); + for directory in ["upper", "work"] { + let path = guest.path(&format!("run/nvx/{point}/{directory}")); + assert!(path.is_dir(), "{case}: {directory}"); + } + root.teardown().unwrap(); + let unmounts: Vec<&str> = ["rootfs"] + .into_iter() + .chain(lowers.iter().copied()) + .chain([point]) + .collect(); + assert_eq!( + guest.kernel.unmounts(), + below(&runtime, &unmounts), + "{case}" + ); } - root.teardown().unwrap(); - assert_eq!( - guest.kernel.unmounts(), - below(&runtime, &["rootfs", "layers/distro", point]), - "{case}" - ); } } } @@ -1275,6 +1583,7 @@ fn overlay_options_that_a_mount_cannot_pass_are_refused_before_any_mount() { mount: Mount::Distro, device: PathBuf::new(), path: format!("/{}", "l".repeat(name)).into(), + partition: None, }; let mode = OverlayMode::TrustedLowers; overlay_options(&[lower], Path::new("/u"), Path::new("/w"), mode) @@ -1316,6 +1625,59 @@ fn overlay_options_that_a_mount_cannot_pass_are_refused_before_any_mount() { assert!(guest.kernel.mounts().is_empty()); } +#[test] +fn up_to_195_layers_fit_in_the_overlay_s_options_below_run_nvx() { + // Below `/run/nvx`, the options of 195 layers, those of partitions 2 through 196, fit + // whatever the mode and the upper, and those of 196 do not. + let lower = |number: u8| Lower { + mount: Mount::Layer { number }, + device: PathBuf::new(), + path: format!("/run/nvx/layers/p{number}").into(), + partition: None, + }; + for point in ["scratch", "tmpfs"] { + for mode in [OverlayMode::TrustedLowers, OverlayMode::UntrustedLowers] { + let upper = Path::new("/run/nvx").join(point); + let options = |layers: u8| { + let lowers: Vec = (2..=layers + 1).map(lower).collect(); + overlay_options(&lowers, &upper.join("upper"), &upper.join("work"), mode) + }; + let longest = options(195).unwrap(); + assert!(longest.len() <= MOUNT_OPTIONS_LIMIT, "{}", longest.len()); + assert!( + matches!(options(196), Err(Error::OverlayOptions { length }) + if length > MOUNT_OPTIONS_LIMIT), + "{point}, {mode:?}" + ); + } + } + + // An assembly of a disk with more layers refuses their options before it mounts anything. + let guest = Guest::new(); + guest.insert(&GptDisk::new(b"configuration", 254)); + let mode = OverlayMode::UntrustedLowers; + let error = guest + .assemble(&guest.gpt_assembly(Upper::Tmpfs, mode)) + .unwrap_err(); + let layers: Vec = (2..=255) + .rev() + .map(|number| format!("layers/p{number}")) + .collect(); + let layers: Vec<&str> = layers.iter().map(String::as_str).collect(); + let length = overlay_data(&guest.text("run/nvx"), &layers, "tmpfs", mode).len(); + assert!( + matches!(error.error(), Error::OverlayOptions { length: actual } if *actual == length), + "{error:?}" + ); + assert_eq!( + error.to_string(), + format!( + "the overlay's options take {length} bytes, more than the 4095 that a mount passes" + ) + ); + assert!(guest.kernel.mounts().is_empty()); +} + #[test] fn the_journal_policy_is_checked_in_the_scratch_s_superblock_before_any_mount() { // A scratch without a journal mounts whatever the policy, and one with a journal only where @@ -1495,6 +1857,356 @@ fn a_tmpfs_upper_is_unmounted_last_and_every_failure_reported() { ); } +#[test] +fn a_gpt_disk_that_differs_from_the_expected_identities_is_refused_before_any_mount() { + let disk = GptDisk::new(b"configuration", 2); + let partition = |number: u8| { + ExpectedPartition::new() + .unique_guid(GptDisk::guid(number)) + .size(disk.size(number)) + }; + // The disk meets every expectation that the caller may set. + let guest = Guest::new(); + guest.insert(&disk); + let mut assembly = guest.gpt_assembly(Upper::Tmpfs, OverlayMode::UntrustedLowers); + assembly.image = Image::Gpt { + expected: Expected::new() + .disk_guid(DISK_GUID) + .partitions((1..=3).map(partition).collect()), + first_partition_bound: FIRST_PARTITION_BOUND, + }; + guest.assemble(&assembly).unwrap().teardown().unwrap(); + + let other = Uuid::from_bytes([0x5A; 16]); + let cases = [ + ( + Expected::new().disk_guid(other), + GptError::DiskGuidMismatch { + expected: other, + found: DISK_GUID, + }, + ), + ( + Expected::new().partitions(vec![partition(1), partition(2)]), + GptError::PartitionCountMismatch { + expected: 2, + found: 3, + }, + ), + ( + Expected::new().partitions(vec![ + partition(1), + partition(2), + ExpectedPartition::new().unique_guid(other), + ]), + GptError::PartitionGuidMismatch { + number: 3, + expected: other, + found: GptDisk::guid(3), + }, + ), + ( + Expected::new().partitions(vec![ + partition(1), + ExpectedPartition::new().size(2 * BLOCK as u64), + partition(3), + ]), + GptError::PartitionSizeMismatch { + number: 2, + expected: 2 * BLOCK as u64, + found: (LAYER_BLOCKS * BLOCK) as u64, + }, + ), + ]; + for (expected, failure) in cases { + let guest = Guest::new(); + guest.insert(&disk); + let mut assembly = guest.gpt_assembly(Upper::Tmpfs, OverlayMode::UntrustedLowers); + assembly.image = Image::Gpt { + expected, + first_partition_bound: FIRST_PARTITION_BOUND, + }; + let error = guest.assemble(&assembly).unwrap_err(); + assert!( + matches!(error.error(), Error::Disk { device, error: DeviceError::Table(_) } + if *device == guest.path("dev/vda")), + "{error:?}" + ); + assert_eq!( + error.to_string(), + format!( + "the GPT disk on the distro device {} failed its validation: invalid partition \ + table: {failure}", + guest.text("dev/vda") + ) + ); + assert!(guest.kernel.mounts().is_empty()); + } +} + +#[test] +fn a_gpt_disk_without_a_layer_is_refused_before_any_mount() { + let guest = Guest::new(); + guest.insert(&GptDisk::new(b"configuration", 0)); + let assembly = guest.gpt_assembly(Upper::Tmpfs, OverlayMode::TrustedLowers); + let error = guest.assemble(&assembly).unwrap_err(); + assert!( + matches!( + error.error(), + Error::Disk { + error: DeviceError::Table(GptError::PartitionCount(1)), + .. + } + ), + "{error:?}" + ); + assert_eq!( + error.to_string(), + format!( + "the GPT disk on the distro device {} failed its validation: invalid partition \ + table: the disk has 1 partitions, not 2 through 255: the first partition and at \ + least one layer", + guest.text("dev/vda") + ) + ); + assert!(guest.kernel.mounts().is_empty()); +} + +#[test] +fn a_layer_that_is_not_ext4_or_whose_journal_needs_a_recovery_is_refused_before_any_mount() { + // A layer may have a journal. + let mut disk = GptDisk::new(b"configuration", 3); + disk.layer_mut(2)[1024 + 0x5C] |= 0x04; + let guest = Guest::new(); + guest.insert(&disk); + let assembly = guest.gpt_assembly(Upper::Tmpfs, OverlayMode::TrustedLowers); + guest.assemble(&assembly).unwrap().teardown().unwrap(); + + // But not one that needs a recovery, which its read-only device can't take, whatever its + // superblock's own default mount options, such as `norecovery`, which would skip it. + let layer = disk.layer_mut(3); + layer[1024 + 0x60] |= 0x04; + layer[1024 + 0x200..][..10].copy_from_slice(b"norecovery"); + let guest = Guest::new(); + guest.insert(&disk); + let assembly = guest.gpt_assembly(Upper::Tmpfs, OverlayMode::TrustedLowers); + let error = guest.assemble(&assembly).unwrap_err(); + assert!( + matches!(error.error(), Error::Recovery { number: 3, device } + if *device == guest.path("dev/vda3")), + "{error:?}" + ); + assert_eq!( + error.to_string(), + format!( + "the layer in partition 3, {}, needs a journal recovery, which its read-only device \ + can't take", + guest.text("dev/vda3") + ) + ); + assert!(guest.kernel.mounts().is_empty()); + + // Nor one that holds no ext4 file system. + let mut disk = GptDisk::new(b"configuration", 2); + disk.layer_mut(2)[1024 + 0x38] ^= 1; + let guest = Guest::new(); + guest.insert(&disk); + let assembly = guest.gpt_assembly(Upper::Tmpfs, OverlayMode::TrustedLowers); + let error = guest.assemble(&assembly).unwrap_err(); + assert!( + matches!(error.error(), Error::NotExt4Layer { number: 2, device } + if *device == guest.path("dev/vda2")), + "{error:?}" + ); + assert_eq!( + error.to_string(), + format!( + "the distro's partition 2, {}, holds no ext4 file system", + guest.text("dev/vda2") + ) + ); + assert!(guest.kernel.mounts().is_empty()); + + // Nor one whose superblock can't be read. + let disk = GptDisk::new(b"configuration", 1); + let guest = Guest::new(); + fs::write(guest.path("dev/vda"), disk.image()).unwrap(); + fs::create_dir(guest.path("dev/vda2")).unwrap(); + let assembly = guest.gpt_assembly(Upper::Tmpfs, OverlayMode::TrustedLowers); + let error = guest.assemble(&assembly).unwrap_err(); + assert!( + matches!( + error.error(), + Error::Device { + role: Role::Distro, + step: DeviceStep::Superblock, + device, + error, + } if *device == guest.path("dev/vda2") + && error.raw_os_error() == Some(Errno::EISDIR as i32) + ), + "{error:?}" + ); + assert!(guest.kernel.mounts().is_empty()); +} + +#[test] +fn a_missing_partition_fails_at_once_before_any_mount() { + // The kernel created the partitions' nodes when it found the disk, so the assembly awaits + // none: a partition whose node is missing fails without a wait. + let disk = GptDisk::new(b"configuration", 2); + let guest = Guest::new(); + fs::write(guest.path("dev/vda"), disk.image()).unwrap(); + write(&guest.path("dev/vda2"), disk.layer(2)); + let assembly = guest.gpt_assembly(Upper::Tmpfs, OverlayMode::TrustedLowers); + let error = guest.assemble(&assembly).unwrap_err(); + assert!( + matches!(error.error(), Error::Partition { number: 3, node } + if *node == guest.path("dev/vda3")), + "{error:?}" + ); + assert_eq!( + error.to_string(), + format!( + "the distro's partition 3 has no block device at {}", + guest.text("dev/vda3") + ) + ); + assert_eq!(guest.kernel.sleeps(), 0); + assert!(guest.kernel.mounts().is_empty()); +} + +#[test] +fn the_first_partition_is_returned_as_opaque_bytes_within_its_bound() { + // One whole sector of bytes that are not UTF-8, with no padding. + let sector: Vec = (0..BLOCK).map(|index| 0x80 | index as u8).collect(); + assert!(std::str::from_utf8(§or).is_err()); + // Trailing NUL bytes and ASCII whitespace are padding, and the others are the first + // partition's bytes. + let padded = b"\0 key=value\0\t\n \r\x0c\0\0"; + let cases: [(&[u8], &[u8]); 3] = [ + (§or, §or), + (padded, b"\0 key=value"), + (b"\n\0", b""), + ]; + for (bytes, expected) in cases { + let guest = Guest::new(); + let disk = GptDisk::new(bytes, 1); + guest.insert(&disk); + let mut assembly = guest.gpt_assembly(Upper::Tmpfs, OverlayMode::TrustedLowers); + // The first partition may be as large as the bound. + assembly.image = Image::Gpt { + expected: Expected::new(), + first_partition_bound: disk.first_partition_blocks() * BLOCK, + }; + let root = guest.assemble(&assembly).unwrap(); + assert_eq!(root.first_partition(), Some(expected)); + assert_eq!( + guest.kernel.mounts(), + below(&guest.text("run/nvx"), &["layers/p2", "tmpfs", "rootfs"]) + ); + root.teardown().unwrap(); + } + + // A first partition larger than the bound is refused before any mount. + let guest = Guest::new(); + guest.insert(&GptDisk::new(§or, 1)); + let mut assembly = guest.gpt_assembly(Upper::Tmpfs, OverlayMode::TrustedLowers); + assembly.image = Image::Gpt { + expected: Expected::new(), + first_partition_bound: BLOCK - 1, + }; + let error = guest.assemble(&assembly).unwrap_err(); + assert!( + matches!( + error.error(), + Error::FirstPartition { + device, + error: GptError::FirstPartitionTooLarge { + size: 512, + bound: 511 + }, + } if *device == guest.path("dev/vda") + ), + "{error:?}" + ); + assert_eq!( + error.to_string(), + format!( + "failed to read the first partition of the GPT disk on the distro device {}: the \ + first partition has 512 bytes, more than the bound of 511", + guest.text("dev/vda") + ) + ); + assert!(guest.kernel.mounts().is_empty()); +} + +#[test] +fn a_teardown_unmounts_the_layers_top_first_and_reports_every_failure() { + let guest = Guest::new(); + guest.insert(&GptDisk::new(b"configuration", 3)); + let assembly = guest.gpt_assembly(Upper::Tmpfs, OverlayMode::UntrustedLowers); + let root = guest.assemble(&assembly).unwrap(); + for name in ["run/nvx/layers/p3", "run/nvx/tmpfs"] { + guest + .kernel + .fail(Operation::Unmount, guest.path(name), Errno::EBUSY); + } + let error = root.teardown().unwrap_err(); + let runtime = guest.text("run/nvx"); + assert_eq!( + guest.kernel.unmounts(), + below( + &runtime, + &["rootfs", "layers/p4", "layers/p3", "layers/p2", "tmpfs"] + ) + ); + let failed: Vec<(&Mount, &Path)> = error + .failures() + .iter() + .map(|failure| (failure.mount(), failure.path())) + .collect(); + let (third, tmpfs) = (guest.path("run/nvx/layers/p3"), guest.path("run/nvx/tmpfs")); + assert_eq!( + failed, + [ + (&Mount::Layer { number: 3 }, third.as_path()), + (&Mount::Tmpfs, tmpfs.as_path()) + ] + ); + assert_eq!( + error.to_string(), + format!( + "failed to unmount the layer in partition 3: {}; failed to unmount the tmpfs upper: {}", + os_error(Errno::EBUSY), + os_error(Errno::EBUSY) + ) + ); + + // A layer that fails to mount leaves the layers below it to unmount, and nothing above. + let guest = Guest::new(); + guest.insert(&GptDisk::new(b"configuration", 3)); + guest.kernel.fail( + Operation::Mount, + guest.path("run/nvx/layers/p3"), + Errno::EINVAL, + ); + let assembly = guest.gpt_assembly(Upper::Tmpfs, OverlayMode::UntrustedLowers); + let error = guest.assemble(&assembly).unwrap_err(); + assert!( + matches!( + error.error(), + Error::Mount { + mount: Mount::Layer { number: 3 }, + .. + } + ), + "{error:?}" + ); + assert!(error.cleanup().is_none()); + assert_eq!(guest.kernel.unmounts(), [guest.text("run/nvx/layers/p2")]); +} + /// Returns a child of an aggregate share. fn child(name: &str, target: &str, access: Access, kind: ShareKind) -> ShareChild { ShareChild { From 7ee79f9f75efe11687ecd7ecb99e8892d8b157cf Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Thu, 22 Oct 2026 01:42:00 -0700 Subject: [PATCH 2/4] tests: run the adapter's layered VMs from a GPT disk The runtime adapter's worker refused a GPT image at the layout stage. It now also assembles the layers of a GPT disk, through nvx::guest::root's Image::Gpt, so that every combination of the guest root's images and uppers gets real-kernel coverage: - The root configuration takes an optional first-partition-bound field, which the host program's launch sets from --first-partition-bound, and which only a GPT disk takes. Without it, the guest bounds the disk's first partition at 64 KiB. The worker expects the disk GUID that the boot tokens name, and refuses a configuration that bounds a first partition for an EROFS image. - The first-partition probe reports the bytes of the first partition that the assembly returned, in hexadecimal. - A GPT scenario writes its disk first: a protective MBR, the primary and backup tables, the first partition, of bytes that are not UTF-8 followed by padding, a copy of the scratch template as the base layer, and a small ext2-format layer, which the kernel's ext4 driver mounts, as the top layer, with a file and a directory. The test writes that layer itself, since mkfs.ext4 is not available everywhere that the tests run, and checks the disk with nvx::gpt before it boots it. - New scenarios cover the disk over a scratch and over RAM in both overlay modes: the first partition, intact; each layer, mounted read-only from its partition without noload; the overlay's order of the layers; copy-up and rename in each mode; and the guest's disks. A disk of another GUID than the launch expects, and a first partition one byte larger than its bound, fail the assembly. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 30bab0d7-7c3e-4120-9add-6c9cc1dc9e7e --- tests/runtime-adapter/README.md | 132 ++-- .../src/bin/runtime-adapter-host.rs | 46 +- tests/runtime-adapter/src/root.rs | 27 +- tests/runtime-adapter/src/worker.rs | 68 +- tests/runtime-adapter/src/worker/probes.rs | 74 +- tests/runtime-adapter/tests/guest_session.rs | 43 +- tests/runtime-adapter/tests/layered_launch.rs | 25 +- tests/runtime-adapter/tests/vm_layered.rs | 636 ++++++++++++++++-- 8 files changed, 890 insertions(+), 161 deletions(-) diff --git a/tests/runtime-adapter/README.md b/tests/runtime-adapter/README.md index 596a7f144..612dc1fe3 100644 --- a/tests/runtime-adapter/README.md +++ b/tests/runtime-adapter/README.md @@ -90,6 +90,7 @@ zeros: | `workload-pids-max` | The workload cgroup's `pids.max` | no | | `distro-sectors` | The size of the image's block device, in 512-byte sectors | no | | `scratch-sectors` | The size of the scratch block device, in 512-byte sectors | no | +| `first-partition-bound` | The most bytes that the first partition of a GPT disk may hold | no | | `overlay-mode` | The overlay's mode: `trusted-lowers` or `untrusted-lowers` | no | | `scratch-journal` | Whether an ext4 scratch may have a journal: `allowed` or `forbidden` | no | @@ -100,14 +101,16 @@ component, or a trailing `/`. The runtime directory holds no `,`, `:`, `\`, or whose parent is not `/`. The optional numbers are positive. Without an overlay mode or a journal policy, the guest takes the defaults of `nvx::guest::root`: an overlay that trusts its lowers, and a scratch that may -have a journal. The host program and the guest agent ship together, so the -guest refuses every other configuration, one with an unknown field included, -rather than ignore what it does not know. `RootConfig::default()` holds the -values that the host program sends unless its caller names others: the runtime -directory `/run/runtime-adapter`, the cgroups `/sys/fs/cgroup/agent` and -`/sys/fs/cgroup/workload`, a wait of 30 seconds, an agent's `memory.low` of -16 MiB, as NVX's own guest reserves, and a workload's `memory.max` of 128 MiB -and `pids.max` of 128, with neither an overlay mode nor a journal policy. +have a journal. Without a bound of the first partition, the guest bounds it at +`root::DEFAULT_FIRST_PARTITION_BOUND`, 64 KiB. The host program and the guest +agent ship together, so the guest refuses every other configuration, one with +an unknown field included, rather than ignore what it does not know. +`RootConfig::default()` holds the values that the host program sends unless its +caller names others: the runtime directory `/run/runtime-adapter`, the cgroups +`/sys/fs/cgroup/agent` and `/sys/fs/cgroup/workload`, a wait of 30 seconds, an +agent's `memory.low` of 16 MiB, as NVX's own guest reserves, and a workload's +`memory.max` of 128 MiB and `pids.max` of 128, with neither a bound of the +first partition, an overlay mode, nor a journal policy. A stage of the root that fails is a `root::RootFailure`: the stage's code, one byte, then the problem, as UTF-8 text. The guest's worker, and with it the VM, @@ -140,12 +143,13 @@ arguments: | Probe | Facts | | --- | --- | -| `mounts` | For the mount of each layer, by the name of its mount point, `distro` for an EROFS image, the scratch's, `scratch`, the tmpfs's, `tmpfs`, and the overlay's, `root`: the number of mounts at its mount point, `NAME.count`, and of the topmost, its `NAME.type`, `NAME.source`, `NAME.options`, and `NAME.super`, the file system's own options | +| `mounts` | For the mount of each layer, by the name of its mount point, `distro` for an EROFS image, and `pN` for the layer in partition N of a GPT disk, the scratch's, `scratch`, the tmpfs's, `tmpfs`, and the overlay's, `root`: the number of mounts at its mount point, `NAME.count`, and of the topmost, its `NAME.type`, `NAME.source`, `NAME.options`, and `NAME.super`, the file system's own options | | `contents` | The root's `os-release.id` and `root.mode`. It writes `.runtime-adapter-probe` at the top of the root, and reports what the root, `write.root`, the upper, `write.upper`, and the top layer, `write.lower`, hold at that name, and whether the top layer refuses a new file, `lower.write` | | `copy-up FILE` | It sets the mode of FILE, a regular file of the layers, to the mode that it has, which copies the file up, and reports its size in the topmost layer that holds it, `lower.size`, and in the upper, `upper.size`; whether the upper's copy holds the file's data, `upper.data`: `copied`, `none` once the overlay copied up the file's metadata alone, or `different`; and whether the root still shows the layer's data, `root.data`: `intact` or `different` | | `rename DIRECTORY` | It renames DIRECTORY, a directory of the layers, to `.runtime-adapter-renamed` at the top of the root, and back if it can: `outcome` is `allowed`, `refused` once the overlay refused it with `EXDEV`, or `failed: PROBLEM` | | `fill` | It writes a file in the root until the upper is full, then removes it, and writes a file there again: the upper's size, `total`, and the bytes available in it before, `available`; the bytes that it wrote, `written`; the error that ended the writes, `error`: `ENOSPC` once the upper was full, or the problem; what the second file holds, `after`; and the guest's memory, `memory`, in bytes | | `devices` | The guest's disks, the block devices that a device backs: `disks`, each as `NAME:SECTORS:ACCESS`, its size in 512-byte sectors and its access mode, `ro` or `rw`, in the order of their names, separated by spaces | +| `first-partition` | The bytes of the first partition of a GPT disk that the assembly returned, without their trailing padding: how many, `length`, and the bytes, `bytes`, in hexadecimal, or `none` for both for an EROFS image | | `cgroups` | The file system at the cgroups' parent, `parent.type`, and its `parent.subtree-control`; whether each cgroup exists, `agent.exists` and `workload.exists`; `agent.memory.low`, `workload.memory.max`, and `workload.pids.max`; the number of processes in each, `agent.processes` and `workload.processes`; and `user-namespaces`, the guest's limit of user namespaces, or `unavailable` without them | | `processes` | For PID 1, `pid1`, the worker, `worker`, and PID 1's unrelated child, `sibling`: `NAME.pid`, and while the process exists, `NAME.state`, `NAME.ppid`, `NAME.root`, `NAME.cgroup`, `NAME.command`, and `NAME.namespaces`: `same` if it shares PID 1's, or the ones that differ | | `hold` | It opens the root, and keeps it open until the worker ends, so that the teardown finds the root busy: `held`, the path that it holds | @@ -248,18 +252,20 @@ tty with `nvx::guest::terminal`, as `serve_terminal` does, and then: `RootFailed`, confirms the stop with `Stopped`, and ends. Before the root configuration, the worker accepts only the configuration, or a -`Stop`, which ends it without a root. It assembles an EROFS image, over an ext4 -scratch or a tmpfs in RAM, in the overlay mode of the configuration, and -refuses a GPT image at the layout stage, as it refuses a configuration that -configures a scratch for an upper in RAM. It ends with 0 once the host stopped -it, after a teardown that succeeded, with the stage's status, from 11 to 15, -once a stage failed, and with 1 once its session failed or the broker closed -the tty, each but the first after a diagnostic on standard error. Before it -exits, it closes what its probes hold, then unmounts a root that it still -holds, as `nvx::guest::root::Root` does when it drops, ignoring failures. A -teardown that failed leaves the root mounted until the VM ends. The worker -runs the root's operations through `worker::RootOperations`, so that tests run -it against a fake root. +`Stop`, which ends it without a root. It assembles an EROFS image, or the +layers of a GPT disk, whose first partition holds at most the bound of the +configuration, over an ext4 scratch or a tmpfs in RAM, in the overlay mode of +the configuration. At the layout stage, it refuses a configuration that +configures a scratch for an upper in RAM, or bounds a first partition for an +EROFS image. It ends with 0 once the host stopped it, after a teardown that +succeeded, with the stage's status, from 11 to 15, once a stage failed, and +with 1 once its session failed or the broker closed the tty, each but the first +after a diagnostic on standard error. Before it exits, it closes what its +probes hold, then unmounts a root that it still holds, as +`nvx::guest::root::Root` does when it drops, ignoring failures. A teardown that +failed leaves the root mounted until the VM ends. The worker runs the root's +operations through `worker::RootOperations`, so that tests run it against a +fake root. ## Host program @@ -401,20 +407,20 @@ launch sends, whether or not the guest accepts it. ends its VM. LAYERED, - `--distro FORMAT:PATH [--distro-identity UUID] --upper (scratch:PATH | ram) [--runtime-dir DIR] [--overlay-mode MODE] [--scratch-journal POLICY]`, + `--distro FORMAT:PATH [--distro-identity UUID] --upper (scratch:PATH | ram) [--runtime-dir DIR] [--overlay-mode MODE] [--scratch-journal POLICY] [--first-partition-bound BYTES]`, gives the VM a [layered root](#layered-roots). Its image is the file PATH, an EROFS file system, `erofs`, or a GPT disk, `gpt`, of which the guest expects - the identity UUID, if given. Its upper is the ext4 scratch image PATH, or - RAM. The guest assembles the root below its runtime directory DIR, - `/run/runtime-adapter` by default, in the overlay mode MODE, - `trusted-lowers` or `untrusted-lowers`, and checks a scratch's journal - against the policy POLICY, `allowed` or `forbidden`, which only a scratch - takes, with the guest's defaults unless given. Once attached, the launch - sends the root configuration, and exchanges the `Ping` and the `Echo` once - the guest assembled the root. If the guest reports that a stage of the root - failed instead, the launch waits for the VM to end, abandons the launch, and - checks that the VM ended with the stage's status. The worker assembles only - an EROFS image, and fails a GPT image at the layout stage. + the identity UUID, if given: the file system's UUID, or the disk's GUID. Its + upper is the ext4 scratch image PATH, or RAM. The guest assembles the root + below its runtime directory DIR, `/run/runtime-adapter` by default, in the + overlay mode MODE, `trusted-lowers` or `untrusted-lowers`, checks a + scratch's journal against the policy POLICY, `allowed` or `forbidden`, which + only a scratch takes, and refuses a first partition larger than BYTES, which + only a GPT disk takes, with the guest's defaults unless given. Once attached, + the launch sends the root configuration, and exchanges the `Ping` and the + `Echo` once the guest assembled the root. If the guest reports that a stage + of the root failed instead, the launch waits for the VM to end, abandons the + launch, and checks that the VM ended with the stage's status. - `reattach STORE` takes the sandbox's lock, and reattaches to its VM, which another process launched, and which it holds, although the VM is not its child, through its verified process identity. It reads the capability from @@ -534,22 +540,22 @@ byte value, and that `init` fails at once on a path that is no tty. `tests/guest_session.rs` also runs the worker against a fake root, behind its test broker: a root that the worker assembles, probes, and tears down on a stop, across a reset of the session; an image over RAM, in the overlay mode of -the configuration; the failure of each stage, which the worker reports before -it ends with the stage's status, without the operations of the later stages; -a teardown that fails; before the root, the configuration and the `Stop` that -it accepts, and the messages that it refuses; and a second configuration and a -broker that closes the stream, which leave the root that it assembled to its -operations. Through the binary, it checks that -`runtime-adapter-guest init TTY layered`, which is not PID 1 there, starts the -worker, which fails at the layout stage on a kernel command line without a -layered root, and ends with its status, and checks `worker`, `idle`, and the -usage errors. Each guest binary that a test starts runs in a process group of -its own, with the processes that it starts, and keeps its process ID, and with -it the group's, until the test has killed the group. A binary that exits but -leaves processes of its group running fails its test. A test that fails, or -gives up on the binary, kills the whole group, even once the binary exited, so -that none of them outlives the test or holds its output open, which tests check -too. +the configuration; a GPT disk, with the configuration's bound of the first +partition; the failure of each stage, which the worker reports before it ends +with the stage's status, without the operations of the later stages; a teardown +that fails; before the root, the configuration and the `Stop` that it accepts, +and the messages that it refuses; and a second configuration and a broker that +closes the stream, which leave the root that it assembled to its operations. +Through the binary, it checks that `runtime-adapter-guest init TTY layered`, +which is not PID 1 there, starts the worker, which fails at the layout stage on +a kernel command line without a layered root, and ends with its status, and +checks `worker`, `idle`, and the usage errors. Each guest binary that a test +starts runs in a process group of its own, with the processes that it starts, +and keeps its process ID, and with it the group's, until the test has killed +the group. A binary that exits but leaves processes of its group running fails +its test. A test that fails, or gives up on the binary, kills the whole group, +even once the binary exited, so that none of them outlives the test or holds +its output open, which tests check too. `tests/initramfs.rs` and `tests/runtime.rs` check, in the host-only configuration, on Linux and Windows: @@ -601,22 +607,29 @@ own: | `erofs_over_scratch_with_untrusted_lowers` | The image over the scratch, in the untrusted-lowers mode, with a journal that the launch allows | The root, whose overlay has none of the trusting mode's options, copies up a file's data too, and refuses the rename of a lower directory with `EXDEV` | | `erofs_over_ram` | The image over a tmpfs in RAM | The root, whose upper and work directories are on a tmpfs of the kernel's default size, whose root directory has mode 0755, with no scratch mounted, and no disk but the image's | | `erofs_over_ram_with_untrusted_lowers` | The image over a tmpfs, in the untrusted-lowers mode | The root over the tmpfs, which copies up a file's data too, and refuses the rename | +| `gpt_over_scratch` | A GPT disk that the scenario writes, with its own GUID, over a copy of the scratch template | A root that the worker assembled: the first partition's bytes, as the disk holds them without their padding; each layer, an ext4 file system, mounted once, `ro,nosuid,nodev` and without `noload`, from its partition; the overlay, with partition 2's layer at its base and partition 3's at its top, in the default mode, which copies up a file's metadata alone and renames a lower directory; a file that reaches the upper and not the top layer; and the disk's and the scratch's disks | +| `gpt_over_scratch_with_untrusted_lowers` | The disk over the scratch, in the untrusted-lowers mode | The root, which copies up a file's data too, and refuses the rename | +| `gpt_over_ram` | The disk over a tmpfs, with a bound that the first partition meets exactly | The root over the tmpfs, with no scratch mounted, and no disk but the GPT disk's | +| `gpt_over_ram_with_untrusted_lowers` | The disk over a tmpfs, in the untrusted-lowers mode | The root over the tmpfs, which copies up a file's data too, and refuses the rename | | `full_scratch_refuses_writes` | The image over the scratch | A file that fills the scratch, through the root, until a write fails with `ENOSPC`, and a write that succeeds once the file is gone | | `full_ram_upper_refuses_writes` | The image over a tmpfs | A file that fills the tmpfs, half of the guest's memory, until a write fails with `ENOSPC`, and a write that succeeds once the file is gone | | `forbidden_journal_fails_the_assembly` | The image over the scratch, whose journal the launch forbids | The assembly's failure, which the launch reports, and a VM that ends with 14 before the guest mounts the scratch | | `other_identity_fails_the_assembly` | The image, of which the launch expects another identity | The assembly's failure, which the launch reports, and a VM that ends with 14 before the guest mounts the scratch | +| `other_disk_guid_fails_the_assembly` | The GPT disk, of which the launch expects another GUID | The assembly's failure, which the launch reports, and a VM that ends with 14 before the guest mounts the scratch | +| `first_partition_over_its_bound_fails_the_assembly` | The GPT disk, with a bound one byte smaller than its first partition | The assembly's failure, which the launch reports, and a VM that ends with 14 before the guest mounts the scratch | | `invalid_configuration_fails_its_stage` | A relative runtime directory | The configuration's failure, which the launch reports, and a VM that ends with 11 | | `busy_root_fails_the_teardown` | The image over the scratch, which the `hold` probe keeps open | The teardown's failure, which the stop reports, and a VM that ends with 15, leaving the scratch mounted | | `mappings_apply_beneath_the_root` | The image over the scratch, beneath which the `map` and `map-checks` probes map | Through the root: zero mappings that do no work, then a table mounted in order, each mapping with its access, `nosuid`, and `nodev`, whose writes reach the sources of the writable ones only, on targets that the mapper created on the overlay. Through mappers of the adapter's own: a link or a magic link at each component, the wrong kinds, a target that would be created inside another mapping, and the runtime's, the container root's, and the adapter's own reserved targets refused, and a read-only mapping that is never writable. A teardown that unmounts the mappings, then the scratch, cleanly | | `a_refused_mapping_latches_the_root` | The image over the scratch, in which the `map-refused` probe plants a symbolic link | A mapping through the link that the root refuses, nothing mounted or created, and a root that refuses every later mapping, then tears down cleanly | A row names the root, the image, the identity that the launch expects of it, -the upper, the runtime directory, the overlay mode, and the scratch's journal -policy, and what the scenario expects: a root that the guest assembles, with -a check that probes it, a teardown that fails after a check, or the stage that -fails, with its problem. Each scenario checks the launch's record, and, over a -scratch, whether the VM left the scratch's journal to recover, which tells -whether the guest unmounted the scratch. +the upper, the runtime directory, the overlay mode, the scratch's journal +policy, and the bound of the first partition, and what the scenario expects: a +root that the guest assembles, with a check that probes it, a teardown that +fails after a check, or the stage that fails, with its problem. Each scenario +checks the launch's record, and, over a scratch, whether the VM left the +scratch's journal to recover, which tells whether the guest unmounted the +scratch. The scenarios use the fixtures of NVX's own layered scenarios, from NVX's artifacts: the Ubuntu EROFS image, `ubuntu-distro.erofs`, its manifest, which @@ -631,6 +644,15 @@ mkfs.ext4 -q -F -m 0 -U 4f3c8bf7-1959-47b4-8811-62b17bbddaf4 \ -E lazy_itable_init=0,lazy_journal_init=0 ../../build/ubuntu-smoke-scratch.ext4 ``` +A scenario of a GPT disk writes the disk from the template first: a protective +MBR, and the primary and backup tables of 128 entries, with the first +partition, 4 KiB of bytes that are not UTF-8, then padding; the base layer, in +partition 2, a copy of the template; and the top layer, in partition 3, a small +file system in the ext2 format, which the kernel's ext4 driver mounts, with a +file and a directory. The test writes the top layer itself, since `mkfs.ext4` +is not available everywhere that the tests run, and checks the disk with +`nvx::gpt` before it launches the VM. + ### Running the VM tests The VM tests, `tests/vm_restart.rs` and `tests/vm_layered.rs`, share their diff --git a/tests/runtime-adapter/src/bin/runtime-adapter-host.rs b/tests/runtime-adapter/src/bin/runtime-adapter-host.rs index 51349eda6..6b8c181b2 100644 --- a/tests/runtime-adapter/src/bin/runtime-adapter-host.rs +++ b/tests/runtime-adapter/src/bin/runtime-adapter-host.rs @@ -12,18 +12,19 @@ //! `whp`, or else on the one that `nvx::Hypervisor::from_env_or_default` selects. //! //! LAYERED, `--distro FORMAT:PATH [--distro-identity UUID] --upper (scratch:PATH | ram) -//! [--runtime-dir DIR] [--overlay-mode MODE] [--scratch-journal POLICY]`, gives the VM a layered -//! root, which the guest agent's worker assembles; see `runtime_adapter::layered` and -//! `runtime_adapter::worker`. Its image is the raw file PATH, an EROFS filesystem, `erofs`, or a -//! GPT disk, `gpt`, of which the guest expects the identity UUID, if given. Its upper is the ext4 +//! [--runtime-dir DIR] [--overlay-mode MODE] [--scratch-journal POLICY] +//! [--first-partition-bound BYTES]`, gives the VM a layered root, which the guest agent's worker +//! assembles; see `runtime_adapter::layered` and `runtime_adapter::worker`. Its image is the raw +//! file PATH, an EROFS filesystem, `erofs`, or a GPT disk, `gpt`, of which the guest expects the +//! identity UUID, if given: the file system's UUID, or the disk's GUID. Its upper is the ext4 //! scratch image PATH, or RAM. The guest assembles the root below its runtime directory DIR, //! `/run/runtime-adapter` by default, in the overlay mode MODE, `trusted-lowers` or -//! `untrusted-lowers`, and checks a scratch's journal against the policy POLICY, `allowed` or -//! `forbidden`, with the guest's defaults, `trusted-lowers` and `allowed`, unless given. Once -//! attached, the launch sends the root configuration, and exchanges the `Ping` and the `Echo` -//! once the guest assembled the root. If the guest reports that a stage of the root failed -//! instead, the launch waits for the VM to end, abandons it, and checks that the VM ended with -//! the stage's status. +//! `untrusted-lowers`, checks a scratch's journal against the policy POLICY, `allowed` or +//! `forbidden`, and refuses a first partition of a GPT disk larger than BYTES, with the guest's +//! defaults, `trusted-lowers`, `allowed`, and 64 KiB, unless given. Once attached, the launch +//! sends the root configuration, and exchanges the `Ping` and the `Echo` once the guest assembled +//! the root. If the guest reports that a stage of the root failed instead, the launch waits for +//! the VM to end, abandons it, and checks that the VM ended with the stage's status. //! - `runtime-adapter-host reattach STORE` reattaches to the VM of STORE, which an earlier process //! launched and which it holds although it is not its child, attaches a new session to it, in //! the session's next epoch, and exchanges a `Ping`, leaving the VM running. @@ -119,7 +120,8 @@ const USAGE: &str = concat!( " [--guest-build BUILD]\n", " [--distro FORMAT:PATH [--distro-identity UUID]", " --upper (scratch:PATH | ram) [--runtime-dir DIR]\n", - " [--overlay-mode MODE] [--scratch-journal POLICY]]\n", + " [--overlay-mode MODE] [--scratch-journal POLICY]", + " [--first-partition-bound BYTES]]\n", " runtime-adapter-host (reattach | stop) STORE [--guest-build BUILD]\n", " runtime-adapter-host probe STORE PROBE... [--guest-build BUILD]", ); @@ -175,6 +177,8 @@ struct Layered { overlay_mode: Option, /// Whether the scratch may have a journal, unless the guest's default. scratch_journal: Option, + /// The most bytes that the first partition of a GPT disk may hold, unless the guest's default. + first_partition_bound: Option, } /// The options of a command. @@ -198,6 +202,8 @@ struct Options { overlay_mode: Option, /// `--scratch-journal`. scratch_journal: Option, + /// `--first-partition-bound`. + first_partition_bound: Option, } fn main() -> ExitCode { @@ -289,6 +295,11 @@ fn parse(arguments: Vec) -> Result { (Some(_), Some(None)) if options.scratch_journal.is_some() => { return Err("--scratch-journal needs --upper scratch:PATH".to_owned()); } + (Some((ImageFormat::Erofs, _)), Some(_)) + if options.first_partition_bound.is_some() => + { + return Err("--first-partition-bound needs --distro gpt:PATH".to_owned()); + } (Some((format, distro)), Some(scratch)) => Some(Layered { distro, image: RootImage { @@ -299,6 +310,7 @@ fn parse(arguments: Vec) -> Result { runtime: options.runtime, overlay_mode: options.overlay_mode, scratch_journal: options.scratch_journal, + first_partition_bound: options.first_partition_bound, }), (Some(_), None) => return Err("--distro needs --upper".to_owned()), (None, _) if options.identity.is_some() => { @@ -310,6 +322,10 @@ fn parse(arguments: Vec) -> Result { ("--runtime-dir", options.runtime.is_some()), ("--overlay-mode", options.overlay_mode.is_some()), ("--scratch-journal", options.scratch_journal.is_some()), + ( + "--first-partition-bound", + options.first_partition_bound.is_some(), + ), ]; if let Some((option, _)) = layered.iter().find(|(_, given)| *given) { return Err(format!("{option} needs --distro")); @@ -404,6 +420,13 @@ fn parse_option( })?; options.scratch_journal = Some(policy); } + ("launch", "--first-partition-bound") if options.first_partition_bound.is_none() => { + let value = text(value)?; + let bound = value.parse::().ok().filter(|bound| *bound > 0); + options.first_partition_bound = Some(bound.ok_or_else(|| { + format!("--first-partition-bound takes a positive number of bytes, not {value:?}") + })?); + } (_, "--guest-build") if options.guest_build.is_none() => { options.guest_build = Some(text(value)?); } @@ -441,6 +464,7 @@ fn launch( )?; launch.config.overlay_mode = layered.overlay_mode; launch.config.scratch_journal = layered.scratch_journal; + launch.config.first_partition_bound = layered.first_partition_bound; Ok(launch) }) .transpose() diff --git a/tests/runtime-adapter/src/root.rs b/tests/runtime-adapter/src/root.rs index 72bdbe861..2c12517ef 100644 --- a/tests/runtime-adapter/src/root.rs +++ b/tests/runtime-adapter/src/root.rs @@ -5,8 +5,8 @@ //! format and the identity that the guest expects of it, and the upper. A [`RootConfig`] carries //! the rest that the root needs, which only the adapter defines, after the host attaches: the //! runtime directory, the cgroups, their resource values, the sizes that the block devices must -//! have, how long the guest waits for them, and how the guest assembles the overlay over the -//! upper. +//! have, how long the guest waits for them, the most bytes that the first partition of a GPT disk +//! may hold, and how the guest assembles the overlay over the upper. //! //! A root configuration is UTF-8 text: one `NAME=VALUE` line per field, each ended by a newline, //! in any order, and each at most once. Numbers are decimal, without a sign or leading zeros: @@ -23,6 +23,7 @@ //! | `workload-pids-max` | The workload cgroup's `pids.max` | no | //! | `distro-sectors` | The size of the image's block device, in 512-byte sectors | no | //! | `scratch-sectors` | The size of the scratch block device, in 512-byte sectors | no | +//! | `first-partition-bound` | The most bytes that the first partition of a GPT disk may hold, [`DEFAULT_FIRST_PARTITION_BOUND`] by default | no | //! | `overlay-mode` | The overlay's [mode](OverlayMode): `trusted-lowers`, the guest's default, or `untrusted-lowers` | no | //! | `scratch-journal` | Whether an ext4 scratch may have a journal, a [`JournalPolicy`]: `allowed`, the guest's default, or `forbidden` | no | //! @@ -60,6 +61,9 @@ pub const DEFAULT_WORKLOAD_PIDS_MAX: u64 = 128; pub const DEFAULT_DEVICE_TIMEOUT: Duration = Duration::from_secs(30); /// The longest wait for a block device that a configuration may ask for: ten minutes. pub const MAX_DEVICE_TIMEOUT: Duration = Duration::from_secs(600); +/// The most bytes that the first partition of a GPT disk may hold unless the host names another +/// bound: 64 KiB. +pub const DEFAULT_FIRST_PARTITION_BOUND: u64 = 64 << 10; /// The longest guest path of a configuration, in bytes: Linux's `PATH_MAX`, without the NUL. const MAX_PATH_BYTES: usize = 4095; @@ -87,11 +91,13 @@ const DISTRO_SECTORS: &str = "distro-sectors"; /// See [`VERSION`]. const SCRATCH_SECTORS: &str = "scratch-sectors"; /// See [`VERSION`]. +const FIRST_PARTITION_BOUND: &str = "first-partition-bound"; +/// See [`VERSION`]. const OVERLAY_MODE: &str = "overlay-mode"; /// See [`VERSION`]. const SCRATCH_JOURNAL: &str = "scratch-journal"; /// Every field, in the order in which [`RootConfig::encode`] writes them. -const FIELDS: [&str; 12] = [ +const FIELDS: [&str; 13] = [ VERSION, RUNTIME, AGENT_CGROUP, @@ -102,6 +108,7 @@ const FIELDS: [&str; 12] = [ WORKLOAD_PIDS_MAX, DISTRO_SECTORS, SCRATCH_SECTORS, + FIRST_PARTITION_BOUND, OVERLAY_MODE, SCRATCH_JOURNAL, ]; @@ -188,6 +195,9 @@ pub struct RootConfig { pub distro_sectors: Option, /// The size that the scratch block device must have, in 512-byte sectors, if any. pub scratch_sectors: Option, + /// The most bytes that the first partition of a GPT disk may hold, unless + /// [`DEFAULT_FIRST_PARTITION_BOUND`]. + pub first_partition_bound: Option, /// The overlay's mode, unless the guest's default. pub overlay_mode: Option, /// Whether an ext4 scratch may have a journal, unless the guest's default. @@ -198,7 +208,7 @@ impl Default for RootConfig { /// Returns the configuration of the defaults: [`DEFAULT_RUNTIME`], [`DEFAULT_AGENT_CGROUP`], /// [`DEFAULT_WORKLOAD_CGROUP`], [`DEFAULT_DEVICE_TIMEOUT`], [`DEFAULT_AGENT_MEMORY_LOW`], /// [`DEFAULT_WORKLOAD_MEMORY_MAX`], and [`DEFAULT_WORKLOAD_PIDS_MAX`], with no sizes, and the - /// guest's default overlay mode and journal policy. + /// guest's default bound of the first partition, overlay mode, and journal policy. fn default() -> Self { Self { runtime: DEFAULT_RUNTIME.to_owned(), @@ -210,6 +220,7 @@ impl Default for RootConfig { workload_pids_max: Some(DEFAULT_WORKLOAD_PIDS_MAX), distro_sectors: None, scratch_sectors: None, + first_partition_bound: None, overlay_mode: None, scratch_journal: None, } @@ -233,6 +244,7 @@ impl RootConfig { (WORKLOAD_PIDS_MAX, self.workload_pids_max), (DISTRO_SECTORS, self.distro_sectors), (SCRATCH_SECTORS, self.scratch_sectors), + (FIRST_PARTITION_BOUND, self.first_partition_bound), ]; fields.extend( optional @@ -370,6 +382,7 @@ impl RootConfig { workload_pids_max: number(WORKLOAD_PIDS_MAX)?, distro_sectors: number(DISTRO_SECTORS)?, scratch_sectors: number(SCRATCH_SECTORS)?, + first_partition_bound: number(FIRST_PARTITION_BOUND)?, overlay_mode, scratch_journal, }) @@ -577,6 +590,7 @@ mod tests { RootConfig { distro_sectors: Some(116_960), scratch_sectors: Some(131_072), + first_partition_bound: Some(4096), overlay_mode: Some(OverlayMode::UntrustedLowers), scratch_journal: Some(JournalPolicy::Forbidden), ..RootConfig::default() @@ -597,6 +611,7 @@ mod tests { workload-pids-max=128\n\ distro-sectors=116960\n\ scratch-sectors=131072\n\ + first-partition-bound=4096\n\ overlay-mode=untrusted-lowers\n\ scratch-journal=forbidden\n" ); @@ -770,6 +785,10 @@ mod tests { format!("{base}scratch-sectors= 1\n").into_bytes(), "not a positive decimal number", ), + ( + format!("{base}first-partition-bound=0\n").into_bytes(), + "has a first-partition-bound that is not a positive decimal number: \"0\"", + ), ( format!("{base}overlay-mode=TrustedLowers\n").into_bytes(), "names the unknown overlay-mode \"TrustedLowers\"", diff --git a/tests/runtime-adapter/src/worker.rs b/tests/runtime-adapter/src/worker.rs index 922655632..4b1fe3a1e 100644 --- a/tests/runtime-adapter/src/worker.rs +++ b/tests/runtime-adapter/src/worker.rs @@ -28,9 +28,10 @@ //! confirms the stop with `Stopped`, and ends. //! //! Before the root configuration, it accepts only the configuration, or a `Stop`, which ends it -//! without a root. It assembles an EROFS image, over an ext4 scratch or over RAM, in the overlay -//! mode that the configuration names, and refuses a GPT image at the layout stage, as it refuses a -//! configuration that configures a scratch for a root over RAM. +//! without a root. It assembles an EROFS image, or the layers of a GPT disk, whose first partition +//! holds at most the bound of the configuration, over an ext4 scratch or over RAM, in the overlay +//! mode that the configuration names. At the layout stage, it refuses a configuration that +//! configures a scratch for a root over RAM, or bounds a first partition for an EROFS image. //! //! The worker, and with it the VM, ends with 0 once the host stopped it, after a teardown that //! succeeded, with the stage's [exit status](RootStage::exit_status) once a stage failed, from 11 @@ -50,6 +51,7 @@ use nix::errno::Errno; use nix::sys::signal::{self, Signal}; use nix::sys::wait::{self, WaitPidFlag, WaitStatus}; use nix::unistd::Pid; +use nvx::gpt::Expected; use nvx::guest::cgroup::Layout; use nvx::guest::root::{ self, Assembly, Bootstrap, Device, Image, Journal, Resources, Root, Upper, UpperKind, @@ -61,7 +63,9 @@ use nvx::protocol::root::{ use crate::frame::{MAX_BODY, MAX_TEXT_BODY, Message, truncated}; use crate::guest::{self, Ended, ServeError, Step}; use crate::probe::{Facts, Probe}; -use crate::root::{JournalPolicy, OverlayMode, RootConfig, RootFailure, RootStage}; +use crate::root::{ + DEFAULT_FIRST_PARTITION_BOUND, JournalPolicy, OverlayMode, RootConfig, RootFailure, RootStage, +}; /// The status with which the worker ends once its session failed or the broker closed it, and /// with which PID 1 ends the VM if a child cannot start or the worker's status cannot be collected. @@ -198,18 +202,20 @@ fn assemble(operations: &mut dyn RootOperations, body: &[u8]) -> Result<(), Root } /// Reads the boot layout of the root from the kernel command line, through `operations`, and -/// checks that the worker assembles it as `config` configures it: an EROFS image, over an ext4 -/// scratch, or over RAM if `config` configures no scratch. +/// checks that the worker assembles it as `config` configures it: an EROFS image or a GPT disk, +/// over an ext4 scratch, or over RAM if `config` configures no scratch, with a bound of the first +/// partition for a GPT disk alone. fn layout(operations: &mut dyn RootOperations, config: &RootConfig) -> Result { let command_line = operations.command_line()?; let layout = RootLayout::parse(&command_line) .map_err(|error| format!("the kernel command line has invalid boot tokens: {error}"))? .ok_or("the kernel command line names no layered root")?; - if layout.image.format != ImageFormat::Erofs { - return Err(format!( - "the worker assembles only an EROFS image, not a {} image", - layout.image.format - )); + if layout.image.format == ImageFormat::Erofs && config.first_partition_bound.is_some() { + return Err( + "the root configuration bounds the first partition, but the boot tokens name an \ + EROFS image" + .to_owned(), + ); } if layout.upper == RootUpper::Ram && (config.scratch_sectors.is_some() || config.scratch_journal.is_some()) @@ -294,12 +300,36 @@ impl RootOperations for LibraryRoot { } /// Runs `nvx::guest::root::assemble` on the block devices at the MMIO bases of their roles: - /// the EROFS image, with the identity that the boot tokens expect of it, over the ext4 - /// scratch, with the configuration's journal policy, or over a tmpfs, in the configuration's - /// overlay mode, with the configuration's runtime directory, device sizes, and timeout. A - /// policy or a mode that the configuration does not name is the guest's default: a scratch - /// may have a journal, and the overlay trusts its lowers. + /// the EROFS image, with the identity that the boot tokens expect of it, or the GPT disk, with + /// the disk GUID that they expect of it, whose first partition holds at most the + /// configuration's bound, over the ext4 scratch, with the configuration's journal policy, or + /// over a tmpfs, in the configuration's overlay mode, with the configuration's runtime + /// directory, device sizes, and timeout. A bound, a policy, or a mode that the configuration + /// does not name is the guest's default: the first partition holds at most + /// [`DEFAULT_FIRST_PARTITION_BOUND`] bytes, a scratch may have a journal, and the overlay + /// trusts its lowers. fn assemble(&mut self, config: &RootConfig, layout: &RootLayout) -> Result<(), String> { + let image = match layout.image.format { + ImageFormat::Erofs => Image::Erofs { + uuid: layout.image.identity, + }, + ImageFormat::Gpt => { + let bound = config + .first_partition_bound + .unwrap_or(DEFAULT_FIRST_PARTITION_BOUND); + let first_partition_bound = usize::try_from(bound).map_err(|_| { + format!("the guest cannot hold a first partition of {bound} bytes") + })?; + let expected = match layout.image.identity { + Some(guid) => Expected::new().disk_guid(guid), + None => Expected::new(), + }; + Image::Gpt { + expected, + first_partition_bound, + } + } + }; let upper = match layout.upper { RootUpper::Scratch => Upper::Scratch { device: Device { @@ -324,9 +354,7 @@ impl RootOperations for LibraryRoot { mmio: DISTRO_MMIO_BASE, sectors: config.distro_sectors, }, - image: Image::Erofs { - uuid: layout.image.identity, - }, + image, upper, overlay, timeout: config.device_timeout, @@ -354,11 +382,13 @@ impl RootOperations for LibraryRoot { .iter() .map(|lower| lower.path().to_owned()) .collect::>(); + let first_partition = root.first_partition().map(<[u8]>::to_vec); let context = probes::Context { runtime: &runtime, root: &path, lowers: &lowers, upper: &upper, + first_partition: first_partition.as_deref(), agent: bootstrap.layout().agent(), workload: bootstrap.layout().workload(), sibling: self.sibling, diff --git a/tests/runtime-adapter/src/worker/probes.rs b/tests/runtime-adapter/src/worker/probes.rs index 6a4245409..a3f49c058 100644 --- a/tests/runtime-adapter/src/worker/probes.rs +++ b/tests/runtime-adapter/src/worker/probes.rs @@ -3,12 +3,13 @@ //! //! | Probe | Facts | //! | --- | --- | -//! | `mounts` | For the mount of each layer, by the name of its mount point, `distro` for an EROFS image, the scratch's, `scratch`, the tmpfs's, `tmpfs`, and the overlay's, `root`: the number of mounts at its mount point, `NAME.count`, and of the topmost, its `NAME.type`, `NAME.source`, `NAME.options`, and `NAME.super`, the file system's own options | +//! | `mounts` | For the mount of each layer, by the name of its mount point, `distro` for an EROFS image, and `pN` for the layer in partition N of a GPT disk, the scratch's, `scratch`, the tmpfs's, `tmpfs`, and the overlay's, `root`: the number of mounts at its mount point, `NAME.count`, and of the topmost, its `NAME.type`, `NAME.source`, `NAME.options`, and `NAME.super`, the file system's own options | //! | `contents` | The root's `os-release.id` and `root.mode`. It writes [`PROBE_FILE`] in the root, and reports what the root, `write.root`, the upper, `write.upper`, and the top layer, `write.lower`, hold at that name, and whether the top layer refuses a new file, `lower.write` | //! | `copy-up FILE` | It sets the mode of FILE, a regular file of the layers, to the mode that it has, which copies the file up, and reports its size in the topmost layer that holds it, `lower.size`, and in the upper, `upper.size`; whether the upper's copy holds the file's data, `upper.data`: `copied`, `none` once the overlay copied up the file's metadata alone, or `different`; and whether the root still shows the layer's data, `root.data`: `intact` or `different` | //! | `rename DIRECTORY` | It renames DIRECTORY, a directory of the layers, to `.runtime-adapter-renamed` at the top of the root, and back if it can: `outcome` is `allowed`, `refused` once the overlay refused it with `EXDEV`, or `failed: PROBLEM` | //! | `fill` | It writes a file in the root until the upper is full, then removes it, and writes a file there again: the upper's size, `total`, and the bytes available in it before, `available`, in bytes; the bytes that it wrote, `written`; the error that ended the writes, `error`: `ENOSPC` once the upper was full, or the problem; what the second file holds, `after`; and the guest's memory, `memory`, in bytes | //! | `devices` | The guest's disks, the block devices that a device backs: `disks`, each as `NAME:SECTORS:ACCESS`, its size in 512-byte sectors and its access mode, `ro` or `rw`, in the order of their names, separated by spaces | +//! | `first-partition` | The bytes of the first partition of a GPT disk that the assembly returned, without their trailing padding: how many, `length`, and the bytes, `bytes`, in hexadecimal, or `none` for both for an EROFS image | //! | `cgroups` | The file system at the cgroups' parent, `parent.type`, and its `parent.subtree-control`; whether each cgroup exists, `agent.exists` and `workload.exists`; `agent.memory.low`, `workload.memory.max`, and `workload.pids.max`; the number of processes in each, `agent.processes` and `workload.processes`; and `user-namespaces`, the guest's limit of user namespaces, or `unavailable` without them | //! | `processes` | For PID 1, `pid1`, the worker, `worker`, and PID 1's unrelated child, `sibling`: `NAME.pid`, and while the process exists, `NAME.state`, `NAME.ppid`, `NAME.root`, `NAME.cgroup`, `NAME.command`, and `NAME.namespaces`: `same` if it shares PID 1's, or the ones that differ | //! | `hold` | It opens the root, and keeps it open until the worker ends, so that the teardown finds the root busy: `held`, the path that it holds | @@ -67,10 +68,13 @@ pub struct Context<'a> { /// The root, the overlay's mount point. pub root: &'a Path, /// The mount points of the root's layers, the base first: `layers/distro`, the one of an EROFS - /// image. + /// image, or `layers/p2` to `layers/pN`, those of the layers of a GPT disk. pub lowers: &'a [PathBuf], /// The upper's mount point, which holds the overlay's `upper` directory. pub upper: &'a Path, + /// The bytes of the first partition of a GPT disk that the assembly returned, or `None` for an + /// EROFS image. + pub first_partition: Option<&'a [u8]>, /// The agent's cgroup. pub agent: &'a Path, /// The workload cgroup. @@ -118,6 +122,7 @@ pub fn run(probe: &Probe, context: &Context<'_>, held: &mut Vec) -> Result "rename", "fill", "devices", + "first-partition", "cgroups", "processes", "hold", @@ -138,6 +143,7 @@ pub fn run(probe: &Probe, context: &Context<'_>, held: &mut Vec) -> Result ("rename", Some(directory)) => rename(context, directory), ("fill", _) => fill(context), ("devices", _) => devices(), + ("first-partition", _) => Ok(first_partition(context)), ("cgroups", _) => cgroups(context), ("processes", _) => processes(context), _ => hold(context, held), @@ -347,6 +353,18 @@ fn devices() -> io::Result { Ok(facts) } +/// Runs the `first-partition` probe. +fn first_partition(context: &Context<'_>) -> Facts { + let (length, bytes) = match context.first_partition { + Some(bytes) => (bytes.len().to_string(), hexadecimal(bytes)), + None => ("none".to_owned(), "none".to_owned()), + }; + let mut facts = Facts::new(); + facts.push("length", length); + facts.push("bytes", bytes); + facts +} + /// Runs the `cgroups` probe. fn cgroups(context: &Context<'_>) -> io::Result { let parent = context.agent.parent().unwrap_or(Path::new("/")); @@ -599,6 +617,16 @@ fn os_release_id(text: &str) -> Option<&str> { .map(|value| value.trim().trim_matches(['"', '\''])) } +/// Returns `bytes` in lowercase hexadecimal, two digits each. +fn hexadecimal(bytes: &[u8]) -> String { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + bytes + .iter() + .flat_map(|byte| [byte >> 4, byte & 0xf]) + .map(|digit| char::from(DIGITS[usize::from(digit)])) + .collect() +} + #[cfg(test)] mod tests { use super::*; @@ -688,6 +716,7 @@ rw,lowerdir=/run/a\\040b/layers/distro,metacopy=on,xino=on root: Path::new("/run/runtime-adapter/rootfs"), lowers: &lowers, upper: Path::new("/run/runtime-adapter/scratch"), + first_partition: None, agent: Path::new("/sys/fs/cgroup/agent"), workload: Path::new("/sys/fs/cgroup/workload"), sibling: None, @@ -703,6 +732,11 @@ rw,lowerdir=/run/a\\040b/layers/distro,metacopy=on,xino=on run(&with_argument, &context, &mut held), Err("the hold probe takes no arguments".to_owned()) ); + let with_argument = Probe::parse("first-partition 2").unwrap(); + assert_eq!( + run(&with_argument, &context, &mut held), + Err("the first-partition probe takes no arguments".to_owned()) + ); for (text, problem) in [ ("copy-up", "the copy-up probe takes one argument"), ("rename a b", "the rename probe takes one argument"), @@ -736,4 +770,40 @@ rw,lowerdir=/run/a\\040b/layers/distro,metacopy=on,xino=on ); assert!(held.is_empty()); } + + #[test] + fn the_first_partition_is_reported_in_hexadecimal() { + let lowers = [ + PathBuf::from("/run/runtime-adapter/layers/p2"), + PathBuf::from("/run/runtime-adapter/layers/p3"), + ]; + let bytes = b"\xff\x00a \x80"; + let mut context = Context { + runtime: Path::new("/run/runtime-adapter"), + root: Path::new("/run/runtime-adapter/rootfs"), + lowers: &lowers, + upper: Path::new("/run/runtime-adapter/tmpfs"), + first_partition: Some(bytes), + agent: Path::new("/sys/fs/cgroup/agent"), + workload: Path::new("/sys/fs/cgroup/workload"), + sibling: None, + }; + let probe = Probe::parse("first-partition").unwrap(); + let mut held = Vec::new(); + let mut expected = Facts::new(); + expected.push("length", "5"); + expected.push("bytes", "ff00612080"); + assert_eq!(run(&probe, &context, &mut held), Ok(expected)); + context.first_partition = Some(b""); + let mut expected = Facts::new(); + expected.push("length", "0"); + expected.push("bytes", ""); + assert_eq!(run(&probe, &context, &mut held), Ok(expected)); + context.first_partition = None; + let mut expected = Facts::new(); + expected.push("length", "none"); + expected.push("bytes", "none"); + assert_eq!(run(&probe, &context, &mut held), Ok(expected)); + assert_eq!(hexadecimal(&[0x0f, 0xa0, 0x5c]), "0fa05c"); + } } diff --git a/tests/runtime-adapter/tests/guest_session.rs b/tests/runtime-adapter/tests/guest_session.rs index c597e3d78..81cf1602b 100644 --- a/tests/runtime-adapter/tests/guest_session.rs +++ b/tests/runtime-adapter/tests/guest_session.rs @@ -1188,15 +1188,37 @@ fn the_worker_assembles_an_image_over_ram_in_the_overlay_mode_of_its_configurati assert_eq!(root.calls, expected); } +#[test] +fn the_worker_assembles_a_gpt_disk_with_the_first_partition_bound_of_its_configuration() { + let gpt = RootLayout { + image: RootImage::gpt(Uuid::parse("6c1f5e0a-2b7d-4c39-9e81-3a5d0f7b2c64")), + upper: RootUpper::Scratch, + }; + let config = RootConfig { + first_partition_bound: Some(4096), + ..RootConfig::default() + }; + let (mut broker, working) = start_worker(FakeRoot::booting(&gpt)); + broker.attach(FIRST); + let messages = [Message::RootConfig(config.encode()), Message::Stop]; + let replies = broker.exchange(&messages, OUTER_MAX_PAYLOAD, 2); + assert_eq!(replies, [Message::RootReady, Message::Stopped]); + let (root, worked) = joined(working); + assert_eq!(worked.unwrap(), Worked::Stopped); + let expected = [ + Call::CommandLine, + Call::Bootstrap(config.clone()), + Call::Assemble(config, gpt), + Call::Teardown, + ]; + assert_eq!(root.calls, expected); +} + #[test] fn the_worker_reports_the_stage_that_failed_and_ends_with_its_status() { let config = RootConfig::default(); let read = || vec![Call::CommandLine]; let bootstrapped = || vec![Call::CommandLine, Call::Bootstrap(config.clone())]; - let gpt = RootLayout { - image: RootImage::gpt(None), - upper: RootUpper::Scratch, - }; let ram = RootLayout { upper: RootUpper::Ram, ..layout() @@ -1247,11 +1269,16 @@ fn the_worker_reports_the_stage_that_failed_and_ends_with_its_status() { read(), ), ( - "a GPT image", - FakeRoot::booting(&gpt), - config.encode(), + "a bound of the first partition, for an EROFS image", + FakeRoot::booting(&layout()), + RootConfig { + first_partition_bound: Some(4096), + ..RootConfig::default() + } + .encode(), RootStage::Layout, - "the worker assembles only an EROFS image, not a gpt image", + "the root configuration bounds the first partition, but the boot tokens name an EROFS \ + image", read(), ), ( diff --git a/tests/runtime-adapter/tests/layered_launch.rs b/tests/runtime-adapter/tests/layered_launch.rs index f6b83520d..fdfff7f8b 100644 --- a/tests/runtime-adapter/tests/layered_launch.rs +++ b/tests/runtime-adapter/tests/layered_launch.rs @@ -199,7 +199,7 @@ fn a_launch_records_what_it_launched() { fn the_host_program_refuses_options_that_do_not_make_a_command() { let launch = ["launch", "store", "--guest-agent", "agent"]; let uuid = "0a490905-41be-ed5f-984e-63eb7e5e20b1"; - let cases: [(&[&str], &str); 19] = [ + let cases: [(&[&str], &str); 23] = [ (&["--distro", "erofs:image"], "--distro needs --upper"), (&["--upper", "ram"], "--upper needs --distro"), ( @@ -226,6 +226,29 @@ fn the_host_program_refuses_options_that_do_not_make_a_command() { ], "--scratch-journal needs --upper scratch:PATH", ), + ( + &["--first-partition-bound", "4096"], + "--first-partition-bound needs --distro", + ), + ( + &[ + "--distro", + "erofs:image", + "--upper", + "ram", + "--first-partition-bound", + "4096", + ], + "--first-partition-bound needs --distro gpt:PATH", + ), + ( + &["--first-partition-bound", "0"], + "--first-partition-bound takes a positive number of bytes, not \"0\"", + ), + ( + &["--first-partition-bound", "64K"], + "--first-partition-bound takes a positive number of bytes, not \"64K\"", + ), ( &["--overlay-mode", "trusted"], "--overlay-mode takes trusted-lowers or untrusted-lowers", diff --git a/tests/runtime-adapter/tests/vm_layered.rs b/tests/runtime-adapter/tests/vm_layered.rs index 95b851c09..ac49d0cda 100644 --- a/tests/runtime-adapter/tests/vm_layered.rs +++ b/tests/runtime-adapter/tests/vm_layered.rs @@ -14,6 +14,12 @@ //! - the scratch template, `ubuntu-smoke-scratch.ext4`, an empty ext4 file system that NVX's CI //! makes with the guest artifacts, of which each scenario boots a copy. //! +//! A scenario of a GPT disk writes the disk first, from the scratch template: its first +//! partition, opaque bytes, then a copy of the template, the base layer, and a small layer that the +//! test writes, the top one, which holds a file and a directory. The test writes it in the ext2 +//! format, which the kernel's ext4 driver mounts, since `mkfs.ext4` is not available everywhere +//! that the tests run. +//! //! ```text //! cargo test --locked --features host --test vm_layered -- --ignored //! ``` @@ -25,10 +31,12 @@ use std::collections::BTreeSet; use std::env; use std::ffi::{OsStr, OsString}; use std::fs::{self, File}; -use std::io::{Read, Seek, SeekFrom}; +use std::io::{self, Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; +use nvx::gpt::{Disk, Expected, ExpectedPartition}; use nvx::host::artifacts::{ArtifactDir, Artifacts, Guest}; +use nvx::uuid::Uuid; use runtime_adapter::layered::RECORD_NAME; use runtime_adapter::root::{DEFAULT_RUNTIME, JournalPolicy, OverlayMode, RootConfig, RootStage}; use serde_json::{Value, json}; @@ -45,6 +53,42 @@ const PAGE: u64 = 4096; /// How many of the bytes available in an upper a file that fills it may leave, which the file's /// own blocks of metadata take. const FULL_SLACK: u64 = 1 << 20; +/// The size of a sector of a disk, its logical block, in bytes. +const SECTOR: u64 = 512; +/// The disk GUID of the scenarios' GPT disks. +const DISK_GUID: &str = "6c1f5e0a-2b7d-4c39-9e81-3a5d0f7b2c64"; +/// The unique GUIDs of the partitions of the scenarios' GPT disks, the first partition's first. +const PARTITION_GUIDS: [&str; 3] = [ + "0b7e4a2c-5d16-4f83-a9c0-1e2d3f4a5b61", + "0b7e4a2c-5d16-4f83-a9c0-1e2d3f4a5b62", + "0b7e4a2c-5d16-4f83-a9c0-1e2d3f4a5b63", +]; +/// The type GUID of every partition of the scenarios' GPT disks: Linux's file system data. +const LINUX_DATA: &str = "0fc63daf-8483-4772-8e79-3d69d8477de4"; +/// The size of the first partition of the scenarios' GPT disks, in bytes. +const FIRST_PARTITION_SIZE: u64 = 4096; +/// The bytes of the first partition that the guest returns: bytes that are not UTF-8, with a NUL +/// and whitespace among them, but not at their end. +const FIRST_PARTITION: &[u8] = b"\xff\xferuntime-adapter\0\t\x80\xc3"; +/// What the first partition holds after [`FIRST_PARTITION`], which the guest trims as the padding +/// that ends its bytes, as it trims the NUL bytes that fill the rest of the partition. +const FIRST_PARTITION_PADDING: &[u8] = b" \0\n\t\r\x0c\0 "; +/// The first sector of each partition of the scenarios' GPT disks is a multiple of this many: 1 +/// MiB, as partitioning tools align them. +const ALIGNMENT: u64 = 2048; +/// The regular file that the top layer of the scenarios' GPT disks holds. +const LAYER_FILE: &str = "layer-file"; +/// The size of [`LAYER_FILE`], in bytes, which spans three of its layer's blocks. +const LAYER_FILE_SIZE: usize = 2500; +/// The directory that the top layer of the scenarios' GPT disks holds. +const LAYER_DIRECTORY: &str = "layer-directory"; +/// The size of a block of the layer that [`small_layer`] writes, in bytes. +const LAYER_BLOCK: usize = 1024; +/// The number of blocks of that layer: 128 KiB. +const LAYER_BLOCKS: usize = 128; +/// The number of inodes of that layer, of which the first ten are reserved, as in every ext2 file +/// system, and the second is its top directory. +const LAYER_INODES: usize = 16; /// The fixtures of the scenarios, among NVX's artifacts. struct Fixtures { @@ -90,10 +134,12 @@ impl Fixtures { } /// The image of a scenario's root. -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] enum Image { /// NVX's Ubuntu EROFS image. Erofs, + /// A GPT disk that the scenario writes; see the [module documentation](self). + Gpt, } /// The identity that a scenario's launch expects of the image. @@ -129,6 +175,8 @@ struct Root { overlay: Option, /// The scratch's journal policy, as the launch names it, unless the guest's default. journal: Option, + /// The bound of the first partition, as the launch names it, unless the guest's default. + first_partition_bound: Option, } /// The root of the scenarios unless they say otherwise: NVX's EROFS image, with its own identity, @@ -140,24 +188,54 @@ const ROOT: Root = Root { runtime: None, overlay: None, journal: None, + first_partition_bound: None, }; +/// The root of the scenarios of a GPT disk unless they say otherwise: the disk, with its own GUID, +/// over a copy of the scratch template, with the guest's defaults. +const GPT: Root = Root { + image: Image::Gpt, + ..ROOT +}; + +/// The image file of a scenario's launch. +struct Distro { + /// The file. + path: PathBuf, + /// The image's format, as the launch names it. + format: &'static str, + /// The identity that the launch expects of the image. + identity: String, +} + impl Root { - /// Returns the identity that the launch expects of the image of `fixtures`. - fn identity(&self, fixtures: &Fixtures) -> String { - match self.identity { - Identity::Own => fixtures.identity.clone(), - Identity::Other => other_identity(&fixtures.identity), + /// Returns the image file of the launch, from the `fixtures`, which writes a GPT disk into + /// `directory` first. + fn distro(&self, fixtures: &Fixtures, directory: &Path) -> Distro { + let (path, format, own) = match self.image { + Image::Erofs => (fixtures.distro.clone(), "erofs", fixtures.identity.clone()), + Image::Gpt => { + let path = directory.join("distro.gpt"); + write_gpt_disk(&path, &fixtures.scratch); + (path, "gpt", DISK_GUID.to_owned()) + } + }; + let identity = match self.identity { + Identity::Own => own, + Identity::Other => other_identity(&own), + }; + Distro { + path, + format, + identity, } } - /// Returns the launch's options of the root, with the `fixtures` and the `scratch` image, for a - /// root over a scratch. - fn options(&self, fixtures: &Fixtures, scratch: &Path) -> Vec { - let Image::Erofs = self.image; - let mut distro = OsString::from("erofs:"); - distro.push(&fixtures.distro); - let identity = self.identity(fixtures); + /// Returns the launch's options of the root, with the image `distro`, and the `scratch` image + /// for a root over a scratch. + fn options(&self, distro: &Distro, scratch: &Path) -> Vec { + let mut image = OsString::from(format!("{}:", distro.format)); + image.push(&distro.path); let upper = match self.upper { Upper::Scratch => { let mut upper = OsString::from("scratch:"); @@ -168,9 +246,9 @@ impl Root { }; let mut options = [ "--distro".into(), - distro, + image, "--distro-identity".into(), - identity.into(), + distro.identity.clone().into(), "--upper".into(), upper, ] @@ -184,6 +262,9 @@ impl Root { if let Some(policy) = self.journal { options.extend(["--scratch-journal".into(), policy.as_str().into()]); } + if let Some(bound) = self.first_partition_bound { + options.extend(["--first-partition-bound".into(), bound.to_string().into()]); + } options } @@ -292,6 +373,42 @@ scenarios! { }, expect: Expect::Ready(check_root), }; + /// A GPT disk over an ext4 scratch, of the disk GUID that the launch expects: its first + /// partition, intact; its layers, each mounted read-only from its partition, with partition + /// 2's at the overlay's base and partition 3's at its top; a copy-up of metadata alone, the + /// rename of a lower directory, and the guest's disks. + gpt_over_scratch: Scenario { + root: GPT, + expect: Expect::Ready(check_root), + }; + /// The disk over the scratch, in the untrusted-lowers mode: a copy-up of the data too, and a + /// rename of a lower directory that fails. + gpt_over_scratch_with_untrusted_lowers: Scenario { + root: Root { + overlay: Some(OverlayMode::UntrustedLowers), + ..GPT + }, + expect: Expect::Ready(check_root), + }; + /// The disk over a tmpfs in RAM, without a scratch device, with a bound that the first + /// partition meets exactly. + gpt_over_ram: Scenario { + root: Root { + upper: Upper::Ram, + first_partition_bound: Some(FIRST_PARTITION_SIZE), + ..GPT + }, + expect: Expect::Ready(check_root), + }; + /// The disk over a tmpfs in RAM, in the untrusted-lowers mode. + gpt_over_ram_with_untrusted_lowers: Scenario { + root: Root { + upper: Upper::Ram, + overlay: Some(OverlayMode::UntrustedLowers), + ..GPT + }, + expect: Expect::Ready(check_root), + }; /// A scratch that a file fills, which then refuses writes until the file is gone. full_scratch_refuses_writes: Scenario { root: ROOT, @@ -316,6 +433,23 @@ scenarios! { root: Root { identity: Identity::Other, ..ROOT }, expect: Expect::Fails(RootStage::Assembly, "the distro layer's UUID is "), }; + /// A GPT disk of another GUID than the launch expects, which the assembly refuses. + other_disk_guid_fails_the_assembly: Scenario { + root: Root { identity: Identity::Other, ..GPT }, + expect: Expect::Fails( + RootStage::Assembly, + "invalid partition table: the disk GUID is 6c1f5e0a-2b7d-4c39-9e81-3a5d0f7b2c64, not \ + the expected ", + ), + }; + /// A first partition one byte larger than the launch's bound, which the assembly refuses. + first_partition_over_its_bound_fails_the_assembly: Scenario { + root: Root { first_partition_bound: Some(FIRST_PARTITION_SIZE - 1), ..GPT }, + expect: Expect::Fails( + RootStage::Assembly, + "the first partition has 4096 bytes, more than the bound of 4095", + ), + }; /// A relative runtime directory, which the guest refuses with the configuration. invalid_configuration_fails_its_stage: Scenario { root: Root { runtime: Some("run/runtime-adapter"), ..ROOT }, @@ -355,7 +489,8 @@ fn run(name: &str, scenario: Scenario) { if over_scratch { fs::copy(&fixtures.scratch, &scratch).unwrap(); } - let options = root.options(&fixtures, &scratch); + let distro = root.distro(&fixtures, &sandbox.root); + let options = root.options(&distro, &scratch); let options = options.iter().map(OsString::as_os_str).collect::>(); let (check, failure) = match scenario.expect { Expect::Fails(stage, problem) => { @@ -364,7 +499,7 @@ fn run(name: &str, scenario: Scenario) { assert_eq!(launched["stopped"], "graceful"); // The launch recorded what it launched, with the root configuration that it sent, which // the guest refused, or accepted before a later stage failed. - check_record(&sandbox, &fixtures, &root, &scratch); + check_record(&sandbox, &root, &distro, &scratch); // The launch ended: no runtime record names the VM. assert_eq!(sandbox.record(), None); assert!( @@ -378,7 +513,7 @@ fn run(name: &str, scenario: Scenario) { }; let launched = sandbox.launch(&options, 0); assert_eq!(launched["root"], "ready"); - let config = check_record(&sandbox, &fixtures, &root, &scratch); + let config = check_record(&sandbox, &root, &distro, &scratch); let vmm = sandbox .vmm() .expect("the VM runs after its launcher exited"); @@ -433,21 +568,20 @@ fn assert_failed(printed: &Printed, stage: RootStage, problem: &str) { assert_eq!(printed["teardown"], "complete", "{printed:?}"); } -/// Checks what the launch recorded in the sandbox's store: a VM with the layered `root` of -/// `fixtures`, over `scratch` or RAM, the image of the identity that the launch expects, the boot -/// tokens of its plan, and the root configuration that the host sent, whether or not the guest -/// accepted it, which it returns. -fn check_record(sandbox: &Sandbox, fixtures: &Fixtures, root: &Root, scratch: &Path) -> RootConfig { +/// Checks what the launch recorded in the sandbox's store: a VM with the layered `root` of the +/// image `distro`, over `scratch` or RAM, the image of the identity that the launch expects, the +/// boot tokens of its plan, and the root configuration that the host sent, whether or not the +/// guest accepted it, which it returns. +fn check_record(sandbox: &Sandbox, root: &Root, distro: &Distro, scratch: &Path) -> RootConfig { let record = fs::read_to_string(sandbox.store.join(RECORD_NAME)).unwrap(); let record: Value = serde_json::from_str(&record).unwrap(); assert_eq!(record["format"], 1); assert_eq!(record["mode"], "layered"); let plan = &record["plan"]; - let identity = root.identity(fixtures); - assert_eq!(plan["distro"]["format"], "erofs"); - assert_eq!(plan["distro"]["identity"], identity.as_str()); + assert_eq!(plan["distro"]["format"], distro.format); + assert_eq!(plan["distro"]["identity"], distro.identity.as_str()); let path = |value: &Value| PathBuf::from(value.as_str().unwrap()); - assert_eq!(path(&plan["distro"]["path"]), fixtures.distro); + assert_eq!(path(&plan["distro"]["path"]), distro.path); let upper = match root.upper { Upper::Scratch => { assert_eq!(plan["upper"]["kind"], "scratch"); @@ -464,7 +598,10 @@ fn check_record(sandbox: &Sandbox, fixtures: &Fixtures, root: &Root, scratch: &P .unwrap() .split(' ') .collect::>(); - let layer = format!("nvx_layer=distro,0xd0003000,erofs:{identity}"); + let layer = format!( + "nvx_layer=distro,0xd0003000,{}:{}", + distro.format, distro.identity + ); for token in ["nvx_sandbox=1", &layer, upper] { assert!(tokens.contains(token), "{token} in {tokens:?}"); } @@ -478,10 +615,11 @@ fn check_record(sandbox: &Sandbox, fixtures: &Fixtures, root: &Root, scratch: &P let sectors = |path: &Path| Some(fs::metadata(path).unwrap().len() / 512); let config = RootConfig { runtime: root.runtime.unwrap_or(DEFAULT_RUNTIME).to_owned(), - distro_sectors: sectors(&fixtures.distro), + distro_sectors: sectors(&distro.path), scratch_sectors: (root.upper == Upper::Scratch) .then(|| sectors(scratch)) .flatten(), + first_partition_bound: root.first_partition_bound, overlay_mode: root.overlay, scratch_journal: root.journal, ..RootConfig::default() @@ -514,15 +652,24 @@ fn options(list: &str) -> BTreeSet<&str> { list.split(',').collect() } -/// Checks the root of a scenario's VM, which the guest assembled, as its upper and its overlay -/// mode make it: its mounts, its contents, a copy-up, and the guest's disks. +/// Checks the root of a scenario's VM, which the guest assembled, as its image, its upper, and its +/// overlay mode make it: its mounts, its contents, its first partition, a copy-up, a rename, and +/// the guest's disks. fn check_root(vm: &mut Vm<'_>) { + let image = vm.root.image; + let (lowers, file, directory) = match image { + Image::Erofs => (&["distro"][..], "usr/lib/os-release", "mnt"), + Image::Gpt => (&["p2", "p3"][..], LAYER_FILE, LAYER_DIRECTORY), + }; + let copy_up = format!("copy-up {file}"); + let rename = format!("rename {directory}"); let facts = vm.probe(&[ "mounts", "contents", - "copy-up usr/lib/os-release", - "rename mnt", + ©_up, + &rename, "devices", + "first-partition", ]); let runtime = vm.runtime; let (upper, kind, other) = match vm.root.upper { @@ -530,13 +677,39 @@ fn check_root(vm: &mut Vm<'_>) { Upper::Ram => ("tmpfs", "tmpfs", "scratch"), }; - // The image, read-only, under the overlay, whose upper and work directories are on the upper, - // all once, below the runtime directory, where no other upper is mounted. - for (mount, kind, flags) in [ - ("distro", "erofs", &["ro"][..]), - (upper, kind, &["rw", "nosuid", "nodev"]), - ("root", "overlay", &["rw", "nosuid", "nodev"]), - ] { + // The guest's disks, whatever their names: the image's, read-only, and the scratch's, + // writable, unless the upper is in RAM. + let disks = facts["devices.disks"].split(' ').collect::>(); + let mut sizes = disks + .iter() + .map(|disk| disk.split_once(':').map_or(*disk, |(_, size)| size)) + .collect::>(); + sizes.sort_unstable(); + let mut expected = vec![format!("{}:ro", vm.config.distro_sectors.unwrap())]; + expected.extend( + vm.config + .scratch_sectors + .map(|sectors| format!("{sectors}:rw")), + ); + expected.sort_unstable(); + assert_eq!(sizes, expected, "{facts:?}"); + + // The image's layers, read-only, under the overlay, whose upper and work directories are on + // the upper, all once, below the runtime directory, where no other upper is mounted. The + // layers of a GPT disk are ext4 file systems on its partitions, which mount without `noload`, + // whatever their own default options. + let layer = match image { + Image::Erofs => ("erofs", &["ro"][..]), + Image::Gpt => ("ext4", &["ro", "nosuid", "nodev"][..]), + }; + let mounts = lowers + .iter() + .map(|lower| (*lower, layer.0, layer.1)) + .chain([ + (upper, kind, &["rw", "nosuid", "nodev"][..]), + ("root", "overlay", &["rw", "nosuid", "nodev"]), + ]); + for (mount, kind, flags) in mounts { assert_eq!(facts[&format!("mounts.{mount}.count")], "1", "{mount}"); assert_eq!(facts[&format!("mounts.{mount}.type")], kind, "{mount}"); let found = options(&facts[&format!("mounts.{mount}.options")]); @@ -546,6 +719,24 @@ fn check_root(vm: &mut Vm<'_>) { ); } assert_eq!(facts[&format!("mounts.{other}.count")], "0", "{other}"); + if image == Image::Gpt { + let disk = disks + .iter() + .find_map(|disk| disk.strip_suffix(":ro")) + .and_then(|disk| disk.split_once(':')) + .map(|(name, _)| name) + .expect("the guest has the disk's device"); + for lower in lowers { + let partition = lower.trim_start_matches('p'); + let source = &facts[&format!("mounts.{lower}.source")]; + assert_eq!(source, &format!("/dev/{disk}{partition}"), "{lower}"); + for fact in ["options", "super"] { + let found = options(&facts[&format!("mounts.{lower}.{fact}")]); + let skipping = ["noload", "norecovery"].map(|option| found.contains(option)); + assert_eq!(skipping, [false; 2], "{lower}: {found:?}"); + } + } + } if vm.root.upper == Upper::Ram { // A tmpfs of the kernel's default size, whose root directory has mode 0755. let tmpfs = options(&facts["mounts.tmpfs.super"]); @@ -553,8 +744,15 @@ fn check_root(vm: &mut Vm<'_>) { let sized = tmpfs.iter().any(|option| option.starts_with("size=")); assert!(!sized, "{tmpfs:?}"); } + // The overlay stacks the layers with the base last, after the top. + let stacked = lowers + .iter() + .rev() + .map(|lower| format!("{runtime}/layers/{lower}")) + .collect::>() + .join(":"); let layers = [ - format!("lowerdir={runtime}/layers/distro"), + format!("lowerdir={stacked}"), format!("upperdir={runtime}/{upper}/upper"), format!("workdir={runtime}/{upper}/work"), ]; @@ -564,7 +762,7 @@ fn check_root(vm: &mut Vm<'_>) { } // The overlay's mode. Trusting its lowers, the overlay copies up a file's metadata alone, - // which leaves the data in the image, and renames a lower directory, with the options that + // which leaves the data in its layer, and renames a lower directory, with the options that // the mode and metadata copy-up set. Otherwise, it copies up the data too, and refuses the // rename, with these options off. let trusting = ["metacopy=on", "redirect_dir=on", "xino=on"]; @@ -583,30 +781,30 @@ fn check_root(vm: &mut Vm<'_>) { assert_eq!(facts["rename.outcome"], rename, "{facts:?}"); // The merged root: the image's files, and a file that the worker wrote, which reaches the - // upper and not the image, which refuses writes. - assert_eq!(facts["contents.os-release.id"], "ubuntu"); + // upper and not the top layer, which refuses writes. A GPT disk's layers hold no + // `os-release`. + let identifier = match image { + Image::Erofs => "ubuntu", + Image::Gpt => "absent", + }; + assert_eq!(facts["contents.os-release.id"], identifier); assert_eq!(facts["contents.root.mode"], "755"); assert_eq!(facts["contents.write.root"], "runtime-adapter"); assert_eq!(facts["contents.write.upper"], "runtime-adapter"); assert_eq!(facts["contents.write.lower"], "absent"); assert_eq!(facts["contents.lower.write"], "refused"); - // The guest's disks, whatever their names: the image's, read-only, and the scratch's, - // writable, unless the upper is in RAM. - let mut disks = facts["devices.disks"] - .split(' ') - .map(|disk| disk.split_once(':').map_or(disk, |(_, size)| size)) - .collect::>(); - disks.sort_unstable(); - let distro = format!("{}:ro", vm.config.distro_sectors.unwrap()); - let mut expected = vec![distro]; - expected.extend( - vm.config - .scratch_sectors - .map(|sectors| format!("{sectors}:rw")), - ); - expected.sort_unstable(); - assert_eq!(disks, expected, "{facts:?}"); + // The first partition of a GPT disk, which the assembly returned as it was, without its + // padding. + let (length, bytes) = match image { + Image::Erofs => ("none".to_owned(), "none".to_owned()), + Image::Gpt => ( + FIRST_PARTITION.len().to_string(), + hexadecimal(FIRST_PARTITION), + ), + }; + assert_eq!(facts["first-partition.length"], length, "{facts:?}"); + assert_eq!(facts["first-partition.bytes"], bytes, "{facts:?}"); } /// Fills the upper of a scenario's root through the root, and checks that the upper refuses @@ -807,3 +1005,319 @@ fn check_refused_mapping(vm: &mut Vm<'_>) { assert_eq!(fact("mounts"), "0"); assert_eq!(fact("outside"), "0"); } + +/// Writes the GPT disk of the scenarios at `path`, as UEFI lays out a disk, with a protective MBR, +/// the primary GPT header and its array of 128 entries, and the backup array and header at the +/// disk's end, of its partitions in this order, each aligned to [`ALIGNMENT`]: +/// +/// 1. the first partition, of [`FIRST_PARTITION_SIZE`] bytes, which holds [`FIRST_PARTITION`], +/// then [`FIRST_PARTITION_PADDING`], then NUL bytes; +/// 2. the base layer, a copy of the scratch template `template`, an ext4 file system that +/// `mkfs.ext4` made, with a journal; +/// 3. the top layer, a [`small_layer`] that holds [`LAYER_FILE`] and [`LAYER_DIRECTORY`]. +/// +/// Then checks the disk as the guest's assembly does, through `nvx::gpt`, with the GUIDs and +/// sizes that it wrote. +fn write_gpt_disk(path: &Path, template: &Path) { + let template_size = fs::metadata(template).unwrap().len(); + assert_eq!( + template_size % SECTOR, + 0, + "the scratch template is whole sectors" + ); + let contents = b"runtime-adapter " + .iter() + .copied() + .cycle() + .take(LAYER_FILE_SIZE) + .collect::>(); + let top = small_layer(&[(LAYER_FILE, contents.as_slice())], &[LAYER_DIRECTORY]); + let sizes = [ + FIRST_PARTITION_SIZE, + template_size, + u64::try_from(top.len()).unwrap(), + ]; + let mut extents = Vec::new(); + let mut next = ALIGNMENT; + for size in sizes { + let last = next + size / SECTOR - 1; + extents.push((next, last)); + next = (last + 1).next_multiple_of(ALIGNMENT); + } + // The backup array's 32 sectors and the backup header end the disk, after the usable sectors. + let sectors = next + ALIGNMENT; + let (first_usable, last_usable) = (34_u64, sectors - 34); + let (backup_entries, backup_header) = (sectors - 33, sectors - 1); + let guid = |text: &str| Uuid::parse(text).unwrap().to_bytes_le(); + + let mut entries = vec![0; 128 * 128]; + for (entry, ((first, last), unique)) in entries + .chunks_exact_mut(128) + .zip(extents.iter().zip(PARTITION_GUIDS)) + { + entry[0..16].copy_from_slice(&guid(LINUX_DATA)); + entry[16..32].copy_from_slice(&guid(unique)); + entry[32..40].copy_from_slice(&first.to_le_bytes()); + entry[40..48].copy_from_slice(&last.to_le_bytes()); + } + let entries_crc = crc32(&entries); + let header = |own: u64, alternate: u64, array: u64| { + let mut header = vec![0; 92]; + header[0..8].copy_from_slice(b"EFI PART"); + header[8..12].copy_from_slice(&0x0001_0000_u32.to_le_bytes()); + header[12..16].copy_from_slice(&92_u32.to_le_bytes()); + header[24..32].copy_from_slice(&own.to_le_bytes()); + header[32..40].copy_from_slice(&alternate.to_le_bytes()); + header[40..48].copy_from_slice(&first_usable.to_le_bytes()); + header[48..56].copy_from_slice(&last_usable.to_le_bytes()); + header[56..72].copy_from_slice(&guid(DISK_GUID)); + header[72..80].copy_from_slice(&array.to_le_bytes()); + header[80..84].copy_from_slice(&128_u32.to_le_bytes()); + header[84..88].copy_from_slice(&128_u32.to_le_bytes()); + header[88..92].copy_from_slice(&entries_crc.to_le_bytes()); + let crc = crc32(&header); + header[16..20].copy_from_slice(&crc.to_le_bytes()); + header + }; + // The protective MBR: one record of type 0xEE, from LBA 1 over the rest of the disk, with the + // addresses that UEFI gives it. + let mut mbr = vec![0; 512]; + let protective = u32::try_from(sectors - 1).unwrap_or(u32::MAX); + mbr[446..450].copy_from_slice(&[0x00, 0x00, 0x02, 0x00]); + mbr[450..454].copy_from_slice(&[0xee, 0xff, 0xff, 0xff]); + mbr[454..458].copy_from_slice(&1_u32.to_le_bytes()); + mbr[458..462].copy_from_slice(&protective.to_le_bytes()); + mbr[510..512].copy_from_slice(&[0x55, 0xaa]); + + let mut disk = File::create(path).unwrap(); + disk.set_len(sectors * SECTOR).unwrap(); + let parts = [ + (0, mbr), + (1, header(1, backup_header, 2)), + (2, entries.clone()), + (backup_entries, entries), + (backup_header, header(backup_header, 1, backup_entries)), + ( + extents[0].0, + [FIRST_PARTITION, FIRST_PARTITION_PADDING].concat(), + ), + (extents[2].0, top), + ]; + for (sector, bytes) in parts { + disk.seek(SeekFrom::Start(sector * SECTOR)).unwrap(); + disk.write_all(&bytes).unwrap(); + } + disk.seek(SeekFrom::Start(extents[1].0 * SECTOR)).unwrap(); + io::copy(&mut File::open(template).unwrap(), &mut disk).unwrap(); + disk.sync_all().unwrap(); + drop(disk); + + let disk = File::open(path).unwrap(); + let partitions = PARTITION_GUIDS + .iter() + .zip(sizes) + .map(|(unique, size)| { + ExpectedPartition::new() + .unique_guid(Uuid::parse(unique).unwrap()) + .size(size) + }) + .collect(); + let expected = Expected::new() + .disk_guid(Uuid::parse(DISK_GUID).unwrap()) + .partitions(partitions); + let read = Disk::read(&disk, 512, sectors * SECTOR, &expected).unwrap(); + let first_partition = read + .read_first_partition(&disk, usize::try_from(FIRST_PARTITION_SIZE).unwrap()) + .unwrap(); + assert_eq!(first_partition, FIRST_PARTITION); +} + +/// Returns a small file system in the ext2 format, of [`LAYER_BLOCKS`] blocks of [`LAYER_BLOCK`] +/// bytes, which the kernel's ext4 driver mounts as an ext4 file system without features: its top +/// directory holds the regular `files`, each a name and its bytes, of at most 12 blocks each, +/// then the empty `directories`, each with mode 0755, and at most [`LAYER_INODES`] inodes in all. +/// +/// Block 1 holds the superblock, 2 the group descriptor, 3 and 4 the block and inode bitmaps, 5 +/// and 6 the inode table, 7 the top directory, and the blocks after them the files' bytes and the +/// directories. +fn small_layer(files: &[(&str, &[u8])], directories: &[&str]) -> Vec { + /// Writes `value` at `offset` of `image`, in little-endian order. + fn put(image: &mut [u8], offset: usize, value: [u8; N]) { + image[offset..offset + N].copy_from_slice(&value); + } + /// Writes the inode `number` of `image`, of `mode`, `size` bytes, and `links` links, whose data + /// is in `blocks`, at most the 12 that an inode addresses directly. + fn inode( + image: &mut [u8], + number: usize, + mode: u16, + size: usize, + links: usize, + blocks: &[usize], + ) { + assert!( + blocks.len() <= 12, + "an inode of the layer addresses 12 blocks" + ); + let offset = 5 * LAYER_BLOCK + (number - 1) * 128; + put(image, offset, mode.to_le_bytes()); + put( + image, + offset + 4, + u32::try_from(size).unwrap().to_le_bytes(), + ); + put( + image, + offset + 26, + u16::try_from(links).unwrap().to_le_bytes(), + ); + // The number of 512-byte sectors that the data takes. + let sectors = u32::try_from(blocks.len() * LAYER_BLOCK / 512).unwrap(); + put(image, offset + 28, sectors.to_le_bytes()); + for (index, block) in blocks.iter().enumerate() { + let block = u32::try_from(*block).unwrap(); + put(image, offset + 40 + 4 * index, block.to_le_bytes()); + } + } + /// Writes the directory `entries` into `block` of `image`, each an inode, a name, and a file + /// type, the last of which spans the rest of the block. + fn directory(image: &mut [u8], block: usize, entries: &[(usize, &str, u8)]) { + let (mut offset, end) = (block * LAYER_BLOCK, (block + 1) * LAYER_BLOCK); + for (index, (inode, name, kind)) in entries.iter().enumerate() { + let length = if index + 1 == entries.len() { + end - offset + } else { + (8 + name.len()).next_multiple_of(4) + }; + put(image, offset, u32::try_from(*inode).unwrap().to_le_bytes()); + put( + image, + offset + 4, + u16::try_from(length).unwrap().to_le_bytes(), + ); + image[offset + 6] = u8::try_from(name.len()).unwrap(); + image[offset + 7] = *kind; + image[offset + 8..offset + 8 + name.len()].copy_from_slice(name.as_bytes()); + offset += length; + } + } + /// Sets the bits `bits` of the bitmap in `block` of `image`. + fn set(image: &mut [u8], block: usize, bits: std::ops::Range) { + for bit in bits { + image[block * LAYER_BLOCK + bit / 8] |= 1 << (bit % 8); + } + } + + let (top, regular, directory_type) = (2, 1, 2); + let mut image = vec![0; LAYER_BLOCK * LAYER_BLOCKS]; + // The first block and the first inode that are free. + let (mut block, mut number) = (8, 11); + let mut entries = vec![(top, ".", directory_type), (top, "..", directory_type)]; + for (name, bytes) in files { + let blocks = (block..block + bytes.len().div_ceil(LAYER_BLOCK)).collect::>(); + image[block * LAYER_BLOCK..block * LAYER_BLOCK + bytes.len()].copy_from_slice(bytes); + inode(&mut image, number, 0o100_644, bytes.len(), 1, &blocks); + entries.push((number, *name, regular)); + block += blocks.len(); + number += 1; + } + for name in directories { + directory( + &mut image, + block, + &[(number, ".", directory_type), (top, "..", directory_type)], + ); + inode(&mut image, number, 0o040_755, LAYER_BLOCK, 2, &[block]); + entries.push((number, *name, directory_type)); + block += 1; + number += 1; + } + directory(&mut image, 7, &entries); + inode( + &mut image, + top, + 0o040_755, + LAYER_BLOCK, + 2 + directories.len(), + &[7], + ); + assert!( + block <= LAYER_BLOCKS && number <= LAYER_INODES + 1, + "the layer is too small" + ); + + // The bitmaps of the used blocks, from block 1, and inodes, from inode 1, whose bits past the + // last block and inode are set, as `mke2fs` sets them. + set(&mut image, 3, 0..block - 1); + set(&mut image, 3, LAYER_BLOCKS - 1..LAYER_BLOCK * 8); + set(&mut image, 4, 0..number - 1); + set(&mut image, 4, LAYER_INODES..LAYER_BLOCK * 8); + let free_blocks = LAYER_BLOCKS - block; + let free_inodes = LAYER_INODES + 1 - number; + + // The superblock: one group of blocks, inodes of 128 bytes, the first of them not reserved, + // 11, and only the `filetype` feature. + let superblock = LAYER_BLOCK; + for (offset, value) in [ + (0x00, LAYER_INODES), + (0x04, LAYER_BLOCKS), + (0x0c, free_blocks), + (0x10, free_inodes), + (0x14, 1), + (0x20, LAYER_BLOCK * 8), + (0x24, LAYER_BLOCK * 8), + (0x28, LAYER_INODES), + (0x4c, 1), + (0x54, 11), + (0x60, 0x2), + ] { + let value = u32::try_from(value).unwrap(); + put(&mut image, superblock + offset, value.to_le_bytes()); + } + for (offset, value) in [ + (0x36, 0xffff), + (0x38, 0xef53), + (0x3a, 1), + (0x3c, 1), + (0x58, 128), + ] { + put(&mut image, superblock + offset, u16::to_le_bytes(value)); + } + let uuid = Uuid::parse("2d4e6f80-91a2-4b3c-8d5e-6f708192a3b4").unwrap(); + image[superblock + 0x68..superblock + 0x78].copy_from_slice(uuid.as_bytes()); + + // The group descriptor: where the bitmaps and the inode table are, and what is free. + let descriptor = 2 * LAYER_BLOCK; + for (offset, value) in [(0x00, 3), (0x04, 4), (0x08, 5)] { + put(&mut image, descriptor + offset, u32::to_le_bytes(value)); + } + for (offset, value) in [ + (0x0c, free_blocks), + (0x0e, free_inodes), + (0x10, 1 + directories.len()), + ] { + let value = u16::try_from(value).unwrap(); + put(&mut image, descriptor + offset, value.to_le_bytes()); + } + image +} + +/// Returns the CRC-32 of `bytes`, which GPT's headers and entry arrays hold: of the reflected +/// polynomial 0xEDB88320, from all ones, inverted. +fn crc32(bytes: &[u8]) -> u32 { + !bytes.iter().fold(!0, |crc, byte| { + (0..8).fold(crc ^ u32::from(*byte), |crc: u32, _| { + (crc >> 1) ^ (0xedb8_8320 & (crc & 1).wrapping_neg()) + }) + }) +} + +/// Returns `bytes` in lowercase hexadecimal, as the `first-partition` probe reports them. +fn hexadecimal(bytes: &[u8]) -> String { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + bytes + .iter() + .flat_map(|byte| [byte >> 4, byte & 0xf]) + .map(|digit| char::from(DIGITS[usize::from(digit)])) + .collect() +} From e8c330a3e8f1777c79d10c722b329920e81efe10 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Thu, 22 Oct 2026 16:50:55 -0700 Subject: [PATCH 3/4] nvx-rs: refuse a runtime directory where a layer of either image is mounted A root of an EROFS image mounts its layer at layers/distro, and one of a GPT disk mounts a layer at layers/pN for each partition after the first. An assembly checked only its own layers' mount points, so a layer that a failed teardown left mounted, of the other image or of a disk with more layers, went unnoticed, as an upper of the other kind did before. The assembly now also checks every entry of layers, in order, and refuses the runtime directory when a file system, or a bind mount of a directory, is mounted at one. Tests cover a GPT disk's top layer that refuses an EROFS image's root and a root of a disk with fewer layers, and an EROFS image that refuses a GPT disk's root. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 30bab0d7-7c3e-4120-9add-6c9cc1dc9e7e --- nvx-rs/src/guest/root.rs | 29 +++++++++++++++---- nvx-rs/src/guest/root/tests.rs | 51 ++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/nvx-rs/src/guest/root.rs b/nvx-rs/src/guest/root.rs index e81653295..acce186cf 100644 --- a/nvx-rs/src/guest/root.rs +++ b/nvx-rs/src/guest/root.rs @@ -52,8 +52,9 @@ //! an assembly refuses a runtime directory that another root holds, in this process or another, //! and one where a file system, or a bind mount of a directory, is mounted already at any of these //! mount points but `staging`, whether or not the new root mounts there, such as an upper of -//! either kind that a failed teardown left; a live share refuses a `staging` where one is mounted -//! already. The lock is advisory: it keeps out the roots of these operations, not other mounts. +//! either kind, or a layer of either image, that a failed teardown left; a live share refuses a +//! `staging` where one is mounted already. The lock is advisory: it keeps out the roots of these +//! operations, not other mounts. //! //! The overlay's options name its layers' and its upper's directories by these paths, and //! `mount(2)` passes a file system at most [`MOUNT_OPTIONS_LIMIT`] bytes of options. An assembly @@ -412,9 +413,9 @@ impl OverlayMode { /// `tmpfs`, and of the root, `rootfs`, a component at a time, and refuses a component that /// exists as anything but a directory, a symbolic link included; takes the runtime directory's /// exclusive lock, which the root holds from then on, and refuses the runtime directory if -/// another root holds it, or if one of these, the other upper's, or `shares`, where a root -/// mounts an aggregate live share, is a mount point already, whether of a file system or of a -/// bind mount of a directory; +/// another root holds it, or if one of these, the other upper's, any entry of `layers`, or +/// `shares`, where a root mounts an aggregate live share, is a mount point already, whether of +/// a file system or of a bind mount of a directory; /// 4. finds the distro device, polling for it every 50 ms until `timeout` passes, checks that it /// is read-only and, if the caller expects a size, of that size, and flushes its buffers; /// 5. for an EROFS image, checks the expected UUID, if any, in its superblock, and reads nothing @@ -668,6 +669,24 @@ impl Root { for point in [other_upper, AGGREGATE_DIRECTORY] { self.check_unoccupied(&runtime.join(point), true)?; } + // Or a layer of either image, at any mount point below `layers`: an EROFS image's, or a + // GPT disk's, which may have had more layers than this root's. + let layers = runtime.join(LAYERS_DIRECTORY); + let mut entries = Vec::new(); + let listing = fs::read_dir(&layers).and_then(|listing| { + for entry in listing { + entries.push(entry?.path()); + } + Ok(()) + }); + listing.map_err(|error| Error::MountPoint { + path: layers.clone(), + error, + })?; + entries.sort(); + for entry in &entries { + self.check_unoccupied(entry, true)?; + } let device = block::prepare( self.kernel.as_mut(), paths, diff --git a/nvx-rs/src/guest/root/tests.rs b/nvx-rs/src/guest/root/tests.rs index 0bf28d3dd..85d7319c8 100644 --- a/nvx-rs/src/guest/root/tests.rs +++ b/nvx-rs/src/guest/root/tests.rs @@ -2207,6 +2207,57 @@ fn a_teardown_unmounts_the_layers_top_first_and_reports_every_failure() { assert_eq!(guest.kernel.unmounts(), [guest.text("run/nvx/layers/p2")]); } +#[test] +fn a_layer_that_a_failed_unmount_left_keeps_the_runtime_directory_occupied() { + // A GPT disk's top layer, left mounted, refuses a root of an EROFS image, and one of a disk + // with fewer layers, which mounts nothing there itself. + let guest = Guest::new(); + guest.insert(&GptDisk::new(b"configuration", 3)); + let top = guest.path("run/nvx/layers/p4"); + guest + .kernel + .fail(Operation::Unmount, top.clone(), Errno::EBUSY); + let assembly = guest.gpt_assembly(Upper::Tmpfs, OverlayMode::TrustedLowers); + guest.assemble(&assembly).unwrap().teardown().unwrap_err(); + guest.insert(&GptDisk::new(b"configuration", 1)); + let (mounts, calls) = (guest.kernel.mounts().len(), guest.kernel.calls()); + let erofs = Assembly { + upper: Upper::Tmpfs, + ..guest.assembly() + }; + for assembly in [erofs, assembly] { + let error = guest.assemble(&assembly).unwrap_err(); + assert!( + matches!(error.error(), Error::Occupied { path } if *path == top), + "{error:?}" + ); + assert_eq!(guest.kernel.mounts().len(), mounts); + assert_eq!(guest.kernel.calls(), calls); + } + + // An EROFS image, left mounted, refuses a root of a GPT disk. + let guest = Guest::new(); + let distro = guest.path("run/nvx/layers/distro"); + guest + .kernel + .fail(Operation::Unmount, distro.clone(), Errno::EBUSY); + let assembly = Assembly { + upper: Upper::Tmpfs, + ..guest.assembly() + }; + guest.assemble(&assembly).unwrap().teardown().unwrap_err(); + guest.insert(&GptDisk::new(b"configuration", 1)); + let (mounts, calls) = (guest.kernel.mounts().len(), guest.kernel.calls()); + let assembly = guest.gpt_assembly(Upper::Tmpfs, OverlayMode::TrustedLowers); + let error = guest.assemble(&assembly).unwrap_err(); + assert!( + matches!(error.error(), Error::Occupied { path } if *path == distro), + "{error:?}" + ); + assert_eq!(guest.kernel.mounts().len(), mounts); + assert_eq!(guest.kernel.calls(), calls); +} + /// Returns a child of an aggregate share. fn child(name: &str, target: &str, access: Access, kind: ShareKind) -> ShareChild { ShareChild { From f7cd8f3da1d79c30540888ec997aa827fc0e5843 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Fri, 23 Oct 2026 00:26:15 -0700 Subject: [PATCH 4/4] nvx-rs: keep a GPT layer's errors from panicking the guest A layer's ext4 superblock chooses how the kernel responds to an error that it finds in the layer: its error policy, or errors=panic among its default mount options, makes the kernel panic, read-only device or not. A corrupt layer would then end the guest instead of failing the assembly with a typed error. The assembly now mounts each layer of a GPT disk with errors=remount-ro, which overrides both, so an error fails the operation that found it. A new scenario of the adapter's layered VMs boots a GPT disk whose top layer's top directory has no mode and whose superblock asks for a panic: the kernel refuses to mount the layer, and the assembly fails, while the guest keeps running. Without this change, the guest panics. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 30bab0d7-7c3e-4120-9add-6c9cc1dc9e7e --- nvx-rs/README.md | 2 +- nvx-rs/src/guest/root.rs | 16 ++++++-- nvx-rs/src/guest/root/tests.rs | 2 +- tests/runtime-adapter/README.md | 23 ++++++----- tests/runtime-adapter/tests/vm_layered.rs | 48 ++++++++++++++++++++--- 5 files changed, 71 insertions(+), 20 deletions(-) diff --git a/nvx-rs/README.md b/nvx-rs/README.md index 4e4f5eac9..97e35ef02 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -99,7 +99,7 @@ nvx-rs/ | `nvx::guest::exec::binding` | `Binding`: an `nvx::exec::Runner` bound to an `Execution`, the glue that a guest adapter would otherwise write. The runner's read allowances become grants, its discards discards, the input that it accepted writes and then a close, its terminations a cancellation, a disconnection, or a termination, and its settlement a kill of the execution's cgroup and checks until no process occupies it; without a cgroup, nothing proves that the program's descendants are gone, so the execution is fenced. It feeds the execution's events back to the runner, statuses that a caller who reaps hands over with `observe` included, and never blocks: the caller polls its descriptors and its input writer until its deadline, and each call takes a bounded number of steps, so that no execution holds the caller's loop | | `nvx::guest::time` | x86_64: `TimeAbi`, the steps of NVX time ABI v1 as typed calls in the caller's process, for any guest init or agent: the cold-boot time step and, separately, the start of the time daemon in a child process that the caller supervises, the VM generation ID, a `CaptureRequest` that repairs a `Restore`, and a `RestoreFinish`. A call takes the permission for OpenVMM's ports on the calling thread for each port transaction alone, and fails with `TimeError::PortRelease` if the thread may still hold some of it, which a process that it started would inherit, and from which `nvx::guest::spawn` starts none. No call waits for a child that it did not start. `nvx-time` runs its steps through them | | `nvx::guest::reseed` | `Seed::read` and `reseed`: a restore's fresh entropy, which starts with its `GenerationId`, and the reseed of the kernel's random number generator with it, in the caller's process. `nvx-reseed` runs them | -| `nvx::guest::root` | The operations of a layered root, for a guest init or agent, whether or not it is PID 1. `bootstrap`: the agent's and the workload's sibling cgroups of a `Layout`, with every process of their parent moved into the agent's, the agent's `memory.low`, and the workload cgroup prepared through `ExecutionCgroup::prepare`, with its `memory.max` and `pids.max`, and user namespaces kept unavailable. `assemble`: the `Assembly`'s block devices, found by their MMIO windows, checked against their roles' access modes and their expected sizes, and flushed, and the `Root` that it mounts, at a runtime directory, a canonical path that no symbolic link leads through, that it holds exclusively until its teardown, where nothing is mounted already, once it has checked everything: the read-only layers of the `Image` that the caller names, which it never probes: an EROFS image, whose expected `nvx::uuid::Uuid` it checks in its superblock, or a GPT disk whose partition table it reads and checks with `nvx::guest::gpt::read_disk`, against the caller's `nvx::gpt::Expected` values, whose first partition it reads, up to the caller's bound, as opaque bytes that it never mounts, and whose later partitions, at least one, hold ext4 layers that it mounts `ro,nosuid,nodev`, from the block devices that the kernel created when it found the disk, once their superblocks show that no journal needs a recovery, which their read-only device can't take, partition 2's at the overlay's base and the last partition's at its top, under an overlay mounted `nosuid,nodev` with the one option set of the caller's `OverlayMode`, `metacopy=on,xino=on` or, for lowers that may carry untrusted overlay xattrs, `metacopy=off,redirect_dir=nofollow`, with no fallback, whose upper and work directories are on the `Upper`: an ext4 scratch mounted `nosuid,nodev`, whose superblock it checks for a journal that the caller's `Journal` policy forbids, or a tmpfs mounted `nosuid,nodev` at the kernel's default size. It refuses overlay options longer than `MOUNT_OPTIONS_LIMIT`, the most that a mount passes, which, below `/run/nvx`, fit up to 195 layers of a GPT disk, and an overlay that the kernel's mount table shows read-only or without the mode's options in effect, since the kernel mounts one so, rather than fail, when its upper can't hold its work directory or its extended attributes. The `Root` reports the first partition's bytes, its `Lower` layers, its `UpperKind`, and its `OverlayMode`. `Root::mount_share` and `Root::mount_aggregate`: virtio-fs live shares at targets in the root that it checks, against the targets that `protocol::share` reserves among others, with one aggregate per root, which it attaches through its `mappings::Mapper` beneath the overlay, with NVX's container root's reserved targets: a share's device mounted out of the root, then moved onto its target, and a copy of each child's entry of the aggregate. `Root::map`: a caller's host mappings, through the same mapper. A share, a child, or a mapping that fails latches the root. `Root::teardown`: the shares and mappings in reverse order, then an aggregate's own mount, the overlay, the layers from the top down, and the upper, with every failed unmount reported. Each failure is a typed `Error`. The operations collect no child's status, change neither the caller's root nor its credentials, start no async runtime, and never power the VM off | +| `nvx::guest::root` | The operations of a layered root, for a guest init or agent, whether or not it is PID 1. `bootstrap`: the agent's and the workload's sibling cgroups of a `Layout`, with every process of their parent moved into the agent's, the agent's `memory.low`, and the workload cgroup prepared through `ExecutionCgroup::prepare`, with its `memory.max` and `pids.max`, and user namespaces kept unavailable. `assemble`: the `Assembly`'s block devices, found by their MMIO windows, checked against their roles' access modes and their expected sizes, and flushed, and the `Root` that it mounts, at a runtime directory, a canonical path that no symbolic link leads through, that it holds exclusively until its teardown, where nothing is mounted already, once it has checked everything: the read-only layers of the `Image` that the caller names, which it never probes: an EROFS image, whose expected `nvx::uuid::Uuid` it checks in its superblock, or a GPT disk whose partition table it reads and checks with `nvx::guest::gpt::read_disk`, against the caller's `nvx::gpt::Expected` values, whose first partition it reads, up to the caller's bound, as opaque bytes that it never mounts, and whose later partitions, at least one, hold ext4 layers that it mounts `ro,nosuid,nodev` with `errors=remount-ro`, so that an error that the kernel finds in a layer fails the operation that found it, rather than panic the guest, as the layer's superblock may ask, from the block devices that the kernel created when it found the disk, once their superblocks show that no journal needs a recovery, which their read-only device can't take, partition 2's at the overlay's base and the last partition's at its top, under an overlay mounted `nosuid,nodev` with the one option set of the caller's `OverlayMode`, `metacopy=on,xino=on` or, for lowers that may carry untrusted overlay xattrs, `metacopy=off,redirect_dir=nofollow`, with no fallback, whose upper and work directories are on the `Upper`: an ext4 scratch mounted `nosuid,nodev`, whose superblock it checks for a journal that the caller's `Journal` policy forbids, or a tmpfs mounted `nosuid,nodev` at the kernel's default size. It refuses overlay options longer than `MOUNT_OPTIONS_LIMIT`, the most that a mount passes, which, below `/run/nvx`, fit up to 195 layers of a GPT disk, and an overlay that the kernel's mount table shows read-only or without the mode's options in effect, since the kernel mounts one so, rather than fail, when its upper can't hold its work directory or its extended attributes. The `Root` reports the first partition's bytes, its `Lower` layers, its `UpperKind`, and its `OverlayMode`. `Root::mount_share` and `Root::mount_aggregate`: virtio-fs live shares at targets in the root that it checks, against the targets that `protocol::share` reserves among others, with one aggregate per root, which it attaches through its `mappings::Mapper` beneath the overlay, with NVX's container root's reserved targets: a share's device mounted out of the root, then moved onto its target, and a copy of each child's entry of the aggregate. `Root::map`: a caller's host mappings, through the same mapper. A share, a child, or a mapping that fails latches the root. `Root::teardown`: the shares and mappings in reverse order, then an aggregate's own mount, the overlay, the layers from the top down, and the upper, with every failed unmount reported. Each failure is a typed `Error`. The operations collect no child's status, change neither the caller's root nor its credentials, start no async runtime, and never power the VM off | | `nvx::guest::gpt` | `read_disk`: the partition table of a GPT disk, read and checked with `nvx::gpt`'s `Disk::read` through the disk's block device, which the caller opened, at the logical block size and size that `BLKSSZGET` and `BLKGETSIZE64` report, with the caller's `Expected` values, such as the disk GUID that a boot parameter names. It only reads the device, at explicit offsets, so it never moves the file's position. It neither compares the partitions that the kernel created with the table nor asks the kernel to reread the table. Each failure is a typed `DeviceError` | Lowering functions take the names by which messages refer to their inputs, such diff --git a/nvx-rs/src/guest/root.rs b/nvx-rs/src/guest/root.rs index acce186cf..456a48da7 100644 --- a/nvx-rs/src/guest/root.rs +++ b/nvx-rs/src/guest/root.rs @@ -191,6 +191,11 @@ const SCRATCH_DIRECTORY_MODE: u32 = 0o755; /// The tmpfs upper's options: the mode of its root directory, which an ext4 scratch's root has /// too, rather than tmpfs's world-writable default. Its size is the kernel's default. const TMPFS_OPTIONS: &[u8] = b"mode=0755"; +/// The options of a GPT disk's layers, ext4 file systems: an error that the kernel finds in a +/// layer fails the operation that found it, rather than panic the guest, as the layer's +/// superblock may ask, with its error policy or among its default mount options, both of which +/// these options override. A layer is read-only already. +const LAYER_OPTIONS: &[u8] = b"errors=remount-ro"; /// The characters that a runtime directory may not hold: the separators and escape of the /// overlay's options, and the C string's terminator. const RUNTIME_RESERVED: &[u8] = b",:\\=\0"; @@ -311,7 +316,9 @@ pub enum Image { /// asks the kernel to reread it. Its first partition holds opaque bytes, which the assembly /// returns through [`Root::first_partition`], and never probes or mounts. Each later /// partition, from partition 2 on, holds a layer, an ext4 file system that the assembly - /// mounts at `layers/pN`, for the partition's number N, `ro,nosuid,nodev`, without `noload`. + /// mounts at `layers/pN`, for the partition's number N, `ro,nosuid,nodev`, without `noload`, + /// and with `errors=remount-ro`: an error that the kernel finds in a layer fails the + /// operation that found it, rather than panic the guest, as the layer's superblock may ask. /// It refuses, before it mounts anything, a layer whose superblock has the `needs_recovery` /// feature: the layer's read-only device can't take the recovery, which `noload` or /// `norecovery` among the superblock's own default mount options would skip. The overlay @@ -729,11 +736,12 @@ impl Root { let work = upper_point.join(WORK_DIRECTORY); let options = overlay_options(&lowers, &upper, &work, assembly.overlay)?; for lower in lowers { - let (filesystem, flags) = match lower.partition { - None => (b"erofs".as_slice(), MountFlags::MS_RDONLY), + let (filesystem, flags, options) = match lower.partition { + None => (b"erofs".as_slice(), MountFlags::MS_RDONLY, None), Some(_) => ( b"ext4".as_slice(), MountFlags::MS_RDONLY | MountFlags::MS_NOSUID | MountFlags::MS_NODEV, + Some(LAYER_OPTIONS), ), }; self.mount_at( @@ -742,7 +750,7 @@ impl Root { &lower.path, Some(filesystem), flags, - None, + options, )?; self.lowers.push(lower); } diff --git a/nvx-rs/src/guest/root/tests.rs b/nvx-rs/src/guest/root/tests.rs index 85d7319c8..e9e54582d 100644 --- a/nvx-rs/src/guest/root/tests.rs +++ b/nvx-rs/src/guest/root/tests.rs @@ -1136,7 +1136,7 @@ fn a_gpt_disk_s_layers_are_mounted_under_the_overlay_top_first() { &format!("{runtime}/layers/p{number}"), Some("ext4"), MountFlags::MS_RDONLY | MountFlags::MS_NOSUID | MountFlags::MS_NODEV, - None, + Some("errors=remount-ro"), ) }; // The disk is flushed, then its table read and checked, and the scratch checked, before any diff --git a/tests/runtime-adapter/README.md b/tests/runtime-adapter/README.md index 612dc1fe3..9487bb42a 100644 --- a/tests/runtime-adapter/README.md +++ b/tests/runtime-adapter/README.md @@ -617,19 +617,20 @@ own: | `other_identity_fails_the_assembly` | The image, of which the launch expects another identity | The assembly's failure, which the launch reports, and a VM that ends with 14 before the guest mounts the scratch | | `other_disk_guid_fails_the_assembly` | The GPT disk, of which the launch expects another GUID | The assembly's failure, which the launch reports, and a VM that ends with 14 before the guest mounts the scratch | | `first_partition_over_its_bound_fails_the_assembly` | The GPT disk, with a bound one byte smaller than its first partition | The assembly's failure, which the launch reports, and a VM that ends with 14 before the guest mounts the scratch | +| `corrupt_layer_fails_the_assembly` | The GPT disk, whose top layer is corrupt and asks the kernel to panic on an error | The top layer's mount, which the kernel refuses without a panic, the assembly's failure, which the launch reports, and a VM that ends with 14 before the guest mounts the scratch | | `invalid_configuration_fails_its_stage` | A relative runtime directory | The configuration's failure, which the launch reports, and a VM that ends with 11 | | `busy_root_fails_the_teardown` | The image over the scratch, which the `hold` probe keeps open | The teardown's failure, which the stop reports, and a VM that ends with 15, leaving the scratch mounted | | `mappings_apply_beneath_the_root` | The image over the scratch, beneath which the `map` and `map-checks` probes map | Through the root: zero mappings that do no work, then a table mounted in order, each mapping with its access, `nosuid`, and `nodev`, whose writes reach the sources of the writable ones only, on targets that the mapper created on the overlay. Through mappers of the adapter's own: a link or a magic link at each component, the wrong kinds, a target that would be created inside another mapping, and the runtime's, the container root's, and the adapter's own reserved targets refused, and a read-only mapping that is never writable. A teardown that unmounts the mappings, then the scratch, cleanly | | `a_refused_mapping_latches_the_root` | The image over the scratch, in which the `map-refused` probe plants a symbolic link | A mapping through the link that the root refuses, nothing mounted or created, and a root that refuses every later mapping, then tears down cleanly | -A row names the root, the image, the identity that the launch expects of it, -the upper, the runtime directory, the overlay mode, the scratch's journal -policy, and the bound of the first partition, and what the scenario expects: a -root that the guest assembles, with a check that probes it, a teardown that -fails after a check, or the stage that fails, with its problem. Each scenario -checks the launch's record, and, over a scratch, whether the VM left the -scratch's journal to recover, which tells whether the guest unmounted the -scratch. +A row names the root, the image, whether a GPT disk's top layer is corrupt, the +identity that the launch expects of the image, the upper, the runtime +directory, the overlay mode, the scratch's journal policy, and the bound of the +first partition, and what the scenario expects: a root that the guest +assembles, with a check that probes it, a teardown that fails after a check, or +the stage that fails, with its problem. Each scenario checks the launch's +record, and, over a scratch, whether the VM left the scratch's journal to +recover, which tells whether the guest unmounted the scratch. The scenarios use the fixtures of NVX's own layered scenarios, from NVX's artifacts: the Ubuntu EROFS image, `ubuntu-distro.erofs`, its manifest, which @@ -651,7 +652,11 @@ partition 2, a copy of the template; and the top layer, in partition 3, a small file system in the ext2 format, which the kernel's ext4 driver mounts, with a file and a directory. The test writes the top layer itself, since `mkfs.ext4` is not available everywhere that the tests run, and checks the disk with -`nvx::gpt` before it launches the VM. +`nvx::gpt` before it launches the VM. For `corrupt_layer_fails_the_assembly`, +it corrupts the top layer: the inode of the layer's top directory loses its +mode, an error that the kernel finds when it mounts the layer, and the error +policy of its superblock asks the kernel to panic on an error, which the +assembly's `errors=remount-ro` overrides. ### Running the VM tests diff --git a/tests/runtime-adapter/tests/vm_layered.rs b/tests/runtime-adapter/tests/vm_layered.rs index ac49d0cda..5ab55ba32 100644 --- a/tests/runtime-adapter/tests/vm_layered.rs +++ b/tests/runtime-adapter/tests/vm_layered.rs @@ -18,7 +18,8 @@ //! partition, opaque bytes, then a copy of the template, the base layer, and a small layer that the //! test writes, the top one, which holds a file and a directory. The test writes it in the ext2 //! format, which the kernel's ext4 driver mounts, since `mkfs.ext4` is not available everywhere -//! that the tests run. +//! that the tests run. One scenario corrupts that layer, whose superblock then asks the kernel to +//! panic on an error, as [`corrupt`] describes. //! //! ```text //! cargo test --locked --features host --test vm_layered -- --ignored @@ -142,6 +143,16 @@ enum Image { Gpt, } +/// The top layer of a scenario's GPT disk. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum TopLayer { + /// A [`small_layer`] that holds [`LAYER_FILE`] and [`LAYER_DIRECTORY`]. + Sound, + /// That layer, corrupt: its top directory's inode has no mode, an error that the kernel finds + /// when it mounts the layer, and its superblock asks the kernel to panic on an error. + Corrupt, +} + /// The identity that a scenario's launch expects of the image. #[derive(Clone, Copy, Debug)] enum Identity { @@ -165,6 +176,8 @@ enum Upper { struct Root { /// The image. image: Image, + /// The top layer of a GPT disk. + top_layer: TopLayer, /// The identity that the launch expects of the image. identity: Identity, /// The upper. @@ -183,6 +196,7 @@ struct Root { /// over a copy of the scratch template, with the guest's defaults. const ROOT: Root = Root { image: Image::Erofs, + top_layer: TopLayer::Sound, identity: Identity::Own, upper: Upper::Scratch, runtime: None, @@ -216,7 +230,7 @@ impl Root { Image::Erofs => (fixtures.distro.clone(), "erofs", fixtures.identity.clone()), Image::Gpt => { let path = directory.join("distro.gpt"); - write_gpt_disk(&path, &fixtures.scratch); + write_gpt_disk(&path, &fixtures.scratch, self.top_layer); (path, "gpt", DISK_GUID.to_owned()) } }; @@ -450,6 +464,15 @@ scenarios! { "the first partition has 4096 bytes, more than the bound of 4095", ), }; + /// A GPT disk whose top layer is corrupt and asks the kernel to panic on an error, which the + /// kernel refuses to mount, without a panic, and the assembly with it. + corrupt_layer_fails_the_assembly: Scenario { + root: Root { top_layer: TopLayer::Corrupt, ..GPT }, + expect: Expect::Fails( + RootStage::Assembly, + "failed to mount the layer in partition 3: ", + ), + }; /// A relative runtime directory, which the guest refuses with the configuration. invalid_configuration_fails_its_stage: Scenario { root: Root { runtime: Some("run/runtime-adapter"), ..ROOT }, @@ -1014,11 +1037,12 @@ fn check_refused_mapping(vm: &mut Vm<'_>) { /// then [`FIRST_PARTITION_PADDING`], then NUL bytes; /// 2. the base layer, a copy of the scratch template `template`, an ext4 file system that /// `mkfs.ext4` made, with a journal; -/// 3. the top layer, a [`small_layer`] that holds [`LAYER_FILE`] and [`LAYER_DIRECTORY`]. +/// 3. the top layer, a [`small_layer`] that holds [`LAYER_FILE`] and [`LAYER_DIRECTORY`], which +/// [`corrupt`] corrupts for a `top_layer` that is [`TopLayer::Corrupt`]. /// /// Then checks the disk as the guest's assembly does, through `nvx::gpt`, with the GUIDs and /// sizes that it wrote. -fn write_gpt_disk(path: &Path, template: &Path) { +fn write_gpt_disk(path: &Path, template: &Path, top_layer: TopLayer) { let template_size = fs::metadata(template).unwrap().len(); assert_eq!( template_size % SECTOR, @@ -1031,7 +1055,10 @@ fn write_gpt_disk(path: &Path, template: &Path) { .cycle() .take(LAYER_FILE_SIZE) .collect::>(); - let top = small_layer(&[(LAYER_FILE, contents.as_slice())], &[LAYER_DIRECTORY]); + let mut top = small_layer(&[(LAYER_FILE, contents.as_slice())], &[LAYER_DIRECTORY]); + if top_layer == TopLayer::Corrupt { + corrupt(&mut top); + } let sizes = [ FIRST_PARTITION_SIZE, template_size, @@ -1302,6 +1329,17 @@ fn small_layer(files: &[(&str, &[u8])], directories: &[&str]) -> Vec { image } +/// Corrupts `layer`, a [`small_layer`]: its top directory's inode, inode 2, loses its mode, while +/// it keeps its links, an error that the kernel finds when it mounts the layer, and the error +/// policy of its superblock, `s_errors`, becomes 3, which asks the kernel to panic on an error. +fn corrupt(layer: &mut [u8]) { + let policy = LAYER_BLOCK + 0x3c; + layer[policy..policy + 2].copy_from_slice(&3_u16.to_le_bytes()); + // The inode table starts at block 5, with inodes of 128 bytes, from inode 1. + let mode = 5 * LAYER_BLOCK + 128; + layer[mode..mode + 2].fill(0); +} + /// Returns the CRC-32 of `bytes`, which GPT's headers and entry arrays hold: of the reflected /// polynomial 0xEDB88320, from all ones, inverted. fn crc32(bytes: &[u8]) -> u32 {