tests: launch the runtime adapter's VMs and reattach after a restart

.github/workflows/ci.yml

@@ -505,6 +505,30 @@ jobs:
           if-no-files-found: error
           retention-days: 1
 
+      # The Windows guest and sandbox job runs the runtime adapter's scenarios
+      # with its guest agent, a static musl executable that Linux builds, as
+      # it builds the images that the job reads. The Linux jobs build their
+      # own.
+      - name: Build the runtime adapter's guest agent
+        shell: bash
+        working-directory: tests/runtime-adapter
+        run: |
+          set -euo pipefail
+          toolchain=$(sed -n 's/^channel = "\([^"]*\)".*/\1/p' ../../rust-toolchain.toml)
+          test -n "${toolchain}"
+          rustup toolchain install "${toolchain}" --profile minimal --no-self-update \
+            --target x86_64-unknown-linux-musl
+          cargo "+${toolchain}" build --locked --release --features guest \
+            --target x86_64-unknown-linux-musl --bin runtime-adapter-guest
+
+      - name: Upload the runtime adapter's guest agent
+        uses: actions/upload-artifact@v7
+        with:
+          name: runtime-adapter-guest
+          path: tests/runtime-adapter/target/x86_64-unknown-linux-musl/release/runtime-adapter-guest
+          if-no-files-found: error
+          retention-days: 1
+
   # A dev push reuses the guest artifact components that the pull request it
   # merges built (see build-guest-artifacts). A same-repository pull request
   # rehearses that selection and check on the artifacts that its own run

.github/workflows/run-nvx-microvm-tests.yml

@@ -1248,24 +1248,49 @@ jobs:
           --backend "${{ inputs.backend }}"
           --output-dir "build/test-results/aci-edge-sandboxes-${{ inputs.backend }}"
 
-      # The runtime adapter's hypervisor scenarios on this backend: any ignored
+      # The runtime adapter's hypervisor scenarios on this backend: the ignored
       # tests of its host-only configuration, which read the backend through
-      # `nvx::Hypervisor::from_env_or_default`. The step builds the adapter
-      # whether or not it declares any.
+      # `nvx::Hypervisor::from_env_or_default`. Its host program launches a VM
+      # of the artifacts above, with the runtime identity that their build
+      # provenance gives, whose initramfs runs the adapter's guest agent in
+      # place of NVX's managed agent. Later processes of the host program
+      # reattach to the VM, as after a restart of the orchestrator, and stop
+      # it. The guest agent is a static musl executable: Linux builds it, and
+      # Windows takes the one that distro-artifacts builds on Linux, as it
+      # takes the Ubuntu and Azure Linux images.
+      - name: Build the runtime adapter's guest agent on Linux
+        if: runner.os != 'Windows' && inputs.guests-and-sandboxes && !inputs.debug-kernel
+        shell: bash
+        working-directory: tests/runtime-adapter
+        run: >-
+          cargo build --locked --release --features guest
+          --target x86_64-unknown-linux-musl --bin runtime-adapter-guest
+
       - name: Run runtime adapter scenarios on Linux
         if: runner.os != 'Windows' && inputs.guests-and-sandboxes && !inputs.debug-kernel
         shell: bash
         working-directory: tests/runtime-adapter
         env:
           NVX_HYPERVISOR: ${{ inputs.backend }}
+          NVX_ARTIFACTS_DIR: ${{ github.workspace }}
+          RUNTIME_ADAPTER_GUEST: ${{ github.workspace }}/tests/runtime-adapter/target/x86_64-unknown-linux-musl/release/runtime-adapter-guest
         run: cargo test --locked --features host -- --ignored
 
+      - name: Download the runtime adapter's guest agent on Windows
+        if: runner.os == 'Windows' && inputs.guests-and-sandboxes && !inputs.debug-kernel
+        uses: actions/download-artifact@v8
+        with:
+          name: runtime-adapter-guest
+          path: build/runtime-adapter
+
       - name: Run runtime adapter scenarios on Windows
         if: runner.os == 'Windows' && inputs.guests-and-sandboxes && !inputs.debug-kernel
         shell: powershell
         working-directory: tests/runtime-adapter
         env:
           NVX_HYPERVISOR: ${{ inputs.backend }}
+          NVX_ARTIFACTS_DIR: ${{ github.workspace }}
+          RUNTIME_ADAPTER_GUEST: ${{ github.workspace }}\build\runtime-adapter\runtime-adapter-guest
         run: cargo test --locked --features host -- --ignored
 
       - name: Upload NVX microVM failure logs

doc/ci.md

@@ -739,6 +739,12 @@ reads, or waits for a guest artifact job that uploads none of its components.
 It also checks that each guest artifact job builds exactly the components
 that it uploads, and that no component builds in two jobs.
 
+`distro-artifacts` also builds the runtime adapter's guest agent, which is no
+guest component: a static musl executable that it uploads as
+`runtime-adapter-guest` for the Windows guest and sandbox job, which downloads
+it into `build/runtime-adapter/`. The Linux guest and sandbox jobs build their
+own (see [Rust crates](#rust-crates)).
+
 Each job restores its components from their caches, and builds, verifies, and
 saves those that miss, as one job did before. The Ubuntu determinism check,
 the provenance that each build records, and the cache keys are unchanged. A
@@ -1295,9 +1301,12 @@ guest binary is a static executable. They also check the adapter's own framing
 over the control session between the host and the guest: in the host-only
 configuration, on Linux and Windows, that the adapter's host program attaches
 to the session and exchanges the framing with the guest agent, and in the
-guest-only configuration, that its guest agent serves the session. Each runs
-against a test broker that sends it messages that span records, resets,
-disconnects, and malformed records.
+guest-only configuration, that its guest agent serves the session, also on a
+tty that it puts in raw mode. Each runs against a test broker that sends it
+messages that span records, resets, disconnects, and malformed records. The
+host-only tests also check the initramfs that runs the guest agent in place of
+NVX's managed agent, and the runtime identity that the host program takes from
+NVX's identity of its artifacts.
 
 The development release job depends on `aci-edge-sandboxes` and `nvx-rs` and
 requires their combined Linux/Windows results to be successful; failed,
@@ -1317,12 +1326,19 @@ it keeps the OpenVMM log under `build/test-results/aci-edge-sandboxes-<backend>`
 uploaded with the other microVM logs. Changes under `aci_edge_sandboxes/` or `nvx-rs/`
 therefore trigger the NVX microVM test matrix.
 
-The same jobs build the runtime adapter's
-host-only configuration on their backend and run any ignored tests that it
-declares, its hypervisor scenarios, with `NVX_HYPERVISOR` naming the backend.
-The adapter's other tests, including those of its guest agent, run in the
-`nvx-rs` job. Changes under `tests/runtime-adapter/` therefore trigger the NVX
-microVM test matrix too.
+The same jobs build the runtime adapter's host-only configuration on their
+backend and run its ignored tests, its VM scenarios, with `NVX_HYPERVISOR`
+naming the backend. Through NVX's supervisor, the adapter's host program
+launches a VM of the job's artifacts, whose initramfs runs the adapter's guest
+agent in place of NVX's managed agent, with the runtime identity that the
+artifacts' build provenance gives. Later processes reattach to the VM, as after
+a restart of the orchestrator, find it mismatched under another runtime
+identity without signaling it, and stop it. The Linux jobs build the guest
+agent, a static musl executable, first, and the Windows job takes the one that
+`distro-artifacts` builds on Linux, as it takes the Ubuntu and Azure Linux
+images. The adapter's other tests, including those of its guest agent, run in
+the `nvx-rs` job. Changes under `tests/runtime-adapter/` therefore trigger the
+NVX microVM test matrix too.
 
 ## Adversarial campaigns
 

doc/project-structure.md

@@ -106,7 +106,7 @@ nvx/
 |   |   |-- gpt/                 Strict reading of GPT disks whose first partition configures an adapter and whose other partitions hold a root's read-only layers, shared by the host and the guest
 |   |   |-- host/                Artifacts, launch, network and egress-policy lowering, sessions, VM instances, sandbox stores, the sandbox launch profile, and the `nvx-rs` CLI (feature `cli`)
 |   |   |-- testing/             Fake OpenVMM (feature `testing`)
-|   |   |-- guest/               Programs that run in Linux guests, the guest's NVXS connection, the libraries through which any guest init or agent starts a program, collects its children's statuses, settles its executions' cgroups, drives each execution, bind-mounts host mappings below the roots that it names, assembles and tears down a layered root, and reads a GPT disk's partition table through its block device, and the typed calls through which it runs the time ABI's steps and the reseed (feature `guest`)
+|   |   |-- guest/               Programs that run in Linux guests, the guest's NVXS connection, the libraries through which any guest init or agent opens the terminal on which it serves its session, starts a program, collects its children's statuses, settles its executions' cgroups, drives each execution, bind-mounts host mappings below the roots that it names, assembles and tears down a layered root, and reads a GPT disk's partition table through its block device, and the typed calls through which it runs the time ABI's steps and the reseed (feature `guest`)
 |   |   `-- bin/                 nvx-rs CLI, nvx-fake-openvmm test double, and the guest binaries
 |   |-- tests/                   Fake-OpenVMM instance lifecycle, sandbox store, CLI, NVXS session, guest-binary, and hypervisor managed-lifecycle tests, the store's record fixtures, and the golden fixtures that Python recorded
 |   |-- build.rs                 Stages the bundled artifacts (feature `bundled`)
@@ -342,7 +342,10 @@ off and uses its public API only. Its protocol-only, host-only, and guest-only
 configurations build in CI, and its tests check that each one builds as an
 independent adapter's does. Its host program and its guest agent exchange the
 adapter's own framing over the control session between the host and the
-guest, and its tests check each of them against a test broker. See its
+guest, and its tests check each of them against a test broker. Its host
+program also runs the guest agent in VMs of NVX's artifacts through NVX's
+supervisor, and reattaches to them from other processes, which its ignored
+tests check on each backend in CI. See its
 [README](../tests/runtime-adapter/README.md).
 
 ## Root files

nvx-rs/README.md

@@ -43,12 +43,12 @@ nvx-rs/
 │   │                with feature `cli`, the `nvx-rs` CLI
 │   ├── testing/     feature `testing`: the fake OpenVMM
 │   ├── guest/       feature `guest`: the programs that run in Linux guests, the guest's NVXS
-│   │                connection, the libraries through which a guest init or agent starts a
-│   │                program, collects its children's statuses, settles its executions'
-│   │                cgroups, bind-mounts host mappings below the roots that it names,
-│   │                assembles and tears down a layered root, and reads a GPT disk's partition
-│   │                table through its block device, and the typed calls through which it runs
-│   │                the time ABI's steps and the reseed
+│   │                connection, the libraries through which a guest init or agent opens the
+│   │                terminal on which it serves its session, starts a program, collects its
+│   │                children's statuses, settles its executions' cgroups, bind-mounts host
+│   │                mappings below the roots that it names, assembles and tears down a layered
+│   │                root, and reads a GPT disk's partition table through its block device, and
+│   │                the typed calls through which it runs the time ABI's steps and the reseed
 │   └── bin/         nvx-rs, nvx-fake-openvmm, and the guest binaries
 ├── tests/           fake-OpenVMM, CLI, NVXS session, process-supervision, store, and guest-binary
 │                    tests, the store's record fixtures, and the golden fixtures that Python
@@ -87,6 +87,7 @@ nvx-rs/
 | `nvx::guest` | The programs that run in Linux guests; see [Guest binaries](#guest-binaries) |
 | `nvx::guest::mappings` | `Mapping`: one host mapping for any guest init or agent, a relative source below an export root, where the host's shares appear, and a target below a target root, the tree that the mappings go into, with its `Access`. `Mapping::apply` takes both roots explicitly: it creates the target as the kind of its source, a directory or a regular file, bind-mounts the source on it, and remounts the bind `nosuid` and `nodev`, and read-only when asked. A `MappingFailure` tells which `MappingStage` failed and its `errno`. The caller keeps the order of its mappings and what a failure means for the rest. NVX's managed agent applies its host mapping table through it, and a layered `Root` binds the children of its aggregate live share through it |
 | `nvx::guest::nvxs` | `GuestConnection`: the guest's role of the NVXS session over a byte stream, such as the control console, on which NVX's managed agent and other guest agents run their sessions. It never reads past the record in progress, so a caller that polls the stream between records sees every record that it has not returned |
+| `nvx::guest::terminal` | `Terminal`: the terminal on which a guest agent serves its session, such as the VM's control console, as a byte stream. It opens the terminal, waiting while the kernel sets it up during boot, puts it in raw mode, so that it passes the session's bytes as they are, and drains it, so that it sends what the agent wrote before the agent ends the VM. NVX's managed agent and other guest agents serve their sessions on it |
 | `nvx::guest::spawn` | `Command`: starts a program in a new process for any guest init or agent, with its arguments, environment, root and working directories, standard streams, inherited descriptors, cgroup, and process group, without NVX's control protocol, managed agent, or workload identity. Its `LaunchReport` tells whether the program started, or which `LaunchStage` failed and why. NVX's managed agent is one caller |
 | `nvx::guest::process` | `Child`: the status of one child that the caller names, by its pidfd or its process ID. It waits for that child alone, so another child's status stays for the caller; a caller that collects every status itself, such as a PID 1 that adopts orphans, hands each one to its `Child` with `observe`. Statuses are `std::process::ExitStatus`, real-time signals included |
 | `nvx::guest::cgroup` | `Layout`: the agent's cgroup and the workload cgroup, which the caller names, and which may not hold the agent's. `ExecutionCgroup`: the cgroup of one execution, the workload cgroup itself or a child of it, which it prepares with the caller's resource values, places processes in, checks for occupancy through `pids.current`, kills with `cgroup.kill`, and settles, over the execution's cgroup or the whole workload subtree, as the caller's `Scope` chooses. It holds its cgroups' directories open, and the process's removals and preparations of cgroups take turns, so that it never reaches another cgroup that takes one of their paths. Settling calls the caller's status collector between its checks, and collects no status itself |
@@ -591,14 +592,15 @@ as `nvx/`.
 The `guest` feature adds the programs that run in NVX's Linux guests: the
 `nvx::guest` module and one binary per program. The Alpine, Ubuntu, and Azure
 Linux images ship all nine at their established `/sbin/nvx-*` paths. The
-module's libraries are not programs: `spawn` starts a program in a new
-process, `process` collects the status of a child that the caller names,
-`cgroup` places, kills, and settles the processes of an execution, `exec`
-drives one program as an execution, `mappings` bind-mounts host mappings below
-the roots that its caller names, and `root` bootstraps the guest for a layered
-root through `cgroup`, assembles the root, mounts live shares in it, binding
-the children of an aggregate through `mappings`, and tears it down.
-`nvx-managed-agent` launches, drives, collects, and settles its workloads
+module's libraries are not programs: `terminal` opens the terminal on which an
+agent serves its session, `spawn` starts a program in a new process, `process`
+collects the status of a child that the caller names, `cgroup` places, kills,
+and settles the processes of an execution, `exec` drives one program as an
+execution, `mappings` bind-mounts host mappings below the roots that its caller
+names, and `root` bootstraps the guest for a layered root through `cgroup`,
+assembles the root, mounts live shares in it, binding the children of an
+aggregate through `mappings`, and tears it down. `nvx-managed-agent` opens its
+control terminal, and launches, drives, collects, and settles its workloads,
 through them, and applies its host mapping table through `mappings`, with its
 mount of the host's export and `/` as the roots, and any other guest init or
 agent can use them too. Any guest init or agent can also run the steps of

nvx-rs/src/guest/managed_agent/tty.rs

@@ -1,32 +1,14 @@
 use std::ffi::CStr;
-use std::time::Duration;
 
-use crate::guest::sys::{self, Errno, FileDescriptor, Mode, OpenFlags};
+use crate::guest::sys::{Errno, FileDescriptor};
+use crate::guest::terminal::Terminal;
 
 use super::seam::System;
 
-/// Number of attempts allowed while the control terminal is still appearing.
-const OPEN_ATTEMPTS: usize = 600;
-/// Delay between retryable control-terminal open failures.
-const OPEN_DELAY: Duration = Duration::from_millis(50);
-
-/// Opens the control terminal, retrying transient boot-time failures.
+/// Opens the control terminal, waiting while the kernel sets it up during boot, as
+/// [`Terminal::open`] does.
 pub(crate) fn open(path: &CStr) -> Result<FileDescriptor, Errno> {
-    for attempt in 0..OPEN_ATTEMPTS {
-        match sys::open(
-            path,
-            OpenFlags::O_RDWR | OpenFlags::O_CLOEXEC,
-            Mode::empty(),
-        ) {
-            Ok(descriptor) => return Ok(descriptor),
-            Err(Errno::ENOENT | Errno::ENXIO | Errno::ENODEV | Errno::EIO) => {}
-            Err(error) => return Err(error),
-        }
-        if attempt + 1 != OPEN_ATTEMPTS {
-            std::thread::sleep(OPEN_DELAY);
-        }
-    }
-    Err(Errno::ETIMEDOUT)
+    Terminal::open_c(path).map(Terminal::into_descriptor)
 }
 
 /// Applies raw local-terminal settings through the replaceable seam.