Ship nvx-rs in NVX releases

.gitattributes

@@ -17,4 +17,12 @@ kernel/config-microvm-debug text eol=lf
 # Linux hosts read the setup tool manifest, including copies staged from Windows.
 scripts/setup/tool-versions.conf text eol=lf
 
-.github/workflows/*.lock.yml linguist-generated=true
+.github/workflows/*.lock.yml linguist-generated=true
+# Recorded from the Python code that nvx-rs replaced, by scripts/nvx_rs_fixtures.py,
+# which microsoft/nvx#359 retired with that code; they are frozen golden data.
+nvx-rs/src/host/python/unicode.rs linguist-generated=true
+nvx-rs/tests/fixtures/cli.json linguist-generated=true
+nvx-rs/tests/fixtures/egress-policy.json linguist-generated=true
+nvx-rs/tests/fixtures/python.json linguist-generated=true
+nvx-rs/tests/fixtures/sandbox.json linguist-generated=true
+nvx-rs/tests/fixtures/sandbox_options.json linguist-generated=true

.github/actions/build-guest-artifacts/action.yml

@@ -73,21 +73,23 @@ runs:
         AZURELINUX_VERSION: ${{ steps.guest-versions.outputs.azurelinux-version }}
         KERNEL_INPUT_HASH: ${{ hashFiles('SOURCE-MANIFEST.json', 'kernel/config-microvm', 'kernel/patches/**', 'docker/Dockerfile', '.dockerignore', 'scripts/nvx_tools/build.py', 'scripts/nvx_tools/build_config.py', 'scripts/nvx_tools/build_constants.py') }}
         DEBUG_KERNEL_INPUT_HASH: ${{ hashFiles('SOURCE-MANIFEST.json', 'kernel/config-microvm', 'kernel/config-microvm-debug', 'kernel/patches/**', 'docker/Dockerfile', '.dockerignore', 'scripts/nvx_tools/build.py', 'scripts/nvx_tools/build_config.py', 'scripts/nvx_tools/build_constants.py') }}
-        ALPINE_INPUT_HASH: ${{ hashFiles('SOURCE-MANIFEST.json', 'guest/common/**', 'guest/alpine/**', 'docker/Dockerfile', '.dockerignore', 'scripts/nvx_tools/build.py', 'scripts/nvx_tools/build_config.py', 'scripts/nvx_tools/build_constants.py', 'scripts/nvx_tools/common.py', 'scripts/nvx_tools/guests.py') }}
-        UBUNTU_INPUT_HASH: ${{ hashFiles('SOURCE-MANIFEST.json', 'guest/common/**', 'guest/ubuntu/**', 'ubuntu/**', 'docker/Dockerfile', '.dockerignore', 'scripts/nvx_tools/build.py', 'scripts/nvx_tools/build_config.py', 'scripts/nvx_tools/build_constants.py', 'scripts/nvx_tools/common.py', 'scripts/nvx_tools/guests.py', 'scripts/nvx_tools/ubuntu.py') }}
-        AZURELINUX_INPUT_HASH: ${{ hashFiles('SOURCE-MANIFEST.json', 'guest/common/**', 'azurelinux/**', 'docker/Dockerfile', '.dockerignore', 'scripts/nvx_tools/azurelinux.py', 'scripts/nvx_tools/build.py', 'scripts/nvx_tools/build_config.py', 'scripts/nvx_tools/build_constants.py', 'scripts/nvx_tools/common.py', 'scripts/nvx_tools/guests.py') }}
+        ALPINE_INPUT_HASH: ${{ hashFiles('SOURCE-MANIFEST.json', 'guest/common/**', 'guest/alpine/**', 'nvx-rs/**', '!nvx-rs/target/**', 'rust-toolchain.toml', 'docker/Dockerfile', '.dockerignore', 'scripts/nvx_tools/build.py', 'scripts/nvx_tools/build_config.py', 'scripts/nvx_tools/build_constants.py', 'scripts/nvx_tools/common.py', 'scripts/nvx_tools/elf.py', 'scripts/nvx_tools/guest_binaries.py', 'scripts/nvx_tools/guests.py') }}
+        UBUNTU_INPUT_HASH: ${{ hashFiles('SOURCE-MANIFEST.json', 'guest/common/**', 'guest/ubuntu/**', 'ubuntu/**', 'nvx-rs/**', '!nvx-rs/target/**', 'rust-toolchain.toml', 'docker/Dockerfile', '.dockerignore', 'scripts/nvx_tools/build.py', 'scripts/nvx_tools/build_config.py', 'scripts/nvx_tools/build_constants.py', 'scripts/nvx_tools/common.py', 'scripts/nvx_tools/elf.py', 'scripts/nvx_tools/guest_binaries.py', 'scripts/nvx_tools/guests.py', 'scripts/nvx_tools/ubuntu.py') }}
+        AZURELINUX_INPUT_HASH: ${{ hashFiles('SOURCE-MANIFEST.json', 'guest/common/**', 'azurelinux/**', 'nvx-rs/**', '!nvx-rs/target/**', 'rust-toolchain.toml', 'docker/Dockerfile', '.dockerignore', 'scripts/nvx_tools/azurelinux.py', 'scripts/nvx_tools/build.py', 'scripts/nvx_tools/build_config.py', 'scripts/nvx_tools/build_constants.py', 'scripts/nvx_tools/common.py', 'scripts/nvx_tools/elf.py', 'scripts/nvx_tools/guest_binaries.py', 'scripts/nvx_tools/guests.py') }}
       run: |
         set -euo pipefail
         # hashFiles hashes contents only, so the kernel keys also name each patch
-        # path and its order.
+        # path and its order, and the guest image keys the path-aware digest of
+        # the Rust guest sources that their package manifests record.
         patch_identity=$(python3 scripts/nvx.py kernel-patch-identity)
+        rust_source_identity=$(python3 scripts/nvx.py guest-binary-source-identity)
         {
           echo "kernel=linux-kernel-v1-${KERNEL_VERSION}-${KERNEL_INPUT_HASH}-${patch_identity}"
           echo "kernel-debug=linux-kernel-debug-v1-${KERNEL_VERSION}-${DEBUG_KERNEL_INPUT_HASH}-${patch_identity}"
-          echo "alpine=alpine-image-v1-${ALPINE_VERSION}-${ALPINE_INPUT_HASH}"
-          echo "ubuntu=ubuntu-initramfs-v1-${UBUNTU_VERSION}-${UBUNTU_INPUT_HASH}"
-          echo "ubuntu-distro=ubuntu-distro-v1-${UBUNTU_VERSION}-${UBUNTU_INPUT_HASH}"
-          echo "azurelinux=azurelinux-initramfs-v1-${AZURELINUX_VERSION}-${AZURELINUX_INPUT_HASH}"
+          echo "alpine=alpine-image-v2-${ALPINE_VERSION}-${ALPINE_INPUT_HASH}-${rust_source_identity}"
+          echo "ubuntu=ubuntu-initramfs-v2-${UBUNTU_VERSION}-${UBUNTU_INPUT_HASH}-${rust_source_identity}"
+          echo "ubuntu-distro=ubuntu-distro-v2-${UBUNTU_VERSION}-${UBUNTU_INPUT_HASH}-${rust_source_identity}"
+          echo "azurelinux=azurelinux-initramfs-v2-${AZURELINUX_VERSION}-${AZURELINUX_INPUT_HASH}-${rust_source_identity}"
         } >> "${GITHUB_OUTPUT}"
 
     - name: Restore Linux kernel (zstd)

.github/actions/check-nvx-rs/action.yml

@@ -61,26 +61,32 @@ runs:
         cargo "+${{ steps.toolchains.outputs.toolchain }}" test --locked --all-features
         cargo "+${{ steps.toolchains.outputs.toolchain }}" test --locked
 
-    # Stages a synthetic release, which the bundled-artifacts test then requires to find.
+    # Stages a synthetic release in the current layout, with nvx-rs and its
+    # provenance, which the bundled-artifacts test then requires to find.
     - name: Check artifact bundling
       shell: bash
       working-directory: nvx-rs
       run: |
         set -euo pipefail
         bundle=$(mktemp -d)
         executable=openvmm
+        cli=nvx-rs
         bundle_dir="${bundle}"
         if [[ "${RUNNER_OS}" == Windows ]]; then
           executable=openvmm.exe
+          cli=nvx-rs.exe
           bundle_dir=$(cygpath -w "${bundle}")
         fi
-        mkdir -p "${bundle}/bin" "${bundle}/guest"
+        mkdir -p "${bundle}/bin" "${bundle}/guest" "${bundle}/provenance"
         printf openvmm > "${bundle}/bin/${executable}"
+        printf nvx-rs > "${bundle}/bin/${cli}"
+        printf '{}' > "${bundle}/provenance/nvx-rs.provenance.json"
         printf kernel > "${bundle}/guest/vmlinux"
         printf initramfs > "${bundle}/guest/initramfs.cpio.gz"
         cp ../SOURCE-MANIFEST.json "${bundle}/"
-        (cd "${bundle}" && sha256sum "bin/${executable}" guest/vmlinux \
-          guest/initramfs.cpio.gz SOURCE-MANIFEST.json > SHA256SUMS)
+        (cd "${bundle}" && sha256sum "bin/${executable}" "bin/${cli}" guest/vmlinux \
+          guest/initramfs.cpio.gz provenance/nvx-rs.provenance.json \
+          SOURCE-MANIFEST.json > SHA256SUMS)
         NVX_BUNDLE_DIR="${bundle_dir}" cargo "+${{ steps.toolchains.outputs.toolchain }}" test --locked --lib \
           --features bundled artifacts
 

.github/actions/package-release/action.yml

@@ -1,14 +1,40 @@
 name: Package release
-description: Build and upload an NVX binary release for one platform
+description: Build an NVX binary release for one platform, and upload it for publication
 
 inputs:
   platform:
     description: Platform suffix used in the package and artifact names
     required: true
+  nvx-rs-artifact:
+    description: Base name of the nvx-rs artifacts for the runner's OS
+    required: true
+  upload:
+    description: Whether to upload the package for the development release
+    required: true
 
 runs:
   using: composite
   steps:
+    # Every package ships the nvx-rs that the NVX CLI job of its OS built and
+    # tested, with its provenance, which nvx.py download installs where
+    # nvx.py looks for them.
+    - name: Download nvx-rs executable
+      uses: actions/download-artifact@v8
+      with:
+        name: ${{ inputs.nvx-rs-artifact }}-executable
+        path: nvx-rs/target/release
+
+    - name: Download nvx-rs provenance
+      uses: actions/download-artifact@v8
+      with:
+        name: ${{ inputs.nvx-rs-artifact }}-provenance
+        path: build
+
+    - name: Make nvx-rs executable
+      if: runner.os != 'Windows'
+      shell: bash
+      run: chmod +x nvx-rs/target/release/nvx-rs
+
     - name: Rematerialize canonical kernel inputs
       if: runner.os == 'Windows'
       shell: powershell
@@ -52,7 +78,7 @@ runs:
         if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
 
     - name: Upload Linux release
-      if: runner.os != 'Windows'
+      if: runner.os != 'Windows' && inputs.upload == 'true'
       uses: actions/upload-artifact@v7
       with:
         name: release-${{ inputs.platform }}
@@ -62,7 +88,7 @@ runs:
         retention-days: 1
 
     - name: Upload Windows release
-      if: runner.os == 'Windows'
+      if: runner.os == 'Windows' && inputs.upload == 'true'
       uses: actions/upload-artifact@v7
       with:
         name: release-${{ inputs.platform }}

.github/actions/validate-nvx/action.yml

@@ -10,7 +10,7 @@ runs:
       run: |
         set -euo pipefail
         python3 -m compileall -q scripts
-        python3 -m unittest scripts/test_performance.py scripts/test_nvx_tools.py scripts/test_guest_binaries.py scripts/test_microvm_tests.py scripts/test_development_release.py scripts/test_egress_policy.py scripts/test_control_session.py scripts/test_managed_agent.py scripts/test_guest_time.py scripts/test_guest_ingress.py scripts/test_host_telemetry.py scripts/test_time_abi.py scripts/test_tool_versions.py scripts/test_markdown_links.py -v
+        python3 -m unittest scripts/test_performance.py scripts/test_nvx_tools.py scripts/test_guest_binaries.py scripts/test_microvm_tests.py scripts/test_development_release.py scripts/test_host_telemetry.py scripts/test_time_abi.py scripts/test_tool_versions.py scripts/test_markdown_links.py -v
         python3 scripts/test_adversarial.py -v
         python3 .github/skills/nvx-host-connect/scripts/test_hosts.py -v
         python3 scripts/nvx.py --help
@@ -30,7 +30,7 @@ runs:
         $ErrorActionPreference = "Stop"
         python -m compileall -q scripts
         if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
-        python -m unittest scripts/test_performance.py scripts/test_nvx_tools.py scripts/test_guest_binaries.py scripts/test_microvm_tests.py scripts/test_development_release.py scripts/test_egress_policy.py scripts/test_control_session.py scripts/test_managed_agent.py scripts/test_guest_time.py scripts/test_guest_ingress.py scripts/test_host_telemetry.py scripts/test_time_abi.py scripts/test_tool_versions.py scripts/test_markdown_links.py -v
+        python -m unittest scripts/test_performance.py scripts/test_nvx_tools.py scripts/test_guest_binaries.py scripts/test_microvm_tests.py scripts/test_development_release.py scripts/test_host_telemetry.py scripts/test_time_abi.py scripts/test_tool_versions.py scripts/test_markdown_links.py -v
         if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
         python scripts\test_adversarial.py -v
         if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

.github/copilot-instructions.md

@@ -50,9 +50,10 @@
   [copilot-setup-steps](./workflows/copilot-setup-steps.yml) prepares. It
   initializes `openvmm/`; installs `requirements-dev.txt` into a Python virtual
   environment on `PATH`, the Rust release that `rust-toolchain.toml` pins with
-  the OpenVMM test targets, the minimum supported Rust versions of the
-  `aci_edge_sandboxes` and `nvx-rs` crates, cargo-nextest, and the native guest build
-  prerequisites; grants `/dev/kvm` access; pulls the pinned shell linter images;
+  the OpenVMM test targets and the `x86_64-unknown-linux-musl` target used by
+  the Rust guest binaries, the minimum supported Rust versions of the
+  `aci_edge_sandboxes` and `nvx-rs` crates, cargo-nextest, and the remaining
+  native guest build prerequisites; grants `/dev/kvm` access; pulls the pinned shell linter images;
   and stages the pinned Linux and Ubuntu Base archives in `.cache/downloads`.
   When CI has cached them for the checked-out inputs, it also restores the guest
   artifacts into `build/` and the KVM OpenVMM binary into
@@ -65,6 +66,13 @@
   and guest artifacts only when their inputs changed. Leave `test-openvmm-unit`,
   `test-openvmm`, `build-guest --guest all`, `verify-guest-determinism`, and
   `benchmark` to CI.
+- `nvx.py sandbox`, `nvx.py run`, and the microVM scenarios that drive them run
+  `nvx-rs`, which `nvx.py` builds with Cargo into `nvx-rs/target/release/` the
+  first time that a command needs it, in about a minute. Rebuild it with
+  `cargo build --release --locked --features cli --manifest-path nvx-rs/Cargo.toml`
+  after changing `nvx-rs/`, or set `NVX_RS` to another build (see
+  [Usage](../doc/usage.md#nvx-rs)). The `managed-lifecycle` scenario runs an
+  `nvx-rs` test with Cargo.
 - Build trees are large. Check `df -h .` before a build, keep one OpenVMM build
   profile at a time, and delete trees you no longer need with `cargo clean` or
   `docker builder prune -af`. The setup disables incremental compilation and