ci: classify microVM tests, benchmarks, and artifact builds separately

.github/workflows/ci.yml

@@ -269,10 +269,12 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 5
     outputs:
-      run-tests: ${{ steps.changes.outputs.run-tests }}
-      run-workloads: ${{ steps.changes.outputs.run-workloads }}
+      run-guest-artifacts: ${{ steps.changes.outputs.run-guest-artifacts }}
+      run-openvmm-binaries: ${{ steps.changes.outputs.run-openvmm-binaries }}
       run-openvmm-unit-tests: ${{ steps.changes.outputs.run-openvmm-unit-tests }}
       run-openvmm-vmm-tests: ${{ steps.changes.outputs.run-openvmm-vmm-tests }}
+      run-microvm-tests: ${{ steps.changes.outputs.run-microvm-tests }}
+      run-benchmarks: ${{ steps.changes.outputs.run-benchmarks }}
     steps:
       - name: Checkout
         uses: actions/checkout@v5
@@ -293,7 +295,10 @@ jobs:
         run: |
           set -euo pipefail
 
-          outputs=(run-tests run-workloads run-openvmm-unit-tests run-openvmm-vmm-tests)
+          outputs=(
+            run-guest-artifacts run-openvmm-binaries run-openvmm-unit-tests
+            run-openvmm-vmm-tests run-microvm-tests run-benchmarks
+          )
 
           run_everything() {
             for output in "${outputs[@]}"; do
@@ -359,7 +364,7 @@ jobs:
         ) &&
         (
           needs.openvmm-changes.result != 'success' ||
-          needs.openvmm-changes.outputs.run-workloads == 'true'
+          needs.openvmm-changes.outputs.run-guest-artifacts == 'true'
         )
       }}
     runs-on: ubuntu-latest
@@ -414,7 +419,7 @@ jobs:
         ) &&
         (
           needs.openvmm-changes.result != 'success' ||
-          needs.openvmm-changes.outputs.run-workloads == 'true'
+          needs.openvmm-changes.outputs.run-guest-artifacts == 'true'
         )
       }}
     runs-on: ubuntu-latest
@@ -460,8 +465,7 @@ jobs:
         ) &&
         (
           needs.openvmm-changes.result != 'success' ||
-          needs.openvmm-changes.outputs.run-tests == 'true' ||
-          needs.openvmm-changes.outputs.run-workloads == 'true'
+          needs.openvmm-changes.outputs.run-openvmm-binaries == 'true'
         )
       }}
     uses: ./.github/workflows/build-openvmm-binary.yml
@@ -491,8 +495,7 @@ jobs:
         ) &&
         (
           needs.openvmm-changes.result != 'success' ||
-          needs.openvmm-changes.outputs.run-tests == 'true' ||
-          needs.openvmm-changes.outputs.run-workloads == 'true'
+          needs.openvmm-changes.outputs.run-openvmm-binaries == 'true'
         )
       }}
     uses: ./.github/workflows/build-openvmm-binary.yml
@@ -522,8 +525,7 @@ jobs:
         ) &&
         (
           needs.openvmm-changes.result != 'success' ||
-          needs.openvmm-changes.outputs.run-tests == 'true' ||
-          needs.openvmm-changes.outputs.run-workloads == 'true'
+          needs.openvmm-changes.outputs.run-openvmm-binaries == 'true'
         )
       }}
     uses: ./.github/workflows/build-openvmm-binary.yml
@@ -1040,7 +1042,7 @@ jobs:
         needs.nvx-cli-linux.result == 'success' &&
         (
           needs.openvmm-changes.result != 'success' ||
-          needs.openvmm-changes.outputs.run-tests == 'true'
+          needs.openvmm-changes.outputs.run-microvm-tests == 'true'
         )
       }}
     uses: ./.github/workflows/run-nvx-microvm-tests.yml
@@ -1073,7 +1075,7 @@ jobs:
         needs.nvx-cli-linux.result == 'success' &&
         (
           needs.openvmm-changes.result != 'success' ||
-          needs.openvmm-changes.outputs.run-tests == 'true'
+          needs.openvmm-changes.outputs.run-microvm-tests == 'true'
         )
       }}
     uses: ./.github/workflows/run-nvx-microvm-tests.yml
@@ -1106,7 +1108,7 @@ jobs:
         needs.nvx-cli-windows.result == 'success' &&
         (
           needs.openvmm-changes.result != 'success' ||
-          needs.openvmm-changes.outputs.run-tests == 'true'
+          needs.openvmm-changes.outputs.run-microvm-tests == 'true'
         )
       }}
     uses: ./.github/workflows/run-nvx-microvm-tests.yml
@@ -1142,7 +1144,7 @@ jobs:
         needs.nvx-cli-linux.result == 'success' &&
         (
           needs.openvmm-changes.result != 'success' ||
-          needs.openvmm-changes.outputs.run-tests == 'true'
+          needs.openvmm-changes.outputs.run-microvm-tests == 'true'
         )
       }}
     uses: ./.github/workflows/run-nvx-microvm-tests.yml
@@ -1170,7 +1172,7 @@ jobs:
         needs.nvx-cli-linux.result == 'success' &&
         (
           needs.openvmm-changes.result != 'success' ||
-          needs.openvmm-changes.outputs.run-tests == 'true'
+          needs.openvmm-changes.outputs.run-microvm-tests == 'true'
         )
       }}
     uses: ./.github/workflows/run-nvx-microvm-tests.yml
@@ -1198,7 +1200,7 @@ jobs:
         needs.nvx-cli-windows.result == 'success' &&
         (
           needs.openvmm-changes.result != 'success' ||
-          needs.openvmm-changes.outputs.run-tests == 'true'
+          needs.openvmm-changes.outputs.run-microvm-tests == 'true'
         )
       }}
     uses: ./.github/workflows/run-nvx-microvm-tests.yml
@@ -1217,7 +1219,10 @@ jobs:
   # that carries `benchmark` instead of `general`, so that no other job can
   # delay it and every workflow run measures its series on the same VM: the
   # pools mix CPU generations and VMs whose results differ by more than the
-  # run-to-run noise (#383, #415).
+  # run-to-run noise (#383, #415). They also package the release, so a pull
+  # request runs them only when one of their inputs changed, but a dev push,
+  # which publishes the development release and records the history, runs
+  # them for every change but documentation.
   platform-kvm:
     name: Platform / Linux / KVM / Virtual machine
     needs: [artifacts, build-openvmm-linux-gnu, nvx-cli-linux, openvmm-changes]
@@ -1238,7 +1243,7 @@ jobs:
         needs.nvx-cli-linux.result == 'success' &&
         (
           needs.openvmm-changes.result != 'success' ||
-          needs.openvmm-changes.outputs.run-workloads == 'true'
+          needs.openvmm-changes.outputs.run-benchmarks == 'true'
         )
       }}
     uses: ./.github/workflows/run-platform.yml
@@ -1278,7 +1283,7 @@ jobs:
         needs.nvx-cli-linux.result == 'success' &&
         (
           needs.openvmm-changes.result != 'success' ||
-          needs.openvmm-changes.outputs.run-workloads == 'true'
+          needs.openvmm-changes.outputs.run-benchmarks == 'true'
         )
       }}
     uses: ./.github/workflows/run-platform.yml
@@ -1318,7 +1323,7 @@ jobs:
         needs.nvx-cli-windows.result == 'success' &&
         (
           needs.openvmm-changes.result != 'success' ||
-          needs.openvmm-changes.outputs.run-workloads == 'true'
+          needs.openvmm-changes.outputs.run-benchmarks == 'true'
         )
       }}
     uses: ./.github/workflows/run-platform.yml
@@ -1493,10 +1498,12 @@ jobs:
               github.event_name != 'pull_request' ||
               github.event.pull_request.head.repo.full_name == github.repository
             }}
-          RUN_TESTS: ${{ needs.openvmm-changes.outputs.run-tests }}
-          RUN_WORKLOADS: ${{ needs.openvmm-changes.outputs.run-workloads }}
+          RUN_GUEST_ARTIFACTS: ${{ needs.openvmm-changes.outputs.run-guest-artifacts }}
+          RUN_OPENVMM_BINARIES: ${{ needs.openvmm-changes.outputs.run-openvmm-binaries }}
           RUN_OPENVMM_UNIT_TESTS: ${{ needs.openvmm-changes.outputs.run-openvmm-unit-tests }}
           RUN_OPENVMM_VMM_TESTS: ${{ needs.openvmm-changes.outputs.run-openvmm-vmm-tests }}
+          RUN_MICROVM_TESTS: ${{ needs.openvmm-changes.outputs.run-microvm-tests }}
+          RUN_BENCHMARKS: ${{ needs.openvmm-changes.outputs.run-benchmarks }}
           QUALITY_RESULT: ${{ needs.quality.result }}
           ACI_EDGE_SANDBOXES_RESULT: ${{ needs.aci-edge-sandboxes.result }}
           NVX_RS_RESULT: ${{ needs.nvx-rs.result }}
@@ -1524,10 +1531,12 @@ jobs:
           python3 scripts/nvx.py check-required-ci
           --event-name "${{ github.event_name }}"
           --same-repository "${SAME_REPOSITORY}"
-          --run-tests "${RUN_TESTS}"
-          --run-workloads "${RUN_WORKLOADS}"
+          --run-guest-artifacts "${RUN_GUEST_ARTIFACTS}"
+          --run-openvmm-binaries "${RUN_OPENVMM_BINARIES}"
           --run-openvmm-unit-tests "${RUN_OPENVMM_UNIT_TESTS}"
           --run-openvmm-vmm-tests "${RUN_OPENVMM_VMM_TESTS}"
+          --run-microvm-tests "${RUN_MICROVM_TESTS}"
+          --run-benchmarks "${RUN_BENCHMARKS}"
 
   performance-persist:
     name: Persist performance baseline

doc/ci.md

@@ -441,7 +441,9 @@ short-lived workflow artifacts alongside the OpenVMM handoff and benchmark
 results. Caches only accelerate reproducible build inputs and outputs; consumers
 do not depend on them as a handoff.
 Pull requests gate regressions against recent matching-platform history, and
-successful pushes to `dev` append their p50 values under `data/`. Metadata-only
+successful pushes to `dev` append their p50 values under `data/`. A pull
+request that changes no input of the platform jobs skips them and the gate
+(see [Change classification](#change-classification)). Metadata-only
 performance jobs use GitHub-hosted Ubuntu runners. Provisioning instructions
 are in the [runner bootstrap guide](../scripts/setup/README.md).
 
@@ -458,7 +460,8 @@ manually dispatched full run starts these jobs at once: `quality`,
 `aci-edge-sandboxes`, `nvx-rs`, `nvx-cli-linux`, and `nvx-cli-windows` on
 GitHub-hosted runners, and `openvmm-changes`, which classifies a push or pull
 request and selects every suite for a full run (see
-[Change classification](#change-classification)). When it finishes,
+[Change classification](#change-classification)). When it finishes, the jobs
+that it selects start:
 `artifacts` and `debug-kernel` restore or build the guest artifacts on
 GitHub-hosted runners, the three `build-openvmm-*` producers restore or build
 OpenVMM on a KVM, an MSHV, and a WHP runner, and `openvmm-unit-tests` starts on
@@ -539,16 +542,29 @@ and head, or between a push's previous and new commits, and
 `nvx.py classify-ci-changes` selects the suites from their paths. The detector
 uses `git diff --name-only --no-renames -z` and the classifier's `--null` option
 so both sides of a rename and unquoted filenames, including embedded newlines,
-participate in the decision:
-
-- `run-workloads` is false only when every changed file is documentation,
-  under `doc/` or ending in `.md`. It enables the guest artifacts, the OpenVMM
-  producers, and the platform benchmarks.
-- `run-tests` is true when a test input changed: the OpenVMM pin, the guests
-  and their package locks, the CLI and its modules, the workflows and actions
-  that build and run the tests, the runner setup scripts, the Rust pin, the
-  `aci_edge_sandboxes` and `nvx` crates, or the runtime adapter in
-  `tests/runtime-adapter/`. It enables the NVX microVM tests.
+participate in the decision. Documentation, every path under `doc/` or ending
+in `.md`, selects no suite, even in a suite's input directory, such as
+`scripts/setup/README.md`, because no suite builds, boots, or runs it. Every
+other path selects each suite that takes it as an input:
+
+- `run-microvm-tests` enables the NVX microVM tests and their debug-kernel
+  jobs. Their inputs are the OpenVMM pin, the guests and every input of their
+  cache keys, the `nvx` crate, whose executable their scenarios run and whose
+  guest binaries every guest ships, the `aci_edge_sandboxes` crate and the
+  runtime adapter in `tests/runtime-adapter/`, which they test, the workflows,
+  actions, and CLI modules that their jobs and the jobs that build their
+  artifacts run, the runner setup scripts, the Rust pin, and
+  `.gitattributes`, which checks out their scripts on Windows.
+- `run-benchmarks` enables the platform jobs, which benchmark each backend and
+  package its release, and so a pull request's performance regression gate.
+  Their inputs are the OpenVMM pin, the guests and every input of their cache
+  keys, the `nvx` crate, which every package ships, the performance history
+  and expected hosts under `data/`, `VERSION`, `LICENSE`, the workflows,
+  actions, and CLI modules that the platform jobs, the gate, and the jobs that
+  build their artifacts run, the runner setup scripts, the Rust pin, and
+  `.gitattributes`. A pull request sets it only when one of these inputs
+  changed, but a push sets it for every change but documentation; see
+  [Development releases and performance history](#development-releases-and-performance-history).
 - `run-openvmm-unit-tests` and `run-openvmm-vmm-tests` enable the OpenVMM
   suites. A push or pull request sets each only when one of that suite's own
   inputs changed. The unit tests depend on the OpenVMM pin, the Rust pin, the
@@ -558,13 +574,40 @@ participate in the decision:
   the `build-guest-artifacts` action. Of `nvx-rs`, those are only the inputs
   of the guest binaries, which `GuestRustBuildConstants.SOURCE_PATTERN`
   matches, so a change to its host side or tests runs the NVX microVM tests
-  but not the VMM tests.
+  and the platform jobs but not the VMM tests.
+- `run-guest-artifacts` enables `artifacts` and `debug-kernel`, which build
+  the guest artifacts and the CI debug kernel, and `run-openvmm-binaries` the
+  three `build-openvmm-*` producers. Neither has a use of its own, so the
+  classifier sets each exactly when a job that uses its artifacts runs: the
+  guest artifacts for the OpenVMM VMM tests, the NVX microVM tests, or the
+  platform jobs, and the OpenVMM binaries for the microVM tests or the
+  platform jobs. Every input of an artifact is an input of each job that uses
+  it. The debug kernel is built with the guest artifacts, as before, because
+  the development release and performance persistence require its build too.
+
+Every CLI module is an input of the microVM tests, and of the platform jobs,
+unless none of that suite's jobs runs it, so a new module selects both until
+the classifier excludes it. `microvm_tests.py` and the modules that only it,
+`nvx.py sandbox`, or `nvx.py test-aci-edge-sandboxes` use, with the
+`microvm_test_scripts/` that it runs in guests, are inputs of the microVM tests
+alone, and `performance.py` is an input of the platform jobs alone. The
+modules that both kinds of job run, such as `benchmark.py`, which
+`microvm_tests.py` imports, `doctor.py`, which every runner validation runs,
+and `release.py`, whose `verify` command the OpenVMM builds run, select both.
 
 A push or pull request that changes neither OpenVMM suite's inputs, such as
 one that changes only the microVM tests, the benchmarks, the Ubuntu or Azure
 Linux guests, or the host side of `nvx-rs`, therefore skips the two jobs that
-occupy the WHP runners longest. The nightly schedule runs at 05:17 UTC, and a
-manual workflow
+occupy the WHP runners longest. A pull request that changes only the microVM
+tests, such as their scenarios, the `aci_edge_sandboxes` crate, or the runtime
+adapter, also skips the platform jobs, which wait for their pool's single
+benchmark runner, and one that changes only the performance tooling, such as
+`performance.py`, the `run-benchmark` action, or `data/`, skips the microVM
+tests. One that changes the input of no suite, such as the CLI tests, the
+adversarial campaigns, or the GitHub-hosted checks, runs only the
+GitHub-hosted jobs.
+
+The nightly schedule runs at 05:17 UTC, and a manual workflow
 dispatch starts the same full run. Both select the full matrix before the
 detector examines a comparison base. Each full run has a separate concurrency
 key, so it neither cancels nor queues behind a `dev` push. Full runs build,
@@ -580,16 +623,43 @@ misses.
 
 For a push or pull request, the detector runs the full matrix instead when the
 comparison base or a commit is unavailable, when the diff or the classifier
-fails, or when the classifier reports a suite twice or not at all, because a
+fails, or when the classifier reports an output twice or not at all, because a
 missing output would skip its jobs.
 `Required status check` passes the same outputs to `check-required-ci`, which
-expects every scheduled job to succeed and every other job to be skipped.
-Tests in `scripts/test_nvx_tools.py` derive each suite's inputs from the
-actions that its jobs use, the modules that `ci.py` imports, and the guest
-cache keys, including the files of the guest binaries' source identity, and
-fail when a pattern misses one. They also check that no other file of
-`nvx-rs` selects the VMM tests or changes a guest image key, while every one
-selects the NVX microVM tests.
+expects every scheduled job to succeed and every other job to be skipped. As
+in `ci.yml`, it schedules a job only when the jobs that build the artifacts
+that it uses are scheduled too.
+Tests in `scripts/test_nvx_tools.py` derive each suite's inputs and fail when
+a pattern misses one. The OpenVMM suites' come from the actions that their
+jobs use, the modules that `ci.py` imports, and the guest cache keys,
+including the files of the guest binaries' source identity. The microVM
+tests' and the platform jobs' come from those jobs and the jobs that build the
+artifacts that they use: the reusable workflows and actions that these reach,
+the files that the cache keys there hash and the source identity's, the CLI
+modules of the `nvx.py` commands that they run, which the tests map from
+`nvx.py`'s parser, with every module that those import and every file that
+those read beside them, and the `nvx` crate, from which the NVX CLI jobs build
+the executable. The tests also check that no other file of `nvx-rs` selects
+the VMM tests or changes a guest image key, while every one selects the NVX
+microVM tests and the platform jobs.
+
+### Development releases and performance history
+
+A `dev` push publishes its commit's development release and appends the
+platform jobs' results to the performance history. `Publish development
+release` and `Persist performance baseline` require all three platform jobs to
+succeed, so a push that skipped them would publish nothing partial or stale:
+it would publish no release and record no history. Its commit would have no
+release of its own, though, and [`download`](usage.md#download) would install
+an ancestor's, which is older still when that ancestor's release failed. A
+push therefore runs the platform jobs, and the jobs that build their
+artifacts, for every change but documentation, as every push did before the
+classifier selected the platform jobs apart from the tests. Of the 108 merges
+to `dev` from October 10 to 21, 2026 that changed more than documentation, 10
+changed no input of the platform jobs, and each of their pushes spent about
+19 benchmark-runner minutes on them: the KVM, MSHV, and WHP platform jobs take
+a median of 5.3, 5.5, and 8.6 minutes. Pull requests publish and persist
+nothing, so they run the platform jobs only when one of their inputs changed.
 
 ## Host qualification
 

doc/contribute.md

@@ -52,8 +52,8 @@ CI runs for a pull request to `dev` when it is opened, reopened, pushed to,
 or marked ready for review, and skips its lint, build, and test jobs while the
 pull request is a draft. A new push cancels the pull request's run in
 progress; converting it back to a draft does not. A ready pull request that
-changes more than documentation occupies the shared self-hosted KVM, MSHV,
-and WHP runner pools, so:
+changes more than documentation runs the hypervisor jobs that its changes
+select on the shared self-hosted KVM, MSHV, and WHP runner pools, so:
 
 - Open a **Draft** when the change touches more than documentation and has
   not yet passed the [quality gates](#quality-gates) and