diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7db9b76c9..be2857e81 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -269,10 +269,12 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 outputs: - run-tests: ${{ steps.changes.outputs.run-tests }} - run-workloads: ${{ steps.changes.outputs.run-workloads }} + run-guest-artifacts: ${{ steps.changes.outputs.run-guest-artifacts }} + run-openvmm-binaries: ${{ steps.changes.outputs.run-openvmm-binaries }} run-openvmm-unit-tests: ${{ steps.changes.outputs.run-openvmm-unit-tests }} run-openvmm-vmm-tests: ${{ steps.changes.outputs.run-openvmm-vmm-tests }} + run-microvm-tests: ${{ steps.changes.outputs.run-microvm-tests }} + run-benchmarks: ${{ steps.changes.outputs.run-benchmarks }} steps: - name: Checkout uses: actions/checkout@v5 @@ -293,7 +295,10 @@ jobs: run: | set -euo pipefail - outputs=(run-tests run-workloads run-openvmm-unit-tests run-openvmm-vmm-tests) + outputs=( + run-guest-artifacts run-openvmm-binaries run-openvmm-unit-tests + run-openvmm-vmm-tests run-microvm-tests run-benchmarks + ) run_everything() { for output in "${outputs[@]}"; do @@ -359,7 +364,7 @@ jobs: ) && ( needs.openvmm-changes.result != 'success' || - needs.openvmm-changes.outputs.run-workloads == 'true' + needs.openvmm-changes.outputs.run-guest-artifacts == 'true' ) }} runs-on: ubuntu-latest @@ -414,7 +419,7 @@ jobs: ) && ( needs.openvmm-changes.result != 'success' || - needs.openvmm-changes.outputs.run-workloads == 'true' + needs.openvmm-changes.outputs.run-guest-artifacts == 'true' ) }} runs-on: ubuntu-latest @@ -460,8 +465,7 @@ jobs: ) && ( needs.openvmm-changes.result != 'success' || - needs.openvmm-changes.outputs.run-tests == 'true' || - needs.openvmm-changes.outputs.run-workloads == 'true' + needs.openvmm-changes.outputs.run-openvmm-binaries == 'true' ) }} uses: ./.github/workflows/build-openvmm-binary.yml @@ -491,8 +495,7 @@ jobs: ) && ( needs.openvmm-changes.result != 'success' || - needs.openvmm-changes.outputs.run-tests == 'true' || - needs.openvmm-changes.outputs.run-workloads == 'true' + needs.openvmm-changes.outputs.run-openvmm-binaries == 'true' ) }} uses: ./.github/workflows/build-openvmm-binary.yml @@ -522,8 +525,7 @@ jobs: ) && ( needs.openvmm-changes.result != 'success' || - needs.openvmm-changes.outputs.run-tests == 'true' || - needs.openvmm-changes.outputs.run-workloads == 'true' + needs.openvmm-changes.outputs.run-openvmm-binaries == 'true' ) }} uses: ./.github/workflows/build-openvmm-binary.yml @@ -1040,7 +1042,7 @@ jobs: needs.nvx-cli-linux.result == 'success' && ( needs.openvmm-changes.result != 'success' || - needs.openvmm-changes.outputs.run-tests == 'true' + needs.openvmm-changes.outputs.run-microvm-tests == 'true' ) }} uses: ./.github/workflows/run-nvx-microvm-tests.yml @@ -1073,7 +1075,7 @@ jobs: needs.nvx-cli-linux.result == 'success' && ( needs.openvmm-changes.result != 'success' || - needs.openvmm-changes.outputs.run-tests == 'true' + needs.openvmm-changes.outputs.run-microvm-tests == 'true' ) }} uses: ./.github/workflows/run-nvx-microvm-tests.yml @@ -1106,7 +1108,7 @@ jobs: needs.nvx-cli-windows.result == 'success' && ( needs.openvmm-changes.result != 'success' || - needs.openvmm-changes.outputs.run-tests == 'true' + needs.openvmm-changes.outputs.run-microvm-tests == 'true' ) }} uses: ./.github/workflows/run-nvx-microvm-tests.yml @@ -1142,7 +1144,7 @@ jobs: needs.nvx-cli-linux.result == 'success' && ( needs.openvmm-changes.result != 'success' || - needs.openvmm-changes.outputs.run-tests == 'true' + needs.openvmm-changes.outputs.run-microvm-tests == 'true' ) }} uses: ./.github/workflows/run-nvx-microvm-tests.yml @@ -1170,7 +1172,7 @@ jobs: needs.nvx-cli-linux.result == 'success' && ( needs.openvmm-changes.result != 'success' || - needs.openvmm-changes.outputs.run-tests == 'true' + needs.openvmm-changes.outputs.run-microvm-tests == 'true' ) }} uses: ./.github/workflows/run-nvx-microvm-tests.yml @@ -1198,7 +1200,7 @@ jobs: needs.nvx-cli-windows.result == 'success' && ( needs.openvmm-changes.result != 'success' || - needs.openvmm-changes.outputs.run-tests == 'true' + needs.openvmm-changes.outputs.run-microvm-tests == 'true' ) }} uses: ./.github/workflows/run-nvx-microvm-tests.yml @@ -1217,7 +1219,10 @@ jobs: # that carries `benchmark` instead of `general`, so that no other job can # delay it and every workflow run measures its series on the same VM: the # pools mix CPU generations and VMs whose results differ by more than the - # run-to-run noise (#383, #415). + # run-to-run noise (#383, #415). They also package the release, so a pull + # request runs them only when one of their inputs changed, but a dev push, + # which publishes the development release and records the history, runs + # them for every change but documentation. platform-kvm: name: Platform / Linux / KVM / Virtual machine needs: [artifacts, build-openvmm-linux-gnu, nvx-cli-linux, openvmm-changes] @@ -1238,7 +1243,7 @@ jobs: needs.nvx-cli-linux.result == 'success' && ( needs.openvmm-changes.result != 'success' || - needs.openvmm-changes.outputs.run-workloads == 'true' + needs.openvmm-changes.outputs.run-benchmarks == 'true' ) }} uses: ./.github/workflows/run-platform.yml @@ -1278,7 +1283,7 @@ jobs: needs.nvx-cli-linux.result == 'success' && ( needs.openvmm-changes.result != 'success' || - needs.openvmm-changes.outputs.run-workloads == 'true' + needs.openvmm-changes.outputs.run-benchmarks == 'true' ) }} uses: ./.github/workflows/run-platform.yml @@ -1318,7 +1323,7 @@ jobs: needs.nvx-cli-windows.result == 'success' && ( needs.openvmm-changes.result != 'success' || - needs.openvmm-changes.outputs.run-workloads == 'true' + needs.openvmm-changes.outputs.run-benchmarks == 'true' ) }} uses: ./.github/workflows/run-platform.yml @@ -1493,10 +1498,12 @@ jobs: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} - RUN_TESTS: ${{ needs.openvmm-changes.outputs.run-tests }} - RUN_WORKLOADS: ${{ needs.openvmm-changes.outputs.run-workloads }} + RUN_GUEST_ARTIFACTS: ${{ needs.openvmm-changes.outputs.run-guest-artifacts }} + RUN_OPENVMM_BINARIES: ${{ needs.openvmm-changes.outputs.run-openvmm-binaries }} RUN_OPENVMM_UNIT_TESTS: ${{ needs.openvmm-changes.outputs.run-openvmm-unit-tests }} RUN_OPENVMM_VMM_TESTS: ${{ needs.openvmm-changes.outputs.run-openvmm-vmm-tests }} + RUN_MICROVM_TESTS: ${{ needs.openvmm-changes.outputs.run-microvm-tests }} + RUN_BENCHMARKS: ${{ needs.openvmm-changes.outputs.run-benchmarks }} QUALITY_RESULT: ${{ needs.quality.result }} ACI_EDGE_SANDBOXES_RESULT: ${{ needs.aci-edge-sandboxes.result }} NVX_RS_RESULT: ${{ needs.nvx-rs.result }} @@ -1524,10 +1531,12 @@ jobs: python3 scripts/nvx.py check-required-ci --event-name "${{ github.event_name }}" --same-repository "${SAME_REPOSITORY}" - --run-tests "${RUN_TESTS}" - --run-workloads "${RUN_WORKLOADS}" + --run-guest-artifacts "${RUN_GUEST_ARTIFACTS}" + --run-openvmm-binaries "${RUN_OPENVMM_BINARIES}" --run-openvmm-unit-tests "${RUN_OPENVMM_UNIT_TESTS}" --run-openvmm-vmm-tests "${RUN_OPENVMM_VMM_TESTS}" + --run-microvm-tests "${RUN_MICROVM_TESTS}" + --run-benchmarks "${RUN_BENCHMARKS}" performance-persist: name: Persist performance baseline diff --git a/doc/ci.md b/doc/ci.md index 45f0102af..991a66753 100644 --- a/doc/ci.md +++ b/doc/ci.md @@ -441,7 +441,9 @@ short-lived workflow artifacts alongside the OpenVMM handoff and benchmark results. Caches only accelerate reproducible build inputs and outputs; consumers do not depend on them as a handoff. Pull requests gate regressions against recent matching-platform history, and -successful pushes to `dev` append their p50 values under `data/`. Metadata-only +successful pushes to `dev` append their p50 values under `data/`. A pull +request that changes no input of the platform jobs skips them and the gate +(see [Change classification](#change-classification)). Metadata-only performance jobs use GitHub-hosted Ubuntu runners. Provisioning instructions are in the [runner bootstrap guide](../scripts/setup/README.md). @@ -458,7 +460,8 @@ manually dispatched full run starts these jobs at once: `quality`, `aci-edge-sandboxes`, `nvx-rs`, `nvx-cli-linux`, and `nvx-cli-windows` on GitHub-hosted runners, and `openvmm-changes`, which classifies a push or pull request and selects every suite for a full run (see -[Change classification](#change-classification)). When it finishes, +[Change classification](#change-classification)). When it finishes, the jobs +that it selects start: `artifacts` and `debug-kernel` restore or build the guest artifacts on GitHub-hosted runners, the three `build-openvmm-*` producers restore or build OpenVMM on a KVM, an MSHV, and a WHP runner, and `openvmm-unit-tests` starts on @@ -539,16 +542,29 @@ and head, or between a push's previous and new commits, and `nvx.py classify-ci-changes` selects the suites from their paths. The detector uses `git diff --name-only --no-renames -z` and the classifier's `--null` option so both sides of a rename and unquoted filenames, including embedded newlines, -participate in the decision: - -- `run-workloads` is false only when every changed file is documentation, - under `doc/` or ending in `.md`. It enables the guest artifacts, the OpenVMM - producers, and the platform benchmarks. -- `run-tests` is true when a test input changed: the OpenVMM pin, the guests - and their package locks, the CLI and its modules, the workflows and actions - that build and run the tests, the runner setup scripts, the Rust pin, the - `aci_edge_sandboxes` and `nvx` crates, or the runtime adapter in - `tests/runtime-adapter/`. It enables the NVX microVM tests. +participate in the decision. Documentation, every path under `doc/` or ending +in `.md`, selects no suite, even in a suite's input directory, such as +`scripts/setup/README.md`, because no suite builds, boots, or runs it. Every +other path selects each suite that takes it as an input: + +- `run-microvm-tests` enables the NVX microVM tests and their debug-kernel + jobs. Their inputs are the OpenVMM pin, the guests and every input of their + cache keys, the `nvx` crate, whose executable their scenarios run and whose + guest binaries every guest ships, the `aci_edge_sandboxes` crate and the + runtime adapter in `tests/runtime-adapter/`, which they test, the workflows, + actions, and CLI modules that their jobs and the jobs that build their + artifacts run, the runner setup scripts, the Rust pin, and + `.gitattributes`, which checks out their scripts on Windows. +- `run-benchmarks` enables the platform jobs, which benchmark each backend and + package its release, and so a pull request's performance regression gate. + Their inputs are the OpenVMM pin, the guests and every input of their cache + keys, the `nvx` crate, which every package ships, the performance history + and expected hosts under `data/`, `VERSION`, `LICENSE`, the workflows, + actions, and CLI modules that the platform jobs, the gate, and the jobs that + build their artifacts run, the runner setup scripts, the Rust pin, and + `.gitattributes`. A pull request sets it only when one of these inputs + changed, but a push sets it for every change but documentation; see + [Development releases and performance history](#development-releases-and-performance-history). - `run-openvmm-unit-tests` and `run-openvmm-vmm-tests` enable the OpenVMM suites. A push or pull request sets each only when one of that suite's own inputs changed. The unit tests depend on the OpenVMM pin, the Rust pin, the @@ -558,13 +574,40 @@ participate in the decision: the `build-guest-artifacts` action. Of `nvx-rs`, those are only the inputs of the guest binaries, which `GuestRustBuildConstants.SOURCE_PATTERN` matches, so a change to its host side or tests runs the NVX microVM tests - but not the VMM tests. + and the platform jobs but not the VMM tests. +- `run-guest-artifacts` enables `artifacts` and `debug-kernel`, which build + the guest artifacts and the CI debug kernel, and `run-openvmm-binaries` the + three `build-openvmm-*` producers. Neither has a use of its own, so the + classifier sets each exactly when a job that uses its artifacts runs: the + guest artifacts for the OpenVMM VMM tests, the NVX microVM tests, or the + platform jobs, and the OpenVMM binaries for the microVM tests or the + platform jobs. Every input of an artifact is an input of each job that uses + it. The debug kernel is built with the guest artifacts, as before, because + the development release and performance persistence require its build too. + +Every CLI module is an input of the microVM tests, and of the platform jobs, +unless none of that suite's jobs runs it, so a new module selects both until +the classifier excludes it. `microvm_tests.py` and the modules that only it, +`nvx.py sandbox`, or `nvx.py test-aci-edge-sandboxes` use, with the +`microvm_test_scripts/` that it runs in guests, are inputs of the microVM tests +alone, and `performance.py` is an input of the platform jobs alone. The +modules that both kinds of job run, such as `benchmark.py`, which +`microvm_tests.py` imports, `doctor.py`, which every runner validation runs, +and `release.py`, whose `verify` command the OpenVMM builds run, select both. A push or pull request that changes neither OpenVMM suite's inputs, such as one that changes only the microVM tests, the benchmarks, the Ubuntu or Azure Linux guests, or the host side of `nvx-rs`, therefore skips the two jobs that -occupy the WHP runners longest. The nightly schedule runs at 05:17 UTC, and a -manual workflow +occupy the WHP runners longest. A pull request that changes only the microVM +tests, such as their scenarios, the `aci_edge_sandboxes` crate, or the runtime +adapter, also skips the platform jobs, which wait for their pool's single +benchmark runner, and one that changes only the performance tooling, such as +`performance.py`, the `run-benchmark` action, or `data/`, skips the microVM +tests. One that changes the input of no suite, such as the CLI tests, the +adversarial campaigns, or the GitHub-hosted checks, runs only the +GitHub-hosted jobs. + +The nightly schedule runs at 05:17 UTC, and a manual workflow dispatch starts the same full run. Both select the full matrix before the detector examines a comparison base. Each full run has a separate concurrency key, so it neither cancels nor queues behind a `dev` push. Full runs build, @@ -580,16 +623,43 @@ misses. For a push or pull request, the detector runs the full matrix instead when the comparison base or a commit is unavailable, when the diff or the classifier -fails, or when the classifier reports a suite twice or not at all, because a +fails, or when the classifier reports an output twice or not at all, because a missing output would skip its jobs. `Required status check` passes the same outputs to `check-required-ci`, which -expects every scheduled job to succeed and every other job to be skipped. -Tests in `scripts/test_nvx_tools.py` derive each suite's inputs from the -actions that its jobs use, the modules that `ci.py` imports, and the guest -cache keys, including the files of the guest binaries' source identity, and -fail when a pattern misses one. They also check that no other file of -`nvx-rs` selects the VMM tests or changes a guest image key, while every one -selects the NVX microVM tests. +expects every scheduled job to succeed and every other job to be skipped. As +in `ci.yml`, it schedules a job only when the jobs that build the artifacts +that it uses are scheduled too. +Tests in `scripts/test_nvx_tools.py` derive each suite's inputs and fail when +a pattern misses one. The OpenVMM suites' come from the actions that their +jobs use, the modules that `ci.py` imports, and the guest cache keys, +including the files of the guest binaries' source identity. The microVM +tests' and the platform jobs' come from those jobs and the jobs that build the +artifacts that they use: the reusable workflows and actions that these reach, +the files that the cache keys there hash and the source identity's, the CLI +modules of the `nvx.py` commands that they run, which the tests map from +`nvx.py`'s parser, with every module that those import and every file that +those read beside them, and the `nvx` crate, from which the NVX CLI jobs build +the executable. The tests also check that no other file of `nvx-rs` selects +the VMM tests or changes a guest image key, while every one selects the NVX +microVM tests and the platform jobs. + +### Development releases and performance history + +A `dev` push publishes its commit's development release and appends the +platform jobs' results to the performance history. `Publish development +release` and `Persist performance baseline` require all three platform jobs to +succeed, so a push that skipped them would publish nothing partial or stale: +it would publish no release and record no history. Its commit would have no +release of its own, though, and [`download`](usage.md#download) would install +an ancestor's, which is older still when that ancestor's release failed. A +push therefore runs the platform jobs, and the jobs that build their +artifacts, for every change but documentation, as every push did before the +classifier selected the platform jobs apart from the tests. Of the 108 merges +to `dev` from October 10 to 21, 2026 that changed more than documentation, 10 +changed no input of the platform jobs, and each of their pushes spent about +19 benchmark-runner minutes on them: the KVM, MSHV, and WHP platform jobs take +a median of 5.3, 5.5, and 8.6 minutes. Pull requests publish and persist +nothing, so they run the platform jobs only when one of their inputs changed. ## Host qualification diff --git a/doc/contribute.md b/doc/contribute.md index 7cef375a2..7504101f9 100644 --- a/doc/contribute.md +++ b/doc/contribute.md @@ -52,8 +52,8 @@ CI runs for a pull request to `dev` when it is opened, reopened, pushed to, or marked ready for review, and skips its lint, build, and test jobs while the pull request is a draft. A new push cancels the pull request's run in progress; converting it back to a draft does not. A ready pull request that -changes more than documentation occupies the shared self-hosted KVM, MSHV, -and WHP runner pools, so: +changes more than documentation runs the hypervisor jobs that its changes +select on the shared self-hosted KVM, MSHV, and WHP runner pools, so: - Open a **Draft** when the change touches more than documentation and has not yet passed the [quality gates](#quality-gates) and diff --git a/doc/distribution.md b/doc/distribution.md index bb722c9f4..99307ff31 100644 --- a/doc/distribution.md +++ b/doc/distribution.md @@ -130,9 +130,10 @@ consistency before deciding whether the build is compatible. A mismatched executable fails before runtime replacement, even for an incompatible fallback release. -CI follows the binary-only path. On a `dev` push or a ready pull request from -a branch of this repository that changes more than documentation, and on -scheduled or manually dispatched full runs, the `linux-kvm`, `linux-mshv`, +CI follows the binary-only path. On a `dev` push that changes more than +documentation, on a ready pull request from a branch of this repository that +changes an input of the platform jobs, and on scheduled or manually +dispatched full runs, the `linux-kvm`, `linux-mshv`, and `windows-whp` platform jobs each stage `dist/nvx--` with `--binary-only`, then archive it as `.tar.gz` on Linux and `.zip` on Windows through the diff --git a/doc/usage.md b/doc/usage.md index 7e3e62765..18a504b57 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -176,11 +176,15 @@ python3 scripts/nvx.py classify-ci-changes Reads one changed repository path per line by default, or NUL-delimited paths with `--null`, and prints the change detector's -`run-tests`, `run-workloads`, `run-openvmm-unit-tests`, and -`run-openvmm-vmm-tests` outputs as `name=true` or `name=false` lines. Only -documentation changes skip the workloads, and only test inputs run the NVX -microVM tests. Pull requests and pushes run each OpenVMM suite only when one -of that suite's own inputs changed. The summary file receives a Markdown +`run-guest-artifacts`, `run-openvmm-binaries`, `run-openvmm-unit-tests`, +`run-openvmm-vmm-tests`, `run-microvm-tests`, and `run-benchmarks` outputs as +`name=true` or `name=false` lines. Documentation selects no suite. Pull +requests and pushes run each test suite only when one of its own inputs +changed. A pull request runs the platform benchmarks only when one of their +inputs changed, but a push runs them for every change but documentation, +because it publishes the development release and records the performance +history. The guest artifacts and the OpenVMM binaries are built exactly when a +job that uses them runs. The summary file receives a Markdown explanation of each decision. CI runs the full matrix instead when the command fails or omits an output; see [Change classification](ci.md#change-classification). @@ -191,10 +195,12 @@ fails or omits an output; see python3 scripts/nvx.py check-required-ci --event-name {pull_request,push,schedule,workflow_dispatch} --same-repository {false,true} - --run-tests VALUE - --run-workloads VALUE + --run-guest-artifacts VALUE + --run-openvmm-binaries VALUE --run-openvmm-unit-tests VALUE --run-openvmm-vmm-tests VALUE + --run-microvm-tests VALUE + --run-benchmarks VALUE ``` Validates the GitHub Actions result values supplied through the @@ -203,6 +209,8 @@ variables. CI supplies the `true` or `false` suite flags that `classify-ci-changes` reports. The command expects successful results for jobs enabled by the event, repository, and suite flags, and `skipped` for jobs that are not enabled; mismatches are reported as errors and cause a nonzero exit. +As in CI, a job that uses the guest artifacts or the OpenVMM binaries is +enabled only when the flags enable the jobs that build them too. ### `check-static-elf` diff --git a/scripts/nvx.py b/scripts/nvx.py index 04ec46a93..84469e6e0 100644 --- a/scripts/nvx.py +++ b/scripts/nvx.py @@ -334,10 +334,12 @@ def command_check_required_ci(args: argparse.Namespace) -> None: args.event_name, same_repository=args.same_repository == "true", changes=CiChanges( - run_tests=args.run_tests == "true", - run_workloads=args.run_workloads == "true", + run_guest_artifacts=args.run_guest_artifacts == "true", + run_openvmm_binaries=args.run_openvmm_binaries == "true", run_openvmm_unit_tests=args.run_openvmm_unit_tests == "true", run_openvmm_vmm_tests=args.run_openvmm_vmm_tests == "true", + run_microvm_tests=args.run_microvm_tests == "true", + run_benchmarks=args.run_benchmarks == "true", ), results=results, ) @@ -896,9 +898,11 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: choices=("false", "true"), required=True, ) - required_ci.add_argument("--run-tests", choices=("false", "true"), required=True) required_ci.add_argument( - "--run-workloads", choices=("false", "true"), required=True + "--run-guest-artifacts", choices=("false", "true"), required=True + ) + required_ci.add_argument( + "--run-openvmm-binaries", choices=("false", "true"), required=True ) required_ci.add_argument( "--run-openvmm-unit-tests", choices=("false", "true"), required=True @@ -906,6 +910,12 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: required_ci.add_argument( "--run-openvmm-vmm-tests", choices=("false", "true"), required=True ) + required_ci.add_argument( + "--run-microvm-tests", choices=("false", "true"), required=True + ) + required_ci.add_argument( + "--run-benchmarks", choices=("false", "true"), required=True + ) required_ci.set_defaults(handler=command_check_required_ci) static_elf = subparsers.add_parser( diff --git a/scripts/nvx_tools/ci.py b/scripts/nvx_tools/ci.py index fa9b96f16..9423c0b64 100644 --- a/scripts/nvx_tools/ci.py +++ b/scripts/nvx_tools/ci.py @@ -64,7 +64,8 @@ # its result. REQUIRED_CI_CLI_JOBS = ("nvx-cli-linux", "nvx-cli-windows") REQUIRED_CI_OPENVMM_TEST_JOBS = ("openvmm-unit-tests",) -# OpenVMM VMM tests boot the NVX guest artifacts, which only workload runs build. +# OpenVMM VMM tests boot the NVX guest artifacts, so they need the job that +# builds them. REQUIRED_CI_OPENVMM_ARTIFACT_TEST_JOBS = ("openvmm-vmm-tests",) REQUIRED_CI_MICROVM_TEST_JOBS = ( "nvx-microvm-tests-kvm", @@ -102,26 +103,56 @@ # The workflow's change detector emits one output per entry, and runs the full # matrix when it cannot classify a change. CI_CHANGE_OUTPUTS = ( - "run-tests", - "run-workloads", + "run-guest-artifacts", + "run-openvmm-binaries", "run-openvmm-unit-tests", "run-openvmm-vmm-tests", + "run-microvm-tests", + "run-benchmarks", ) CI_DOCUMENTATION_PATHS = re.compile(r"^(doc/|.*\.md$)", re.IGNORECASE) -# Inputs of the hypervisor test suites: OpenVMM, the guests that the microVM -# tests boot, the Rust crates that test-aci-edge-sandboxes runs, the runtime -# adapter whose scenarios the microVM workflow runs, the CLI and workflows that -# drive the suites, and the runners. -CI_TEST_INPUT_PATHS = re.compile( - r"^(openvmm($|/)|aci_edge_sandboxes/|nvx-rs/|azurelinux/|" - r"tests/runtime-adapter/|" - r"\.github/workflows/" - r"(ci|build-openvmm-binary|run-nvx-microvm-tests|run-platform)\.yml$|" +# Inputs of the NVX microVM tests: the OpenVMM pin, the guests and every input +# of their cache keys, the nvx-rs executable that their scenarios run, the +# crates and the runtime adapter that they test, the workflows, actions, and +# CLI modules that their jobs and the jobs that build their artifacts run, the +# runners, and the attributes that check out their scripts on Windows. Every +# module of the CLI is an input but those that none of these jobs runs. +MICROVM_TEST_INPUT_PATHS = re.compile( + r"^(openvmm($|/)|rust-toolchain\.toml$|scripts/setup/|\.gitattributes$|" + r"nvx-rs/|aci_edge_sandboxes/|tests/runtime-adapter/|" + r"kernel/|guest/|ubuntu/|azurelinux/|" + r"docker/|\.dockerignore$|SOURCE-MANIFEST\.json$|" + r"\.github/workflows/(ci|build-openvmm-binary|run-nvx-microvm-tests)\.yml$|" r"\.github/actions/(build-guest-artifacts|build-openvmm|checkout-openvmm|" r"sccache|setup-curl|validate-runner)/|" - r"scripts/nvx\.py$|scripts/nvx_tools/|scripts/setup/|" - r"kernel/|alpine/|guest/|ubuntu/|" - r"docker/Dockerfile$|\.dockerignore$|SOURCE-MANIFEST\.json$|rust-toolchain\.toml$)" + r"scripts/nvx\.py$|" + r"scripts/nvx_tools/(?!(adversarial|adversarial_broker|adversarial_executor|" + r"agent_bundling_tests|create_linux_source_archive|performance)\.py$|" + r"adversarial_cases/))" +) +# Inputs of the platform jobs, which benchmark each backend and package its +# release, and of the regression gate that reads their results: the OpenVMM +# pin, the guests and every input of their cache keys, the nvx-rs executable +# that the packages ship, the performance history and expected hosts, the +# release version and license, the workflows, actions, and CLI modules that +# these jobs and the jobs that build their artifacts run, the runners, and the +# attributes that check out their inputs on Windows. Every module of the CLI is +# an input but those that none of these jobs runs. +BENCHMARK_INPUT_PATHS = re.compile( + r"^(openvmm($|/)|rust-toolchain\.toml$|scripts/setup/|\.gitattributes$|" + r"nvx-rs/|kernel/|guest/|ubuntu/|azurelinux/|" + r"docker/|\.dockerignore$|SOURCE-MANIFEST\.json$|" + r"data/|VERSION$|LICENSE$|" + r"\.github/workflows/(ci|build-openvmm-binary|run-platform)\.yml$|" + r"\.github/actions/(build-guest-artifacts|build-openvmm|checkout-openvmm|" + r"gate-performance|package-release|prepare-performance-results|" + r"run-benchmark|sccache|setup-curl|validate-runner)/|" + r"scripts/nvx\.py$|" + r"scripts/nvx_tools/(?!(aci_edge_sandboxes_tests|adversarial|" + r"adversarial_broker|adversarial_executor|agent_bundling_tests|" + r"create_linux_source_archive|managed_exec_tests|microvm_tests|" + r"openvmm_process|sandbox|sandbox_lifecycle_tests)\.py$|" + r"adversarial_cases/|microvm_test_scripts/))" ) # Inputs of the OpenVMM unit and documentation tests: the pinned revision, the # Rust release, the runner setup, and the code and workflow that run them. @@ -154,20 +185,24 @@ class CiChanges: """The suites that a change runs, as the change detector reports them.""" - run_tests: bool - run_workloads: bool + run_guest_artifacts: bool + run_openvmm_binaries: bool run_openvmm_unit_tests: bool run_openvmm_vmm_tests: bool + run_microvm_tests: bool + run_benchmarks: bool def outputs(self) -> dict[str, bool]: return dict( zip( CI_CHANGE_OUTPUTS, ( - self.run_tests, - self.run_workloads, + self.run_guest_artifacts, + self.run_openvmm_binaries, self.run_openvmm_unit_tests, self.run_openvmm_vmm_tests, + self.run_microvm_tests, + self.run_benchmarks, ), strict=True, ) @@ -197,27 +232,54 @@ def kernel_patch_identity(patch_root: Path) -> str: return digest.hexdigest()[:16] +def _ci_inputs(changed_files: Sequence[str]) -> list[str]: + """Return the changed files that can select a suite: every one but the + documentation, which no suite builds, boots, or runs.""" + return [ + path + for path in changed_files + if path and not CI_DOCUMENTATION_PATHS.match(path) + ] + + def classify_ci_changes(event_name: str, changed_files: Sequence[str]) -> CiChanges: """Select the suites that a change between two commits runs. - Pull requests and pushes run each OpenVMM suite only when tests run and - one of that suite's own inputs changed. + Documentation never selects a suite. Pull requests and pushes run each test + suite only when one of its own inputs changed. A pull request runs the + platform jobs only when one of their inputs changed, but a push runs them + for every change but documentation, so that every such `dev` commit + publishes its development release and records its performance results. The + guest artifacts and the OpenVMM binaries are built exactly when a job that + uses them runs. """ if event_name not in ("pull_request", "push"): raise ValueError(f"unsupported CI event {event_name!r}") files = [path for path in changed_files if path] - run_workloads = not files or not all( - CI_DOCUMENTATION_PATHS.match(path) for path in files + inputs = _ci_inputs(files) + + def changed(pattern: re.Pattern[str]) -> bool: + return any(pattern.match(path) for path in inputs) + + run_openvmm_vmm_tests = changed(OPENVMM_VMM_TEST_INPUT_PATHS) + run_microvm_tests = changed(MICROVM_TEST_INPUT_PATHS) + # An empty diff runs the platform jobs, as it ran every workload before + # the classifier selected them separately. + run_benchmarks = ( + not files + or changed(BENCHMARK_INPUT_PATHS) + or (event_name == "push" and bool(inputs)) ) - run_tests = any(CI_TEST_INPUT_PATHS.match(path) for path in files) return CiChanges( - run_tests=run_tests, - run_workloads=run_workloads, - run_openvmm_unit_tests=run_tests - and any(OPENVMM_UNIT_TEST_INPUT_PATHS.match(path) for path in files), - run_openvmm_vmm_tests=run_tests - and any(OPENVMM_VMM_TEST_INPUT_PATHS.match(path) for path in files), + run_guest_artifacts=run_openvmm_vmm_tests + or run_microvm_tests + or run_benchmarks, + run_openvmm_binaries=run_microvm_tests or run_benchmarks, + run_openvmm_unit_tests=changed(OPENVMM_UNIT_TEST_INPUT_PATHS), + run_openvmm_vmm_tests=run_openvmm_vmm_tests, + run_microvm_tests=run_microvm_tests, + run_benchmarks=run_benchmarks, ) @@ -228,9 +290,10 @@ def describe_ci_changes( ) -> str: """Return the Markdown job summary that explains each decision.""" files = [path for path in changed_files if path] + inputs = _ci_inputs(files) def evidence(pattern: re.Pattern[str]) -> str: - matches = [path for path in files if pattern.match(path)] + matches = [path for path in inputs if pattern.match(path)] shown = ", ".join(f"`{path}`" for path in matches[:3]) if len(matches) > 3: shown += f", and {len(matches) - 3} more" @@ -239,46 +302,82 @@ def evidence(pattern: re.Pattern[str]) -> str: def section(title: str, selected: bool, reason: str) -> str: return f"### {title}: {'run' if selected else 'skip'}\n{reason}\n" - if not files: - workloads_reason = "No file changed, so the workloads run." - elif changes.run_workloads: - workloads_reason = "At least one non-documentation file changed." + def inputs_reason(selected: bool, pattern: re.Pattern[str]) -> str: + if selected and any(pattern.match(path) for path in inputs): + return f"Inputs changed: {evidence(pattern)}." + if not files: + return "No file changed." + if not inputs: + return "Only documentation files changed." + return "No input changed." + + def users_reason(users: dict[str, bool]) -> str: + names = [name for name, runs in users.items() if runs] + if not names: + return "No job that uses them runs." + if len(names) > 2: + names = [", ".join(names[:-1]) + ",", names[-1]] + return f"Jobs that use them run: {' and '.join(names)}." + + if changes.run_benchmarks and not files: + benchmarks_reason = "No file changed, so the platform jobs run." + elif ( + changes.run_benchmarks + and event_name == "push" + and not any(BENCHMARK_INPUT_PATHS.match(path) for path in inputs) + ): + benchmarks_reason = ( + "A push runs the platform jobs for every change but documentation, " + "so that it publishes its development release and records its " + "performance results." + ) else: - workloads_reason = "Only documentation files changed." - sections = [ - section( - "Artifact-backed tests and benchmarks", - changes.run_workloads, - workloads_reason, - ), - section( - "NVX microVM tests", - changes.run_tests, - f"Test inputs changed: {evidence(CI_TEST_INPUT_PATHS)}." - if changes.run_tests - else "No test input changed.", - ), - ] - for title, selected, pattern in ( + benchmarks_reason = inputs_reason(changes.run_benchmarks, BENCHMARK_INPUT_PATHS) + return "".join( ( - "OpenVMM unit tests", - changes.run_openvmm_unit_tests, - OPENVMM_UNIT_TEST_INPUT_PATHS, - ), - ( - "OpenVMM VMM tests", - changes.run_openvmm_vmm_tests, - OPENVMM_VMM_TEST_INPUT_PATHS, - ), - ): - if selected: - reason = f"Suite inputs changed: {evidence(pattern)}." - elif changes.run_tests: - reason = "No input of this suite changed." - else: - reason = "No test input changed." - sections.append(section(title, selected, reason)) - return "".join(sections) + section( + "Guest artifacts", + changes.run_guest_artifacts, + users_reason( + { + "the OpenVMM VMM tests": changes.run_openvmm_vmm_tests, + "the NVX microVM tests": changes.run_microvm_tests, + "the platform benchmarks": changes.run_benchmarks, + }, + ), + ), + section( + "OpenVMM binaries", + changes.run_openvmm_binaries, + users_reason( + { + "the NVX microVM tests": changes.run_microvm_tests, + "the platform benchmarks": changes.run_benchmarks, + }, + ), + ), + section( + "OpenVMM unit tests", + changes.run_openvmm_unit_tests, + inputs_reason( + changes.run_openvmm_unit_tests, OPENVMM_UNIT_TEST_INPUT_PATHS + ), + ), + section( + "OpenVMM VMM tests", + changes.run_openvmm_vmm_tests, + inputs_reason( + changes.run_openvmm_vmm_tests, OPENVMM_VMM_TEST_INPUT_PATHS + ), + ), + section( + "NVX microVM tests", + changes.run_microvm_tests, + inputs_reason(changes.run_microvm_tests, MICROVM_TEST_INPUT_PATHS), + ), + section("Platform benchmarks", changes.run_benchmarks, benchmarks_reason), + ) + ) def required_ci_expected_results( @@ -295,21 +394,29 @@ def required_ci_expected_results( ): raise ValueError(f"unsupported CI event {event_name!r}") - run_tests = changes.run_tests - run_workloads = changes.run_workloads + # Each job runs only when its suite is selected and the jobs that build the + # artifacts that it uses are selected too, as its condition in ci.yml says. + run_guest_artifacts = changes.run_guest_artifacts + run_openvmm_binaries = changes.run_openvmm_binaries + run_microvm_tests = ( + changes.run_microvm_tests and run_guest_artifacts and run_openvmm_binaries + ) + run_benchmarks = ( + changes.run_benchmarks and run_guest_artifacts and run_openvmm_binaries + ) repository_jobs_enabled = event_name != "pull_request" or same_repository expected = { "quality": "success", "aci-edge-sandboxes": "success", "nvx-rs": "success", "openvmm-changes": "success", - REQUIRED_CI_ARTIFACT_JOB: "success" if run_workloads else "skipped", - REQUIRED_CI_DEBUG_KERNEL_JOB: "success" if run_workloads else "skipped", + REQUIRED_CI_ARTIFACT_JOB: "success" if run_guest_artifacts else "skipped", + REQUIRED_CI_DEBUG_KERNEL_JOB: "success" if run_guest_artifacts else "skipped", "performance-gate": ( "success" if event_name == "pull_request" and repository_jobs_enabled - and run_workloads + and run_benchmarks else "skipped" ), } @@ -323,7 +430,7 @@ def required_ci_expected_results( { job: ( "success" - if repository_jobs_enabled and (run_tests or run_workloads) + if repository_jobs_enabled and run_openvmm_binaries else "skipped" ) for job in REQUIRED_CI_BUILD_JOBS @@ -345,7 +452,7 @@ def required_ci_expected_results( "success" if repository_jobs_enabled and changes.run_openvmm_vmm_tests - and run_workloads + and run_guest_artifacts else "skipped" ) for job in REQUIRED_CI_OPENVMM_ARTIFACT_TEST_JOBS @@ -355,7 +462,7 @@ def required_ci_expected_results( { job: ( "success" - if repository_jobs_enabled and run_tests and run_workloads + if repository_jobs_enabled and run_microvm_tests else "skipped" ) for job in ( @@ -368,7 +475,7 @@ def required_ci_expected_results( { job: ( "success" - if event_name != "pull_request" and run_tests and run_workloads + if event_name != "pull_request" and run_microvm_tests else "skipped" ) for job in REQUIRED_CI_MICROVM_DEBUG_PUSH_JOBS @@ -376,7 +483,9 @@ def required_ci_expected_results( ) expected.update( { - job: ("success" if repository_jobs_enabled and run_workloads else "skipped") + job: ( + "success" if repository_jobs_enabled and run_benchmarks else "skipped" + ) for job in REQUIRED_CI_PLATFORM_JOBS } ) diff --git a/scripts/test_nvx_tools.py b/scripts/test_nvx_tools.py index 7a553f8c4..db7d24d52 100644 --- a/scripts/test_nvx_tools.py +++ b/scripts/test_nvx_tools.py @@ -8,6 +8,7 @@ import http.client import http.server import io +import itertools import json import lzma import os @@ -249,6 +250,65 @@ def _expression_conjuncts(expression: str) -> list[str]: return conjuncts +def _nvx_command_modules() -> dict[str, set[str]]: + """Map each nvx.py command to the scripts/nvx_tools modules that its parser + configuration and handler, and the nvx.py functions that they call, name.""" + tree = ast.parse( + (BuildConstants.REPO_ROOT / "scripts" / "nvx.py").read_text(encoding="utf-8") + ) + imported: dict[str, str] = {} + functions: dict[str, ast.FunctionDef] = {} + for node in tree.body: + if isinstance(node, ast.ImportFrom) and node.module is not None: + package, _, module = node.module.partition(".") + if package == "nvx_tools": + for alias in node.names: + imported[alias.asname or alias.name] = module or alias.name + elif isinstance(node, ast.FunctionDef): + functions[node.name] = node + parse_args = functions["parse_args"] + functions.update( + (node.name, node) + for node in ast.walk(parse_args) + if isinstance(node, ast.FunctionDef) and node is not parse_args + ) + + def names(node: ast.AST) -> set[str]: + return {child.id for child in ast.walk(node) if isinstance(child, ast.Name)} + + def modules(node: ast.AST, seen: set[str]) -> set[str]: + found: set[str] = set() + for name in names(node): + if name in imported: + found.add(imported[name]) + elif name in functions and name not in seen: + seen.add(name) + found |= modules(functions[name], seen) + return found + + parsers: dict[str, str] = {} + commands: dict[str, set[str]] = {} + for statement in parse_args.body: + if ( + isinstance(statement, ast.Assign) + and isinstance(statement.value, ast.Call) + and isinstance(statement.value.func, ast.Attribute) + and statement.value.func.attr == "add_parser" + and isinstance(statement.value.func.value, ast.Name) + and statement.value.func.value.id == "subparsers" + ): + [target] = statement.targets + command = statement.value.args[0] + assert isinstance(target, ast.Name) + assert isinstance(command, ast.Constant) + assert isinstance(command.value, str) + parsers[target.id] = command.value + commands[command.value] = set() + for parser in names(statement) & parsers.keys(): + commands[parsers[parser]] |= modules(statement, set()) + return commands + + def _write_release_fixture( root: Path, ) -> tuple[dict[str, Path], dict[str, object], str]: @@ -583,32 +643,25 @@ def section(title: str, level: int) -> str: def test_check_required_ci_rejects_malformed_run_flags(self): # microsoft/nvx#496: an empty or malformed --run-* value must not be # silently accepted as false, which would let required-status checking - # pass with required jobs skipped. + # pass with required jobs skipped. Every output of the change detector + # is a required flag. + flags = tuple(f"--{output}" for output in ci.CI_CHANGE_OUTPUTS) base = [ "check-required-ci", "--event-name", "pull_request", "--same-repository", "true", - "--run-tests", - "true", - "--run-workloads", - "true", - "--run-openvmm-unit-tests", - "true", - "--run-openvmm-vmm-tests", - "true", + *(value for flag in flags for value in (flag, "true")), ] - flags = ( - "--run-tests", - "--run-workloads", - "--run-openvmm-unit-tests", - "--run-openvmm-vmm-tests", - ) for flag in flags: - for bad in ("", "tru", "TRUE", "1", "yes"): + for bad in ("", "tru", "TRUE", "1", "yes", None): argv = list(base) - argv[argv.index(flag) + 1] = bad + index = argv.index(flag) + if bad is None: + del argv[index : index + 2] + else: + argv[index + 1] = bad with self.subTest(flag=flag, value=bad): with ( patch("sys.stderr", new_callable=io.StringIO), @@ -631,14 +684,11 @@ def test_check_required_ci_accepts_every_ci_workflow_event(self): "", "--same-repository", "true", - "--run-tests", - "true", - "--run-workloads", - "true", - "--run-openvmm-unit-tests", - "true", - "--run-openvmm-vmm-tests", - "true", + *( + value + for output in ci.CI_CHANGE_OUTPUTS + for value in (f"--{output}", "true") + ), ] for event_name in ( "pull_request", @@ -651,6 +701,50 @@ def test_check_required_ci_accepts_every_ci_workflow_event(self): with self.subTest(event_name=event_name): self.assertEqual(nvx.parse_args(argv).event_name, event_name) + def test_check_required_ci_passes_each_flag_to_its_suite(self): + # With the results that a selection expects, the command passes; with + # any one flag flipped, the expected results differ, so it fails. A + # flag passed to the wrong suite would fail the first or pass the + # second. + for changes in ( + ci.CiChanges(True, True, True, True, True, True), + ci.CiChanges(True, True, False, False, True, False), + ci.CiChanges(True, True, False, True, False, True), + ): + results = ci.required_ci_expected_results( + "pull_request", same_repository=True, changes=changes + ) + environment = { + ci.REQUIRED_CI_RESULT_ENVIRONMENTS[job]: result + for job, result in results.items() + } + for flipped in (None, *ci.CI_CHANGE_OUTPUTS): + argv = [ + "check-required-ci", + "--event-name", + "pull_request", + "--same-repository", + "true", + ] + for output, selected in changes.outputs().items(): + argv += [ + f"--{output}", + str(selected != (output == flipped)).lower(), + ] + args = nvx.parse_args(argv) + with ( + self.subTest(changes=changes, flipped=flipped), + patch.dict(os.environ, environment), + patch("sys.stdout", new_callable=io.StringIO), + ): + if flipped is None: + args.handler(args) + else: + with self.assertRaisesRegex( + common.ScriptError, "required CI job result" + ): + args.handler(args) + def test_guest_selection_includes_azure_linux(self): args = nvx.parse_args(["build-initramfs", "--guest", "azurelinux"]) self.assertEqual(args.guest, "azurelinux") @@ -3826,92 +3920,141 @@ def test_required_ci_result_policy(self): debug_kernel_push = set(ci.REQUIRED_CI_MICROVM_DEBUG_PUSH_JOBS) artifacts = {ci.REQUIRED_CI_ARTIFACT_JOB, ci.REQUIRED_CI_DEBUG_KERNEL_JOB} platforms = set(ci.REQUIRED_CI_PLATFORM_JOBS) - workloads = artifacts | builds | platforms microvm = microvm_tests | debug_kernel full_matrix = ( repository - | workloads + | artifacts + | builds | openvmm_tests | openvmm_artifact_tests | microvm | debug_kernel_push - ) - # Event, same repository, the change detector's outputs (tests, - # workloads, OpenVMM unit tests, OpenVMM VMM tests), and the jobs that - # must succeed; every other required job must be skipped. + | platforms + ) + gate = {"performance-gate"} + # Event, same repository, the change detector's outputs (guest + # artifacts, OpenVMM binaries, OpenVMM unit tests, OpenVMM VMM tests, + # NVX microVM tests, platform benchmarks), and the jobs that must + # succeed; every other required job must be skipped. + everything = (True, True, True, True, True, True) cases = ( - ("pull_request", True, (False, False, False, False), repository), + # Documentation, or a change to no suite's inputs. + ("pull_request", True, (False,) * 6, repository), + # Only the benchmarks or the packages change. ( "pull_request", True, - (False, True, False, False), - repository | workloads | {"performance-gate"}, + (True, True, False, False, False, True), + repository | artifacts | builds | platforms | gate, ), + # Only the microVM tests change. ( "pull_request", True, - (True, False, True, True), - repository | builds | openvmm_tests, + (True, True, False, False, True, False), + repository | artifacts | builds | microvm, ), + # Only the OpenVMM suites, which build OpenVMM themselves. ( "pull_request", True, - (True, True, True, True), + (False, False, True, False, False, False), + repository | openvmm_tests, + ), + ( + "pull_request", + True, + (True, False, False, True, False, False), + repository | artifacts | openvmm_artifact_tests, + ), + ( + "pull_request", + True, + everything, repository - | workloads + | artifacts + | builds | openvmm_tests | openvmm_artifact_tests | microvm - | {"performance-gate"}, + | platforms + | gate, ), # A pull request that changes no input of either OpenVMM suite. ( "pull_request", True, - (True, True, False, False), - repository | workloads | microvm | {"performance-gate"}, + (True, True, False, False, True, True), + repository | artifacts | builds | microvm | platforms | gate, ), # Guest inputs reach the VMM tests but not the unit tests. ( "pull_request", True, - (True, True, False, True), + (True, True, False, True, True, True), repository - | workloads + | artifacts + | builds | openvmm_artifact_tests | microvm - | {"performance-gate"}, + | platforms + | gate, ), - ("pull_request", False, (False, False, False, False), always_successful), + # A job runs only when the jobs that build its artifacts run, as + # its condition in ci.yml says, whatever the flags claim. + ( + "pull_request", + True, + (False, True, True, True, True, True), + repository | builds | openvmm_tests, + ), + ( + "pull_request", + True, + (True, False, False, True, True, True), + repository | artifacts | openvmm_artifact_tests, + ), + ("pull_request", False, (False,) * 6, always_successful), ( "pull_request", False, - (False, True, False, False), + (True, True, False, False, False, True), always_successful | artifacts, ), - ("pull_request", False, (True, False, True, True), always_successful), ( "pull_request", False, - (True, True, True, True), - always_successful | artifacts, + (False, False, True, False, False, False), + always_successful, ), - ("push", True, (False, False, False, False), repository), - ("push", True, (False, True, False, False), repository | workloads), + ("pull_request", False, everything, always_successful | artifacts), + ("push", True, (False,) * 6, repository), ( "push", True, - (True, False, True, True), - repository | builds | openvmm_tests, + (True, True, False, False, False, True), + repository | artifacts | builds | platforms, + ), + ( + "push", + True, + (True, True, False, False, True, True), + repository + | artifacts + | builds + | microvm + | debug_kernel_push + | platforms, ), ( "push", True, - (True, True, True, True), - full_matrix, + (False, False, True, False, False, False), + repository | openvmm_tests, ), - ("schedule", False, (True, True, True, True), full_matrix), - ("workflow_dispatch", False, (True, True, True, True), full_matrix), + ("push", True, everything, full_matrix), + ("schedule", False, everything, full_matrix), + ("workflow_dispatch", False, everything, full_matrix), ) for event_name, same_repository, flags, successful_jobs in cases: @@ -4010,6 +4153,67 @@ def test_required_ci_job_uses_tested_policy(self): ) self.assertIn(f'--{output} "${{{environment}}}"', job) + def test_ci_jobs_follow_their_change_detector_output(self): + # Each job that the change detector selects runs when its output is + # true, or when the detector did not succeed, so that a failed + # classification runs the full matrix. Required status check passes + # every output to check-required-ci, and no other job reads one. + workflow = ( + BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" + ).read_text(encoding="utf-8") + selections = { + ci.REQUIRED_CI_ARTIFACT_JOB: "run-guest-artifacts", + ci.REQUIRED_CI_DEBUG_KERNEL_JOB: "run-guest-artifacts", + **dict.fromkeys(ci.REQUIRED_CI_BUILD_JOBS, "run-openvmm-binaries"), + **dict.fromkeys(ci.REQUIRED_CI_OPENVMM_TEST_JOBS, "run-openvmm-unit-tests"), + **dict.fromkeys( + ci.REQUIRED_CI_OPENVMM_ARTIFACT_TEST_JOBS, "run-openvmm-vmm-tests" + ), + **dict.fromkeys( + ( + *ci.REQUIRED_CI_MICROVM_TEST_JOBS, + *ci.REQUIRED_CI_MICROVM_DEBUG_JOBS, + *ci.REQUIRED_CI_MICROVM_DEBUG_PUSH_JOBS, + ), + "run-microvm-tests", + ), + **dict.fromkeys(ci.REQUIRED_CI_PLATFORM_JOBS, "run-benchmarks"), + } + self.assertEqual(set(selections.values()), set(ci.CI_CHANGE_OUTPUTS)) + for job_name, output in selections.items(): + with self.subTest(job=job_name): + job = _workflow_job(workflow, job_name) + needs = re.search(r"^ needs: \[(.+)\]$", job, re.MULTILINE) + assert needs is not None + self.assertIn("openvmm-changes", needs.group(1).split(", ")) + self.assertIn( + " (\n" + " needs.openvmm-changes.result != 'success' ||\n" + f" needs.openvmm-changes.outputs.{output} == 'true'\n" + " )\n", + job, + ) + self.assertEqual( + re.findall(r"needs\.openvmm-changes\.outputs\.([a-z-]+)", job), + [output], + ) + jobs = re.findall( + r"^ ([A-Za-z0-9_-]+):$", workflow.split("\njobs:\n", 1)[1], re.MULTILINE + ) + self.assertEqual( + { + job_name + for job_name in jobs + if "needs.openvmm-changes.outputs." in _workflow_job(workflow, job_name) + }, + set(selections) | {"required-status-check"}, + ) + # The detector checks and forwards exactly the classifier's outputs. + script = _workflow_step(workflow, "openvmm-changes", "Detect relevant changes") + listed = re.search(r"\n +outputs=\(([^)]*)\)\n", script) + assert listed is not None + self.assertEqual(tuple(listed.group(1).split()), ci.CI_CHANGE_OUTPUTS) + def test_ci_full_run_triggers_and_concurrency_are_separate(self): workflow = ( BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" @@ -4202,74 +4406,192 @@ def test_ci_full_runs_do_not_publish_persist_or_save_caches(self): self.assertEqual(workflow.count("github.event_name == 'push'"), 13) def test_ci_change_classification_selects_suites_by_input(self): - # The outputs (tests, workloads, OpenVMM unit tests, OpenVMM VMM - # tests) for each class of changed path. - cases = ( - ((), (False, True, False, False)), - (("doc/ci.md", "README.md", "DOC/Guide.MD"), (False, False, False, False)), - (("openvmm",), (True, True, True, True)), - (("rust-toolchain.toml",), (True, True, True, True)), - (("scripts/setup/setup-windows-whp.ps1",), (True, True, True, True)), - (("scripts/setup/azure/build_package.py",), (True, True, True, True)), - (("scripts/setup/README.md",), (True, False, True, True)), - ((".github/workflows/ci.yml",), (True, True, True, True)), - ((".github/actions/validate-runner/action.yml",), (True, True, True, True)), - ((".github/actions/sccache/action.yml",), (True, True, True, True)), - ((".github/actions/setup-curl/curl-shim.rs",), (True, True, True, True)), - (("scripts/nvx.py",), (True, True, True, True)), - (("scripts/nvx_tools/ci.py",), (True, True, True, True)), - ((".dockerignore",), (True, True, False, True)), - (("scripts/nvx_tools/common.py",), (True, True, True, True)), - (("scripts/nvx_tools/build.py",), (True, True, False, True)), - (("scripts/nvx_tools/guests.py",), (True, True, False, True)), - (("kernel/config-microvm",), (True, True, False, True)), - (("kernel/patches/0001-example.patch",), (True, True, False, True)), - (("guest/common/init",), (True, True, False, True)), - (("guest/alpine/nvx-container-enter",), (True, True, False, True)), - (("SOURCE-MANIFEST.json",), (True, True, False, True)), - (("docker/Dockerfile",), (True, True, False, True)), + guest, binaries, _, vmm, microvm, benchmarks = ci.CI_CHANGE_OUTPUTS + + def select(*outputs: str) -> ci.CiChanges: + return ci.CiChanges(*(output in outputs for output in ci.CI_CHANGE_OUTPUTS)) + + everything = select(*ci.CI_CHANGE_OUTPUTS) + without_unit_tests = select(guest, binaries, vmm, microvm, benchmarks) + microvm_and_benchmarks = select(guest, binaries, microvm, benchmarks) + microvm_only = select(guest, binaries, microvm) + benchmarks_only = select(guest, binaries, benchmarks) + # Each class of changed path, the outputs that a pull request selects, + # and those that a push selects when they differ: a push runs the + # platform jobs, and the jobs that build their artifacts, for every + # change but documentation. + cases: tuple[tuple[tuple[str, ...], ci.CiChanges, ci.CiChanges | None], ...] = ( + # An empty diff runs the platform jobs, as it ran every workload. + ((), benchmarks_only, None), + (("doc/ci.md", "README.md", "DOC/Guide.MD"), select(), None), + # Documentation selects no suite, even in a suite's directory. + (("scripts/setup/README.md",), select(), None), + (("nvx-rs/README.md", "tests/runtime-adapter/README.md"), select(), None), + ( + ("scripts/setup/README.md", "scripts/test_nvx_tools.py"), + select(), + benchmarks_only, + ), + # Inputs of every suite: the OpenVMM pin, the Rust pin, the + # runners, the workflow, the actions that every suite's jobs use, + # the CLI, and the modules that the OpenVMM suites run. + (("openvmm",), everything, None), + (("rust-toolchain.toml",), everything, None), + (("scripts/setup/setup-windows-whp.ps1",), everything, None), + (("scripts/setup/azure/build_package.py",), everything, None), + ((".github/workflows/ci.yml",), everything, None), + ((".github/actions/validate-runner/action.yml",), everything, None), + ((".github/actions/checkout-openvmm/action.yml",), everything, None), + ((".github/actions/sccache/action.yml",), everything, None), + ((".github/actions/setup-curl/curl-shim.rs",), everything, None), + (("scripts/nvx.py",), everything, None), + (("scripts/nvx_tools/ci.py",), everything, None), + (("scripts/nvx_tools/common.py",), everything, None), + # Inputs of the kernel and Alpine initramfs that the VMM tests + # boot, and of the other guests. + ((".dockerignore",), without_unit_tests, None), + (("docker/Dockerfile",), without_unit_tests, None), + (("SOURCE-MANIFEST.json",), without_unit_tests, None), + (("scripts/nvx_tools/build.py",), without_unit_tests, None), + (("scripts/nvx_tools/guests.py",), without_unit_tests, None), + (("kernel/config-microvm",), without_unit_tests, None), + (("kernel/patches/0001-example.patch",), without_unit_tests, None), + (("guest/common/init",), without_unit_tests, None), + (("guest/alpine/nvx-container-enter",), without_unit_tests, None), ( (".github/actions/build-guest-artifacts/action.yml",), - (True, True, False, True), + without_unit_tests, + None, + ), + # Of nvx-rs, the guest images, and so the VMM tests, take only the + # inputs of the guest binaries. + (("nvx-rs/src/lib.rs",), without_unit_tests, None), + (("nvx-rs/Cargo.lock",), without_unit_tests, None), + (("nvx-rs/src/guest/exit.rs",), without_unit_tests, None), + (("nvx-rs/src/bin/nvx-time.rs",), without_unit_tests, None), + (("guest/ubuntu/nvx-bashrc",), microvm_and_benchmarks, None), + (("ubuntu/packages.lock.json",), microvm_and_benchmarks, None), + (("azurelinux/packages.lock.json",), microvm_and_benchmarks, None), + # Inputs of the OpenVMM binaries, and the modules and attributes + # that both the microVM tests and the platform jobs take. + ( + (".github/workflows/build-openvmm-binary.yml",), + microvm_and_benchmarks, + None, + ), + ( + (".github/actions/build-openvmm/action.yml",), + microvm_and_benchmarks, + None, + ), + (("scripts/nvx_tools/benchmark.py",), microvm_and_benchmarks, None), + ( + ("scripts/nvx_tools/benchmark_scripts/guest-exit.sh",), + microvm_and_benchmarks, + None, + ), + (("scripts/nvx_tools/doctor.py",), microvm_and_benchmarks, None), + (("scripts/nvx_tools/host_time_probe.rs",), microvm_and_benchmarks, None), + ( + ("scripts/nvx_tools/process_supervisor.py",), + microvm_and_benchmarks, + None, + ), + (("scripts/nvx_tools/release.py",), microvm_and_benchmarks, None), + ((".gitattributes",), microvm_and_benchmarks, None), + # The rest of nvx-rs: the microVM tests run the nvx-rs CLI, and + # every package ships it. + (("nvx-rs/src/host/cli/args.rs",), microvm_and_benchmarks, None), + (("nvx-rs/src/bin/nvx-rs.rs",), microvm_and_benchmarks, None), + (("nvx-rs/tests/cli.rs",), microvm_and_benchmarks, None), + # Inputs of the microVM tests alone. + (("aci_edge_sandboxes/src/lib.rs",), microvm_only, microvm_and_benchmarks), + ( + ("tests/runtime-adapter/src/lib.rs",), + microvm_only, + microvm_and_benchmarks, + ), + ( + ("scripts/nvx_tools/microvm_tests.py",), + microvm_only, + microvm_and_benchmarks, + ), + (("scripts/nvx_tools/sandbox.py",), microvm_only, microvm_and_benchmarks), + ( + ("scripts/nvx_tools/microvm_test_scripts/lifecycle.sh",), + microvm_only, + microvm_and_benchmarks, ), - (("guest/ubuntu/nvx-bashrc",), (True, True, False, False)), - (("ubuntu/packages.lock.json",), (True, True, False, False)), - (("azurelinux/packages.lock.json",), (True, True, False, False)), - (("aci_edge_sandboxes/src/lib.rs",), (True, True, False, False)), - (("nvx-rs/src/lib.rs",), (True, True, False, True)), - (("nvx-rs/Cargo.lock",), (True, True, False, True)), - (("nvx-rs/src/guest/exit.rs",), (True, True, False, True)), - (("nvx-rs/src/bin/nvx-time.rs",), (True, True, False, True)), - (("nvx-rs/src/host/cli/args.rs",), (True, True, False, False)), - (("nvx-rs/src/bin/nvx-rs.rs",), (True, True, False, False)), - (("nvx-rs/tests/cli.rs",), (True, True, False, False)), - (("tests/runtime-adapter/src/lib.rs",), (True, True, False, False)), - (("scripts/nvx_tools/microvm_tests.py",), (True, True, False, False)), ( (".github/workflows/run-nvx-microvm-tests.yml",), - (True, True, False, False), + microvm_only, + microvm_and_benchmarks, + ), + # Inputs of the platform jobs and the regression gate alone. + (("scripts/nvx_tools/performance.py",), benchmarks_only, None), + ((".github/workflows/run-platform.yml",), benchmarks_only, None), + ((".github/actions/run-benchmark/action.yml",), benchmarks_only, None), + ((".github/actions/package-release/action.yml",), benchmarks_only, None), + ((".github/actions/gate-performance/action.yml",), benchmarks_only, None), + ( + (".github/actions/prepare-performance-results/action.yml",), + benchmarks_only, + None, + ), + (("data/linux-kvm-virtual-machine.csv",), benchmarks_only, None), + (("data/windows-whp-virtual-machine-host.json",), benchmarks_only, None), + (("VERSION",), benchmarks_only, None), + (("LICENSE",), benchmarks_only, None), + # Inputs of jobs that every change runs, or that only a push runs + # and only after the platform jobs. + ((".github/actions/validate-nvx/action.yml",), select(), benchmarks_only), + ((".github/actions/check-quality/action.yml",), select(), benchmarks_only), + ( + (".github/actions/persist-performance-results/action.yml",), + select(), + benchmarks_only, + ), + ( + (".github/actions/publish-development-release/action.yml",), + select(), + benchmarks_only, ), - ((".github/actions/build-openvmm/action.yml",), (True, True, False, False)), - ((".github/actions/validate-nvx/action.yml",), (False, True, False, False)), + (("scripts/publish_development_release.py",), select(), benchmarks_only), + (("scripts/test_nvx_tools.py",), select(), benchmarks_only), + (("scripts/nvx_tools/adversarial.py",), select(), benchmarks_only), ( - (".github/actions/run-benchmark/action.yml",), - (False, True, False, False), + ("scripts/nvx_tools/adversarial_cases/guest-isolation.json",), + select(), + benchmarks_only, ), - (("scripts/test_nvx_tools.py",), (False, True, False, False)), - (("data/linux-kvm-virtual-machine.csv",), (False, True, False, False)), + (("scripts/nvx_tools/agent_bundling_tests.py",), select(), benchmarks_only), + (("requirements-dev.txt",), select(), benchmarks_only), + ((".github/workflows/copilot-setup-steps.yml",), select(), benchmarks_only), + # A change selects the union of its paths' suites. ( ("scripts/nvx_tools/microvm_tests.py", "kernel/config-microvm"), - (True, True, False, True), + without_unit_tests, + None, + ), + ( + ( + "scripts/nvx_tools/microvm_tests.py", + "scripts/nvx_tools/performance.py", + ), + microvm_and_benchmarks, + None, ), ) - for changed_files, flags in cases: + for changed_files, pull_request, push in cases: with self.subTest(changed_files=changed_files): - for event_name in ("pull_request", "push"): - with self.subTest(event_name=event_name): - self.assertEqual( - ci.classify_ci_changes(event_name, changed_files), - ci.CiChanges(*flags), - ) + self.assertEqual( + ci.classify_ci_changes("pull_request", changed_files), + pull_request, + ) + self.assertEqual( + ci.classify_ci_changes("push", changed_files), + push or pull_request, + ) self.assertEqual( ci.classify_ci_changes("pull_request", ["openvmm", ""]), ci.classify_ci_changes("pull_request", ["openvmm"]), @@ -4277,22 +4599,43 @@ def test_ci_change_classification_selects_suites_by_input(self): with self.assertRaisesRegex(ValueError, "unsupported CI event"): ci.classify_ci_changes("workflow_dispatch", ["openvmm"]) - def test_push_classifies_each_openvmm_suite_by_its_inputs(self): - self.assertEqual( - ci.classify_ci_changes("push", ["guest/common/init"]), - ci.CiChanges(True, True, False, True), - ) - self.assertEqual( - ci.classify_ci_changes( - "push", ["tests/runtime-adapter/src/bin/runtime-adapter-host.rs"] - ), - ci.CiChanges(True, True, False, False), + def test_ci_change_classification_builds_artifacts_for_the_jobs_that_use_them(self): + # The guest artifacts and the OpenVMM binaries have no use of their + # own: each is built exactly when a job that uses it runs, so no job + # waits for an artifact that its run skips. + paths = ( + "openvmm", + "doc/ci.md", + "kernel/config-microvm", + "guest/ubuntu/nvx-bashrc", + "nvx-rs/src/host/mod.rs", + "scripts/nvx_tools/ci.py", + "scripts/nvx_tools/microvm_tests.py", + "scripts/nvx_tools/performance.py", + "scripts/test_nvx_tools.py", + ".github/actions/validate-nvx/action.yml", ) + for count in range(len(paths) + 1): + for changed_files in itertools.combinations(paths, count): + for event_name in ("pull_request", "push"): + changes = ci.classify_ci_changes(event_name, changed_files) + with self.subTest(event_name=event_name, files=changed_files): + self.assertEqual( + changes.run_guest_artifacts, + changes.run_openvmm_vmm_tests + or changes.run_microvm_tests + or changes.run_benchmarks, + ) + self.assertEqual( + changes.run_openvmm_binaries, + changes.run_microvm_tests or changes.run_benchmarks, + ) def test_ci_change_classification_runs_microvm_tests_for_the_runtime_adapter(self): # The microVM jobs run the runtime adapter's scenarios on every backend, # so a change to the adapter alone runs them. Neither OpenVMM suite - # builds or boots the adapter. + # builds or boots the adapter, and neither the benchmarks nor the + # packages use it, so a pull request skips the platform jobs. workflow = ( BuildConstants.REPO_ROOT / ".github" @@ -4320,10 +4663,12 @@ def test_ci_change_classification_runs_microvm_tests_for_the_runtime_adapter(sel for changed_files in (*((path,) for path in adapter), adapter): with self.subTest(changed_files=changed_files): changes = ci.classify_ci_changes("pull_request", changed_files) - self.assertEqual(changes, ci.CiChanges(True, True, False, False)) + self.assertEqual( + changes, ci.CiChanges(True, True, False, False, True, False) + ) self.assertEqual( ci.classify_ci_changes("push", changed_files), - ci.CiChanges(True, True, False, False), + ci.CiChanges(True, True, False, False, True, True), ) expected = ci.required_ci_expected_results( "pull_request", same_repository=True, changes=changes @@ -4336,6 +4681,8 @@ def test_ci_change_classification_runs_microvm_tests_for_the_runtime_adapter(sel for job in ( *ci.REQUIRED_CI_OPENVMM_TEST_JOBS, *ci.REQUIRED_CI_OPENVMM_ARTIFACT_TEST_JOBS, + *ci.REQUIRED_CI_PLATFORM_JOBS, + "performance-gate", ): self.assertEqual(expected[job], "skipped", job) summary = ci.describe_ci_changes( @@ -4344,11 +4691,12 @@ def test_ci_change_classification_runs_microvm_tests_for_the_runtime_adapter(sel ci.classify_ci_changes("pull_request", adapter), ) self.assertIn( - "### NVX microVM tests: run\nTest inputs changed: " + "### NVX microVM tests: run\nInputs changed: " "`tests/runtime-adapter/Cargo.toml`, `tests/runtime-adapter/Cargo.lock`, " "`tests/runtime-adapter/src/lib.rs`, and 2 more.\n", summary, ) + self.assertIn("### Platform benchmarks: skip\nNo input changed.\n", summary) def test_ci_change_classification_covers_each_suite_input(self): github = BuildConstants.REPO_ROOT / ".github" @@ -4441,28 +4789,195 @@ def imported_modules(module: str) -> set[str]: } vmm_inputs |= guest_inputs["KERNEL"] | guest_inputs["ALPINE"] vmm_inputs |= guest_binary_inputs - test_inputs = vmm_inputs.union(*guest_inputs.values()) | { - ".github/workflows/run-nvx-microvm-tests.yml", - ".github/workflows/build-openvmm-binary.yml", - *( - f".github/actions/{name}/action.yml" - for configuration in ( - (github / "workflows" / "run-nvx-microvm-tests.yml").read_text( - encoding="utf-8" - ), - (github / "workflows" / "build-openvmm-binary.yml").read_text( + + # The NVX microVM tests and the platform jobs take every input of the + # jobs that they run and of the jobs that build the artifacts that they + # use: those jobs' workflows, the reusable workflows and actions that + # they reach, the files that the cache keys there hash, the files of the + # guest binaries' source identity, with which every guest image key + # ends, the modules of the nvx.py commands that they run, with every + # module that those import and every file that those read beside them, + # and what every self-hosted job takes: the OpenVMM pin, the Rust pin + # that validate-runner checks, the runner setup scripts, and the + # attributes that check out their scripts on Windows. + command_modules = _nvx_command_modules() + + def module_files(module: str) -> set[str]: + source = (tools / f"{module}.py").read_text(encoding="utf-8") + files = {f"scripts/nvx_tools/{module}.py"} + for name in re.findall(r'Path\(__file__\)\.with_name\("([^"]+)"\)', source): + data = tools / name + self.assertTrue(data.exists(), data) + files.add( + f"scripts/nvx_tools/{name}{'/example' if data.is_dir() else ''}" + ) + return files + + closures: dict[str, set[str]] = {} + + def command_files(command: str) -> set[str]: + files: set[str] = set() + for module in command_modules[command]: + if module not in closures: + closures[module] = set[str]().union( + *map(module_files, imported_modules(module)) + ) + files |= closures[module] + return files + + def job_inputs( + *jobs: str, tests_only: frozenset[str] = frozenset() + ) -> set[str]: + configurations: list[str] = [] + inputs = { + "scripts/nvx.py", + "scripts/nvx_tools/__init__.py", + ".github/workflows/ci.yml", + } + for job in jobs: + definition = _workflow_job(workflow, job) + configurations.append(definition) + for reusable in re.findall( + r"uses: \./\.github/workflows/([a-z0-9-]+\.yml)", definition + ): + inputs.add(f".github/workflows/{reusable}") + configurations.append( + (github / "workflows" / reusable).read_text(encoding="utf-8") + ) + # Actions whose effect on the job's outputs is none, such as the + # CLI tests, contribute neither their files nor their commands. + actions = set[str]().union(*map(local_actions, configurations)) + for name in actions - tests_only: + inputs.add(f".github/actions/{name}/action.yml") + configurations.append( + (github / "actions" / name / "action.yml").read_text( encoding="utf-8" - ), - _workflow_job(workflow, "debug-kernel"), + ) ) - for name in local_actions(configuration) + if name == "checkout-openvmm": + inputs.add("openvmm") + # Every self-hosted job validates its runner first. + if name == "validate-runner": + inputs |= { + "rust-toolchain.toml", + "scripts/setup/setup-linux-runner.sh", + ".gitattributes", + } + commands: set[str] = set() + for configuration in configurations: + for expression in re.findall(r"hashFiles\(([^)]*)\)", configuration): + inputs |= { + path.replace("**", "example").replace("*", "example") + for path in re.findall(r"'([^']+)'", expression) + if not path.startswith("!") + } + # A PowerShell wrapper passes its commands on as @args, and + # every step that uses one repeats its commands on Linux. + commands.update( + re.findall( + r"scripts[/\\]nvx\.py[ \t]+([a-z][a-z0-9-]*)", configuration + ) + ) + return inputs.union(*map(command_files, commands)) + + # Every job that runs nvx-rs or ships it takes the executable that the + # NVX CLI jobs build from the nvx-rs crate with the pinned Rust + # release, and its provenance. + nvx_rs_inputs = job_inputs( + *ci.REQUIRED_CI_CLI_JOBS, tests_only=frozenset({"validate-nvx"}) + ) | {"nvx-rs/example", "rust-toolchain.toml"} + for job in ci.REQUIRED_CI_CLI_JOBS: + with self.subTest(job=job): + definition = _workflow_job(workflow, job) + self.assertIn("rust-toolchain.toml", definition) + self.assertIn("--manifest-path nvx-rs/Cargo.toml", definition) + all_guest_inputs = ( + set[str]().union(*guest_inputs.values()) | guest_binary_inputs + ) + microvm_workflow = ( + github / "workflows" / "run-nvx-microvm-tests.yml" + ).read_text(encoding="utf-8") + self.assertIn("working-directory: tests/runtime-adapter\n", microvm_workflow) + self.assertIn( + 'REPO_ROOT / "aci_edge_sandboxes"', + (tools / "aci_edge_sandboxes_tests.py").read_text(encoding="utf-8"), + ) + microvm_inputs = ( + job_inputs( + *ci.REQUIRED_CI_MICROVM_TEST_JOBS, + *ci.REQUIRED_CI_MICROVM_DEBUG_JOBS, + *ci.REQUIRED_CI_MICROVM_DEBUG_PUSH_JOBS, + ci.REQUIRED_CI_ARTIFACT_JOB, + ci.REQUIRED_CI_DEBUG_KERNEL_JOB, + *ci.REQUIRED_CI_BUILD_JOBS, + ) + | nvx_rs_inputs + | all_guest_inputs + | {"tests/runtime-adapter/example", "aci_edge_sandboxes/example"} + ) + # The platform jobs check their history's expected host in data/, + # name their packages with VERSION, and ship LICENSE; the regression + # gate reads their results. Pushes alone publish and persist them, + # after every change but documentation. + run_benchmark = (github / "actions" / "run-benchmark" / "action.yml").read_text( + encoding="utf-8" + ) + self.assertIn("--history-dir data", run_benchmark) + self.assertIn( + "cat VERSION", + (github / "actions" / "package-release" / "action.yml").read_text( + encoding="utf-8" ), - } - self.assertNotIn(".github/actions/validate-nvx/action.yml", test_inputs) + ) + self.assertIn( + '("LICENSE", "README.md", "THIRD_PARTY_NOTICES.md")', + (tools / "release.py").read_text(encoding="utf-8"), + ) + benchmark_inputs = ( + job_inputs( + *ci.REQUIRED_CI_PLATFORM_JOBS, + "performance-gate", + ci.REQUIRED_CI_ARTIFACT_JOB, + *ci.REQUIRED_CI_BUILD_JOBS, + ) + | nvx_rs_inputs + | all_guest_inputs + | {"data/example", "VERSION", "LICENSE"} + ) + self.assertLessEqual( + { + "scripts/nvx_tools/microvm_tests.py", + "scripts/nvx_tools/microvm_test_scripts/example", + "scripts/nvx_tools/doctor.py", + "scripts/nvx_tools/host_time_probe.rs", + "scripts/nvx_tools/benchmark_scripts/example", + "scripts/nvx_tools/process_supervisor.py", + "scripts/nvx_tools/release.py", + ".github/actions/build-openvmm/action.yml", + ".github/workflows/build-openvmm-binary.yml", + ".github/workflows/run-nvx-microvm-tests.yml", + }, + microvm_inputs, + ) + self.assertLessEqual( + { + "scripts/nvx_tools/performance.py", + "scripts/nvx_tools/benchmark_scripts/example", + ".github/actions/gate-performance/action.yml", + ".github/actions/package-release/action.yml", + ".github/workflows/run-platform.yml", + }, + benchmark_inputs, + ) + for inputs in (microvm_inputs, benchmark_inputs, nvx_rs_inputs): + self.assertNotIn(".github/actions/validate-nvx/action.yml", inputs) + self.assertNotIn("scripts/nvx_tools/performance.py", microvm_inputs) + self.assertNotIn("scripts/nvx_tools/microvm_tests.py", benchmark_inputs) for pattern, inputs in ( (ci.OPENVMM_UNIT_TEST_INPUT_PATHS, unit_inputs), (ci.OPENVMM_VMM_TEST_INPUT_PATHS, vmm_inputs), - (ci.CI_TEST_INPUT_PATHS, test_inputs), + (ci.MICROVM_TEST_INPUT_PATHS, microvm_inputs), + (ci.BENCHMARK_INPUT_PATHS, benchmark_inputs), ): for path in sorted(inputs): with self.subTest(pattern=pattern.pattern, path=path): @@ -4472,8 +4987,10 @@ def test_only_guest_binary_inputs_rebuild_guests_and_run_vmm_tests(self): # The guest images carry only the Rust guest binaries of nvx-rs, so a # change to a file that their build reads rebuilds every guest image and # runs the OpenVMM VMM tests, which boot the Alpine image, and no other - # nvx-rs change does either. The NVX microVM tests run the nvx-rs CLI, so - # every nvx-rs change runs them. + # nvx-rs change does either. The NVX microVM tests run the nvx-rs CLI, + # and every package ships it, so every nvx-rs change runs them and the + # platform jobs, and so the jobs that build their artifacts, which + # restore unchanged guest images from the cache. root = BuildConstants.REPO_ROOT identity = { path.relative_to(root).as_posix() @@ -4489,7 +5006,8 @@ def test_only_guest_binary_inputs_rebuild_guests_and_run_vmm_tests(self): self.assertLessEqual({"nvx-rs/src/host/mod.rs", "nvx-rs/src/lib.rs"}, crate) for path in sorted(crate | identity): with self.subTest(path=path): - self.assertIsNotNone(ci.CI_TEST_INPUT_PATHS.match(path)) + self.assertIsNotNone(ci.MICROVM_TEST_INPUT_PATHS.match(path)) + self.assertIsNotNone(ci.BENCHMARK_INPUT_PATHS.match(path)) if path in identity: self.assertIsNotNone(ci.OPENVMM_VMM_TEST_INPUT_PATHS.match(path)) else: @@ -4506,7 +5024,8 @@ def test_only_guest_binary_inputs_rebuild_guests_and_run_vmm_tests(self): ("nvx-rs/src/bin/nvx-time.rs.orig", False), ): with self.subTest(path=path): - self.assertIsNotNone(ci.CI_TEST_INPUT_PATHS.match(path)) + self.assertIsNotNone(ci.MICROVM_TEST_INPUT_PATHS.match(path)) + self.assertIsNotNone(ci.BENCHMARK_INPUT_PATHS.match(path)) self.assertEqual( ci.OPENVMM_VMM_TEST_INPUT_PATHS.match(path) is not None, selected ) @@ -4589,12 +5108,18 @@ def test_only_guest_binary_inputs_rebuild_guests_and_run_vmm_tests(self): with self.subTest(changed=paths, event_name=event_name): changes = ci.classify_ci_changes(event_name, paths) self.assertEqual( - changes, ci.CiChanges(True, True, False, guest) + changes, + ci.CiChanges(True, True, False, guest, True, True), ) expected = ci.required_ci_expected_results( event_name, same_repository=True, changes=changes ) - for job in ci.REQUIRED_CI_MICROVM_TEST_JOBS: + for job in ( + ci.REQUIRED_CI_ARTIFACT_JOB, + *ci.REQUIRED_CI_BUILD_JOBS, + *ci.REQUIRED_CI_MICROVM_TEST_JOBS, + *ci.REQUIRED_CI_PLATFORM_JOBS, + ): self.assertEqual(expected[job], "success", job) self.assertEqual( expected["openvmm-vmm-tests"], @@ -4638,46 +5163,102 @@ def test_ci_change_summary_explains_each_decision(self): changed_files = [ "scripts/nvx_tools/microvm_tests.py", "guest/common/init", + "guest/common/README.md", "guest/common/nvx-hostmount", "guest/common/nvx-snapshot", "guest/common/nvx-identity-probe", ] changes = ci.classify_ci_changes("pull_request", changed_files) summary = ci.describe_ci_changes("pull_request", changed_files, changes) - self.assertIn("### Artifact-backed tests and benchmarks: run\n", summary) - self.assertIn("### NVX microVM tests: run\n", summary) + self.assertEqual( + re.findall(r"^### (.+): (?:run|skip)$", summary, re.MULTILINE), + [ + "Guest artifacts", + "OpenVMM binaries", + "OpenVMM unit tests", + "OpenVMM VMM tests", + "NVX microVM tests", + "Platform benchmarks", + ], + ) self.assertIn( - "### OpenVMM unit tests: skip\nNo input of this suite changed.\n", + "### Guest artifacts: run\nJobs that use them run: the OpenVMM VMM " + "tests, the NVX microVM tests, and the platform benchmarks.\n", summary, ) self.assertIn( - "### OpenVMM VMM tests: run\nSuite inputs changed: " + "### OpenVMM binaries: run\nJobs that use them run: the NVX microVM " + "tests and the platform benchmarks.\n", + summary, + ) + self.assertIn("### OpenVMM unit tests: skip\nNo input changed.\n", summary) + self.assertIn( + "### OpenVMM VMM tests: run\nInputs changed: " "`guest/common/init`, `guest/common/nvx-hostmount`, " "`guest/common/nvx-snapshot`, and 1 more.\n", summary, ) + self.assertIn( + "### NVX microVM tests: run\nInputs changed: " + "`scripts/nvx_tools/microvm_tests.py`, `guest/common/init`, " + "`guest/common/nvx-hostmount`, and 2 more.\n", + summary, + ) push = ci.describe_ci_changes( "push", changed_files, ci.classify_ci_changes("push", changed_files) ) self.assertEqual(push, summary) - documentation = ci.describe_ci_changes( - "pull_request", - ["doc/ci.md"], - ci.classify_ci_changes("pull_request", ["doc/ci.md"]), + + def describe(event_name: str, files: list[str]) -> str: + return ci.describe_ci_changes( + event_name, files, ci.classify_ci_changes(event_name, files) + ) + + tests_only = describe("pull_request", ["scripts/test_nvx_tools.py"]) + self.assertIn( + "### Guest artifacts: skip\nNo job that uses them runs.\n", tests_only ) + self.assertIn("### Platform benchmarks: skip\nNo input changed.\n", tests_only) + push = describe("push", ["scripts/test_nvx_tools.py"]) self.assertIn( - "### Artifact-backed tests and benchmarks: skip\n" - "Only documentation files changed.\n", + "### OpenVMM binaries: run\nJobs that use them run: the platform " + "benchmarks.\n", + push, + ) + self.assertIn( + "### Platform benchmarks: run\nA push runs the platform jobs for every " + "change but documentation, so that it publishes its development " + "release and records its performance results.\n", + push, + ) + self.assertIn("### NVX microVM tests: skip\nNo input changed.\n", push) + documentation = describe("pull_request", ["doc/ci.md"]) + self.assertEqual(describe("push", ["doc/ci.md"]), documentation) + self.assertIn( + "### Guest artifacts: skip\nNo job that uses them runs.\n", documentation + ) + self.assertIn( + "### OpenVMM VMM tests: skip\nOnly documentation files changed.\n", + documentation, + ) + self.assertIn( + "### Platform benchmarks: skip\nOnly documentation files changed.\n", documentation, ) + empty = describe("pull_request", []) + self.assertIn("### NVX microVM tests: skip\nNo file changed.\n", empty) self.assertIn( - "### OpenVMM VMM tests: skip\nNo test input changed.\n", documentation + "### Platform benchmarks: run\nNo file changed, so the platform jobs " + "run.\n", + empty, ) def test_classify_ci_changes_command_prints_every_output(self): with tempfile.TemporaryDirectory() as temporary: changed_files = Path(temporary) / "changed-files" - changed_files.write_text("guest/common/init\n", encoding="utf-8") + changed_files.write_text( + "scripts/nvx_tools/microvm_tests.py\n", encoding="utf-8" + ) summary = Path(temporary) / "summary.md" summary.write_text("before\n", encoding="utf-8") args = nvx.parse_args( @@ -4697,15 +5278,22 @@ def test_classify_ci_changes_command_prints_every_output(self): self.assertEqual( stdout.getvalue().splitlines(), [ - "run-tests=true", - "run-workloads=true", + "run-guest-artifacts=true", + "run-openvmm-binaries=true", "run-openvmm-unit-tests=false", - "run-openvmm-vmm-tests=true", + "run-openvmm-vmm-tests=false", + "run-microvm-tests=true", + "run-benchmarks=false", ], ) + self.assertEqual( + [line.split("=", 1)[0] for line in stdout.getvalue().splitlines()], + list(ci.CI_CHANGE_OUTPUTS), + ) text = summary.read_text(encoding="utf-8") self.assertTrue(text.startswith("before\n### ")) - self.assertIn("### OpenVMM VMM tests: run\n", text) + self.assertIn("### NVX microVM tests: run\n", text) + self.assertIn("### Platform benchmarks: skip\n", text) def test_classify_ci_changes_command_reads_null_delimited_paths(self): with tempfile.TemporaryDirectory() as temporary: @@ -4732,10 +5320,12 @@ def test_classify_ci_changes_command_reads_null_delimited_paths(self): self.assertEqual( stdout.getvalue().splitlines(), [ - "run-tests=true", - "run-workloads=true", + "run-guest-artifacts=true", + "run-openvmm-binaries=true", "run-openvmm-unit-tests=false", "run-openvmm-vmm-tests=true", + "run-microvm-tests=true", + "run-benchmarks=true", ], ) summary = (root / "summary").read_bytes().decode("utf-8") @@ -4823,11 +5413,13 @@ def git(*arguments: str, directory: Path = root) -> str: line.split("=", 1) for line in output.read_text(encoding="utf-8").splitlines() ) + # A kernel patch is an input of every suite but the + # OpenVMM unit tests, and documentation of none. expected = { - "run-tests": str(test_input).lower(), - "run-workloads": str(test_input).lower(), - "run-openvmm-unit-tests": "false", - "run-openvmm-vmm-tests": str(test_input).lower(), + output: str( + test_input and output != "run-openvmm-unit-tests" + ).lower() + for output in ci.CI_CHANGE_OUTPUTS } self.assertEqual(actual, expected) @@ -4861,17 +5453,20 @@ def test_ci_change_detector_runs_the_full_matrix_unless_classified(self): return "${CLASSIFIER_STATUS}" } """ - complete = ( - "run-tests=true\\nrun-workloads=true\\n" - "run-openvmm-unit-tests=false\\nrun-openvmm-vmm-tests=false\\n" - ) - everything = dict.fromkeys(ci.CI_CHANGE_OUTPUTS, "true") + # The classification of a change only to the microVM tests. classified = { - "run-tests": "true", - "run-workloads": "true", + "run-guest-artifacts": "true", + "run-openvmm-binaries": "true", "run-openvmm-unit-tests": "false", "run-openvmm-vmm-tests": "false", + "run-microvm-tests": "true", + "run-benchmarks": "false", } + self.assertEqual(list(classified), list(ci.CI_CHANGE_OUTPUTS)) + complete = "".join( + f"{output}={value}\\n" for output, value in classified.items() + ) + everything = dict.fromkeys(ci.CI_CHANGE_OUTPUTS, "true") cases = ( ( "schedule", @@ -4888,15 +5483,25 @@ def test_ci_change_detector_runs_the_full_matrix_unless_classified(self): ("missing commit", {"GIT_CAT_FILE_STATUS": "1"}, everything), ("failed diff", {"GIT_DIFF_STATUS": "128"}, everything), ("failed classifier", {"CLASSIFIER_STATUS": "1"}, everything), - ( - "missing output", - {"CLASSIFICATION": complete.rsplit("run-openvmm-vmm", 1)[0]}, - everything, + # Every output must appear exactly once. + *( + ( + f"missing {output}", + {"CLASSIFICATION": complete.replace(f"{output}={value}\\n", "")}, + everything, + ) + for output, value in classified.items() ), - ( - "duplicate output", - {"CLASSIFICATION": complete + "run-tests=false\\n"}, - everything, + *( + ( + f"duplicate {output}", + { + "CLASSIFICATION": complete + + f"{output}={'false' if value == 'true' else 'true'}\\n" + }, + everything, + ) + for output, value in classified.items() ), ( "malformed output", @@ -6829,7 +7434,8 @@ def test_ci_takes_rust_toolchain_from_repository_pin(self): # A new release reruns every suite that it builds. for pattern in ( - ci.CI_TEST_INPUT_PATHS, + ci.MICROVM_TEST_INPUT_PATHS, + ci.BENCHMARK_INPUT_PATHS, ci.OPENVMM_UNIT_TEST_INPUT_PATHS, ci.OPENVMM_VMM_TEST_INPUT_PATHS, ):