diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1387ed1a1..ee50bd927 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -724,7 +724,9 @@ jobs: # The KVM and MSHV VMM tests run the same Linux build, which boots no guest, # so one job builds it beside the guest artifacts, on a KVM runner whose - # sccache holds it, and hands its test content to both backends' jobs. + # sccache holds it, and hands its test content to both backends' jobs. It + # restores the content instead while nothing that determines it changes + # (doc/ci.md#openvmm-vmm-test-cache). build-openvmm-vmm-tests-linux: name: OpenVMM vmm-tests build / Linux needs: [openvmm-changes] @@ -758,14 +760,12 @@ jobs: with: backend: kvm - - name: Checkout OpenVMM - id: openvmm - uses: ./.github/actions/checkout-openvmm - with: - ssh-key: ${{ secrets.OPENVMM_DEPLOY_KEY }} - - - name: Set up retrying curl - uses: ./.github/actions/setup-curl + # The key covers everything that determines the test content and reads + # the OpenVMM revision from the gitlink, so it needs no OpenVMM checkout. + - name: Prepare OpenVMM test cache key + id: openvmm-tests-cache + shell: bash + run: python3 scripts/nvx.py openvmm-test-cache-key --backend kvm --backend mshv - name: Prepare Linux cache tools id: cache-tools-linux @@ -778,8 +778,47 @@ jobs: ln -sf "$(command -v gzip)" "${cache_tools}/gzip" echo "path=${cache_tools}" >> "${GITHUB_OUTPUT}" + # Each dev push that builds the tests saves them under the key, its run, + # and its attempt, and the restore takes the newest of those entries. + # Full runs build the tests from scratch, so that a breakage of the + # build that the key does not cover shows within a day. A failed + # restore only costs the build, like a miss. + - name: Restore OpenVMM tests on Linux + if: github.event_name == 'pull_request' || github.event_name == 'push' + continue-on-error: true + uses: actions/cache/restore@v5 + env: + PATH: ${{ steps.cache-tools-linux.outputs.path }} + with: + path: ${{ runner.temp }}/openvmm-vmm-tests-linux + key: ${{ steps.openvmm-tests-cache.outputs.key }} + restore-keys: | + ${{ steps.openvmm-tests-cache.outputs.key }}- + + # The reuse keeps only the content that a fresh build makes, and removes + # a refused entry or a partial restore, so that the build starts empty. + - name: Reuse restored OpenVMM tests on Linux + id: reuse-openvmm-tests-linux + shell: bash + run: >- + python3 scripts/nvx.py reuse-openvmm-tests + --backend kvm --backend mshv + --content "${RUNNER_TEMP}/openvmm-vmm-tests-linux" + + - name: Checkout OpenVMM + id: openvmm + if: steps.reuse-openvmm-tests-linux.outputs.reused != 'true' + uses: ./.github/actions/checkout-openvmm + with: + ssh-key: ${{ secrets.OPENVMM_DEPLOY_KEY }} + + - name: Set up retrying curl + if: steps.reuse-openvmm-tests-linux.outputs.reused != 'true' + uses: ./.github/actions/setup-curl + - name: Restore OpenVMM inputs on Linux id: restore-openvmm-inputs-linux + if: steps.reuse-openvmm-tests-linux.outputs.reused != 'true' uses: actions/cache/restore@v5 env: PATH: ${{ steps.cache-tools-linux.outputs.path }} @@ -793,6 +832,7 @@ jobs: - name: Restore shared Cargo cache on Linux id: restore-openvmm-cargo-linux + if: steps.reuse-openvmm-tests-linux.outputs.reused != 'true' uses: actions/cache/restore@v5 env: PATH: ${{ steps.cache-tools-linux.outputs.path }} @@ -805,11 +845,13 @@ jobs: openvmm-cargo-v1-${{ runner.os }}-${{ runner.arch }}- - name: Prepare sccache + if: steps.reuse-openvmm-tests-linux.outputs.reused != 'true' uses: ./.github/actions/sccache with: operation: start - name: Build OpenVMM tests on Linux + if: steps.reuse-openvmm-tests-linux.outputs.reused != 'true' shell: bash env: GH_TOKEN: ${{ github.token }} @@ -819,11 +861,21 @@ jobs: --output "${RUNNER_TEMP}/openvmm-vmm-tests-linux" - name: Report sccache - if: always() + if: always() && steps.reuse-openvmm-tests-linux.outputs.reused != 'true' uses: ./.github/actions/sccache with: operation: stop + # A build that a later reuse would refuse fails here rather than reach + # the cache or the test jobs. + - name: Check built OpenVMM tests on Linux + if: steps.reuse-openvmm-tests-linux.outputs.reused != 'true' + shell: bash + run: >- + python3 scripts/nvx.py check-openvmm-tests + --backend kvm --backend mshv + --content "${RUNNER_TEMP}/openvmm-vmm-tests-linux" + # include-hidden-files keeps the content's .config/nextest.toml. - name: Upload OpenVMM tests uses: actions/upload-artifact@v7 @@ -835,11 +887,25 @@ jobs: overwrite: true retention-days: 1 + - name: Save OpenVMM tests on Linux + if: >- + success() && + github.event_name == 'push' && + github.ref == 'refs/heads/dev' && + steps.reuse-openvmm-tests-linux.outputs.reused != 'true' + uses: actions/cache/save@v5 + env: + PATH: ${{ steps.cache-tools-linux.outputs.path }} + with: + path: ${{ runner.temp }}/openvmm-vmm-tests-linux + key: ${{ steps.openvmm-tests-cache.outputs.key }}-${{ github.run_id }}-${{ github.run_attempt }} + - name: Save OpenVMM inputs on Linux if: >- success() && github.event_name == 'push' && github.ref == 'refs/heads/dev' && + steps.reuse-openvmm-tests-linux.outputs.reused != 'true' && steps.restore-openvmm-inputs-linux.outputs.cache-hit != 'true' uses: actions/cache/save@v5 env: @@ -855,6 +921,7 @@ jobs: success() && github.event_name == 'push' && github.ref == 'refs/heads/dev' && + steps.reuse-openvmm-tests-linux.outputs.reused != 'true' && steps.restore-openvmm-cargo-linux.outputs.cache-hit != 'true' uses: actions/cache/save@v5 env: @@ -965,7 +1032,9 @@ jobs: # The WHP VMM tests boot no guest while they build either, so a WHP runner, # whose sccache holds the Windows test build, builds them beside the guest - # artifacts and hands their test content to the shards of the WHP job. + # artifacts and hands their test content to the shards of the WHP job. As on + # Linux, it restores the content instead while nothing that determines it + # changes. build-openvmm-vmm-tests-windows: name: OpenVMM vmm-tests build / Windows needs: [openvmm-changes] @@ -999,21 +1068,47 @@ jobs: with: backend: whp + - name: Prepare OpenVMM test cache key + id: openvmm-tests-cache + shell: powershell + run: python scripts\nvx.py openvmm-test-cache-key --backend whp + + - name: Prepare Windows cache tools + shell: powershell + run: python scripts\nvx.py setup-cross-os-cache + + - name: Restore OpenVMM tests on Windows + if: github.event_name == 'pull_request' || github.event_name == 'push' + continue-on-error: true + uses: actions/cache/restore@v5 + with: + path: ${{ runner.temp }}/openvmm-vmm-tests-windows + key: ${{ steps.openvmm-tests-cache.outputs.key }} + restore-keys: | + ${{ steps.openvmm-tests-cache.outputs.key }}- + + - name: Reuse restored OpenVMM tests on Windows + id: reuse-openvmm-tests-windows + shell: powershell + run: >- + python scripts\nvx.py reuse-openvmm-tests + --backend whp + --content "$env:RUNNER_TEMP\openvmm-vmm-tests-windows" + - name: Checkout OpenVMM id: openvmm + if: steps.reuse-openvmm-tests-windows.outputs.reused != 'true' uses: ./.github/actions/checkout-openvmm with: ssh-key: ${{ secrets.OPENVMM_DEPLOY_KEY }} - name: Set up retrying curl + if: steps.reuse-openvmm-tests-windows.outputs.reused != 'true' uses: ./.github/actions/setup-curl - - name: Prepare Windows cache tools - shell: powershell - run: python scripts\nvx.py setup-cross-os-cache - - name: Restore OpenVMM inputs on Windows id: restore-openvmm-inputs-windows + if: steps.reuse-openvmm-tests-windows.outputs.reused != 'true' uses: actions/cache/restore@v5 with: path: | @@ -1025,6 +1120,7 @@ jobs: - name: Restore shared Cargo cache on Windows id: restore-openvmm-cargo-windows + if: steps.reuse-openvmm-tests-windows.outputs.reused != 'true' uses: actions/cache/restore@v5 with: path: | @@ -1035,11 +1131,13 @@ jobs: openvmm-cargo-v1-${{ runner.os }}-${{ runner.arch }}- - name: Prepare sccache + if: steps.reuse-openvmm-tests-windows.outputs.reused != 'true' uses: ./.github/actions/sccache with: operation: start - name: Build OpenVMM tests on Windows + if: steps.reuse-openvmm-tests-windows.outputs.reused != 'true' shell: powershell env: GH_TOKEN: ${{ github.token }} @@ -1049,11 +1147,19 @@ jobs: --output "$env:RUNNER_TEMP\openvmm-vmm-tests-windows" - name: Report sccache - if: always() + if: always() && steps.reuse-openvmm-tests-windows.outputs.reused != 'true' uses: ./.github/actions/sccache with: operation: stop + - name: Check built OpenVMM tests on Windows + if: steps.reuse-openvmm-tests-windows.outputs.reused != 'true' + shell: powershell + run: >- + python scripts\nvx.py check-openvmm-tests + --backend whp + --content "$env:RUNNER_TEMP\openvmm-vmm-tests-windows" + # include-hidden-files keeps the content's .config/nextest.toml. - name: Upload OpenVMM tests uses: actions/upload-artifact@v7 @@ -1065,11 +1171,23 @@ jobs: overwrite: true retention-days: 1 + - name: Save OpenVMM tests on Windows + if: >- + success() && + github.event_name == 'push' && + github.ref == 'refs/heads/dev' && + steps.reuse-openvmm-tests-windows.outputs.reused != 'true' + uses: actions/cache/save@v5 + with: + path: ${{ runner.temp }}/openvmm-vmm-tests-windows + key: ${{ steps.openvmm-tests-cache.outputs.key }}-${{ github.run_id }}-${{ github.run_attempt }} + - name: Save OpenVMM inputs on Windows if: >- success() && github.event_name == 'push' && github.ref == 'refs/heads/dev' && + steps.reuse-openvmm-tests-windows.outputs.reused != 'true' && steps.restore-openvmm-inputs-windows.outputs.cache-hit != 'true' uses: actions/cache/save@v5 with: @@ -1083,6 +1201,7 @@ jobs: success() && github.event_name == 'push' && github.ref == 'refs/heads/dev' && + steps.reuse-openvmm-tests-windows.outputs.reused != 'true' && steps.restore-openvmm-cargo-windows.outputs.cache-hit != 'true' uses: actions/cache/save@v5 with: diff --git a/doc/ci.md b/doc/ci.md index 03988c822..298edfc62 100644 --- a/doc/ci.md +++ b/doc/ci.md @@ -510,8 +510,9 @@ that it selects start: `artifacts`, `distro-artifacts`, and `debug-kernel` restore or build the guest artifacts on GitHub-hosted runners (see [Guest artifact handoff](#guest-artifact-handoff)), the three OpenVMM binary producers restore or build OpenVMM on a KVM, an MSHV, and a WHP runner, -`build-openvmm-vmm-tests-linux` and `build-openvmm-vmm-tests-windows` build -the Linux VMM tests on a KVM runner and the WHP ones on a WHP runner, and +`build-openvmm-vmm-tests-linux` and `build-openvmm-vmm-tests-windows` restore +or build the Linux VMM tests on a KVM runner and the WHP ones on a WHP runner +(see [OpenVMM VMM test cache](#openvmm-vmm-test-cache)), and `openvmm-unit-tests` starts on a GitHub-hosted and a WHP runner. `openvmm-vmm-tests-linux` waits for `artifacts`, which builds the kernel and the Alpine initramfs, and the Linux VMM-test build, and `openvmm-vmm-tests-windows`, @@ -618,7 +619,9 @@ and then two for the [shards of the WHP VMM tests](#openvmm-vmm-test-shards). The build ended after the last WHP microVM job in 10 of the 13 runs since [#783](https://github.com/microsoft/nvx/pull/783) that ran both, so the shards usually take runners that the microVM jobs freed, and a run then uses at most -six. When a pool has fewer +six. A run that restores the Windows build from the +[cache](#openvmm-vmm-test-cache) instead ends it within about a minute, so its +shards run beside the microVM jobs, and it may use all seven. When a pool has fewer free general runners than a backend's microVM jobs, the jobs that start last wait for the first to end, so the backend's microVM tests take up to about twice as long as one of its jobs, still less than the unsharded suite. @@ -943,7 +946,9 @@ operating system and hands each build to the jobs of its backends: the one-day `openvmm-vmm-tests-linux` artifact, which keeps the content's hidden `.config` and replaces an earlier attempt's content. It needs no guest artifacts, so it starts with the OpenVMM binary producers instead of - waiting for `artifacts`. + waiting for `artifacts`. While nothing that determines the content changes, + it restores the content from the + [OpenVMM VMM test cache](#openvmm-vmm-test-cache) instead of building it. - `openvmm-vmm-tests-linux` runs a KVM and an MSHV leg once the build and `artifacts` succeed. Each downloads the content, the kernel, and the Alpine initramfs, and runs [`test-openvmm`](usage.md#test-openvmm) on the content, @@ -955,7 +960,8 @@ operating system and hands each build to the jobs of its backends: general WHP runner, whose sccache holds the Windows test build, with the Windows `vmm-tests` caches, and uploads the content as the one-day `openvmm-vmm-tests-windows` artifact. It also starts with the OpenVMM binary - producers. + producers, and also restores the content from the cache instead of + building it while nothing that determines it changes. - `openvmm-vmm-tests-windows` runs the WHP tests, in the [shards](#openvmm-vmm-test-shards) of its matrix, once that build and `artifacts` succeed. Like a Linux leg, each shard downloads the content, the @@ -976,18 +982,104 @@ operating system and hands each build to the jobs of its backends: curl shim nor a GitHub token, which only flowey's downloads use. The content's manifest, `nvx-openvmm-tests.json`, records the OpenVMM revision, -the Rust release, each backend's filter, the runner and its options, and every -file, so that a test job refuses content built from another OpenVMM revision -or for another filter, and so that the content describes the inputs on which -reusing it would depend. The builds and their test jobs run together: a run -that selects the VMM tests, and so builds the guest artifacts, runs all four -VMM jobs, and any other run skips all four. `Required status check` expects +the Rust release, the target and the build options, each backend's filter, the +runner and its options, and every file with its size and SHA-256, so that a +test job refuses content built from another OpenVMM revision or for another +filter, and so that the content describes the inputs on which reusing it would +depend. The builds and their test jobs run together: a run that selects the +VMM tests, and so builds the guest artifacts, runs all four VMM jobs, and any +other run skips all four. `Required status check` expects exactly that, and `Publish development release` and `Persist performance baseline` require the four jobs to succeed, or all four to be skipped because the change selected no VMM tests, so that a failed or unexpectedly skipped build or test blocks them. The WHP job succeeds only when each of its shards does. +## OpenVMM VMM test cache + +What determines the VMM tests' build changes far less often than the tests +run: replaying the change classifier over the 42 `dev` merges through +[#659](https://github.com/microsoft/nvx/pull/659), the VMM tests would have run +on 33 of them, but the OpenVMM pin changed in none. Each producer therefore +caches its test content across runs, and restores it instead of building it +while nothing that determines it changes +([#692](https://github.com/microsoft/nvx/issues/692)). + +[`openvmm-test-cache-key`](usage.md#openvmm-test-cache-key) keys the content +of a producer's backends as `openvmm-vmm-tests-vVERSION-TARGET-REVISION-DIGEST`. +TARGET is the build's target, `linux-x64` or `windows-x64`; REVISION is the +OpenVMM revision that the checkout's gitlink pins; and DIGEST is the SHA-256 +of every field of the content's manifest that describes what makes a clean +build of that revision: the manifest's format, the Rust release that +`rust-toolchain.toml` pins, the target, the `vmm-tests-run` build options, a +release build without optional features, and each backend's filter and their +union. A change to any of them, such as a pin promotion, a new Rust release, +or a test that a filter adds, therefore builds the tests again. VERSION, +`OPENVMM_TEST_CACHE_VERSION` in [`ci.py`](../scripts/nvx_tools/ci.py), covers +what none of them captures: increment it when what a build makes, or what a +reuse checks, changes otherwise, such as the environment in which +`build-openvmm-tests` builds the tests. Like the OpenVMM binary cache, the key +leaves out the C toolchains and other tools of the runners, so a runner +upgrade reuses the content built before it until the key changes, and the +full runs below check the build every day. + +On a push or a pull request, a producer computes the key before it checks out +OpenVMM and restores the newest entry of the key into the directory that it +uploads. [`reuse-openvmm-tests`](usage.md#reuse-openvmm-tests) keeps that +content only if it is exactly what a fresh build of the key makes: its +manifest must record every field that the key digests, for a clean checkout, +and the runner and options that its run script runs, which the command reads +from the script again, and the directory must hold exactly the files that the +manifest lists, with their sizes, SHA-256, and executable bits. The test jobs +therefore run the same nextest archive with the same options and filters, and +so exactly the tests, that a fresh build would give them. On such a hit, the +producer skips the OpenVMM checkout, the retrying curl, the OpenVMM input and +Cargo caches, sccache, and the build, and uploads the restored content as the +same-run artifact that the test jobs download as before; the test jobs never +read the cache. Otherwise, the reuse removes whatever the restore left, with a +warning annotation when it refuses an entry or a partial restore, and the +producer builds the tests as before. The restore step continues on error, so +a miss, an eviction, a failed restore, or a refused entry only costs the +build. + +After a build, [`check-openvmm-tests`](usage.md#check-openvmm-tests) fails the +producer unless the content passes the same check, so that neither the cache +nor a test job gets a build that a reuse would refuse. Only `dev` pushes save +the content, as they save the other OpenVMM caches, and only after a build: +under the key followed by the run ID and attempt, so that each build makes an +entry of its own, and the restore, which takes the newest entry whose key +starts with the key, takes the latest build. An entry that a later reuse +refuses, for example after a change to the check that kept the version, +therefore blocks nothing: the next `dev` push builds the tests and saves a +newer entry. Pull requests restore the entries that `dev` pushes save but save +none, so a pull request that changes the key builds the tests in each of its +runs. Scheduled and manually dispatched full runs neither restore nor save the +content: they build the tests every day, so that a breakage of the build that +the key doesn't cover, such as a toolchain change on the runners, shows within +a day rather than at the next pin promotion. + +This change's validation run, +[39988095577](https://github.com/microsoft/nvx/actions/runs/39988095577), saved +its builds from its pull request through a temporary change. The 228.5 MiB of +Linux content made a 124.0 MiB entry, which the KVM runners compress with gzip, +as they do their other caches, and the 132.6 MiB of Windows content a 37.3 MiB +entry, which the WHP runners compress with zstd: 161.3 MiB a key, 1.6% of the +repository's 10 GB of Actions caches. Those were full, at 10.4 GB in 89 +entries on October 23, 2026, so each new entry displaces the least recently +used ones. These were stale: the guest image entries of merged pull requests, +and the OpenVMM input and Cargo entries of the previous OpenVMM pin, about +2.4 GiB, which no run restores any more. A hit, in turn, restores neither the +`vmm-tests` OpenVMM input entry, about 945 MiB per operating system, nor its +Cargo entry, 161 MiB on Linux and 261 MiB on Windows, so those age out between +pin promotions as well. On a miss, the key, the restore, and the two checks +added under five seconds to each producer, whose builds took 340 and 712 +seconds, and a `dev` push's save adds about 10 and 4 seconds more. Rerun on +other runners, the producers hit, and took 20 and 36 seconds instead of 6.5 +and 14.2 minutes: 3 seconds to restore each entry, at about 118 and 25 MB/s, +under 1 and 2 seconds for the reuse's check, and 8 and 7 seconds to upload the +content. The test jobs then ran the same 61 KVM, 60 MSHV, and 15 and 14 WHP +tests, all passing, from the restored content as from the build. + ## OpenVMM VMM test shards `nvx.py test-openvmm --shard INDEX/COUNT` runs one of COUNT shards of a @@ -1210,7 +1302,9 @@ dispatch starts the same full run. Both select the full matrix before the detector examines a comparison base. Each full run has a separate concurrency key, so it neither cancels nor queues behind a `dev` push. Full runs build, test, benchmark, and package, but publish and persist nothing and save no -caches. Only `dev` pushes save the OpenVMM caches, and only pushes and pull +caches, and they build the OpenVMM VMM tests without restoring them from the +[OpenVMM VMM test cache](#openvmm-vmm-test-cache). Only `dev` pushes save the +OpenVMM caches, and only pushes and pull requests save the guest artifact and pip caches, through the `save-cache` input of the [`build-guest-artifacts`](../.github/actions/build-guest-artifacts/action.yml) diff --git a/doc/usage.md b/doc/usage.md index 847b07857..be7222fe6 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -88,6 +88,9 @@ stale build never ships or runs unnoticed. | `check-static-elf` | Check that executables are static ELF files without an interpreter or an executable stack. | | `test-openvmm-unit` | Run the OpenVMM workspace unit and documentation tests. | | `build-openvmm-tests` | Build OpenVMM Petri VMM tests without running them. | +| `openvmm-test-cache-key` | Print the key under which CI caches the OpenVMM VMM tests that `build-openvmm-tests` builds. | +| `check-openvmm-tests` | Check that a directory holds the OpenVMM VMM tests that `build-openvmm-tests` builds from this checkout. | +| `reuse-openvmm-tests` | Keep OpenVMM VMM tests that a cache restored if `build-openvmm-tests` builds them from this checkout, and remove them otherwise. | | `test-openvmm` | Run OpenVMM Petri VMM tests. | | `plan-openvmm-test-shards` | Split the OpenVMM VMM tests into shards that nextest runs in balanced times for CI. | | `test-microvm` | Run NVX Linux and device correctness tests through OpenVMM. | @@ -539,16 +542,73 @@ Builds the VMM tests that [`test-openvmm`](#test-openvmm) runs for each `--output` names, so that another host can run them without building OpenVMM. The backends must run on this host's operating system: Linux builds the KVM and MSHV tests, and Windows the WHP tests. The command runs OpenVMM's -`vmm-tests-run` in build-only mode on the union of the backends' filters, -which puts the tests' nextest archive, OpenVMM, the guest agents, the test -firmware and kernels, and a `vmm-tests-run-target` runner into the directory. -It then reads the runner's options from the run script that the build writes, -and records them in the directory's `nvx-openvmm-tests.json` with the OpenVMM -revision and whether its checkout was clean, the Rust release, each backend's -filter, and every file's size and executable bit. The tests boot NVX's kernel +`vmm-tests-run` in build-only mode on the union of the backends' filters, for +the `linux-x64` or `windows-x64` target, since NVX supports only x86-64 hosts, +as a release build without optional features whose run script uses the +nextest CI profile, which puts the tests' nextest archive, OpenVMM, the guest +agents, the test firmware and kernels, and a `vmm-tests-run-target` runner into +the directory. It then reads the runner's options from the run script that the +build writes, and records them in the directory's `nvx-openvmm-tests.json` +with the OpenVMM revision and whether its checkout was clean, the Rust +release, the target, the `vmm-tests-run` build options, each backend's filter, +and every file's size, SHA-256, and executable bit. The tests boot NVX's kernel and initramfs only when they run, so the build needs no guest artifacts. On Windows, it appends `/Brepro` to `_LINK_`, as `test-openvmm` does. +### `openvmm-test-cache-key` + +```text +python3 scripts/nvx.py openvmm-test-cache-key + --backend {kvm,mshv,whp} [--backend {kvm,mshv,whp} ...] +``` + +Prints the key under which CI caches the test content that +[`build-openvmm-tests`](#build-openvmm-tests) builds for the backends from this +checkout (see [OpenVMM VMM test cache](ci.md#openvmm-vmm-test-cache)): +`openvmm-vmm-tests-vVERSION-TARGET-REVISION-DIGEST`. VERSION is the +cache's own version, which changes when what a build makes changes in a way +that nothing else in the key captures. TARGET is the build's target, REVISION +the OpenVMM revision that the checkout's index pins, and DIGEST the SHA-256 of +the fields of the manifest of a clean build of that revision that describe +what built it: the manifest's format, the Rust release, the target, the build +options, and each backend's filter and their union. The backends must run on +this host's operating system, as for `build-openvmm-tests`, and the command +needs neither the OpenVMM submodule's files nor Cargo. In GitHub Actions, it +also gives its step the `key` output. + +### `check-openvmm-tests` + +```text +python3 scripts/nvx.py check-openvmm-tests + --backend {kvm,mshv,whp} [--backend {kvm,mshv,whp} ...] + --content PATH +``` + +Fails unless the directory that `--content` names holds exactly the test +content of the backends' cache key: a manifest that records every field that +[`openvmm-test-cache-key`](#openvmm-test-cache-key) digests, for a clean +OpenVMM checkout, and the runner and options that the content's run script +runs, and exactly the files that the manifest lists, with their sizes, +SHA-256, and executable bits. CI checks each build this way before it caches +it. + +### `reuse-openvmm-tests` + +```text +python3 scripts/nvx.py reuse-openvmm-tests + --backend {kvm,mshv,whp} [--backend {kvm,mshv,whp} ...] + --content PATH +``` + +Keeps the directory that `--content` names, into which a cache restored test +content, if it passes the checks of +[`check-openvmm-tests`](#check-openvmm-tests), and otherwise removes it with a +warning that gives the reason, so that a build can take its place. A +directory that doesn't exist, as after a cache miss, needs no removal. The +command fails only when it cannot compute the key or remove the directory, +never because of the content. In GitHub Actions, it gives its step the +`reused` output, `true` or `false`. + ### `test-openvmm` ```text diff --git a/scripts/nvx.py b/scripts/nvx.py index dadd53711..ce4e12b7f 100644 --- a/scripts/nvx.py +++ b/scripts/nvx.py @@ -53,10 +53,13 @@ REQUIRED_CI_RESULT_ENVIRONMENTS, CiChanges, build_openvmm_tests, + check_openvmm_tests, classify_ci_changes, describe_ci_changes, kernel_patch_identity, + openvmm_test_cache_key, required_ci_failures, + reuse_openvmm_tests, run_openvmm_tests, run_openvmm_unit_tests, setup_cross_os_cache, @@ -359,6 +362,18 @@ def command_build_openvmm_tests(args: argparse.Namespace) -> None: build_openvmm_tests(args.backend, args.output) +def command_openvmm_test_cache_key(args: argparse.Namespace) -> None: + openvmm_test_cache_key(args.backend, os.environ) + + +def command_check_openvmm_tests(args: argparse.Namespace) -> None: + check_openvmm_tests(args.backend, args.content) + + +def command_reuse_openvmm_tests(args: argparse.Namespace) -> None: + reuse_openvmm_tests(args.backend, args.content, os.environ) + + def command_test_openvmm(args: argparse.Namespace) -> None: shard = ( None @@ -870,6 +885,66 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: ) openvmm_test_build.set_defaults(handler=command_build_openvmm_tests) + openvmm_test_cache_key = subparsers.add_parser( + "openvmm-test-cache-key", + help=( + "print the key under which CI caches the OpenVMM VMM tests that " + "build-openvmm-tests builds" + ), + ) + openvmm_test_cache_key.add_argument( + "--backend", + choices=OPENVMM_TEST_BACKENDS, + action="append", + required=True, + help="key this backend's tests; repeat for each backend that the build builds", + ) + openvmm_test_cache_key.set_defaults(handler=command_openvmm_test_cache_key) + + openvmm_test_check = subparsers.add_parser( + "check-openvmm-tests", + help=( + "check that a directory holds the OpenVMM VMM tests that " + "build-openvmm-tests builds from this checkout" + ), + ) + openvmm_test_check.add_argument( + "--backend", + choices=OPENVMM_TEST_BACKENDS, + action="append", + required=True, + help="check for this backend's tests; repeat for each backend of the build", + ) + openvmm_test_check.add_argument( + "--content", + type=Path, + required=True, + help="directory of the test content to check", + ) + openvmm_test_check.set_defaults(handler=command_check_openvmm_tests) + + openvmm_test_reuse = subparsers.add_parser( + "reuse-openvmm-tests", + help=( + "keep OpenVMM VMM tests that a cache restored if build-openvmm-tests " + "builds them from this checkout, and remove them otherwise" + ), + ) + openvmm_test_reuse.add_argument( + "--backend", + choices=OPENVMM_TEST_BACKENDS, + action="append", + required=True, + help="reuse this backend's tests; repeat for each backend of the build", + ) + openvmm_test_reuse.add_argument( + "--content", + type=Path, + required=True, + help="directory into which the cache restored the test content", + ) + openvmm_test_reuse.set_defaults(handler=command_reuse_openvmm_tests) + openvmm_tests = subparsers.add_parser( "test-openvmm", help="run OpenVMM Petri VMM tests", diff --git a/scripts/nvx_tools/ci.py b/scripts/nvx_tools/ci.py index 3921e05ab..8bf481564 100644 --- a/scripts/nvx_tools/ci.py +++ b/scripts/nvx_tools/ci.py @@ -32,6 +32,7 @@ bytes_to_mib, download, github_job, + is_link_or_junction, openvmm_git_state, path_exists, require_file, @@ -39,6 +40,7 @@ require_tool, run_capture, run_checked, + sha256_file, strict_json_object, ) @@ -46,9 +48,31 @@ # The operating system that builds and runs each backend's VMM tests. OPENVMM_TEST_OPERATING_SYSTEMS = {"kvm": "linux", "mshv": "linux", "whp": "windows"} # `build-openvmm-tests` describes the test content that it builds in this file, -# which `test-openvmm --content` reads. +# which `test-openvmm --content` reads. Format 2 added the target, the build +# options, and the SHA-256 of each file. OPENVMM_TEST_CONTENT_MANIFEST = "nvx-openvmm-tests.json" -OPENVMM_TEST_CONTENT_FORMAT = 1 +OPENVMM_TEST_CONTENT_FORMAT = 2 +# The vmm-tests-run target of each operating system's content. NVX supports +# only x86-64 hosts, and the WHP tests run only windows-x64 content, so the +# build names its target rather than taking the host's. +OPENVMM_TEST_TARGETS = {"linux": "linux-x64", "windows": "windows-x64"} +# The vmm-tests-run options that build the content: a release build without +# optional features, whose run script runs the tests with the nextest CI +# profile and without the interactive VHD prompt. +OPENVMM_TEST_BUILD_OPTIONS = ( + "--release", + "--ci-profile", + "--skip-vhd-prompt", + "--build-only", +) +# CI caches the test content of each operating system across runs under a key +# of everything that determines it (doc/ci.md#openvmm-vmm-test-cache): this +# version, the target, the OpenVMM revision, and the SHA-256 of every field of +# the content's manifest that describes what built it. Increment the version +# when what a build makes, or what a reuse of the content checks, changes in a +# way that none of those fields captures, such as the environment in which the +# tests build. +OPENVMM_TEST_CACHE_VERSION = 1 # The script that `vmm-tests-run --build-only` writes to run the content. OPENVMM_TEST_RUN_SCRIPTS = {"linux": "run.sh", "windows": "run.ps1"} # The nextest archive of the tests, in the content's directory of its target. @@ -1112,18 +1136,66 @@ def _openvmm_test_content_files(directory: Path) -> list[dict[str, object]]: # Artifact uploads drop file modes, so the consumer restores # these. "executable": os.name != "nt" and bool(metadata.st_mode & 0o111), + "sha256": sha256_file(path), } ) return files +def _openvmm_gitlink() -> str: + """Return the OpenVMM revision that the checkout's index pins.""" + gitlink = run_capture( + ["git", "-C", BuildConstants.REPO_ROOT, "rev-parse", ":openvmm"] + ) + require_success(gitlink, "OpenVMM gitlink query") + return gitlink.stdout.decode("ascii").strip() + + +def _read_openvmm_test_manifest(path: Path) -> dict[str, object]: + try: + document = cast( + object, + json.loads( + path.read_text(encoding="utf-8"), + object_pairs_hook=strict_json_object, + ), + ) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ScriptError(f"{path} is not valid JSON: {error}") from error + if not isinstance(document, dict): + raise ScriptError(f"{path} is not a JSON object") + return cast(dict[str, object], document) + + +def _openvmm_test_content_inputs( + selected: Sequence[str], revision: str, clean: bool +) -> dict[str, object]: + """Return the fields of the manifest of the content that + ``build_openvmm_tests`` builds for the selected backends from an OpenVMM + checkout at ``revision`` that describe what built it: everything that + determines the content but the build itself.""" + operating_system = OPENVMM_TEST_OPERATING_SYSTEMS[selected[0]] + return { + "format": OPENVMM_TEST_CONTENT_FORMAT, + "openvmm_revision": revision, + "openvmm_clean": clean, + "rust_toolchain": OpenVMMBuildConstants.RUST_TOOLCHAIN, + "operating_system": operating_system, + "target": OPENVMM_TEST_TARGETS[operating_system], + "build_options": list(OPENVMM_TEST_BUILD_OPTIONS), + "filters": {backend: OPENVMM_TEST_FILTERS[backend] for backend in selected}, + "build_filter": openvmm_test_build_filter(selected), + } + + def build_openvmm_tests(backends: Sequence[str], output: Path) -> None: """Build, without running them, the VMM tests of every named backend into a test content directory that ``run_openvmm_tests`` runs on another host. The directory holds what OpenVMM's `vmm-tests-run-target` needs, and a - manifest that binds it to the OpenVMM revision, the Rust release, and the - backends' filters that built it. + manifest that binds it to the OpenVMM revision, the Rust release, the + target, the build options, and the backends' filters that built it, and + lists the size and SHA-256 of each of its files. """ selected = _openvmm_test_content_backends(backends) require_file( @@ -1134,6 +1206,8 @@ def build_openvmm_tests(backends: Sequence[str], output: Path) -> None: cargo = require_tool("cargo") rustup = require_tool("rustup") revision, clean = openvmm_git_state(OpenVMMBuildConstants.DIRECTORY) + operating_system = OPENVMM_TEST_OPERATING_SYSTEMS[selected[0]] + target = OPENVMM_TEST_TARGETS[operating_system] build_filter = openvmm_test_build_filter(selected) if "'" in build_filter: raise ScriptError("OpenVMM run scripts cannot quote a filter with a quote") @@ -1150,10 +1224,9 @@ def build_openvmm_tests(backends: Sequence[str], output: Path) -> None: cargo, "xflowey", "vmm-tests-run", - "--release", - "--ci-profile", - "--skip-vhd-prompt", - "--build-only", + "--target", + target, + *OPENVMM_TEST_BUILD_OPTIONS, "--filter", build_filter, "--dir", @@ -1163,19 +1236,13 @@ def build_openvmm_tests(backends: Sequence[str], output: Path) -> None: env=environment, ) - operating_system = OPENVMM_TEST_OPERATING_SYSTEMS[selected[0]] - runner, arguments = _openvmm_test_run_target( - output / OPENVMM_TEST_RUN_SCRIPTS[operating_system], build_filter - ) + script = output / OPENVMM_TEST_RUN_SCRIPTS[operating_system] + runner, arguments = _openvmm_test_run_target(script, build_filter) + if arguments[arguments.index("--target") + 1] != target: + raise ScriptError(f"{script} does not run the {target} tests that it built") files = _openvmm_test_content_files(output) manifest = { - "format": OPENVMM_TEST_CONTENT_FORMAT, - "openvmm_revision": revision, - "openvmm_clean": clean, - "rust_toolchain": OpenVMMBuildConstants.RUST_TOOLCHAIN, - "operating_system": operating_system, - "filters": {backend: OPENVMM_TEST_FILTERS[backend] for backend in selected}, - "build_filter": build_filter, + **_openvmm_test_content_inputs(selected, revision, clean), "runner": runner, "arguments": arguments, "files": files, @@ -1190,6 +1257,185 @@ def build_openvmm_tests(backends: Sequence[str], output: Path) -> None: ) +@dataclass(frozen=True) +class _OpenVMMTestCacheEntry: + """The content of some backends that CI caches: a clean build of the + OpenVMM revision that the checkout pins, the fields of its manifest that + describe what built it, and its cache key.""" + + backends: tuple[str, ...] + inputs: Mapping[str, object] + key: str + + def describe(self) -> str: + return ", ".join(backend.upper() for backend in self.backends) + + +def _openvmm_test_cache_entry(backends: Sequence[str]) -> _OpenVMMTestCacheEntry: + selected = _openvmm_test_content_backends(backends) + inputs = _openvmm_test_content_inputs(selected, _openvmm_gitlink(), True) + canonical = json.dumps(inputs, sort_keys=True, separators=(",", ":")) + digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + return _OpenVMMTestCacheEntry( + backends=selected, + inputs=inputs, + key=( + f"openvmm-vmm-tests-v{OPENVMM_TEST_CACHE_VERSION}-{inputs['target']}-" + f"{inputs['openvmm_revision']}-{digest}" + ), + ) + + +def _set_step_output(environment: Mapping[str, str], name: str, value: str) -> None: + """Give the GitHub Actions step that runs the command an output.""" + path = environment.get("GITHUB_OUTPUT") + if path: + with Path(path).open("a", encoding="utf-8", newline="\n") as stream: + stream.write(f"{name}={value}\n") + + +def openvmm_test_cache_key( + backends: Sequence[str], environment: Mapping[str, str] | None = None +) -> str: + """Print and return the key under which CI caches the content that + ``build_openvmm_tests`` builds for the backends from this checkout, and + give the step that asks for it the `key` output.""" + key = _openvmm_test_cache_entry(backends).key + print(key) + _set_step_output(environment or {}, "key", key) + return key + + +def _brief(value: object) -> str: + text = json.dumps(value) + return text if len(text) <= 80 else f"{text[:77]}..." + + +def _openvmm_test_file_refusal( + listed: object, files: Sequence[Mapping[str, object]] +) -> str | None: + """Return how a manifest's list of files differs from the files of its + content, as _openvmm_test_content_files lists them, or None if it lists + exactly those files.""" + if listed == files: + return None + present = {str(entry["path"]): entry for entry in files} + recorded = { + str(cast(dict[str, object], entry).get("path")): cast(dict[str, object], entry) + for entry in (cast(list[object], listed) if isinstance(listed, list) else []) + if isinstance(entry, dict) + } + missing = sorted(recorded.keys() - present.keys()) + if missing: + return f"it lacks {missing[0]}, which its manifest lists" + unlisted = sorted(present.keys() - recorded.keys()) + if unlisted: + return f"its manifest does not list {unlisted[0]}" + for path, entry in sorted(present.items()): + for field in ("size", "sha256", "executable"): + if recorded[path].get(field) != entry[field]: + return f"the {field} of {path} differs from its manifest's" + return "its manifest does not list exactly its files" + + +def _openvmm_test_content_refusal( + entry: _OpenVMMTestCacheEntry, content: Path +) -> str | None: + """Return why ``content`` does not hold the test content that ``entry`` + describes, or None if it holds exactly that: a manifest that records the + entry's fields, and the runner and options that the content's run script + runs, and exactly the files that the manifest lists, with their sizes, + SHA-256, and executable bits.""" + path = content / OPENVMM_TEST_CONTENT_MANIFEST + if not path.is_file(): + return f"it has no {OPENVMM_TEST_CONTENT_MANIFEST}" + try: + manifest = _read_openvmm_test_manifest(path) + for field, expected in entry.inputs.items(): + if manifest.get(field) != expected: + return ( + f"its manifest records the {field} {_brief(manifest.get(field))}" + f", not {_brief(expected)}" + ) + operating_system = cast(str, entry.inputs["operating_system"]) + script = content / OPENVMM_TEST_RUN_SCRIPTS[operating_system] + runner, arguments = _openvmm_test_run_target( + script, cast(str, entry.inputs["build_filter"]) + ) + if arguments[arguments.index("--target") + 1] != entry.inputs["target"]: + return f"{script.name} runs the tests of another target" + if manifest.get("runner") != runner or manifest.get("arguments") != arguments: + return ( + f"its manifest records another runner or other options than " + f"{script.name} runs" + ) + files = _openvmm_test_content_files(content) + except (OSError, ScriptError, UnicodeDecodeError) as error: + return str(error) + return _openvmm_test_file_refusal(manifest.get("files"), files) + + +def check_openvmm_tests(backends: Sequence[str], content: Path) -> None: + """Fail unless ``content`` holds exactly the content that + ``build_openvmm_tests`` builds for the backends from this checkout, which + CI caches under ``openvmm_test_cache_key``.""" + entry = _openvmm_test_cache_entry(backends) + refusal = _openvmm_test_content_refusal(entry, content) + if refusal is not None: + raise ScriptError( + f"{content} does not hold the {entry.describe()} OpenVMM tests of " + f"cache key {entry.key}: {refusal}" + ) + print( + f">> {content} holds the {entry.describe()} OpenVMM tests of cache key " + f"{entry.key}" + ) + + +def _annotation(text: str) -> str: + return text.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + + +def reuse_openvmm_tests( + backends: Sequence[str], + content: Path, + environment: Mapping[str, str] | None = None, +) -> bool: + """Keep ``content``, which a cache restored, if it holds exactly the + content that ``build_openvmm_tests`` builds for the backends from this + checkout, and remove whatever it holds otherwise, so that a build can take + its place. Report the outcome, warn of a refusal, give the step the + `reused` output, and return whether the content is reused.""" + entry = _openvmm_test_cache_entry(backends) + if not path_exists(content): + reused = False + print( + f">> {content} holds no {entry.describe()} OpenVMM tests of cache key " + f"{entry.key} to reuse" + ) + else: + refusal = _openvmm_test_content_refusal(entry, content) + reused = refusal is None + if reused: + print( + f">> reusing the {entry.describe()} OpenVMM tests of cache key " + f"{entry.key} in {content}" + ) + else: + detail = ( + f"{content} does not hold the {entry.describe()} OpenVMM tests of " + f"cache key {entry.key}: {refusal}. Removed it, so that the tests " + "build instead." + ) + print(f"::warning title=OpenVMM test cache::{_annotation(detail)}") + if content.is_dir() and not is_link_or_junction(content): + shutil.rmtree(content) + else: + content.unlink() + _set_step_output(environment or {}, "reused", "true" if reused else "false") + return reused + + @dataclass(frozen=True) class _OpenVMMTestContent: """What runs the tests of a checked test content directory.""" @@ -1205,29 +1451,13 @@ def _openvmm_test_content(content: Path, backend: str) -> _OpenVMMTestContent: path = require_file( content / OPENVMM_TEST_CONTENT_MANIFEST, "OpenVMM test content manifest" ) - try: - document = cast( - object, - json.loads( - path.read_text(encoding="utf-8"), - object_pairs_hook=strict_json_object, - ), - ) - except (UnicodeDecodeError, json.JSONDecodeError) as error: - raise ScriptError(f"{path} is not valid JSON: {error}") from error - if not isinstance(document, dict): - raise ScriptError(f"{path} is not a JSON object") - manifest = cast(dict[str, object], document) + manifest = _read_openvmm_test_manifest(path) if manifest.get("format") != OPENVMM_TEST_CONTENT_FORMAT: raise ScriptError( f"{path} has format {manifest.get('format')!r}, " f"expected {OPENVMM_TEST_CONTENT_FORMAT}" ) - gitlink = run_capture( - ["git", "-C", BuildConstants.REPO_ROOT, "rev-parse", ":openvmm"] - ) - require_success(gitlink, "OpenVMM gitlink query") - revision = gitlink.stdout.decode("ascii").strip() + revision = _openvmm_gitlink() if manifest.get("openvmm_revision") != revision: raise ScriptError( f"{path} describes tests of OpenVMM " diff --git a/scripts/test_nvx_tools.py b/scripts/test_nvx_tools.py index a3de327ae..ee4cbc13b 100644 --- a/scripts/test_nvx_tools.py +++ b/scripts/test_nvx_tools.py @@ -1134,6 +1134,37 @@ def test_release_commands_keep_their_cli_contract(self): ): nvx.parse_args(["build-openvmm-tests", "--output", "content"]) + cache_key = nvx.parse_args( + ["openvmm-test-cache-key", "--backend", "kvm", "--backend", "mshv"] + ) + self.assertEqual(cache_key.backend, ["kvm", "mshv"]) + self.assertIs(cache_key.handler, nvx.command_openvmm_test_cache_key) + for command, handler in ( + ("check-openvmm-tests", nvx.command_check_openvmm_tests), + ("reuse-openvmm-tests", nvx.command_reuse_openvmm_tests), + ): + with self.subTest(command=command): + parsed = nvx.parse_args( + [command, "--backend", "whp", "--content", "content"] + ) + self.assertEqual(parsed.backend, ["whp"]) + self.assertEqual(parsed.content, Path("content")) + self.assertIs(parsed.handler, handler) + for arguments in ( + ["openvmm-test-cache-key"], + ["openvmm-test-cache-key", "--backend", "arm"], + ["check-openvmm-tests", "--backend", "whp"], + ["check-openvmm-tests", "--content", "content"], + ["reuse-openvmm-tests", "--backend", "whp"], + ["reuse-openvmm-tests", "--content", "content"], + ): + with ( + self.subTest(arguments=arguments), + patch("sys.stderr", new_callable=io.StringIO), + self.assertRaises(SystemExit), + ): + nvx.parse_args(arguments) + def test_openvmm_help_describes_petri_vmm_tests(self): with ( patch("sys.stdout", new_callable=io.StringIO) as output, @@ -3589,6 +3620,81 @@ def _openvmm_run_script(operating_system: str, test_filter: str) -> str: "--artifacts VmgsWithBootEntry --skip-vhd-prompt --ci-profile" ) + def _build_openvmm_test_content( + self, + root: Path, + backends: Sequence[str], + *, + revision: str = "a" * 40, + clean: bool = True, + script: str | None = None, + ) -> tuple[Path, MagicMock, MagicMock, str]: + """Build the backends' test content into ``root / "content"`` with + build_openvmm_tests, with an OpenVMM checkout at ``revision`` and a + flowey that writes a runner, a kernel, the nextest configuration, and + ``script`` or the run script that the pinned flowey writes. Return the + content, the mocks of openvmm_git_state and run_checked, and what the + build printed.""" + operating_system = ci.OPENVMM_TEST_OPERATING_SYSTEMS[backends[0]] + build_filter = ci.openvmm_test_build_filter(backends) + runner = ( + "x86_64-pc-windows-msvc/flowey_hvlite.exe" + if operating_system == "windows" + else "x86_64-unknown-linux-gnu/flowey_hvlite" + ) + targets = sorted( + { + target + for backend in backends + for target in OpenVMMBuildConstants.TEST_RUST_TARGETS[backend] + } + ) + installed_targets = common.CommandResult( + args=("rustup", "target", "list"), + returncode=0, + stdout="".join(f"{target}\n" for target in targets).encode(), + stderr=b"", + ) + + def build(command: list[str], **_: object) -> None: + if "--build-only" not in command: + return + content = Path(command[command.index("--dir") + 1]) + files = { + runner: b"runner", + "x86_64/vmlinux": b"kernel", + ".config/nextest.toml": b"[profile.ci]\n", + } + for relative, data in files.items(): + path = content / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + path.chmod(0o755 if relative != ".config/nextest.toml" else 0o644) + run_script = content / ci.OPENVMM_TEST_RUN_SCRIPTS[operating_system] + run_script.write_text( + script or self._openvmm_run_script(operating_system, build_filter), + encoding="utf-8", + ) + run_script.chmod(0o755) + + openvmm = root / "openvmm" + openvmm.mkdir(exist_ok=True) + (openvmm / "Cargo.toml").touch() + output = root / "content" + with ( + patch.object(OpenVMMBuildConstants, "DIRECTORY", openvmm), + patch.object(ci, "require_tool", side_effect=["cargo", "rustup"]), + patch.object(ci, "run_capture", return_value=installed_targets), + patch.object( + ci, "openvmm_git_state", return_value=(revision, clean) + ) as git_state, + patch.object(ci, "run_checked", side_effect=build) as run_checked, + patch.dict(os.environ, {"RUNNER_TEMP": os.fspath(root)}), + patch("sys.stdout", new_callable=io.StringIO) as stdout, + ): + ci.build_openvmm_tests(backends, output) + return output, git_state, run_checked, stdout.getvalue() + def test_openvmm_test_content_paths_stay_inside_the_directory(self): for value in ("x86_64/vmlinux", ".config/nextest.toml", "run.sh"): with self.subTest(value=value): @@ -3713,64 +3819,19 @@ def test_build_openvmm_tests_builds_only_and_describes_the_content(self): backends = ("whp",) if os.name == "nt" else ("mshv", "kvm") operating_system = "windows" if os.name == "nt" else "linux" build_filter = ci.openvmm_test_build_filter(backends) - targets = { - target - for backend in backends - for target in OpenVMMBuildConstants.TEST_RUST_TARGETS[backend] - } - installed_targets = common.CommandResult( - args=("rustup", "target", "list"), - returncode=0, - stdout="".join(f"{target}\n" for target in sorted(targets)).encode(), - stderr=b"", - ) runner = ( "x86_64-pc-windows-msvc/flowey_hvlite.exe" if os.name == "nt" else "x86_64-unknown-linux-gnu/flowey_hvlite" ) - def build(command: list[str], **_: object) -> None: - if "--build-only" not in command: - return - content = Path(command[command.index("--dir") + 1]) - files = { - runner: b"runner", - "x86_64/vmlinux": b"kernel", - ".config/nextest.toml": b"[profile.ci]\n", - } - for relative, data in files.items(): - path = content / relative - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(data) - path.chmod(0o755 if relative != ".config/nextest.toml" else 0o644) - script = content / ci.OPENVMM_TEST_RUN_SCRIPTS[operating_system] - script.write_text( - self._openvmm_run_script(operating_system, build_filter), - encoding="utf-8", - ) - script.chmod(0o755) - with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) openvmm = root / "openvmm" - openvmm.mkdir() - (openvmm / "Cargo.toml").touch() - output = root / "content" revision = "a" * 40 - - with ( - patch.object(OpenVMMBuildConstants, "DIRECTORY", openvmm), - patch.object(ci, "require_tool", side_effect=["cargo", "rustup"]), - patch.object(ci, "run_capture", return_value=installed_targets), - patch.object( - ci, "openvmm_git_state", return_value=(revision, True) - ) as git_state, - patch.object(ci, "run_checked", side_effect=build) as run_checked, - patch.dict(os.environ, {"RUNNER_TEMP": os.fspath(root)}), - patch("sys.stdout", new_callable=io.StringIO) as stdout, - ): - ci.build_openvmm_tests(backends, output) + output, git_state, run_checked, printed = self._build_openvmm_test_content( + root, backends, revision=revision + ) git_state.assert_called_once_with(openvmm) restore, tests = run_checked.call_args_list @@ -3778,12 +3839,18 @@ def build(command: list[str], **_: object) -> None: restore.args[0], ["cargo", "xflowey", "restore-packages", "--no-compat-igvm"], ) + target = "windows-x64" if os.name == "nt" else "linux-x64" + self.assertEqual(target, ci.OPENVMM_TEST_TARGETS[operating_system]) + # The build names its target and its build mode, without optional + # features. self.assertEqual( tests.args[0], [ "cargo", "xflowey", "vmm-tests-run", + "--target", + target, "--release", "--ci-profile", "--skip-vhd-prompt", @@ -3814,14 +3881,25 @@ def build(command: list[str], **_: object) -> None: ) script = ci.OPENVMM_TEST_RUN_SCRIPTS[operating_system] executable = os.name != "nt" + + def digest(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + self.assertEqual( manifest, { - "format": 1, + "format": 2, "openvmm_revision": revision, "openvmm_clean": True, "rust_toolchain": OpenVMMBuildConstants.RUST_TOOLCHAIN, "operating_system": operating_system, + "target": target, + "build_options": [ + "--release", + "--ci-profile", + "--skip-vhd-prompt", + "--build-only", + ], "filters": { backend: ci.OPENVMM_TEST_FILTERS[backend] for backend in sorted(backends) @@ -3830,7 +3908,7 @@ def build(command: list[str], **_: object) -> None: "runner": runner, "arguments": [ "--target", - "windows-x64" if os.name == "nt" else "linux-x64", + target, "--repetitions", "1", "--artifacts", @@ -3844,24 +3922,377 @@ def build(command: list[str], **_: object) -> None: "path": ".config/nextest.toml", "size": 13, "executable": False, + "sha256": digest(b"[profile.ci]\n"), }, { "path": script, "size": (output / script).stat().st_size, "executable": executable, + "sha256": digest((output / script).read_bytes()), + }, + { + "path": runner, + "size": 6, + "executable": executable, + "sha256": digest(b"runner"), }, - {"path": runner, "size": 6, "executable": executable}, { "path": "x86_64/vmlinux", "size": 6, "executable": executable, + "sha256": digest(b"kernel"), }, ), key=lambda entry: str(entry["path"]), ), }, ) - self.assertIn("4 files", stdout.getvalue()) + self.assertIn("4 files", printed) + + def test_build_openvmm_tests_refuse_a_run_script_of_another_target(self): + backends = ("whp",) if os.name == "nt" else ("kvm", "mshv") + operating_system = ci.OPENVMM_TEST_OPERATING_SYSTEMS[backends[0]] + target = ci.OPENVMM_TEST_TARGETS[operating_system] + other = "linux-x64" if target == "windows-x64" else "windows-x64" + script = self._openvmm_run_script( + operating_system, ci.openvmm_test_build_filter(backends) + ).replace(f"--target {target} ", f"--target {other} ") + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + with self.assertRaisesRegex( + common.ScriptError, f"does not run the {target} tests that it built" + ): + self._build_openvmm_test_content(root, backends, script=script) + self.assertFalse( + (root / "content" / ci.OPENVMM_TEST_CONTENT_MANIFEST).exists() + ) + + @staticmethod + def _openvmm_gitlink(revision: str) -> common.CommandResult: + return common.CommandResult( + args=("git", "rev-parse", ":openvmm"), + returncode=0, + stdout=f"{revision}\n".encode(), + stderr=b"", + ) + + def test_openvmm_test_cache_key_covers_every_input_of_the_content(self): + # #692: CI caches each OS's build under a key of everything that + # determines it, so that a change to any of them builds the tests + # again, and only such a change does. + backends = ("whp",) if os.name == "nt" else ("kvm", "mshv") + operating_system = ci.OPENVMM_TEST_OPERATING_SYSTEMS[backends[0]] + target = ci.OPENVMM_TEST_TARGETS[operating_system] + revision = "a" * 40 + + def key( + selected: Sequence[str] = backends, + gitlink: str = revision, + environment: Mapping[str, str] | None = None, + ) -> str: + with ( + patch.object( + ci, "run_capture", return_value=self._openvmm_gitlink(gitlink) + ) as run_capture, + patch("sys.stdout", new_callable=io.StringIO) as stdout, + ): + value = ci.openvmm_test_cache_key(selected, environment) + # The key needs only the checkout's gitlink, not OpenVMM's files. + run_capture.assert_called_once_with( + ["git", "-C", BuildConstants.REPO_ROOT, "rev-parse", ":openvmm"] + ) + self.assertEqual(stdout.getvalue(), f"{value}\n") + return value + + base = key() + self.assertRegex( + base, rf"^openvmm-vmm-tests-v1-{target}-{revision}-[0-9a-f]{{64}}$" + ) + # GitHub rejects longer keys and keys with a comma. + self.assertLessEqual(len(base), 512) + self.assertNotIn(",", base) + self.assertEqual(key(tuple(reversed(backends))), base) + keys = {base} + for name, change, gitlink in ( + ("OpenVMM revision", contextlib.nullcontext(), "b" * 40), + ( + "cache version", + patch.object(ci, "OPENVMM_TEST_CACHE_VERSION", 2), + revision, + ), + ( + "content format", + patch.object(ci, "OPENVMM_TEST_CONTENT_FORMAT", 3), + revision, + ), + ( + "Rust release", + patch.object(OpenVMMBuildConstants, "RUST_TOOLCHAIN", "1.0.0"), + revision, + ), + ( + "target", + patch.dict(ci.OPENVMM_TEST_TARGETS, {operating_system: "other-x64"}), + revision, + ), + ( + "build options", + patch.object( + ci, + "OPENVMM_TEST_BUILD_OPTIONS", + (*ci.OPENVMM_TEST_BUILD_OPTIONS, "--copy-extras"), + ), + revision, + ), + ( + "filter", + patch.dict(ci.OPENVMM_TEST_FILTERS, {backends[0]: "all()"}), + revision, + ), + ): + with self.subTest(input=name), change: + changed = key(gitlink=gitlink) + self.assertNotIn(changed, keys) + keys.add(changed) + # A build of fewer backends has a key of its own. + if len(backends) > 1: + self.assertNotIn(key(backends[:1]), keys) + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) / "github-output" + self.assertEqual( + key(environment={"GITHUB_OUTPUT": os.fspath(output)}), base + ) + self.assertEqual(output.read_text(encoding="utf-8"), f"key={base}\n") + + def test_openvmm_test_cache_takes_only_a_clean_build_of_its_key(self): + # #692: CI checks a build before it caches it, and reuses restored + # content only when the same check passes, so that a hit runs exactly + # the tests that a fresh build runs. A fresh build passes; any + # difference in what the key digests, in the runner and options that + # the run script runs, or in the files fails, and the reuse then + # removes the content, so that the tests build in its place. + backends = ("whp",) if os.name == "nt" else ("kvm", "mshv") + names = ", ".join(backend.upper() for backend in backends) + operating_system = ci.OPENVMM_TEST_OPERATING_SYSTEMS[backends[0]] + target = ci.OPENVMM_TEST_TARGETS[operating_system] + script = ci.OPENVMM_TEST_RUN_SCRIPTS[operating_system] + runner = ( + "x86_64-pc-windows-msvc/flowey_hvlite.exe" + if operating_system == "windows" + else "x86_64-unknown-linux-gnu/flowey_hvlite" + ) + revision = "a" * 40 + + def set_field(field: str, value: object) -> Callable[[Path], None]: + def edit(content: Path) -> None: + path = content / ci.OPENVMM_TEST_CONTENT_MANIFEST + manifest = json.loads(path.read_text(encoding="utf-8")) + manifest[field] = value + path.write_text(json.dumps(manifest), encoding="utf-8") + + return edit + + def write(relative: str, data: bytes) -> Callable[[Path], object]: + return lambda content: (content / relative).write_bytes(data) + + def replace_in_script(old: str, new: str) -> Callable[[Path], None]: + def edit(content: Path) -> None: + path = content / script + text = path.read_text(encoding="utf-8") + self.assertIn(old, text) + path.write_text(text.replace(old, new), encoding="utf-8") + + return edit + + def run( + content: Path, gitlink: str = revision + ) -> tuple[str | None, bool, str, str]: + """Check and then reuse the content, and return the check's error, + whether the reuse kept it, what both printed, and the step + output.""" + output = content.parent / "github-output" + output.unlink(missing_ok=True) + with ( + patch.object( + ci, "run_capture", return_value=self._openvmm_gitlink(gitlink) + ), + patch("sys.stdout", new_callable=io.StringIO) as stdout, + ): + try: + ci.check_openvmm_tests(backends, content) + except common.ScriptError as error: + refusal: str | None = str(error) + else: + refusal = None + reused = ci.reuse_openvmm_tests( + backends, content, {"GITHUB_OUTPUT": os.fspath(output)} + ) + return refusal, reused, stdout.getvalue(), output.read_text("utf-8") + + with tempfile.TemporaryDirectory() as temporary: + content, *_ = self._build_openvmm_test_content( + Path(temporary), backends, revision=revision + ) + with ( + patch.object( + ci, "run_capture", return_value=self._openvmm_gitlink(revision) + ), + patch("sys.stdout", new_callable=io.StringIO), + ): + key = ci.openvmm_test_cache_key(backends) + refusal, reused, printed, output = run(content) + self.assertIsNone(refusal) + self.assertTrue(reused) + self.assertEqual(output, "reused=true\n") + self.assertIn( + f"{content} holds the {names} OpenVMM tests of cache key {key}", + printed, + ) + self.assertIn(f"reusing the {names} OpenVMM tests of cache key", printed) + self.assertNotIn("::warning", printed) + self.assertTrue((content / ci.OPENVMM_TEST_CONTENT_MANIFEST).is_file()) + + # A cache miss leaves nothing to reuse or to warn of. + missing = Path(temporary) / "missing" + refusal, reused, printed, output = run(missing) + self.assertIn("has no nvx-openvmm-tests.json", refusal or "") + self.assertFalse(reused) + self.assertEqual(output, "reused=false\n") + self.assertIn("holds no", printed) + self.assertNotIn("::warning", printed) + + cases: list[tuple[str, Callable[[Path], object], str, str]] = [ + ( + "no manifest", + lambda content: (content / ci.OPENVMM_TEST_CONTENT_MANIFEST).unlink(), + revision, + f"it has no {ci.OPENVMM_TEST_CONTENT_MANIFEST}", + ), + ( + "malformed manifest", + write(ci.OPENVMM_TEST_CONTENT_MANIFEST, b"{"), + revision, + "is not valid JSON", + ), + ( + "OpenVMM revision", + lambda _: None, + "b" * 40, + f'the openvmm_revision "{revision}", not "{"b" * 40}"', + ), + ( + "dirty checkout", + set_field("openvmm_clean", False), + revision, + "the openvmm_clean false, not true", + ), + ("content format", set_field("format", 1), revision, "the format 1, not 2"), + ( + "Rust release", + set_field("rust_toolchain", "1.0.0"), + revision, + 'the rust_toolchain "1.0.0"', + ), + ( + "target", + set_field("target", "other-x64"), + revision, + 'the target "other-x64"', + ), + ( + "build options", + set_field("build_options", ["--build-only"]), + revision, + 'the build_options ["--build-only"]', + ), + ( + "filter", + set_field("filters", {backends[0]: "all()"}), + revision, + f'the filters {{"{backends[0]}": "all()"}}', + ), + ( + "run script options", + replace_in_script( + "--ci-profile", "--ci-profile --no-reuse-prepped-vhds" + ), + revision, + f"another runner or other options than {script} runs", + ), + ( + "run script target", + replace_in_script(f"--target {target} ", "--target other-x64 "), + revision, + f"{script} runs the tests of another target", + ), + ( + "missing file", + lambda content: (content / "x86_64" / "vmlinux").unlink(), + revision, + "it lacks x86_64/vmlinux, which its manifest lists", + ), + ( + "unlisted file", + write("stray", b"stray"), + revision, + "its manifest does not list stray", + ), + ( + "changed file", + write("x86_64/vmlinux", b"KERNEL"), + revision, + "the sha256 of x86_64/vmlinux differs", + ), + ( + "truncated file", + write("x86_64/vmlinux", b"kern"), + revision, + "the size of x86_64/vmlinux differs", + ), + ] + if os.name != "nt": + + def drop_executable_bit(content: Path) -> None: + (content / runner).chmod(0o644) + + def link(content: Path) -> None: + (content / "link").symlink_to("x86_64/vmlinux") + + cases += [ + ( + "executable bit", + drop_executable_bit, + revision, + f"the executable of {runner} differs", + ), + ("symbolic link", link, revision, "a non-regular file link"), + ] + for name, edit, gitlink, message in cases: + with self.subTest(case=name), tempfile.TemporaryDirectory() as temporary: + content, *_ = self._build_openvmm_test_content( + Path(temporary), backends, revision=revision + ) + edit(content) + refusal, reused, printed, output = run(content, gitlink) + self.assertIn(message, refusal or "") + self.assertIn( + f"does not hold the {names} OpenVMM tests of cache key", + refusal or "", + ) + self.assertFalse(reused) + self.assertFalse(common.path_exists(content)) + self.assertEqual(output, "reused=false\n") + self.assertIn("::warning title=OpenVMM test cache::", printed) + self.assertIn(message, printed) + + # The reuse also removes a file where it expects the directory. + with tempfile.TemporaryDirectory() as temporary: + content = Path(temporary) / "content" + content.write_bytes(b"content") + refusal, reused, printed, _ = run(content) + self.assertFalse(reused) + self.assertFalse(common.path_exists(content)) + self.assertIn("::warning title=OpenVMM test cache::", printed) def test_build_openvmm_tests_refuse_other_hosts_and_used_directories(self): other = ("kvm",) if os.name == "nt" else ("whp",) @@ -3899,16 +4330,25 @@ def _openvmm_test_content( runner.chmod(0o644) arguments = ["--target", "linux-x64", "--ci-profile"] manifest = { - "format": 1, + "format": ci.OPENVMM_TEST_CONTENT_FORMAT, "openvmm_revision": revision, "openvmm_clean": True, "rust_toolchain": OpenVMMBuildConstants.RUST_TOOLCHAIN, "operating_system": ci.OPENVMM_TEST_OPERATING_SYSTEMS[backend], + "target": "linux-x64", + "build_options": list(ci.OPENVMM_TEST_BUILD_OPTIONS), "filters": {backend: ci.OPENVMM_TEST_FILTERS[backend]}, "build_filter": ci.OPENVMM_TEST_FILTERS[backend], "runner": "target/flowey_hvlite", "arguments": arguments, - "files": [{"path": "target/flowey_hvlite", "size": 6, "executable": True}], + "files": [ + { + "path": "target/flowey_hvlite", + "size": 6, + "executable": True, + "sha256": hashlib.sha256(b"runner").hexdigest(), + } + ], } (content / ci.OPENVMM_TEST_CONTENT_MANIFEST).write_text( json.dumps(manifest), encoding="utf-8" @@ -4027,17 +4467,24 @@ def _openvmm_windows_test_content( path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(relative.encode()) manifest = { - "format": 1, + "format": ci.OPENVMM_TEST_CONTENT_FORMAT, "openvmm_revision": revision, "openvmm_clean": True, "rust_toolchain": OpenVMMBuildConstants.RUST_TOOLCHAIN, "operating_system": "windows", + "target": "windows-x64", + "build_options": list(ci.OPENVMM_TEST_BUILD_OPTIONS), "filters": {"whp": ci.OPENVMM_TEST_FILTERS["whp"]}, "build_filter": ci.OPENVMM_TEST_FILTERS["whp"], "runner": "x86_64-pc-windows-msvc/flowey_hvlite.exe", "arguments": list(arguments), "files": [ - {"path": relative, "size": len(relative), "executable": False} + { + "path": relative, + "size": len(relative), + "executable": False, + "sha256": hashlib.sha256(relative.encode()).hexdigest(), + } for relative in files ], } @@ -4591,7 +5038,8 @@ def test_openvmm_tests_refuse_content_that_is_not_this_checkouts(self): def edit(manifest: dict[str, object], change: str, content: Path) -> None: if change == "format": - manifest["format"] = 2 + # Content of the previous format, such as an earlier run built. + manifest["format"] = ci.OPENVMM_TEST_CONTENT_FORMAT - 1 elif change == "revision": manifest["openvmm_revision"] = "d" * 40 elif change == "backend": @@ -4616,7 +5064,7 @@ def edit(manifest: dict[str, object], change: str, content: Path) -> None: (content / "target" / "flowey_hvlite").write_bytes(b"run") for change, message in ( - ("format", "has format 2, expected 1"), + ("format", "has format 1, expected 2"), ("revision", f"of OpenVMM d{{40}}, but this checkout pins {revision}"), ("backend", "no tests of this checkout's"), ("filter", "no tests of this checkout's"), @@ -7222,8 +7670,12 @@ def load(name: str) -> Any: "setup-cross-os-cache": set[str](), "verify": set[str](), "test-openvmm": kernel_alpine, - # The VMM tests' build boots no guest. + # The VMM tests' build, its cache key, and the checks of what it + # builds or a cache restores boot no guest. "build-openvmm-tests": set[str](), + "openvmm-test-cache-key": set[str](), + "check-openvmm-tests": set[str](), + "reuse-openvmm-tests": set[str](), "test-aci-edge-sandboxes": kernel_alpine, # A sandbox boots the Alpine control initramfs under its layers. "sandbox": kernel_alpine, @@ -8152,7 +8604,7 @@ def test_ci_full_runs_do_not_publish_persist_or_save_caches(self): self.assertEqual( writes, { - "workflows/ci.yml": 8, + "workflows/ci.yml": 10, "actions/build-openvmm/action.yml": 2, "actions/build-guest-artifacts/action.yml": 6, "actions/check-quality/action.yml": 1, @@ -8175,10 +8627,10 @@ def test_ci_full_runs_do_not_publish_persist_or_save_caches(self): with self.subTest(call=name): self.assertEqual(arguments.get("save-cache"), save_cache) # No other push-only job condition is allowed in this workflow: only - # its eight cache saves, the release, performance persistence, the - # four save-cache arguments, and the two reuse arguments name the push - # event. - self.assertEqual(workflow.count("github.event_name == 'push'"), 16) + # its ten cache saves, the release, performance persistence, the four + # save-cache arguments, the two reuse arguments, and the two restores + # of the OpenVMM VMM tests, which full runs skip, name the push event. + self.assertEqual(workflow.count("github.event_name == 'push'"), 20) def test_ci_change_classification_selects_suites_by_input(self): guest, binaries, _, vmm, microvm, benchmarks = ci.CI_CHANGE_OUTPUTS @@ -11321,6 +11773,154 @@ def test_ci_hands_each_vmm_test_build_to_the_jobs_of_its_os(self): self.assertEqual(job.count(gate), 1) self.assertNotIn("needs.openvmm-vmm-tests.result", job) + def test_ci_reuses_the_cached_vmm_test_build_of_each_os(self): + # #692: each VMM-test build restores the newest cached content of its + # key and builds the tests only when the reuse refuses what the + # restore left. Only a dev push saves, after a build that passes the + # check, under the key, its run, and its attempt; full runs neither + # restore nor save; and the test jobs take only the same-run artifact, + # which test_ci_shares_openvmm_inputs_and_binary_artifacts checks. + jobs = yaml.safe_load( + (BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml").read_text( + encoding="utf-8" + ) + )["jobs"] + producers = { + "Linux": ( + "build-openvmm-vmm-tests-linux", + "python3 scripts/nvx.py", + "--backend kvm --backend mshv", + '"${RUNNER_TEMP}/openvmm-vmm-tests-linux"', + "Prepare Linux cache tools", + ), + "Windows": ( + "build-openvmm-vmm-tests-windows", + "python scripts\\nvx.py", + "--backend whp", + '"$env:RUNNER_TEMP\\openvmm-vmm-tests-windows"', + "Prepare Windows cache tools", + ), + } + key = "${{ steps.openvmm-tests-cache.outputs.key }}" + dev_push = ( + "success() && github.event_name == 'push' && github.ref == 'refs/heads/dev'" + ) + + def words(text: object) -> str: + return " ".join(str(text).split()) + + for system, (job_name, nvx_py, backends, content, tools) in producers.items(): + order = [step["name"] for step in jobs[job_name]["steps"]] + steps = {step["name"]: step for step in jobs[job_name]["steps"]} + path = f"${{{{ runner.temp }}}}/openvmm-vmm-tests-{system.lower()}" + reuse_id = f"reuse-openvmm-tests-{system.lower()}" + built = f"steps.{reuse_id}.outputs.reused != 'true'" + with self.subTest(job=job_name): + # The key covers exactly the backends of the build. + key_step = steps["Prepare OpenVMM test cache key"] + self.assertEqual(key_step["id"], "openvmm-tests-cache") + self.assertEqual( + words(key_step["run"]), + f"{nvx_py} openvmm-test-cache-key {backends}", + ) + self.assertIn( + f"build-openvmm-tests {backends} --output {content}", + words(steps[f"Build OpenVMM tests on {system}"]["run"]), + ) + + # Pushes and pull requests restore the newest entry of the key + # into the directory that the job uploads. + restore = steps[f"Restore OpenVMM tests on {system}"] + self.assertEqual(restore["uses"], "actions/cache/restore@v5") + self.assertEqual( + restore["if"], + "github.event_name == 'pull_request' || " + "github.event_name == 'push'", + ) + self.assertEqual( + restore["with"], + {"path": path, "key": key, "restore-keys": f"{key}-\n"}, + ) + # A failed restore falls back to the build, as a miss does. + self.assertIs(restore["continue-on-error"], True) + self.assertEqual(steps["Upload OpenVMM tests"]["with"]["path"], path) + self.assertNotIn("if", steps["Upload OpenVMM tests"]) + + # Every run decides on the reuse, which removes whatever it + # refuses, and only a refusal runs the steps of the build. + reuse = steps[f"Reuse restored OpenVMM tests on {system}"] + self.assertEqual(reuse["id"], reuse_id) + self.assertNotIn("if", reuse) + self.assertNotIn("continue-on-error", reuse) + self.assertEqual( + words(reuse["run"]), + f"{nvx_py} reuse-openvmm-tests {backends} --content {content}", + ) + for name in ( + "Checkout OpenVMM", + "Set up retrying curl", + f"Restore OpenVMM inputs on {system}", + f"Restore shared Cargo cache on {system}", + "Prepare sccache", + f"Build OpenVMM tests on {system}", + f"Check built OpenVMM tests on {system}", + ): + self.assertEqual(steps[name]["if"], built, name) + self.assertEqual(steps["Report sccache"]["if"], f"always() && {built}") + + # A build that a reuse would refuse fails the job. + check = steps[f"Check built OpenVMM tests on {system}"] + self.assertNotIn("continue-on-error", check) + self.assertEqual( + words(check["run"]), + f"{nvx_py} check-openvmm-tests {backends} --content {content}", + ) + + # Only a dev push that built the tests saves them, as an entry + # of its own. + save = steps[f"Save OpenVMM tests on {system}"] + self.assertEqual(save["uses"], "actions/cache/save@v5") + self.assertEqual(words(save["if"]), f"{dev_push} && {built}") + self.assertEqual( + save["with"], + { + "path": path, + "key": f"{key}-${{{{ github.run_id }}}}-" + "${{ github.run_attempt }}", + }, + ) + for name in ( + f"Save OpenVMM inputs on {system}", + f"Save shared Cargo cache on {system}", + ): + self.assertTrue( + words(steps[name]["if"]).startswith(f"{dev_push} && {built}"), + name, + ) + # The Linux caches compress with gzip, as the job's others do. + if system == "Linux": + for step in (restore, save): + self.assertEqual( + step["env"], + {"PATH": "${{ steps.cache-tools-linux.outputs.path }}"}, + ) + + for earlier, later in itertools.pairwise( + ( + "Validate runner", + "Prepare OpenVMM test cache key", + tools, + f"Restore OpenVMM tests on {system}", + f"Reuse restored OpenVMM tests on {system}", + "Checkout OpenVMM", + f"Build OpenVMM tests on {system}", + f"Check built OpenVMM tests on {system}", + "Upload OpenVMM tests", + f"Save OpenVMM tests on {system}", + ) + ): + self.assertLess(order.index(earlier), order.index(later)) + def test_ci_runs_the_whp_vmm_tests_in_their_shards(self): # #691: the WHP tests run in the shards that openvmm_test_shards.json # plans, one leg of the WHP job's matrix each, from the content of