From 3c75a463361672fb78e4b2fd150da02b3f3ca716 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sat, 17 Oct 2026 19:50:09 -0400 Subject: [PATCH] ci(release): gate releases on a hosted Linux Xvfb lane The release gate required live-qualification.yml on a self-hosted qualified Linux desktop. No such runner was ever registered, so no release could pass (1.3.0 release run 39075912841 failed closed). Replace that requirement with a GitHub-hosted ubuntu-latest job in production-qualification.yml. It runs three counted trials under Xvfb with X RECORD and XTEST, openbox, AT-SPI, and a focused GTK window: the structural observation test and the slow recorder and input tests, 8 tests per trial, no skips. The gate no longer proves more than one monitor, real input hardware, a compositor, or real window moves. live-qualification.yml keeps those lanes weekly and outside the gate. Signed-off-by: Richard Abrich --- .github/workflows/live-qualification.yml | 17 +- .../workflows/production-qualification.yml | 233 +++++++++++++++++- .github/workflows/release.yml | 4 +- CHANGELOG.md | 11 +- docs/DESIGN.md | 28 ++- docs/WINDOW_CAPTURE.md | 9 +- scripts/check_release_ci.py | 98 +------- tests/test_release_qualification_gates.py | 103 +++----- tests/test_repository_contracts.py | 34 ++- 9 files changed, 330 insertions(+), 207 deletions(-) diff --git a/.github/workflows/live-qualification.yml b/.github/workflows/live-qualification.yml index 35429fa..bb418af 100644 --- a/.github/workflows/live-qualification.yml +++ b/.github/workflows/live-qualification.yml @@ -1,8 +1,13 @@ # Live interactive qualification of the native recorder. # -# These lanes require physically qualified hosts. The release gate requires -# the exact-SHA Linux lane. The weekly schedule maintains the macOS and Windows -# evidence when their qualified runners exist. +# These lanes require physically qualified hosts. They are NOT part of the +# release gate. Until 2026-10-17 the release required the exact-SHA Linux lane, +# but no qualified Linux desktop was ever registered, so no release could pass. +# The release now requires the hosted Linux Xvfb lane in +# production-qualification.yml instead, which proves less: one virtual monitor +# and injected XTEST input, not two physical monitors and real input hardware. +# The weekly schedule runs these lanes when their qualified runners exist, and +# fails when none does. A manual dispatch can still qualify a chosen SHA. # # The lanes prove: # @@ -18,9 +23,9 @@ # - live native structural observation through AX, UIA and AT-SPI; # - three counted trials of the above per operating system. # -# Each operating system has a separate runner variable. A release dispatch -# selects Linux only. This prevents an available Linux runner from queueing -# work for missing macOS or Windows runners. A scheduled or "all" run warns +# Each operating system has a separate runner variable. A dispatch for one +# platform selects that platform only, so an available Linux runner never +# queues work for missing macOS or Windows runners. A scheduled or "all" run warns # for each lane it skips and fails when no lane can run. name: Live qualification diff --git a/.github/workflows/production-qualification.yml b/.github/workflows/production-qualification.yml index fd5397a..0623f51 100644 --- a/.github/workflows/production-qualification.yml +++ b/.github/workflows/production-qualification.yml @@ -3,11 +3,13 @@ # Every job here runs on a GitHub-hosted runner. openadapt-capture is public, # so those runners are free and always available. # -# The physical-desktop lanes live in live-qualification.yml. The release -# orchestrator requires its exact-SHA Linux lane in addition to this hosted -# workflow. Scheduled live runs maintain the macOS and Windows evidence. +# The physical-desktop lanes live in live-qualification.yml. Since 2026-10-17 +# the release no longer requires any of them: no qualified desktop was ever +# registered, so the exact-SHA Linux lane the release required could never +# run. This workflow's hosted Linux Xvfb lane took its place in the gate. The +# physical lanes run weekly, outside the gate, once a desktop is registered. # -# The live lanes prove: +# The physical lanes prove, and this hosted gate therefore does NOT prove: # # - that global input injected through the operating system reaches the # native listeners and is written into the capture; @@ -19,8 +21,11 @@ # Each of those needs a physical desktop session with two monitors and real # input permissions. Measured on 2026-08-28, every GitHub-hosted runner # reports exactly one monitor (Xvfb with two screens and +xinerama, and a RandR -# --setmonitor split, both still report one to MSS), and injected input never -# reaches the native hooks on any of the three hosted operating systems. +# --setmonitor split, both still report one to MSS; rechecked 2026-10-17), and +# injected input never reaches the native hooks on hosted macOS or Windows. On +# Linux, XTEST input into Xvfb does reach the X RECORD listener, and AT-SPI +# works against a focused GTK window, so the hosted Linux lane runs those +# tests; that is weaker evidence than physical input on a real desktop. # # What this gate does prove, all of it against one exact commit: # @@ -34,7 +39,10 @@ # - the real recorder starts against a real display on hosted macOS and # Windows, produces a real first frame, creates its capture database, # holds memory bounded, and shuts down cleanly, three counted trials per -# operating system. +# operating system; +# - on hosted Linux under Xvfb, the same recorder lifecycle plus throughput, +# capture round-trip, XTEST input reaching X RECORD, and AT-SPI structural +# evidence for a focused element, three counted trials of eight tests. # # scripts/check_release_ci.py additionally requires a successful test.yml run # on the same exact commit, which is where the complete headless suite and the @@ -257,7 +265,7 @@ jobs: # A GitHub-hosted runner exposes exactly one virtual display. The # multiple-monitor contract is proven only by the self-hosted lanes in - # live-qualification.yml. This step still records the exact topology the + # live-qualification.yml, which the release no longer requires. This step still records the exact topology the # trials ran against, and still fails on a topology that changes during # qualification. - name: Record a stable single-monitor desktop @@ -277,8 +285,9 @@ jobs: # # The three input-injection tests of this file are deliberately NOT here. # Injected input does not reach the native low-level hooks in a hosted - # runner session, measured on all three hosted operating systems on - # 2026-08-28. live-qualification.yml runs them on a qualified host. + # macOS or Windows session, measured on 2026-08-28. The hosted Linux + # Xvfb lane below runs them against X RECORD; live-qualification.yml + # runs them on a qualified host, outside the release gate. # Three trials take under three minutes on a healthy runner, so the step # carries its own bound well under the job's. A step that stops making # progress reports a failure in twelve minutes instead of holding the @@ -332,3 +341,207 @@ jobs: path: evidence/ if-no-files-found: error retention-days: 30 + + # The release's Linux lane. It replaced the self-hosted "Interactive + # qualification (Linux X64)" lane as a release requirement on 2026-10-17, + # because no qualified Linux desktop was ever registered and the release + # could not pass. That lane stays in live-qualification.yml, weekly and + # non-blocking, for when a desktop exists. + # + # What this lane proves, on a GitHub-hosted ubuntu-latest runner with an + # Xvfb X server, the openbox window manager, the AT-SPI bus, and a focused + # GTK text field: + # + # - the exact wheel with its [linux] extra installs against the host AT-SPI + # runtime; + # - the real recorder starts, produces a first frame, creates its database, + # holds memory bounded, sustains throughput, round-trips a capture, and + # shuts down cleanly; + # - input injected through XTEST reaches the X RECORD listener and lands in + # the capture; + # - the AT-SPI structural provider returns exact evidence for the focused + # element; + # - all of it three counted times, eight named tests per trial, no skips. + # + # What it does NOT prove, and the self-hosted lane did: + # + # - more than one monitor. Xvfb exposes exactly one, so the multiple-monitor + # desktop, its coordinate translation, and a negative-origin secondary + # monitor go unchecked; + # - real input hardware, a real compositor, or a signed-in desktop session. + # Injected XTEST events into a virtual server are not the same as a + # person's keyboard and mouse on physical displays; + # - real window movement and resize under a production window manager. + hosted-linux-live: + name: Hosted Linux live qualification (ubuntu-latest, Xvfb) + needs: build-candidate + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - name: Checkout the exact candidate source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Install exact uv + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + with: + version: "0.11.29" + + - name: Download the exact candidate + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: capture-candidate-${{ github.sha }} + path: dist + + - name: Install the X server, window manager, AT-SPI, GTK, and video tools + shell: bash + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y \ + at-spi2-core \ + dbus \ + ffmpeg \ + gcc \ + gir1.2-atspi-2.0 \ + gir1.2-gtk-3.0 \ + libcairo2-dev \ + libgirepository1.0-dev \ + openbox \ + pkg-config \ + python3-dev \ + x11-utils \ + xvfb + + - name: Install the exact wheel into an isolated environment + shell: bash + run: | + set -euo pipefail + qualification_root="${RUNNER_TEMP}/capture-qualification-${GITHUB_RUN_ID}" + qualification_python="${qualification_root}/bin/python" + candidate_wheels=(dist/*.whl) + if [ "${#candidate_wheels[@]}" -ne 1 ]; then + echo "Expected exactly one candidate wheel." + exit 1 + fi + uv venv --clear --python 3.12 "${qualification_root}" + uv pip install --python "${qualification_python}" \ + "${candidate_wheels[0]}[linux]" \ + pytest==9.1.1 pytest-timeout==2.4.0 pynput==1.8.2 + echo "QUALIFICATION_PYTHON=${qualification_python}" >> "${GITHUB_ENV}" + + - name: Require the reviewed external video tools + shell: bash + run: | + set -euo pipefail + mkdir -p evidence + command -v ffmpeg + command -v ffprobe + ffmpeg -version > evidence/ffmpeg-version.txt + ffprobe -version > evidence/ffprobe-version.txt + + # One step, one D-Bus session: the AT-SPI bus, the GTK fixture, and the + # trials must share it. Xvfb runs at 1280x1024 with RECORD and XTEST. + # The display check still records the exact topology and fails if it + # changes, but it can only require one monitor here. + # + # Each trial starts a fresh, focused fixture window and runs the AT-SPI + # test first, while the fixture holds focus. The input tests then click + # and type into the session. Three trials take about two minutes. + - name: Run three counted Xvfb live qualification trials + timeout-minutes: 15 + shell: bash + env: + OPENADAPT_CAPTURE_PRODUCTION_QUALIFICATION: "1" + QUALIFICATION_DISPLAY: ":99" + run: | + set -euo pipefail + cat > "${RUNNER_TEMP}/fixture_window.py" <<'PY' + """A focused, accessible GTK text field for the AT-SPI structural test.""" + import gi + + gi.require_version("Gtk", "3.0") + from gi.repository import Gtk + + window = Gtk.Window(title="OpenAdapt Capture qualification fixture") + entry = Gtk.Entry() + entry.set_text("qualification fixture") + window.add(entry) + window.set_default_size(480, 120) + window.connect("destroy", Gtk.main_quit) + window.show_all() + window.present() + entry.grab_focus() + Gtk.main() + PY + cat > "${RUNNER_TEMP}/xvfb_session.sh" <<'SH' + set -euo pipefail + export DISPLAY="${QUALIFICATION_DISPLAY}" + export GTK_MODULES=gail:atk-bridge NO_AT_BRIDGE=0 + evidence="${GITHUB_WORKSPACE}/evidence" + Xvfb "${DISPLAY}" -screen 0 1280x1024x24 -nolisten tcp \ + +extension RECORD +extension XTEST > "${evidence}/xvfb.log" 2>&1 & + for _ in $(seq 1 50); do xdpyinfo >/dev/null 2>&1 && break; sleep 0.2; done + xdpyinfo > "${evidence}/xdpyinfo.txt" + grep -q RECORD "${evidence}/xdpyinfo.txt" + grep -q XTEST "${evidence}/xdpyinfo.txt" + /usr/libexec/at-spi-bus-launcher --launch-immediately > "${evidence}/at-spi.log" 2>&1 & + openbox > "${evidence}/openbox.log" 2>&1 & + sleep 1 + "${QUALIFICATION_PYTHON}" "${GITHUB_WORKSPACE}/scripts/check_display_topology.py" \ + --minimum-monitors 1 \ + --output "${evidence}/display-topology.json" + cd "${RUNNER_TEMP}" + for trial in 1 2 3; do + echo "::group::Linux Xvfb trial ${trial}" + "${QUALIFICATION_PYTHON}" "${RUNNER_TEMP}/fixture_window.py" \ + >> "${evidence}/fixture.log" 2>&1 & + fixture_pid=$! + sleep 3 + "${QUALIFICATION_PYTHON}" -m pytest \ + "${GITHUB_WORKSPACE}/tests/test_structural_observation.py" \ + "${GITHUB_WORKSPACE}/tests/test_performance.py" \ + -m slow -v --timeout=300 --import-mode=importlib -p no:cacheprovider \ + "--junitxml=${evidence}/trial-${trial}-ubuntu-latest.xml" + kill "${fixture_pid}" 2>/dev/null || true + wait "${fixture_pid}" 2>/dev/null || true + echo "::endgroup::" + done + SH + dbus-run-session -- bash "${RUNNER_TEMP}/xvfb_session.sh" + + - name: Reject skipped, incomplete, or shrunken qualification trials + shell: bash + run: | + set -euo pipefail + for trial in 1 2 3; do + python scripts/check_junit_no_skips.py \ + --expected-tests 8 \ + "evidence/trial-${trial}-ubuntu-latest.xml" + done + + - name: Aggregate the counted trial evidence + shell: bash + run: | + set -euo pipefail + python scripts/aggregate_qualification_trials.py \ + --os linux \ + --candidate-sha "${GITHUB_SHA}" \ + --expected-trials 3 \ + evidence/trial-1-ubuntu-latest.xml \ + evidence/trial-2-ubuntu-latest.xml \ + evidence/trial-3-ubuntu-latest.xml \ + --output evidence/qualification-summary-ubuntu-latest.json + + - name: Upload hosted Linux live evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: capture-hosted-live-linux-${{ github.sha }} + path: evidence/ + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 994586d..b443544 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -130,7 +130,7 @@ jobs: ) PY - - name: Require exact-main test, hosted, and live Linux evidence + - name: Require exact-main test and hosted qualification evidence env: GH_TOKEN: ${{ github.token }} shell: bash @@ -279,7 +279,7 @@ jobs: exit 1 fi - - name: Require exact-tag test, hosted, and live Linux evidence + - name: Require exact-tag test and hosted qualification evidence env: GH_TOKEN: ${{ github.token }} run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 725a769..7aeb276 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -176,9 +176,14 @@ _This release uses the MIT License._ and recheck protected `main` immediately before the release tag. Clean-wheel qualification now installs and probes the pinned FFmpeg runtime on all three operating systems. The Linux lane also installs the wheel's `linux` extra. - Publication requires three live trials on a qualified Linux desktop for the - same commit. Separate runner flags prevent a Linux release check from waiting - for absent macOS or Windows runners. + Separate runner flags prevent a Linux live check from waiting for absent + macOS or Windows runners. +- **release**: Gate releases on a hosted Linux lane instead of a self-hosted + Linux desktop that was never registered. Three trials on Xvfb with openbox, + AT-SPI, and a focused GTK window run the recorder, XTEST input reaching X + RECORD, and AT-SPI structural tests. The gate no longer proves more than one + monitor or real input hardware. The physical-desktop lanes still run weekly, + outside the gate, once a qualified desktop is registered. - **windows**: Fail when ffmpeg install is absent ([#99](https://github.com/OpenAdaptAI/openadapt-capture/pull/99), [`682f77d`](https://github.com/OpenAdaptAI/openadapt-capture/commit/682f77d168bdff60d55285a1682a7c0ebf726847)) diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 519cf04..689f1ad 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -290,11 +290,15 @@ must: hosted Windows runner, prove a real first frame, prove the capture database, prove bounded memory, and prove a clean shutdown, in three counted trials per operating system with no skip; +- on a hosted Ubuntu runner under Xvfb, with openbox, the AT-SPI bus, and a + focused GTK text field, run the same recorder lifecycle plus throughput, + capture round-trip, XTEST input reaching the X RECORD listener, and AT-SPI + structural evidence, in three counted trials of eight tests each; - record the exact display topology each trial ran against; and - retain machine-readable test and topology evidence. -`scripts/check_release_ci.py` also requires a successful `test.yml` run and a -successful live Linux qualification on the same exact commit. `test.yml` runs +`scripts/check_release_ci.py` also requires a successful `test.yml` run on the +same exact commit. `test.yml` runs the complete headless suite and the byte-for-byte synthetic fixture check. The release workflow accepts only a successful, complete job set for its exact @@ -303,10 +307,13 @@ commit. Missing, stale, skipped, partial, or failed evidence blocks publication. ## Live qualification `live-qualification.yml` runs the interactive lanes that need a physical -desktop. The release gate requires the exact-commit Linux lane. Weekly runs -maintain the macOS and Windows evidence when those qualified runners exist. +desktop. They are not part of the release gate. Until 2026-10-17 the release +required the exact-commit Linux lane, but no qualified Linux desktop was ever +registered, so no release could pass. The hosted Linux Xvfb lane in +`production-qualification.yml` replaced it in the gate. Weekly runs exercise +the physical lanes when qualified runners exist, and fail when none does. -Those lanes prove: +Those lanes prove, and a release therefore no longer proves: - that global input injected through the operating system reaches the native listeners and is written into the capture; @@ -315,9 +322,14 @@ Those lanes prove: - live window movement and resize under window-scoped capture; and - live native structural observation through AX, UIA, and AT-SPI. -No GitHub-hosted runner can prove these. Measured on 2026-08-28, every hosted -runner reports exactly one monitor, and Xvfb with two screens and `+xinerama`, -and a RandR `--setmonitor` split, both still report one monitor to MSS. +No GitHub-hosted runner can prove all of these. Measured on 2026-08-28 and +again on 2026-10-17, every hosted runner reports exactly one monitor, and Xvfb +with two screens and `+xinerama`, and a RandR `--setmonitor` split, both still +report one monitor to MSS. On Linux, XTEST input into Xvfb reaches X RECORD +and AT-SPI works against a GTK window, so the hosted Linux lane covers input +delivery and structural observation, but only through a virtual X server with +injected events: not real input hardware, a real compositor, or real window +movement and resize. Injected input reaches no native hook on any of the three hosted operating systems. Each lane needs a host with two monitors, a logged-in desktop session, and real input and screen permissions. diff --git a/docs/WINDOW_CAPTURE.md b/docs/WINDOW_CAPTURE.md index 166e0d3..7390414 100644 --- a/docs/WINDOW_CAPTURE.md +++ b/docs/WINDOW_CAPTURE.md @@ -116,10 +116,11 @@ count, and privacy-safe monitor rectangles as combined-frame coordinate space. This includes a secondary monitor whose native coordinates have a negative origin. -The multiple-monitor path is qualified by `live-qualification.yml`, which needs -a host with two real monitors. A GitHub-hosted runner reports one monitor, so -the release gate records the single-monitor topology its trials ran against and -does not prove the multiple-monitor contract. Downstream converters must not +The multiple-monitor path is qualified only by `live-qualification.yml`, which +needs a host with two real monitors and is not part of the release gate. A +GitHub-hosted runner reports one monitor, so the release gate records the +single-monitor topology its trials ran against and does not prove the +multiple-monitor contract. Downstream converters must not apply the legacy display-ratio scale when `coordinate_space == "virtual_desktop_pixels"`. diff --git a/scripts/check_release_ci.py b/scripts/check_release_ci.py index 9a3acc8..d32a837 100644 --- a/scripts/check_release_ci.py +++ b/scripts/check_release_ci.py @@ -13,8 +13,14 @@ from dataclasses import dataclass from typing import Any, Callable -# The hosted workflow proves the portable package lifecycle. The live workflow -# separately proves the exact candidate on a qualified Linux desktop. +# Every release requirement runs on GitHub-hosted runners. The hosted workflow +# proves the portable package lifecycle, the live recorder on macOS and +# Windows, and the Linux recorder, X RECORD input, and AT-SPI structure on +# Xvfb. Since 2026-10-17 the release no longer requires the self-hosted +# live-qualification.yml lanes: no qualified desktop was ever registered, so a +# release could not pass. The release therefore no longer proves multiple +# monitors or real input hardware; live-qualification.yml still does, weekly +# and outside the gate, once a qualified desktop exists. EXPECTED_QUALIFICATION_JOBS = frozenset( { "Build candidate distributions", @@ -23,15 +29,7 @@ "Clean candidate wheel (windows-latest)", "Hosted live recorder qualification (macos-latest)", "Hosted live recorder qualification (windows-latest)", - } -) -EXPECTED_LIVE_QUALIFICATION_JOBS = frozenset( - { - "Select live qualification platforms", - "Build candidate distributions", - "Interactive qualification (Linux X64)", - "Interactive qualification (macOS ARM64)", - "Interactive qualification (Windows X64)", + "Hosted Linux live qualification (ubuntu-latest, Xvfb)", } ) ACTIVE_STATES = frozenset({"queued", "in_progress", "waiting", "pending", "requested"}) @@ -59,11 +57,6 @@ class WorkflowRequirement: ("workflow_dispatch",), {"candidate_sha": "{sha}"}, ), - WorkflowRequirement( - "live-qualification.yml", - ("workflow_dispatch",), - {"candidate_sha": "{sha}", "platform": "linux"}, - ), ) @@ -180,58 +173,6 @@ def validate_test_jobs(jobs: list[dict[str, Any]]) -> None: ) -def validate_live_linux_jobs(jobs: list[dict[str, Any]]) -> None: - """Require exact-candidate evidence from the qualified Linux runner.""" - - names = [str(job.get("name")) for job in jobs] - duplicates = sorted({name for name in names if names.count(name) > 1}) - if duplicates: - raise ReleaseEvidenceError(f"live qualification has duplicate jobs: {duplicates}") - actual = set(names) - if actual != EXPECTED_LIVE_QUALIFICATION_JOBS: - missing = sorted(EXPECTED_LIVE_QUALIFICATION_JOBS - actual) - unexpected = sorted(actual - EXPECTED_LIVE_QUALIFICATION_JOBS) - raise ReleaseEvidenceError( - "live qualification job set differs from the release contract: " - f"missing={missing}, unexpected={unexpected}" - ) - required = { - "Select live qualification platforms", - "Build candidate distributions", - "Interactive qualification (Linux X64)", - } - incomplete = [ - f"{job.get('name')}={job.get('status')}/{job.get('conclusion')}" - for job in jobs - if job.get("name") in required - and (job.get("status") != "completed" or job.get("conclusion") != "success") - ] - if incomplete: - raise ReleaseEvidenceError( - "live Linux qualification has incomplete jobs: " + ", ".join(incomplete) - ) - - -def _run_has_live_linux_job( - *, - base: str, - run: dict[str, Any], - token: str, - get_json: Callable[[str, str], dict[str, Any]], -) -> bool: - """Return true when a run selected Linux instead of another live platform.""" - - payload = get_json( - f"{base}/actions/runs/{run['id']}/jobs?filter=latest&per_page=100", - token, - ) - jobs = payload.get("jobs") - if not isinstance(jobs, list): - return False - linux = [job for job in jobs if job.get("name") == "Interactive qualification (Linux X64)"] - return len(linux) == 1 and linux[0].get("conclusion") != "skipped" - - def _workflow_runs( *, base: str, @@ -276,17 +217,6 @@ def dispatch_missing_qualifications( for run in runs if run.get("head_sha") == sha and run.get("event") in requirement.events ] - if requirement.file_name == "live-qualification.yml": - matching = [ - run - for run in matching - if _run_has_live_linux_job( - base=base, - run=run, - token=token, - get_json=get_json, - ) - ] if not any( run.get("head_sha") == sha and run.get("event") in requirement.events for run in matching @@ -341,16 +271,6 @@ def check_once( if not isinstance(jobs, list): raise EvidencePending("GitHub returned no production qualification jobs list") validate_qualification_jobs(jobs) - - live = selected["live-qualification.yml"] - live_jobs_payload = get_json( - f"{base}/actions/runs/{live['id']}/jobs?filter=latest&per_page=100", - token, - ) - live_jobs = live_jobs_payload.get("jobs") - if not isinstance(live_jobs, list): - raise EvidencePending("GitHub returned no live qualification jobs list") - validate_live_linux_jobs(live_jobs) return {name: int(run["id"]) for name, run in selected.items()} diff --git a/tests/test_release_qualification_gates.py b/tests/test_release_qualification_gates.py index b6dcd5f..342bda6 100644 --- a/tests/test_release_qualification_gates.py +++ b/tests/test_release_qualification_gates.py @@ -16,12 +16,10 @@ from scripts.check_display_topology import DisplayTopologyError, qualify_topology from scripts.check_junit_no_skips import JUnitQualificationError, check_reports from scripts.check_release_ci import ( - EXPECTED_LIVE_QUALIFICATION_JOBS, EXPECTED_QUALIFICATION_JOBS, ReleaseEvidenceError, check_once, dispatch_missing_qualifications, - validate_live_linux_jobs, validate_qualification_jobs, validate_test_jobs, ) @@ -153,22 +151,6 @@ def _successful_jobs() -> list[dict[str, str]]: ] -def _successful_live_linux_jobs() -> list[dict[str, str]]: - required = { - "Select live qualification platforms", - "Build candidate distributions", - "Interactive qualification (Linux X64)", - } - return [ - { - "name": name, - "status": "completed", - "conclusion": "success" if name in required else "skipped", - } - for name in sorted(EXPECTED_LIVE_QUALIFICATION_JOBS) - ] - - def test_release_gate_rejects_missing_qualification_job() -> None: jobs = _successful_jobs()[:-1] @@ -198,14 +180,27 @@ def test_release_gate_requires_successful_package_contract() -> None: ) -def test_release_gate_requires_successful_live_linux_qualification() -> None: - jobs = _successful_live_linux_jobs() - validate_live_linux_jobs(jobs) +def test_release_gate_requires_the_hosted_linux_xvfb_lane() -> None: + """The Linux lane replaced the self-hosted desktop lane in the release gate.""" + assert "Hosted Linux live qualification (ubuntu-latest, Xvfb)" in EXPECTED_QUALIFICATION_JOBS - linux = next(job for job in jobs if job["name"] == "Interactive qualification (Linux X64)") + jobs = [ + job + for job in _successful_jobs() + if job["name"] != "Hosted Linux live qualification (ubuntu-latest, Xvfb)" + ] + with pytest.raises(ReleaseEvidenceError, match="job set differs"): + validate_qualification_jobs(jobs) + + jobs = _successful_jobs() + linux = next( + job + for job in jobs + if job["name"] == "Hosted Linux live qualification (ubuntu-latest, Xvfb)" + ) linux["conclusion"] = "skipped" - with pytest.raises(ReleaseEvidenceError, match="live Linux qualification"): - validate_live_linux_jobs(jobs) + with pytest.raises(ReleaseEvidenceError, match="incomplete jobs"): + validate_qualification_jobs(jobs) def test_release_gate_binds_both_workflows_and_jobs_to_exact_sha() -> None: @@ -238,19 +233,6 @@ def get_json(url: str, _token: str): } ] } - if "/live-qualification.yml/runs?" in url: - return { - "workflow_runs": [ - { - "id": 30, - "head_sha": sha, - "event": "workflow_dispatch", - "created_at": "2026-08-18T00:02:00Z", - "status": "completed", - "conclusion": "success", - } - ] - } if "/actions/runs/20/jobs?" in url: return {"jobs": _successful_jobs()} if "/actions/runs/10/jobs?" in url: @@ -263,8 +245,6 @@ def get_json(url: str, _token: str): } ] } - if "/actions/runs/30/jobs?" in url: - return {"jobs": _successful_live_linux_jobs()} raise AssertionError(f"unexpected URL {url}") assert check_once( @@ -275,7 +255,6 @@ def get_json(url: str, _token: str): ) == { "test.yml": 10, "production-qualification.yml": 20, - "live-qualification.yml": 30, } @@ -289,16 +268,13 @@ def get_json(url: str, _token: str): if "/production-qualification.yml/runs?" in url: return {"workflow_runs": []} if "/live-qualification.yml/runs?" in url: - return {"workflow_runs": []} + raise AssertionError("the release must not query the self-hosted live workflow") raise AssertionError(f"unexpected URL {url}") def dispatch(repository, requirement, *, ref, sha, token): # type: ignore[no-untyped-def] calls.append((repository, requirement.file_name, ref, sha)) assert token == "token" - expected_inputs = {"candidate_sha": "{sha}"} - if requirement.file_name == "live-qualification.yml": - expected_inputs["platform"] = "linux" - assert requirement.dispatch_inputs == expected_inputs + assert requirement.dispatch_inputs == {"candidate_sha": "{sha}"} started = dispatch_missing_qualifications( repository="OpenAdaptAI/openadapt-capture", @@ -309,39 +285,16 @@ def dispatch(repository, requirement, *, ref, sha, token): # type: ignore[no-un dispatch=dispatch, ) - assert started == ("production-qualification.yml", "live-qualification.yml") + assert started == ("production-qualification.yml",) assert calls == [ ("OpenAdaptAI/openadapt-capture", "production-qualification.yml", "main", sha), - ("OpenAdaptAI/openadapt-capture", "live-qualification.yml", "main", sha), ] -def test_release_orchestrator_does_not_accept_a_non_linux_live_run() -> None: - sha = "a" * 40 - dispatched: list[str] = [] - - def get_json(url: str, _token: str): - if "/test.yml/runs?" in url or "/production-qualification.yml/runs?" in url: - return {"workflow_runs": [{"head_sha": sha, "event": "workflow_dispatch", "id": 10}]} - if "/live-qualification.yml/runs?" in url: - return {"workflow_runs": [{"head_sha": sha, "event": "workflow_dispatch", "id": 30}]} - if "/actions/runs/30/jobs?" in url: - jobs = _successful_live_linux_jobs() - next(job for job in jobs if job["name"] == "Interactive qualification (Linux X64)")[ - "conclusion" - ] = "skipped" - return {"jobs": jobs} - raise AssertionError(f"unexpected URL {url}") +def test_release_requirements_never_include_a_self_hosted_workflow() -> None: + from scripts.check_release_ci import REQUIREMENTS - dispatch_missing_qualifications( - repository="OpenAdaptAI/openadapt-capture", - sha=sha, - ref="main", - token="token", - get_json=get_json, - dispatch=lambda _repository, requirement, **_kwargs: dispatched.append( - requirement.file_name - ), - ) - - assert dispatched == ["live-qualification.yml"] + assert [requirement.file_name for requirement in REQUIREMENTS] == [ + "test.yml", + "production-qualification.yml", + ] diff --git a/tests/test_repository_contracts.py b/tests/test_repository_contracts.py index 589f7c3..a4c8d00 100644 --- a/tests/test_repository_contracts.py +++ b/tests/test_repository_contracts.py @@ -43,7 +43,7 @@ def test_release_tag_requires_reviewed_exact_main_and_release_app() -> None: "- name: Require the reviewed release candidate on current main" ) qualification_index = release.index( - "- name: Require exact-main test, hosted, and live Linux evidence" + "- name: Require exact-main test and hosted qualification evidence" ) artifact_index = release.index("- name: Build and verify the exact release artifacts") tag_index = release.index("- name: Create and push one annotated release tag") @@ -171,14 +171,30 @@ def test_the_release_gate_job_set_matches_the_qualification_workflow() -> None: assert produced == set(EXPECTED_QUALIFICATION_JOBS) -def test_live_release_gate_job_set_matches_the_live_workflow() -> None: - """The exact-SHA Linux release check must name every live workflow job.""" - from scripts.check_release_ci import EXPECTED_LIVE_QUALIFICATION_JOBS +def test_hosted_linux_release_lane_runs_input_and_structural_trials() -> None: + """The hosted Linux lane must keep every test the self-hosted lane ran. - workflow = (ROOT / ".github/workflows/live-qualification.yml").read_text(encoding="utf-8") - produced = set(re.findall(r"^ name: (.+)$", workflow, flags=re.M)) + It replaced the self-hosted Linux lane in the release gate on 2026-10-17. + It runs the same two test files, eight named tests per trial, three + trials, under Xvfb with X RECORD, XTEST, AT-SPI, and a focused GTK window. + """ + workflow = (ROOT / ".github/workflows/production-qualification.yml").read_text(encoding="utf-8") + linux = workflow[workflow.index("\n hosted-linux-live:") :] + release = (ROOT / ".github/workflows/release.yml").read_text(encoding="utf-8") - assert produced == set(EXPECTED_LIVE_QUALIFICATION_JOBS) + assert "runs-on: ubuntu-latest" in linux + assert '"${candidate_wheels[0]}[linux]"' in linux + assert 'tests/test_structural_observation.py"' in linux + assert 'tests/test_performance.py"' in linux + assert "-k " not in linux + assert "+extension RECORD +extension XTEST" in linux + assert "at-spi-bus-launcher" in linux + assert "dbus-run-session" in linux + assert "entry.grab_focus()" in linux + assert "--expected-tests 8" in linux + assert "--expected-trials 3" in linux + assert 'OPENADAPT_CAPTURE_PRODUCTION_QUALIFICATION: "1"' in linux + assert "live-qualification.yml" not in release def _step_script(workflow: str, step_name: str) -> str: @@ -234,9 +250,7 @@ def _select_live_lanes(tmp_path: Path, event: str, platform: str = "", **runners @live_lane_selection -@pytest.mark.parametrize( - ("event", "platform"), [("schedule", ""), ("workflow_dispatch", "all")] -) +@pytest.mark.parametrize(("event", "platform"), [("schedule", ""), ("workflow_dispatch", "all")]) def test_live_qualification_fails_when_no_lane_can_run(tmp_path, event, platform) -> None: """A run that qualifies nothing must not conclude success.""" completed, outputs = _select_live_lanes(tmp_path, event, platform)