Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 86 additions & 2 deletions .github/workflows/desktop-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,11 @@
#
# If any build fails, nothing is published. If a later job fails, re-run the
# failed jobs of the same run: it reuses the attested bytes. A new run refuses a
# version that is half-published instead of rebuilding it.
# version that is half-published instead of rebuilding it, with one exception:
# when PyPI already serves the version and its GitHub release is a draft, a
# dispatch resumes. It builds nothing; the publish job downloads the draft,
# reverifies every byte against its attestations and the tag's commit, requires
# PyPI to serve the attested wheel and sdist, and publishes the draft.
#
# Signing: macOS DMGs are ad-hoc signed unless Developer ID credentials exist.
# Windows goes through SignPath when the four SIGNPATH_* secrets are set (one
Expand Down Expand Up @@ -99,6 +103,8 @@ jobs:
PACKAGES_HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
outputs:
release: ${{ steps.fresh.outputs.release }}
resume: ${{ steps.state.outputs.resume }}
resume-commit: ${{ steps.state.outputs.resume-commit }}
version: ${{ steps.state.outputs.version }}
tag: ${{ steps.state.outputs.tag }}
sbom: ${{ steps.state.outputs.sbom }}
Expand Down Expand Up @@ -176,6 +182,30 @@ jobs:
elif [ "${pypi}" = 404 ] && [ "${release}" = none ] && \
! git ls-remote --exit-code --tags origin "refs/tags/${tag}" >/dev/null; then
echo "release=true" >> "${GITHUB_OUTPUT}"
elif [ "${pypi}" = 200 ] && [ "${release}" != false ] && \
git ls-remote --exit-code --tags origin "refs/tags/${tag}" >/dev/null; then
# PyPI took the wheel and sdist, the tag exists, and no published
# release does: the staged draft was never published. (This token
# cannot see drafts, so the tags endpoint reports none; the publish
# job requires the draft with the release App's token.) Only a
# dispatch resumes; any other run builds nothing and says so.
if [ "${EVENT_NAME}" != workflow_dispatch ]; then
echo "::warning::${tag} is on PyPI and tagged, but its release is not published. Dispatch this workflow on main to resume it; this run builds nothing."
echo "release=false" >> "${GITHUB_OUTPUT}"
exit 0
fi
# Resume with the bytes already attested and staged; build nothing.
git fetch --no-tags origin \
"+refs/tags/${tag}:refs/tags/${tag}" \
+refs/heads/main:refs/remotes/origin/main
commit="$(git rev-list -n 1 "refs/tags/${tag}")"
git merge-base --is-ancestor "${commit}" refs/remotes/origin/main
echo "${tag} is on PyPI and tagged at ${commit} with no published release; resuming its publication."
{
echo "release=false"
echo "resume=true"
echo "resume-commit=${commit}"
} >> "${GITHUB_OUTPUT}"
elif [ "${pypi}" = 200 ] || [ "${pypi}" = 404 ]; then
echo "${tag} is partly published (PyPI ${pypi}, GitHub ${release})." >&2
echo "Re-run the failed jobs of the run that started it; do not rebuild." >&2
Expand Down Expand Up @@ -1211,18 +1241,29 @@ jobs:
VERSION: ${{ needs.gate.outputs.version }}
run: |
python scripts/release_publication.py verify-pypi \
--directory pypi-dist --version "${VERSION}" --wait-seconds 300
--directory pypi-dist --checksums release-assets/SHA256SUMS \
--version "${VERSION}" --wait-seconds 300

publish:
name: Publish the immutable release
needs: [gate, stage, publish-pypi]
# A release publishes after PyPI. A resume (stage and publish-pypi
# skipped) publishes the draft an earlier run staged and PyPI already
# serves.
if: >-
!cancelled() && needs.gate.result == 'success' && (
(needs.gate.outputs.release == 'true' && needs.publish-pypi.result == 'success') ||
needs.gate.outputs.resume == 'true'
)
runs-on: ubuntu-22.04
timeout-minutes: 20
environment: release-identity
permissions:
attestations: read
contents: read
env:
TAG: ${{ needs.gate.outputs.tag }}
VERSION: ${{ needs.gate.outputs.version }}
EXPECTED_AUTHOR: openadapt-release[bot]
steps:
- name: Create a repository-scoped release App token
Expand All @@ -1236,6 +1277,17 @@ jobs:
permission-contents: write
permission-metadata: read

- name: Require the exact release App identity
env:
ACTUAL_APP_SLUG: ${{ steps.release-app.outputs.app-slug }}
# Runs before the checkout below, so packages/desktop doesn't exist yet.
working-directory: ${{ github.workspace }}
run: |
if [ "${ACTUAL_APP_SLUG}" != openadapt-release ]; then
echo "Publishing requires the openadapt-release App, not ${ACTUAL_APP_SLUG}." >&2
exit 1
fi

- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.sha }}
Expand All @@ -1261,11 +1313,43 @@ jobs:
echo "path=${cli}" >> "${GITHUB_OUTPUT}"

- name: Download the exact attested release set
if: needs.gate.outputs.resume != 'true'
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
name: desktop-release-${{ needs.gate.outputs.tag }}
path: packages/desktop/upload

- name: Resume from the staged draft, reverified, and the PyPI files
if: needs.gate.outputs.resume == 'true'
env:
GH_TOKEN: ${{ steps.release-app.outputs.token }}
GH_CLI: ${{ steps.release-cli.outputs.path }}
ATTESTATION_TOKEN: ${{ github.token }}
RESUME_COMMIT: ${{ needs.gate.outputs.resume-commit }}
run: |
set -euo pipefail
state="$(python scripts/release_publication.py state --repository "${GITHUB_REPOSITORY}" --tag "${TAG}" --cli "${GH_CLI}" --metadata "${RUNNER_TEMP}/release.json")"
if [ "${state}" != draft ]; then
echo "Refusing to resume: ${TAG} is ${state}, not a draft." >&2
exit 1
fi
mkdir "${RUNNER_TEMP}/staged"
"${GH_CLI}" release download "${TAG}" --repo "${GITHUB_REPOSITORY}" --dir "${RUNNER_TEMP}/staged"
# The same checks the stage job ran before it uploaded these files:
# SHA256SUMS names exactly this set, and every attestation names
# this workflow on main and the tag's commit.
GH_TOKEN="${ATTESTATION_TOKEN}" python scripts/release_publication.py reverify \
--directory "${RUNNER_TEMP}/staged" \
--repository "${GITHUB_REPOSITORY}" \
--tag "${TAG}" \
--source-commit "${RESUME_COMMIT}" \
--upload upload
python scripts/release_publication.py python-dist \
--directory upload --version "${VERSION}" --output pypi-dist
python scripts/release_publication.py verify-pypi \
--directory pypi-dist --checksums upload/SHA256SUMS \
--version "${VERSION}" --wait-seconds 0

- name: Publish the verified draft once, then verify the immutable release
env:
GH_TOKEN: ${{ steps.release-app.outputs.token }}
Expand Down
6 changes: 6 additions & 0 deletions packages/desktop/RELEASES.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,12 @@ launcher's.
through Trusted Publishing, and the draft is published once. If any build
fails, nothing is published. A later failure resumes by re-running the
failed jobs of the same run; a new run refuses a half-published version.
One state resumes from a new run instead: PyPI serves the version, the tag
exists, and the release is still a draft. Dispatch the workflow on main
(`gh workflow run desktop-release.yml --ref main`). It builds nothing; the
publish job downloads the draft, reverifies every file against
`SHA256SUMS` and its attestations for the tag's commit, requires PyPI to
serve the attested wheel and sdist, and publishes the draft.
5. Signing is stated, never implied. macOS DMGs are ad-hoc signed until
Developer ID credentials exist. Windows goes through SignPath when the four
`SIGNPATH_*` secrets are set in `native-release`: one request deep-signs the
Expand Down
53 changes: 45 additions & 8 deletions packages/desktop/scripts/release_publication.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
notes release notes that state each platform's signing
state whether the GitHub release is absent, a draft, or published
python-dist the wheel and sdist, checked against SHA256SUMS
verify-pypi require PyPI to serve exactly those bytes
verify-pypi require PyPI to serve exactly the attested wheel and sdist
"""

from __future__ import annotations
Expand Down Expand Up @@ -171,14 +171,47 @@ def pypi_digests(version: str) -> dict[str, str]:
return {item["filename"]: item["digests"]["sha256"] for item in data.get("urls", [])}


# pypa/gh-action-pypi-publish writes one of these next to each file it uploads
# with attestations enabled. They are PyPI's upload attestations, not
# distributions, and PyPI never lists them as files of the release.
PUBLISH_ATTESTATION_SUFFIX = ".publish.attestation"


def verify_pypi(
directory: Path, *, version: str, wait_seconds: int, fetch=pypi_digests, sleep=time.sleep
directory: Path,
*,
version: str,
checksums: Path,
wait_seconds: int,
fetch=None,
sleep=time.sleep,
) -> dict[str, str]:
"""Require PyPI to serve exactly the files in ``directory``, byte for byte."""

expected = {path.name: _sha256(path) for path in directory.iterdir() if path.is_file()}
if set(expected) != native_release.python_distribution_names(version):
raise ValueError(f"unexpected distribution set: {sorted(expected)}")
"""Require PyPI to serve exactly the attested wheel and sdist, byte for byte.

The expected digests come from the attested ``checksums`` (SHA256SUMS), and
the files in ``directory`` must be those bytes. The publish action's
``*.publish.attestation`` files beside them are ignored; anything else is
refused.
"""

fetch = fetch or pypi_digests
names = native_release.python_distribution_names(version)
attested = native_release.read_checksums(checksums)
missing = sorted(names - set(attested))
if missing:
raise ValueError(f"the attested SHA256SUMS does not name {missing}")
expected = {name: attested[name] for name in sorted(names)}
present = {
path.name
for path in directory.iterdir()
if not (path.name.endswith(PUBLISH_ATTESTATION_SUFFIX) and path.is_file())
}
if present != names:
raise ValueError(f"unexpected distribution set: {sorted(present)}")
for name, digest in expected.items():
path = directory / name
if not path.is_file() or path.is_symlink() or _sha256(path) != digest:
raise ValueError(f"{name} differs from the attested SHA256SUMS")
deadline = time.monotonic() + wait_seconds
while True:
try:
Expand Down Expand Up @@ -223,6 +256,7 @@ def main(argv: list[str] | None = None) -> int:
p = sub.add_parser("verify-pypi")
p.add_argument("--directory", type=Path, required=True)
p.add_argument("--version", required=True)
p.add_argument("--checksums", type=Path, required=True)
p.add_argument("--wait-seconds", type=int, default=300)
args = parser.parse_args(argv)
try:
Expand Down Expand Up @@ -260,7 +294,10 @@ def main(argv: list[str] | None = None) -> int:
print(path)
elif args.command == "verify-pypi":
served = verify_pypi(
args.directory, version=args.version, wait_seconds=args.wait_seconds
args.directory,
version=args.version,
checksums=args.checksums,
wait_seconds=args.wait_seconds,
)
print(f"PyPI serves exactly {sorted(served)}")
except (OSError, ValueError, subprocess.CalledProcessError) as exc:
Expand Down
106 changes: 98 additions & 8 deletions packages/desktop/tests/test_release_publication.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,18 +59,25 @@ def test_python_dist_takes_only_the_attested_wheel_and_sdist(tmp_path: Path) ->
assert checksums.is_file()


def _dist(tmp_path: Path, version: str) -> tuple[Path, dict[str, str]]:
def _dist(tmp_path: Path, version: str) -> tuple[Path, Path, dict[str, str]]:
"""A pypi-dist directory and the attested SHA256SUMS that names its files."""

directory = tmp_path / "pypi-dist"
directory.mkdir()
digests = {}
for name in python_distribution_names(version):
(directory / name).write_bytes(name.encode())
digests[name] = hashlib.sha256(name.encode()).hexdigest()
return directory, digests
checksums = tmp_path / "SHA256SUMS"
lines = [f"{digest} {name}" for name, digest in sorted(digests.items())]
# The attested set names every release file, not only the wheel and sdist.
lines.append(f"{'2' * 64} OpenAdapt-Desktop-{version}-x86_64.AppImage")
checksums.write_text("\n".join(lines) + "\n", encoding="utf-8")
return directory, checksums, digests


def test_verify_pypi_waits_for_and_requires_the_exact_bytes(tmp_path: Path) -> None:
directory, digests = _dist(tmp_path, "1.2.3")
directory, checksums, digests = _dist(tmp_path, "1.2.3")
answers = [urllib.error.HTTPError("u", 404, "missing", None, None), {}, digests]

def fetch(_version: str) -> dict[str, str]:
Expand All @@ -80,25 +87,108 @@ def fetch(_version: str) -> dict[str, str]:
return answer

served = release_publication.verify_pypi(
directory, version="1.2.3", wait_seconds=60, fetch=fetch, sleep=lambda _s: None
directory,
version="1.2.3",
checksums=checksums,
wait_seconds=60,
fetch=fetch,
sleep=lambda _s: None,
)
assert served == digests


def test_verify_pypi_ignores_the_publish_actions_attestation_files(tmp_path: Path) -> None:
# pypa/gh-action-pypi-publish with attestations writes one beside each
# upload; Desktop 0.22.0 run 39374653331 refused them as unexpected.
directory, checksums, digests = _dist(tmp_path, "1.2.3")
for name in digests:
(directory / f"{name}.publish.attestation").write_text("{}", encoding="utf-8")
served = release_publication.verify_pypi(
directory, version="1.2.3", checksums=checksums, wait_seconds=0, fetch=lambda _v: digests
)
assert served == digests


def test_verify_pypi_compares_against_the_attested_sha256sums(tmp_path: Path) -> None:
directory, checksums, digests = _dist(tmp_path, "1.2.3")
wheel = next(name for name in digests if name.endswith(".whl"))
(directory / wheel).write_bytes(b"swapped after attestation")
tampered = {**digests, wheel: hashlib.sha256(b"swapped after attestation").hexdigest()}
with pytest.raises(ValueError, match="differs from the attested SHA256SUMS"):
release_publication.verify_pypi(
directory,
version="1.2.3",
checksums=checksums,
wait_seconds=0,
fetch=lambda _v: tampered,
)


def test_verify_pypi_refuses_any_other_extra_local_file(tmp_path: Path) -> None:
directory, checksums, digests = _dist(tmp_path, "1.2.3")
(directory / "openadapt_desktop-1.2.3-cp312-win_amd64.whl").write_bytes(b"extra")
with pytest.raises(ValueError, match="unexpected distribution set"):
release_publication.verify_pypi(
directory,
version="1.2.3",
checksums=checksums,
wait_seconds=0,
fetch=lambda _v: digests,
)
(directory / "openadapt_desktop-1.2.3-cp312-win_amd64.whl").unlink()
# The exemption is for the attestation suffix only, not anything nearby.
(directory / "openadapt_desktop-1.2.3.tar.gz.attestation").write_text("{}", encoding="utf-8")
with pytest.raises(ValueError, match="unexpected distribution set"):
release_publication.verify_pypi(
directory,
version="1.2.3",
checksums=checksums,
wait_seconds=0,
fetch=lambda _v: digests,
)


def test_verify_pypi_requires_sha256sums_to_name_the_wheel_and_sdist(tmp_path: Path) -> None:
directory, checksums, digests = _dist(tmp_path, "1.2.3")
checksums.write_text(f"{'2' * 64} OpenAdapt-Desktop-1.2.3-x86_64.AppImage\n", encoding="utf-8")
with pytest.raises(ValueError, match="does not name"):
release_publication.verify_pypi(
directory,
version="1.2.3",
checksums=checksums,
wait_seconds=0,
fetch=lambda _v: digests,
)


def test_verify_pypi_refuses_different_or_extra_files(tmp_path: Path) -> None:
directory, digests = _dist(tmp_path, "1.2.3")
directory, checksums, digests = _dist(tmp_path, "1.2.3")
other = {name: "0" * 64 for name in digests}
with pytest.raises(ValueError, match="different bytes"):
release_publication.verify_pypi(
directory, version="1.2.3", wait_seconds=0, fetch=lambda _v: other
directory, version="1.2.3", checksums=checksums, wait_seconds=0, fetch=lambda _v: other
)
extra = {**digests, "openadapt_desktop-1.2.3-cp312-win_amd64.whl": "1" * 64}
with pytest.raises(ValueError, match="unexpected files"):
release_publication.verify_pypi(
directory, version="1.2.3", wait_seconds=0, fetch=lambda _v: extra
directory, version="1.2.3", checksums=checksums, wait_seconds=0, fetch=lambda _v: extra
)
with pytest.raises(ValueError, match="in time"):
release_publication.verify_pypi(
directory, version="1.2.3", wait_seconds=0, fetch=lambda _v: {}
directory, version="1.2.3", checksums=checksums, wait_seconds=0, fetch=lambda _v: {}
)
assert json.dumps(digests)


def test_verify_pypi_command_takes_the_attested_checksums(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
directory, checksums, digests = _dist(tmp_path, "1.2.3")
(directory / "openadapt_desktop-1.2.3.tar.gz.publish.attestation").write_text("{}")
monkeypatch.setattr(release_publication, "pypi_digests", lambda _v: digests)
argv = ["verify-pypi", "--directory", str(directory), "--version", "1.2.3"]
with pytest.raises(SystemExit):
release_publication.main(argv)
assert (
release_publication.main([*argv, "--checksums", str(checksums), "--wait-seconds", "0"]) == 0
)
Loading
Loading