diff --git a/.github/workflows/trigger-site-rebuild.yml b/.github/workflows/trigger-site-rebuild.yml index 3cf7e86..fb9ba45 100644 --- a/.github/workflows/trigger-site-rebuild.yml +++ b/.github/workflows/trigger-site-rebuild.yml @@ -1,6 +1,8 @@ name: Trigger site rebuild on: + schedule: + - cron: '0 2 1 * *' workflow_dispatch: inputs: full_rebuild: @@ -8,6 +10,12 @@ on: required: false default: true type: boolean + engine_channel: + description: 'Defold channel for the build' + required: true + default: beta + type: choice + options: [beta, stable] push: branches: - 'master' @@ -17,10 +25,21 @@ on: - '.github/workflows/trigger-site-rebuild.yml' - 'validate_examples.py' - 'test_validate_examples.py' + - 'test_build_examples.py' + - 'test_build_snapshot.py' - 'build_examples.py' + - 'build_snapshot.py' - 'download_bob.py' - 'create_archives.py' +permissions: + contents: write + actions: read + +concurrency: + group: examples-publish + cancel-in-progress: false + jobs: validate: runs-on: ubuntu-latest @@ -30,7 +49,7 @@ jobs: { name: 'Install Python', uses: actions/setup-python@v4, with: { python-version: 3.10.22, architecture: x64 } }, { name: 'Test example validation', - run: 'python -m unittest test_validate_examples.py' + run: 'python -m unittest test_validate_examples.py test_build_examples.py test_build_snapshot.py' }, { name: 'Validate examples', @@ -39,7 +58,7 @@ jobs: AFTER_SHA: '${{ github.sha }}', FULL_REBUILD: '${{ github.event_name == ''workflow_dispatch'' && inputs.full_rebuild }}' }, - run: 'if [ "$FULL_REBUILD" = "true" ]; then python validate_examples.py; else python validate_examples.py --changed-from "$BEFORE_SHA" --changed-to "$AFTER_SHA"; fi' + run: 'python validate_examples.py' } ] @@ -47,47 +66,55 @@ jobs: runs-on: ubuntu-latest needs: validate if: "github.event_name != 'push' || !contains(github.event.head_commit.message, 'skip-ci')" - outputs: - example_count: ${{ steps.count.outputs.example_count }} - changed_examples: ${{ steps.count.outputs.changed_examples }} - steps: [ - { name: 'Checkout', uses: actions/checkout@v4, with: { fetch-depth: 0 } }, - { name: 'Install Python', uses: actions/setup-python@v4, with: { python-version: 3.10.22, architecture: x64 } }, - { - name: 'Install Java', - uses: actions/setup-java@v4, - with: { distribution: 'temurin', java-version: '25' } - }, - { - name: 'Download Bob', - run: 'python download_bob.py --channel alpha --output bob.jar' - }, - { - id: 'count', - name: 'Count touched examples', - env: { - BEFORE_SHA: '${{ github.event.before }}', - AFTER_SHA: '${{ github.sha }}', - FULL_REBUILD: '${{ github.event_name == ''workflow_dispatch'' && inputs.full_rebuild }}' - }, - run: 'if [ "$FULL_REBUILD" = "true" ]; then echo "example_count=$(python build_examples.py --dry-run)" >> "$GITHUB_OUTPUT"; echo "changed_examples=$(python build_examples.py --list-json)" >> "$GITHUB_OUTPUT"; else echo "example_count=$(python build_examples.py --changed-from "$BEFORE_SHA" --changed-to "$AFTER_SHA" --dry-run)" >> "$GITHUB_OUTPUT"; echo "changed_examples=$(python build_examples.py --changed-from "$BEFORE_SHA" --changed-to "$AFTER_SHA" --list-json)" >> "$GITHUB_OUTPUT"; fi' - }, - { - name: 'Build touched examples', - if: steps.count.outputs.example_count != '0', - env: { - BEFORE_SHA: '${{ github.event.before }}', - AFTER_SHA: '${{ github.sha }}', - FULL_REBUILD: '${{ github.event_name == ''workflow_dispatch'' && inputs.full_rebuild }}' - }, - run: 'if [ "$FULL_REBUILD" = "true" ]; then python build_examples.py --bob-jar bob.jar; else python build_examples.py --bob-jar bob.jar --changed-from "$BEFORE_SHA" --changed-to "$AFTER_SHA"; fi' - } - ] + steps: + - name: Checkout + uses: actions/checkout@v4 + with: { fetch-depth: 0 } + - uses: actions/setup-python@v4 + with: { python-version: '3.10.22', architecture: x64 } + - uses: actions/setup-java@v4 + with: { distribution: temurin, java-version: '25' } + - name: Download Bob + env: + ENGINE_CHANNEL: ${{ github.event_name == 'schedule' && 'stable' || inputs.engine_channel || 'beta' }} + run: python download_bob.py --channel "$ENGINE_CHANNEL" --output bob.jar + - name: Restore latest successful snapshot + if: github.event_name != 'schedule' && !(github.event_name == 'workflow_dispatch' && inputs.full_rebuild) + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + mkdir -p previous + gh run list --workflow trigger-site-rebuild.yml --branch master --status success --limit 50 --json databaseId --jq '.[].databaseId' > previous/runs.txt + while read -r run_id; do + artifact_id=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/artifacts" --jq '.artifacts[] | select(.name == "examples-bundles" and .expired == false) | .id') + if [ -n "$artifact_id" ]; then + gh run download "$run_id" --name examples-bundles --dir previous + break + fi + done < previous/runs.txt + - name: Build complete bundle snapshot + env: + FULL_REBUILD: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.full_rebuild) }} + KEEP_GOING: ${{ github.event_name == 'schedule' }} + run: | + args=() + if [ "$FULL_REBUILD" = true ]; then args+=(--full); elif [ -f previous/examples-bundles.zip ]; then args+=(--baseline previous/examples-bundles.zip); fi + if [ "$KEEP_GOING" = true ]; then args+=(--keep-going); fi + python build_snapshot.py --bob-jar bob.jar "${args[@]}" + - name: Upload bundle snapshot + uses: actions/upload-artifact@v4 + with: + name: examples-bundles + path: examples-bundles.zip + retention-days: 30 + compression-level: 0 + if-no-files-found: error publish: runs-on: ubuntu-latest needs: build - if: "needs.build.outputs.example_count != '0' && (github.event_name != 'push' || !contains(github.event.head_commit.message, 'skip-ci'))" + if: "github.event_name != 'push' || !contains(github.event.head_commit.message, 'skip-ci')" steps: [ { name: 'Checkout', uses: actions/checkout@v4, with: { fetch-depth: 1 } }, { name: 'Install Python', uses: actions/setup-python@v4, with: { python-version: 3.10.22, architecture: x64 } }, @@ -97,7 +124,7 @@ jobs: }, { name: "Create tag", - run: 'git tag v${{ github.sha }}' + run: 'git rev-parse v${{ github.sha }} >/dev/null 2>&1 || git tag v${{ github.sha }}' }, { name: "Push to tag", @@ -112,8 +139,8 @@ jobs: name: 'Repository dispatch', env: { DISPATCH_TOKEN: '${{ secrets.SERVICES_GITHUB_TOKEN }}', - CHANGED_EXAMPLES: '${{ needs.build.outputs.changed_examples }}' + EXAMPLES_RUN_ID: '${{ github.run_id }}' }, - run: 'python -c ''import json, os; print(json.dumps({"event_type":"examples","client_payload":{"examples_sha":os.environ["GITHUB_SHA"],"changed_examples":json.loads(os.environ.get("CHANGED_EXAMPLES") or "[]")}}))'' > dispatch.json && curl --fail --request POST --url https://api.github.com/repos/defold/defold.github.io/dispatches --header "Accept: application/vnd.github+json" --header "Authorization: Bearer ${DISPATCH_TOKEN}" --header "X-GitHub-Api-Version: 2022-11-28" --data @dispatch.json' + run: 'python -c ''import json, os; print(json.dumps({"event_type":"examples","client_payload":{"examples_sha":os.environ["GITHUB_SHA"],"examples_run_id":os.environ["EXAMPLES_RUN_ID"]}}))'' > dispatch.json && curl --fail --request POST --url https://api.github.com/repos/defold/defold.github.io/dispatches --header "Accept: application/vnd.github+json" --header "Authorization: Bearer ${DISPATCH_TOKEN}" --header "X-GitHub-Api-Version: 2022-11-28" --data @dispatch.json' } ] diff --git a/.github/workflows/validate-examples.yml b/.github/workflows/validate-examples.yml index 7bac107..46c44ab 100644 --- a/.github/workflows/validate-examples.yml +++ b/.github/workflows/validate-examples.yml @@ -8,6 +8,9 @@ on: - '.github/workflows/trigger-site-rebuild.yml' - 'validate_examples.py' - 'test_validate_examples.py' + - 'test_build_examples.py' + - 'test_build_snapshot.py' + - 'build_snapshot.py' - 'build_examples.py' - 'download_bob.py' @@ -19,7 +22,7 @@ jobs: { name: 'Install Python', uses: actions/setup-python@v4, with: { python-version: 3.10.22, architecture: x64 } }, { name: 'Test example validation', - run: 'python -m unittest test_validate_examples.py' + run: 'python -m unittest test_validate_examples.py test_build_examples.py test_build_snapshot.py' }, { name: 'Validate examples', @@ -32,7 +35,7 @@ jobs: }, { name: 'Download Bob', - run: 'python download_bob.py --channel alpha --output bob.jar' + run: 'python download_bob.py --channel beta --output bob.jar' }, { name: 'Build touched examples', diff --git a/README.md b/README.md index 4ca07c1..35a2087 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,12 @@ This repository includes the Defold examples used in the examples section on https://defold.com/examples +## CI and website publication + +Pull requests build changed examples using the latest Defold beta. Changes to master build the examples changed since the latest successful bundle snapshot and produce a complete snapshot for the website. If there is no usable previous snapshot, all examples are built. A failed build never dispatches a site update. The snapshot is a GitHub Actions artifact retained for 30 days; expired snapshots are automatically removed and the next publication starts with a full build. + +The website workflow downloads the snapshot for the dispatched run and imports its matching source revision. It can be retried manually with the `examples` action, which uses the latest successful available snapshot. On the first day of each month, the publication workflow rebuilds every example using the latest stable release and publishes the complete result only if all builds succeed. The publication workflow can also be run manually with a choice of beta or stable. + ## Adding more examples Examples are grouped by category, for instance "physics", "sprite" or "collection". Each group of examples has a folder in /examples. Here's how to add a new example named "foobar" to the "sprite" category: diff --git a/build_examples.py b/build_examples.py index 4ff9ad8..bdfd7c3 100644 --- a/build_examples.py +++ b/build_examples.py @@ -73,6 +73,8 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--changed-to", default="") parser.add_argument("--dry-run", action="store_true") parser.add_argument("--list-json", action="store_true") + parser.add_argument("--keep-going", action="store_true", help="Build remaining examples after a failure") + parser.add_argument("--bundle-dir", type=Path, help="Copy successful web bundles into this directory") return parser.parse_args() @@ -98,7 +100,7 @@ def prepare_project_copy(project_dir: Path) -> Path: return project_copy -def build_project(bob_jar: str, project_dir: Path) -> None: +def build_project(bob_jar: str, project_dir: Path, bundle_dir: Path | None = None) -> None: print(f"Building {project_dir}") bob_jar_path = Path(bob_jar).resolve() project_copy = prepare_project_copy(project_dir) @@ -124,6 +126,15 @@ def build_project(bob_jar: str, project_dir: Path) -> None: check=True, cwd=project_copy, ) + if bundle_dir is not None: + indexes = list((project_copy / "build" / "default").rglob("index.html")) + if len(indexes) != 1: + raise RuntimeError(f"Expected one bundle index for {project_dir}, found {len(indexes)}") + destination = bundle_dir / project_dir + destination.parent.mkdir(parents=True, exist_ok=True) + if destination.exists(): + shutil.rmtree(destination) + shutil.copytree(indexes[0].parent, destination) finally: shutil.rmtree(project_copy.parent) @@ -147,9 +158,21 @@ def main() -> int: print(json.dumps([str(project) for project in projects])) return 0 + failed = [] for project_dir in projects: if (project_dir / "game.project").is_file(): - build_project(args.bob_jar, project_dir) + try: + build_project(args.bob_jar, project_dir, args.bundle_dir) + except (subprocess.CalledProcessError, RuntimeError): + if not args.keep_going: + raise + failed.append(project_dir) + + if failed: + print("Failed example builds:", file=sys.stderr) + for project_dir in failed: + print(f" {project_dir}", file=sys.stderr) + return 1 print("Example builds passed.") return 0 diff --git a/build_snapshot.py b/build_snapshot.py new file mode 100644 index 0000000..3062491 --- /dev/null +++ b/build_snapshot.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Create a complete, reusable snapshot of successfully built examples.""" +import argparse +import hashlib +import json +import shutil +import subprocess +import sys +import tempfile +import zipfile +from pathlib import Path + +from build_examples import build_project, touched_example_projects, tracked_example_projects + + +def build_snapshot(bob, output, baseline=None, full=False, keep_going=False): + head = subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip() + projects = set(tracked_example_projects()) + build_config = hashlib.sha256(Path('build_examples.py').read_bytes()).hexdigest() + with tempfile.TemporaryDirectory(prefix='example-snapshot-') as temp: + root = Path(temp) + bundles = root / 'bundles' + bundles.mkdir() + base_sha = None + if baseline and not full: + with zipfile.ZipFile(baseline) as archive: + # Only snapshots produced by this workflow are accepted. + for entry in archive.infolist(): + path = Path(entry.filename) + if path.is_absolute() or '..' in path.parts: + raise ValueError('Invalid snapshot path') + archive.extractall(root) + manifest = json.loads((root / 'manifest.json').read_text()) + base_sha = manifest['examples_sha'] + if subprocess.run(['git', 'merge-base', '--is-ancestor', base_sha, head]).returncode != 0: + raise RuntimeError('Snapshot revision is not an ancestor of HEAD') + if manifest.get('build_config') != build_config: + base_sha = None + + changed = set(touched_example_projects(base_sha, head)) if base_sha else set(projects) + for category in bundles.iterdir(): + if category.is_dir(): + for example in category.iterdir(): + if Path(category.name, example.name) not in projects: + shutil.rmtree(example) + failed = [] + for project in sorted(projects): + if project in changed or not (bundles / project / 'index.html').is_file(): + try: + build_project(bob, project, bundles) + except (subprocess.CalledProcessError, RuntimeError): + if not keep_going: + raise + failed.append(project) + if failed: + print('Failed example builds:', file=sys.stderr) + for project in failed: + print(f' {project}', file=sys.stderr) + raise RuntimeError(f'{len(failed)} example builds failed; snapshot not published') + (root / 'manifest.json').write_text(json.dumps({'examples_sha': head, 'build_config': build_config}, sort_keys=True) + '\n') + output = Path(output).resolve() + with zipfile.ZipFile(output, 'w', compression=zipfile.ZIP_DEFLATED, compresslevel=6) as archive: + for file in sorted(root.rglob('*')): + if file.is_file(): + archive.write(file, file.relative_to(root)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--bob-jar', default='bob.jar') + parser.add_argument('--output', default='examples-bundles.zip') + parser.add_argument('--baseline') + parser.add_argument('--full', action='store_true') + parser.add_argument('--keep-going', action='store_true') + args = parser.parse_args() + build_snapshot(args.bob_jar, args.output, args.baseline, args.full, args.keep_going) diff --git a/download_bob.py b/download_bob.py index 15b6164..bfe89d9 100644 --- a/download_bob.py +++ b/download_bob.py @@ -4,6 +4,7 @@ import argparse import json +import sys import urllib.request from pathlib import Path @@ -14,7 +15,7 @@ def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() - parser.add_argument("--channel", default="alpha") + parser.add_argument("--channel", default="beta") parser.add_argument("--output", default="bob.jar") return parser.parse_args() @@ -48,6 +49,7 @@ def download_asset(url: str, output: Path) -> None: def main() -> int: args = parse_args() sha1 = fetch_sha1(args.channel) + print(f"Defold {args.channel} SHA: {sha1}", file=sys.stderr) download_asset(BOB_URL_TEMPLATE.format(sha1=sha1), Path(args.output)) print(args.output) return 0 diff --git a/test_build_examples.py b/test_build_examples.py new file mode 100644 index 0000000..51141a8 --- /dev/null +++ b/test_build_examples.py @@ -0,0 +1,43 @@ +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import build_examples + + +class BundleExportTests(unittest.TestCase): + def test_exports_bundle_from_successful_build(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + project = root / "sprite" / "example" + project.mkdir(parents=True) + (project / "game.project").write_text("title = Example\n") + output = root / "output" + + def fake_build(*args, cwd, **kwargs): + bundle = cwd / "build" / "default" / "Example" + bundle.mkdir(parents=True) + (bundle / "index.html").write_text("index") + (bundle / "game.arcd").write_text("bundle") + + with patch.object(build_examples.subprocess, "run", side_effect=fake_build): + # The exported layout uses project paths relative to the repository root. + with patch.object(build_examples, "prepare_project_copy") as prepare: + copy = root / "temporary" / "example" + copy.mkdir(parents=True) + prepare.return_value = copy + build_examples.build_project("bob.jar", Path("sprite/example"), output) + self.assertEqual((output / "sprite/example/game.arcd").read_text(), "bundle") + + def test_missing_bundle_fails(self): + with tempfile.TemporaryDirectory() as tmp: + copy = Path(tmp) / "temporary" / "example" + copy.mkdir(parents=True) + with patch.object(build_examples, "prepare_project_copy", return_value=copy), patch.object(build_examples.subprocess, "run"): + with self.assertRaises(RuntimeError): + build_examples.build_project("bob.jar", Path("sprite/example"), Path(tmp) / "output") + + +if __name__ == "__main__": + unittest.main() diff --git a/test_build_snapshot.py b/test_build_snapshot.py new file mode 100644 index 0000000..822b443 --- /dev/null +++ b/test_build_snapshot.py @@ -0,0 +1,70 @@ +import json +import tempfile +import unittest +import zipfile +from pathlib import Path +from unittest.mock import patch + +import build_snapshot + + +class SnapshotTests(unittest.TestCase): + def test_full_build_reports_failures_without_writing_snapshot(self): + with tempfile.TemporaryDirectory() as temp: + output = Path(temp) / 'snapshot.zip' + built = [] + + def build(bob, project, destination): + built.append(project) + if project == Path('sprite/a'): + raise RuntimeError('failed') + + with patch.object(build_snapshot, 'tracked_example_projects', return_value={Path('sprite/a'), Path('sprite/b')}), \ + patch.object(build_snapshot.subprocess, 'check_output', return_value='abc\n'), \ + patch.object(build_snapshot, 'build_project', side_effect=build): + with self.assertRaises(RuntimeError): + build_snapshot.build_snapshot('bob.jar', output, full=True, keep_going=True) + self.assertEqual(built, [Path('sprite/a'), Path('sprite/b')]) + self.assertFalse(output.exists()) + + def test_reuses_unchanged_bundles_and_preserves_previous_on_failure(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + first, second = root / 'first.zip', root / 'second.zip' + projects = {Path('sprite/a'), Path('sprite/b')} + built = [] + + def build(bob, project, destination): + built.append(project) + bundle = destination / project + bundle.mkdir(parents=True, exist_ok=True) + (bundle / 'index.html').write_text(str(project)) + + with patch.object(build_snapshot, 'tracked_example_projects', return_value=projects), \ + patch.object(build_snapshot.subprocess, 'check_output', return_value='abc\n'), \ + patch.object(build_snapshot, 'build_project', side_effect=build): + build_snapshot.build_snapshot('bob.jar', first) + self.assertEqual(set(built), projects) + built.clear() + with patch.object(build_snapshot, 'tracked_example_projects', return_value=projects), \ + patch.object(build_snapshot.subprocess, 'check_output', return_value='def\n'), \ + patch.object(build_snapshot.subprocess, 'run') as ancestry, \ + patch.object(build_snapshot, 'touched_example_projects', return_value=[Path('sprite/a')]), \ + patch.object(build_snapshot, 'build_project', side_effect=build): + ancestry.return_value.returncode = 0 + build_snapshot.build_snapshot('bob.jar', second, first) + self.assertEqual(built, [Path('sprite/a')]) + with zipfile.ZipFile(second) as archive: + self.assertEqual(archive.read('bundles/sprite/b/index.html'), b'sprite/b') + self.assertEqual(json.loads(archive.read('manifest.json'))['examples_sha'], 'def') + original = first.read_bytes() + with patch.object(build_snapshot, 'tracked_example_projects', return_value=projects), \ + patch.object(build_snapshot.subprocess, 'check_output', return_value='def\n'), \ + patch.object(build_snapshot, 'build_project', side_effect=RuntimeError('failed')): + with self.assertRaises(RuntimeError): + build_snapshot.build_snapshot('bob.jar', second, full=True) + self.assertEqual(first.read_bytes(), original) + + +if __name__ == '__main__': + unittest.main()