Build and share example bundles from CI

.github/workflows/trigger-site-rebuild.yml

@@ -1,13 +1,21 @@
 name: Trigger site rebuild
 
 on:
+  schedule:
+    - cron: '0 2 1 * *'
   workflow_dispatch:
     inputs:
       full_rebuild:
         description: 'Validate, build, archive, and publish all examples'
         required: false
         default: true
         type: boolean
+      engine_channel:
+        description: 'Defold channel for the build'
+        required: true
+        default: beta
+        type: choice
+        options: [beta, stable]
   push:
     branches:
       - 'master'
@@ -17,10 +25,21 @@ on:
       - '.github/workflows/trigger-site-rebuild.yml'
       - 'validate_examples.py'
       - 'test_validate_examples.py'
+      - 'test_build_examples.py'
+      - 'test_build_snapshot.py'
       - 'build_examples.py'
+      - 'build_snapshot.py'
       - 'download_bob.py'
       - 'create_archives.py'
 
+permissions:
+  contents: write
+  actions: read
+
+concurrency:
+  group: examples-publish
+  cancel-in-progress: false
+
 jobs:
   validate:
     runs-on: ubuntu-latest
@@ -30,7 +49,7 @@ jobs:
       { name: 'Install Python', uses: actions/setup-python@v4, with: { python-version: 3.10.22, architecture: x64 } },
       {
         name: 'Test example validation',
-        run: 'python -m unittest test_validate_examples.py'
+        run: 'python -m unittest test_validate_examples.py test_build_examples.py test_build_snapshot.py'
       },
       {
         name: 'Validate examples',
@@ -39,55 +58,63 @@ jobs:
           AFTER_SHA: '${{ github.sha }}',
           FULL_REBUILD: '${{ github.event_name == ''workflow_dispatch'' && inputs.full_rebuild }}'
         },
-        run: 'if [ "$FULL_REBUILD" = "true" ]; then python validate_examples.py; else python validate_examples.py --changed-from "$BEFORE_SHA" --changed-to "$AFTER_SHA"; fi'
+        run: 'python validate_examples.py'
       }
     ]
 
   build:
     runs-on: ubuntu-latest
     needs: validate
     if: "github.event_name != 'push' || !contains(github.event.head_commit.message, 'skip-ci')"
-    outputs:
-      example_count: ${{ steps.count.outputs.example_count }}
-      changed_examples: ${{ steps.count.outputs.changed_examples }}
-    steps: [
-      { name: 'Checkout', uses: actions/checkout@v4, with: { fetch-depth: 0 } },
-      { name: 'Install Python', uses: actions/setup-python@v4, with: { python-version: 3.10.22, architecture: x64 } },
-      {
-        name: 'Install Java',
-        uses: actions/setup-java@v4,
-        with: { distribution: 'temurin', java-version: '25' }
-      },
-      {
-        name: 'Download Bob',
-        run: 'python download_bob.py --channel alpha --output bob.jar'
-      },
-      {
-        id: 'count',
-        name: 'Count touched examples',
-        env: {
-          BEFORE_SHA: '${{ github.event.before }}',
-          AFTER_SHA: '${{ github.sha }}',
-          FULL_REBUILD: '${{ github.event_name == ''workflow_dispatch'' && inputs.full_rebuild }}'
-        },
-        run: 'if [ "$FULL_REBUILD" = "true" ]; then echo "example_count=$(python build_examples.py --dry-run)" >> "$GITHUB_OUTPUT"; echo "changed_examples=$(python build_examples.py --list-json)" >> "$GITHUB_OUTPUT"; else echo "example_count=$(python build_examples.py --changed-from "$BEFORE_SHA" --changed-to "$AFTER_SHA" --dry-run)" >> "$GITHUB_OUTPUT"; echo "changed_examples=$(python build_examples.py --changed-from "$BEFORE_SHA" --changed-to "$AFTER_SHA" --list-json)" >> "$GITHUB_OUTPUT"; fi'
-      },
-      {
-        name: 'Build touched examples',
-        if: steps.count.outputs.example_count != '0',
-        env: {
-          BEFORE_SHA: '${{ github.event.before }}',
-          AFTER_SHA: '${{ github.sha }}',
-          FULL_REBUILD: '${{ github.event_name == ''workflow_dispatch'' && inputs.full_rebuild }}'
-        },
-        run: 'if [ "$FULL_REBUILD" = "true" ]; then python build_examples.py --bob-jar bob.jar; else python build_examples.py --bob-jar bob.jar --changed-from "$BEFORE_SHA" --changed-to "$AFTER_SHA"; fi'
-      }
-    ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with: { fetch-depth: 0 }
+      - uses: actions/setup-python@v4
+        with: { python-version: '3.10.22', architecture: x64 }
+      - uses: actions/setup-java@v4
+        with: { distribution: temurin, java-version: '25' }
+      - name: Download Bob
+        env:
+          ENGINE_CHANNEL: ${{ github.event_name == 'schedule' && 'stable' || inputs.engine_channel || 'beta' }}
+        run: python download_bob.py --channel "$ENGINE_CHANNEL" --output bob.jar
+      - name: Restore latest successful snapshot
+        if: github.event_name != 'schedule' && !(github.event_name == 'workflow_dispatch' && inputs.full_rebuild)
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          mkdir -p previous
+          gh run list --workflow trigger-site-rebuild.yml --branch master --status success --limit 50 --json databaseId --jq '.[].databaseId' > previous/runs.txt
+          while read -r run_id; do
+            artifact_id=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/artifacts" --jq '.artifacts[] | select(.name == "examples-bundles" and .expired == false) | .id')
+            if [ -n "$artifact_id" ]; then
+              gh run download "$run_id" --name examples-bundles --dir previous
+              break
+            fi
+          done < previous/runs.txt
+      - name: Build complete bundle snapshot
+        env:
+          FULL_REBUILD: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.full_rebuild) }}
+          KEEP_GOING: ${{ github.event_name == 'schedule' }}
+        run: |
+          args=()
+          if [ "$FULL_REBUILD" = true ]; then args+=(--full); elif [ -f previous/examples-bundles.zip ]; then args+=(--baseline previous/examples-bundles.zip); fi
+          if [ "$KEEP_GOING" = true ]; then args+=(--keep-going); fi
+          python build_snapshot.py --bob-jar bob.jar "${args[@]}"
+      - name: Upload bundle snapshot
+        uses: actions/upload-artifact@v4
+        with:
+          name: examples-bundles
+          path: examples-bundles.zip
+          retention-days: 30
+          compression-level: 0
+          if-no-files-found: error
 
   publish:
     runs-on: ubuntu-latest
     needs: build
-    if: "needs.build.outputs.example_count != '0' && (github.event_name != 'push' || !contains(github.event.head_commit.message, 'skip-ci'))"
+    if: "github.event_name != 'push' || !contains(github.event.head_commit.message, 'skip-ci')"
     steps: [
       { name: 'Checkout', uses: actions/checkout@v4, with: { fetch-depth: 1 } },
       { name: 'Install Python', uses: actions/setup-python@v4, with: { python-version: 3.10.22, architecture: x64 } },
@@ -97,7 +124,7 @@ jobs:
       },
       {
         name: "Create tag",
-        run: 'git tag v${{ github.sha }}'
+        run: 'git rev-parse v${{ github.sha }} >/dev/null 2>&1 || git tag v${{ github.sha }}'
       },
       {
         name: "Push to tag",
@@ -112,8 +139,8 @@ jobs:
         name: 'Repository dispatch',
         env: {
           DISPATCH_TOKEN: '${{ secrets.SERVICES_GITHUB_TOKEN }}',
-          CHANGED_EXAMPLES: '${{ needs.build.outputs.changed_examples }}'
+          EXAMPLES_RUN_ID: '${{ github.run_id }}'
         },
-        run: 'python -c ''import json, os; print(json.dumps({"event_type":"examples","client_payload":{"examples_sha":os.environ["GITHUB_SHA"],"changed_examples":json.loads(os.environ.get("CHANGED_EXAMPLES") or "[]")}}))'' > dispatch.json && curl --fail --request POST --url https://api.github.com/repos/defold/defold.github.io/dispatches --header "Accept: application/vnd.github+json" --header "Authorization: Bearer ${DISPATCH_TOKEN}" --header "X-GitHub-Api-Version: 2022-11-28" --data @dispatch.json'
+        run: 'python -c ''import json, os; print(json.dumps({"event_type":"examples","client_payload":{"examples_sha":os.environ["GITHUB_SHA"],"examples_run_id":os.environ["EXAMPLES_RUN_ID"]}}))'' > dispatch.json && curl --fail --request POST --url https://api.github.com/repos/defold/defold.github.io/dispatches --header "Accept: application/vnd.github+json" --header "Authorization: Bearer ${DISPATCH_TOKEN}" --header "X-GitHub-Api-Version: 2022-11-28" --data @dispatch.json'
       }
     ]

.github/workflows/validate-examples.yml

@@ -8,6 +8,9 @@ on:
       - '.github/workflows/trigger-site-rebuild.yml'
       - 'validate_examples.py'
       - 'test_validate_examples.py'
+      - 'test_build_examples.py'
+      - 'test_build_snapshot.py'
+      - 'build_snapshot.py'
       - 'build_examples.py'
       - 'download_bob.py'
 
@@ -19,7 +22,7 @@ jobs:
       { name: 'Install Python', uses: actions/setup-python@v4, with: { python-version: 3.10.22, architecture: x64 } },
       {
         name: 'Test example validation',
-        run: 'python -m unittest test_validate_examples.py'
+        run: 'python -m unittest test_validate_examples.py test_build_examples.py test_build_snapshot.py'
       },
       {
         name: 'Validate examples',
@@ -32,7 +35,7 @@ jobs:
       },
       {
         name: 'Download Bob',
-        run: 'python download_bob.py --channel alpha --output bob.jar'
+        run: 'python download_bob.py --channel beta --output bob.jar'
       },
       {
         name: 'Build touched examples',

README.md

@@ -2,6 +2,12 @@
 
 This repository includes the Defold examples used in the examples section on https://defold.com/examples
 
+## CI and website publication
+
+Pull requests build changed examples using the latest Defold beta. Changes to master build the examples changed since the latest successful bundle snapshot and produce a complete snapshot for the website. If there is no usable previous snapshot, all examples are built. A failed build never dispatches a site update. The snapshot is a GitHub Actions artifact retained for 30 days; expired snapshots are automatically removed and the next publication starts with a full build.
+
+The website workflow downloads the snapshot for the dispatched run and imports its matching source revision. It can be retried manually with the `examples` action, which uses the latest successful available snapshot. On the first day of each month, the publication workflow rebuilds every example using the latest stable release and publishes the complete result only if all builds succeed. The publication workflow can also be run manually with a choice of beta or stable.
+
 ## Adding more examples
 Examples are grouped by category, for instance "physics", "sprite" or "collection". Each group of examples has a folder in /examples. Here's how to add a new example named "foobar" to the "sprite" category:
 

build_examples.py

@@ -73,6 +73,8 @@ def parse_args() -> argparse.Namespace:
 	parser.add_argument("--changed-to", default="")
 	parser.add_argument("--dry-run", action="store_true")
 	parser.add_argument("--list-json", action="store_true")
+	parser.add_argument("--keep-going", action="store_true", help="Build remaining examples after a failure")
+	parser.add_argument("--bundle-dir", type=Path, help="Copy successful web bundles into this directory")
 	return parser.parse_args()
 
 
@@ -98,7 +100,7 @@ def prepare_project_copy(project_dir: Path) -> Path:
 	return project_copy
 
 
-def build_project(bob_jar: str, project_dir: Path) -> None:
+def build_project(bob_jar: str, project_dir: Path, bundle_dir: Path | None = None) -> None:
 	print(f"Building {project_dir}")
 	bob_jar_path = Path(bob_jar).resolve()
 	project_copy = prepare_project_copy(project_dir)
@@ -124,6 +126,15 @@ def build_project(bob_jar: str, project_dir: Path) -> None:
 			check=True,
 			cwd=project_copy,
 		)
+		if bundle_dir is not None:
+			indexes = list((project_copy / "build" / "default").rglob("index.html"))
+			if len(indexes) != 1:
+				raise RuntimeError(f"Expected one bundle index for {project_dir}, found {len(indexes)}")
+			destination = bundle_dir / project_dir
+			destination.parent.mkdir(parents=True, exist_ok=True)
+			if destination.exists():
+				shutil.rmtree(destination)
+			shutil.copytree(indexes[0].parent, destination)
 	finally:
 		shutil.rmtree(project_copy.parent)
 
@@ -147,9 +158,21 @@ def main() -> int:
 		print(json.dumps([str(project) for project in projects]))
 		return 0
 
+	failed = []
 	for project_dir in projects:
 		if (project_dir / "game.project").is_file():
-			build_project(args.bob_jar, project_dir)
+			try:
+				build_project(args.bob_jar, project_dir, args.bundle_dir)
+			except (subprocess.CalledProcessError, RuntimeError):
+				if not args.keep_going:
+					raise
+				failed.append(project_dir)
+
+	if failed:
+		print("Failed example builds:", file=sys.stderr)
+		for project_dir in failed:
+			print(f"  {project_dir}", file=sys.stderr)
+		return 1
 
 	print("Example builds passed.")
 	return 0

build_snapshot.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+"""Create a complete, reusable snapshot of successfully built examples."""
+import argparse
+import hashlib
+import json
+import shutil
+import subprocess
+import sys
+import tempfile
+import zipfile
+from pathlib import Path
+
+from build_examples import build_project, touched_example_projects, tracked_example_projects
+
+
+def build_snapshot(bob, output, baseline=None, full=False, keep_going=False):
+    head = subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip()
+    projects = set(tracked_example_projects())
+    build_config = hashlib.sha256(Path('build_examples.py').read_bytes()).hexdigest()
+    with tempfile.TemporaryDirectory(prefix='example-snapshot-') as temp:
+        root = Path(temp)
+        bundles = root / 'bundles'
+        bundles.mkdir()
+        base_sha = None
+        if baseline and not full:
+            with zipfile.ZipFile(baseline) as archive:
+                # Only snapshots produced by this workflow are accepted.
+                for entry in archive.infolist():
+                    path = Path(entry.filename)
+                    if path.is_absolute() or '..' in path.parts:
+                        raise ValueError('Invalid snapshot path')
+                archive.extractall(root)
+            manifest = json.loads((root / 'manifest.json').read_text())
+            base_sha = manifest['examples_sha']
+            if subprocess.run(['git', 'merge-base', '--is-ancestor', base_sha, head]).returncode != 0:
+                raise RuntimeError('Snapshot revision is not an ancestor of HEAD')
+            if manifest.get('build_config') != build_config:
+                base_sha = None
+
+        changed = set(touched_example_projects(base_sha, head)) if base_sha else set(projects)
+        for category in bundles.iterdir():
+            if category.is_dir():
+                for example in category.iterdir():
+                    if Path(category.name, example.name) not in projects:
+                        shutil.rmtree(example)
+        failed = []
+        for project in sorted(projects):
+            if project in changed or not (bundles / project / 'index.html').is_file():
+                try:
+                    build_project(bob, project, bundles)
+                except (subprocess.CalledProcessError, RuntimeError):
+                    if not keep_going:
+                        raise
+                    failed.append(project)
+        if failed:
+            print('Failed example builds:', file=sys.stderr)
+            for project in failed:
+                print(f'  {project}', file=sys.stderr)
+            raise RuntimeError(f'{len(failed)} example builds failed; snapshot not published')
+        (root / 'manifest.json').write_text(json.dumps({'examples_sha': head, 'build_config': build_config}, sort_keys=True) + '\n')
+        output = Path(output).resolve()
+        with zipfile.ZipFile(output, 'w', compression=zipfile.ZIP_DEFLATED, compresslevel=6) as archive:
+            for file in sorted(root.rglob('*')):
+                if file.is_file():
+                    archive.write(file, file.relative_to(root))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--bob-jar', default='bob.jar')
+    parser.add_argument('--output', default='examples-bundles.zip')
+    parser.add_argument('--baseline')
+    parser.add_argument('--full', action='store_true')
+    parser.add_argument('--keep-going', action='store_true')
+    args = parser.parse_args()
+    build_snapshot(args.bob_jar, args.output, args.baseline, args.full, args.keep_going)

download_bob.py

@@ -4,6 +4,7 @@
 
 import argparse
 import json
+import sys
 import urllib.request
 from pathlib import Path
 
@@ -14,7 +15,7 @@
 
 def parse_args() -> argparse.Namespace:
 	parser = argparse.ArgumentParser()
-	parser.add_argument("--channel", default="alpha")
+	parser.add_argument("--channel", default="beta")
 	parser.add_argument("--output", default="bob.jar")
 	return parser.parse_args()
 
@@ -48,6 +49,7 @@ def download_asset(url: str, output: Path) -> None:
 def main() -> int:
 	args = parse_args()
 	sha1 = fetch_sha1(args.channel)
+	print(f"Defold {args.channel} SHA: {sha1}", file=sys.stderr)
 	download_asset(BOB_URL_TEMPLATE.format(sha1=sha1), Path(args.output))
 	print(args.output)
 	return 0