Import prebuilt example bundles instead of building here again

.github/workflows/update_site.yml

@@ -24,7 +24,8 @@ jobs:
     if: ${{ !contains(github.event.head_commit.message || '', 'skip-ci') }}
     env:
       ACTION: ${{ github.event.action || github.event.inputs.action }}
-      EXAMPLES_REF: ${{ github.event.client_payload.examples_sha || github.event.client_payload.after || github.event.client_payload.sha || 'master' }}
+      EXAMPLES_REF: ${{ github.event.client_payload.examples_sha || '' }}
+      EXAMPLES_RUN_ID: ${{ github.event.client_payload.examples_run_id || '' }}
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
@@ -50,7 +51,7 @@ jobs:
           architecture: x64
 
       - name: Install Java
-        if: env.ACTION == 'examples' || env.ACTION == 'codepad' || env.ACTION == 'all'
+        if: env.ACTION == 'codepad' || env.ACTION == 'all'
         uses: actions/setup-java@f2beeb24e141e01a676f977032f5a29d81c9e27e
         with:
           java-version: '25'
@@ -78,6 +79,32 @@ jobs:
         name: Update refdoc
         run: python update.py --download refdoc
 
+      - name: Download built example bundles
+        if: env.ACTION == 'examples' || env.ACTION == 'all'
+        env:
+          GH_TOKEN: ${{ secrets.SERVICES_GITHUB_TOKEN }}
+        run: |
+          set -euo pipefail
+          run_id="$EXAMPLES_RUN_ID"
+          if [ -z "$run_id" ]; then
+            run_id=$(gh run list --repo defold/examples --workflow trigger-site-rebuild.yml --branch master --status success --limit 50 --json databaseId --jq '.[].databaseId' | while read -r candidate; do
+              if gh api "repos/defold/examples/actions/runs/$candidate/artifacts" --jq '.artifacts[] | select(.name == "examples-bundles" and .expired == false) | .id' | grep -q .; then echo "$candidate"; break; fi
+            done)
+          fi
+          test -n "$run_id"
+          gh run download "$run_id" --repo defold/examples --name examples-bundles --dir .
+          snapshot_sha=$(python -c 'import json, zipfile; print(json.loads(zipfile.ZipFile("examples-bundles.zip").read("manifest.json"))["examples_sha"])')
+          if [ -n "$EXAMPLES_REF" ] && [ "$EXAMPLES_REF" != "$snapshot_sha" ]; then
+            echo "Snapshot SHA does not match the dispatch" >&2; exit 1
+          fi
+          if [ -f _data/examples_build.json ]; then
+            previous_sha=$(python -c 'import json; print(json.load(open("_data/examples_build.json"))["examples_sha"])')
+            status=$(gh api "repos/defold/examples/compare/$previous_sha...$snapshot_sha" --jq .status)
+            if [ "$status" != ahead ] && [ "$status" != identical ]; then
+              echo "Refusing to publish an older or divergent examples snapshot" >&2; exit 1
+            fi
+          fi
+
       - if: env.ACTION == 'examples'
         name: Update examples
         run: python update.py --download --examples-ref "$EXAMPLES_REF" examples

_config.yml

@@ -30,6 +30,7 @@ exclude:
     # Downloaded inputs and tooling used to generate the site.
     - doc-master.zip
     - examples-master.zip
+    - examples-bundles.zip
     - codepad-master.zip
     - asset-portal-master.zip
     - games-showcase-master.zip

scripts/example_bundle_validation.py

@@ -0,0 +1,73 @@
+"""Validate source revisions and complete Defold HTML5 bundle inputs."""
+import json
+import re
+import zipfile
+from pathlib import Path, PurePosixPath
+
+
+REQUIRED_ARCHIVES = {'game.projectc', 'game.arci', 'game.arcd', 'game.dmanifest'}
+REQUIRED_BUNDLE_FILES = ('index.html', 'dmloader.js', 'Defoldexamples.wasm', 'Defoldexamples_wasm.js')
+
+
+def validate_source_archive(archive_path, sha):
+    if not isinstance(sha, str) or not re.fullmatch(r'[0-9a-f]{40}', sha):
+        raise RuntimeError('Invalid examples revision in snapshot manifest')
+    expected_root = 'examples-' + sha
+    with zipfile.ZipFile(archive_path) as archive:
+        entries = archive.infolist()
+        if not entries:
+            raise RuntimeError('Examples source archive is empty')
+        for entry in entries:
+            path = PurePosixPath(entry.filename)
+            if path.is_absolute() or '..' in path.parts or not path.parts or path.parts[0] != expected_root:
+                raise RuntimeError('Examples source archive does not match snapshot revision {}'.format(sha))
+    return expected_root
+
+
+def validate_example_bundle(bundle_dir, example_path):
+    bundle_dir = Path(bundle_dir)
+    def fail(detail):
+        raise RuntimeError('Invalid bundle for {}: {}'.format(example_path, detail))
+    for name in REQUIRED_BUNDLE_FILES:
+        file = bundle_dir / name
+        if not file.is_file() or file.stat().st_size == 0:
+            fail('missing or empty {}'.format(name))
+    threaded = ('Defoldexamples_pthread.wasm', 'Defoldexamples_pthread_wasm.js')
+    if any((bundle_dir / name).exists() for name in threaded):
+        for name in threaded:
+            file = bundle_dir / name
+            if not file.is_file() or file.stat().st_size == 0:
+                fail('missing or empty {}'.format(name))
+    archive_dir = bundle_dir / 'archive'
+    manifest_path = archive_dir / 'archive_files.json'
+    try:
+        manifest = json.loads(manifest_path.read_text(encoding='utf-8'))
+        content = manifest['content']
+        if not isinstance(content, list):
+            raise ValueError('content must be an array')
+        names = set()
+        for item in content:
+            name, size, pieces = item['name'], item['size'], item['pieces']
+            if not isinstance(name, str) or name in names or not isinstance(size, int) or isinstance(size, bool) or size < 0:
+                raise ValueError('invalid archive entry')
+            names.add(name)
+            if not isinstance(pieces, list) or not pieces:
+                raise ValueError('missing pieces for {}'.format(name))
+            offset = 0
+            for piece in pieces:
+                piece_name = piece['name']
+                if not isinstance(piece_name, str):
+                    raise ValueError('invalid piece name')
+                path = PurePosixPath(piece_name)
+                if path.is_absolute() or len(path.parts) != 1 or path.name in ('.', '..') or '\\' in piece_name:
+                    raise ValueError('invalid archive piece path')
+                file = archive_dir / piece_name
+                if not file.is_file() or piece['offset'] != offset:
+                    raise ValueError('missing or invalid piece {}'.format(piece_name))
+                offset += file.stat().st_size
+            if offset != size:
+                raise ValueError('incorrect size for {}'.format(name))
+        if not REQUIRED_ARCHIVES.issubset(names):
+            raise ValueError('missing required game archives: {}'.format(', '.join(sorted(REQUIRED_ARCHIVES - names))))
+    except (OSError, ValueError, TypeError, KeyError) as error:
+        fail('invalid archive manifest or pieces ({})'.format(error))

tests/test_example_bundle_import.py

@@ -0,0 +1,142 @@
+import ast
+import json
+import os
+import re
+import shutil
+import tempfile
+import unittest
+import zipfile
+from contextlib import contextmanager
+from pathlib import Path
+
+from scripts.example_bundle_validation import validate_example_bundle, validate_source_archive
+
+
+SHA_A = 'a' * 40
+SHA_B = 'b' * 40
+
+
+def bundle_files():
+    entries = []
+    result = {'index.html': b'index', 'dmloader.js': b'loader',
+              'Defoldexamples.wasm': b'wasm', 'Defoldexamples_wasm.js': b'engine'}
+    for suffix in ('projectc', 'arci', 'arcd', 'dmanifest'):
+        data = ('data-' + suffix).encode()
+        name = 'game0.' + suffix
+        result['archive/' + name] = data
+        entries.append({'name': 'game.' + suffix, 'size': len(data), 'pieces': [{'name': name, 'offset': 0}]})
+    result['archive/archive_files.json'] = json.dumps({'content': entries}).encode()
+    return result
+
+
+class ExampleBundleImportTest(unittest.TestCase):
+    def setUp(self):
+        self.original = os.getcwd()
+        self.temp = tempfile.TemporaryDirectory()
+        os.chdir(self.temp.name)
+        for path in ('_includes/examples', '_data', 'examples/sprite/demo', 'examples/sprite/other'):
+            Path(path).mkdir(parents=True)
+        Path('_data/examplesindex.json').write_text('[]')
+        Path('_data/examples_build.json').write_text('{"examples_sha":"old"}')
+        Path('_includes/examples/keep.txt').write_text('unchanged')
+        for example in ('demo', 'other'):
+            Path('examples/sprite', example, 'keep.txt').write_text('unchanged')
+        self.make_source(SHA_A)
+        self.make_snapshot()
+
+    def tearDown(self):
+        os.chdir(self.original)
+        self.temp.cleanup()
+
+    def make_source(self, sha):
+        with zipfile.ZipFile('examples-master.zip', 'w') as archive:
+            for example in ('demo', 'other'):
+                archive.writestr('examples-{}/sprite/{}/example.md'.format(sha, example), '---\ntitle: Demo\n---\n')
+
+    def make_snapshot(self, remove=None):
+        with zipfile.ZipFile('examples-bundles.zip', 'w') as archive:
+            archive.writestr('manifest.json', json.dumps({'examples_sha': SHA_A}))
+            for example in ('demo', 'other'):
+                for name, content in bundle_files().items():
+                    if (example, name) != remove:
+                        archive.writestr('bundles/sprite/{}/{}'.format(example, name), content)
+
+    def importer(self):
+        source = (Path(__file__).resolve().parents[1] / 'update.py').read_text()
+        tree = ast.parse(source)
+        function = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == 'process_examples')
+        @contextmanager
+        def tmpdir():
+            with tempfile.TemporaryDirectory() as path:
+                yield path
+        def rmtree(path):
+            if os.path.isdir(path): shutil.rmtree(path)
+        def rmmkdir(path):
+            rmtree(path)
+            os.makedirs(path)
+        def unzip(path, destination):
+            with zipfile.ZipFile(path) as archive: archive.extractall(destination)
+        class Dedupe:
+            @staticmethod
+            def run(): pass
+        namespace = dict(os=os, Path=Path, json=json, zipfile=zipfile, shutil=shutil, re=re,
+            EXAMPLES_ZIP='examples-master.zip', tmpdir=tmpdir, rmtree=rmtree, rmmkdir=rmmkdir,
+            validate_source_archive=validate_source_archive, validate_example_bundle=validate_example_bundle,
+            unzip=unzip, read_as_json=lambda p: json.loads(Path(p).read_text()),
+            load_frontmatter=lambda p: {'title': 'Demo'}, find_example_scripts=lambda p: [],
+            resolve_example_scripts=lambda a, b: [], makedirs=lambda p: os.makedirs(p, exist_ok=True),
+            copy_example_scripts=lambda *a: None, replace_frontmatter=lambda *a: None,
+            read_as_string=lambda p: Path(p).read_text(), copy_documentation_images=lambda *a: None,
+            write_as_json=lambda p, d: Path(p).write_text(json.dumps(d)),
+            generate_llms_examples=lambda: None, dedupe_examples_wasm=Dedupe(), normalize_example_path=lambda p: p)
+        exec(compile(ast.Module(body=[function], type_ignores=[]), 'update.py', 'exec'), namespace)
+        return namespace['process_examples']
+
+    def assert_preserved(self):
+        self.assertEqual(Path('_includes/examples/keep.txt').read_text(), 'unchanged')
+        for example in ('demo', 'other'):
+            self.assertEqual(Path('examples/sprite', example, 'keep.txt').read_text(), 'unchanged')
+        self.assertEqual(json.loads(Path('_data/examples_build.json').read_text())['examples_sha'], 'old')
+
+    def test_imports_bundle_matching_source_snapshot(self):
+        self.importer()()
+        self.assertEqual(Path('examples/sprite/demo/archive/game0.arcd').read_bytes(), b'data-arcd')
+        self.assertEqual(json.loads(Path('_data/examples_build.json').read_text())['examples_sha'], SHA_A)
+
+    def test_mismatched_cached_source_does_not_change_existing_examples(self):
+        self.make_source(SHA_B)
+        with self.assertRaisesRegex(RuntimeError, 'does not match snapshot revision'):
+            self.importer()()
+        self.assert_preserved()
+
+    def test_incomplete_bundle_does_not_change_any_examples(self):
+        for missing in ('dmloader.js', 'Defoldexamples.wasm', 'Defoldexamples_wasm.js', 'archive/game0.arcd', 'archive/archive_files.json'):
+            with self.subTest(missing=missing):
+                self.make_snapshot(remove=('other', missing))
+                with self.assertRaisesRegex(RuntimeError, 'Invalid bundle for sprite/other'):
+                    self.importer()()
+                self.assert_preserved()
+
+    def test_split_archive_pieces_are_checked(self):
+        root = Path('split')
+        root.mkdir()
+        files = bundle_files()
+        manifest = json.loads(files['archive/archive_files.json'])
+        arcd = next(item for item in manifest['content'] if item['name'] == 'game.arcd')
+        data = files.pop('archive/game0.arcd')
+        files['archive/game0.arcd'] = data[:2]
+        files['archive/game1.arcd'] = data[2:]
+        arcd['pieces'] = [{'name': 'game0.arcd', 'offset': 0}, {'name': 'game1.arcd', 'offset': 2}]
+        files['archive/archive_files.json'] = json.dumps(manifest).encode()
+        for name, content in files.items():
+            path = root / name
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_bytes(content)
+        validate_example_bundle(root, 'sprite/demo')
+        (root / 'archive/game1.arcd').unlink()
+        with self.assertRaisesRegex(RuntimeError, 'game1.arcd'):
+            validate_example_bundle(root, 'sprite/demo')
+
+
+if __name__ == '__main__':
+    unittest.main()