Import example bundles built by examples CI

.github/workflows/update_site.yml

@@ -24,7 +24,8 @@ jobs:
     if: ${{ !contains(github.event.head_commit.message || '', 'skip-ci') }}
     env:
       ACTION: ${{ github.event.action || github.event.inputs.action }}
-      EXAMPLES_REF: ${{ github.event.client_payload.examples_sha || github.event.client_payload.after || github.event.client_payload.sha || 'master' }}
+      EXAMPLES_REF: ${{ github.event.client_payload.examples_sha || '' }}
+      EXAMPLES_RUN_ID: ${{ github.event.client_payload.examples_run_id || '' }}
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
@@ -50,7 +51,7 @@ jobs:
           architecture: x64
 
       - name: Install Java
-        if: env.ACTION == 'examples' || env.ACTION == 'codepad' || env.ACTION == 'all'
+        if: env.ACTION == 'codepad' || env.ACTION == 'all'
         uses: actions/setup-java@f2beeb24e141e01a676f977032f5a29d81c9e27e
         with:
           java-version: '25'
@@ -78,6 +79,32 @@ jobs:
         name: Update refdoc
         run: python update.py --download refdoc
 
+      - name: Download built example bundles
+        if: env.ACTION == 'examples' || env.ACTION == 'all'
+        env:
+          GH_TOKEN: ${{ secrets.SERVICES_GITHUB_TOKEN }}
+        run: |
+          set -euo pipefail
+          run_id="$EXAMPLES_RUN_ID"
+          if [ -z "$run_id" ]; then
+            run_id=$(gh run list --repo defold/examples --workflow trigger-site-rebuild.yml --branch master --status success --limit 50 --json databaseId --jq '.[].databaseId' | while read -r candidate; do
+              if gh api "repos/defold/examples/actions/runs/$candidate/artifacts" --jq '.artifacts[] | select(.name == "examples-bundles" and .expired == false) | .id' | grep -q .; then echo "$candidate"; break; fi
+            done)
+          fi
+          test -n "$run_id"
+          gh run download "$run_id" --repo defold/examples --name examples-bundles --dir .
+          snapshot_sha=$(python -c 'import json, zipfile; print(json.loads(zipfile.ZipFile("examples-bundles.zip").read("manifest.json"))["examples_sha"])')
+          if [ -n "$EXAMPLES_REF" ] && [ "$EXAMPLES_REF" != "$snapshot_sha" ]; then
+            echo "Snapshot SHA does not match the dispatch" >&2; exit 1
+          fi
+          if [ -f _data/examples_build.json ]; then
+            previous_sha=$(python -c 'import json; print(json.load(open("_data/examples_build.json"))["examples_sha"])')
+            status=$(gh api "repos/defold/examples/compare/$previous_sha...$snapshot_sha" --jq .status)
+            if [ "$status" != ahead ] && [ "$status" != identical ]; then
+              echo "Refusing to publish an older or divergent examples snapshot" >&2; exit 1
+            fi
+          fi
+
       - if: env.ACTION == 'examples'
         name: Update examples
         run: python update.py --download --examples-ref "$EXAMPLES_REF" examples

tests/test_example_bundle_import.py

@@ -0,0 +1,72 @@
+import ast
+import json
+import os
+import re
+import shutil
+import tempfile
+import unittest
+import zipfile
+from contextlib import contextmanager
+from pathlib import Path
+
+
+class ExampleBundleImportTest(unittest.TestCase):
+    def test_imports_bundle_matching_source_snapshot(self):
+        source = (Path(__file__).resolve().parents[1] / 'update.py').read_text()
+        tree = ast.parse(source)
+        function = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == 'process_examples')
+        original = os.getcwd()
+        with tempfile.TemporaryDirectory() as temp:
+            try:
+                os.chdir(temp)
+                for path in ('_includes', '_data', 'examples'):
+                    Path(path).mkdir()
+                Path('_data/examplesindex.json').write_text('[]')
+                with zipfile.ZipFile('examples-master.zip', 'w') as archive:
+                    archive.writestr('examples-abc/sprite/demo/example.md', '---\ntitle: Demo\n---\n')
+                with zipfile.ZipFile('examples-bundles.zip', 'w') as archive:
+                    archive.writestr('manifest.json', json.dumps({'examples_sha': 'abc'}))
+                    archive.writestr('bundles/sprite/demo/index.html', 'index')
+                    archive.writestr('bundles/sprite/demo/game.arcd', 'bundle')
+
+                @contextmanager
+                def tmpdir():
+                    with tempfile.TemporaryDirectory() as path:
+                        yield path
+
+                def rmtree(path):
+                    if os.path.isdir(path):
+                        shutil.rmtree(path)
+
+                def rmmkdir(path):
+                    rmtree(path)
+                    os.makedirs(path)
+
+                def unzip(path, destination):
+                    with zipfile.ZipFile(path) as archive:
+                        archive.extractall(destination)
+
+                class Dedupe:
+                    @staticmethod
+                    def run():
+                        pass
+
+                namespace = dict(os=os, Path=Path, json=json, zipfile=zipfile, shutil=shutil, re=re,
+                    EXAMPLES_ZIP='examples-master.zip', tmpdir=tmpdir, rmtree=rmtree, rmmkdir=rmmkdir,
+                    unzip=unzip, find_unzipped_repo_dir=lambda root, prefix: next(str(p) for p in Path(root).iterdir() if p.is_dir() and p.name.startswith(prefix)),
+                    read_as_json=lambda p: json.loads(Path(p).read_text()), load_frontmatter=lambda p: {'title': 'Demo'},
+                    find_example_scripts=lambda p: [], resolve_example_scripts=lambda a, b: [],
+                    makedirs=lambda p: os.makedirs(p, exist_ok=True), copy_example_scripts=lambda *a: None,
+                    replace_frontmatter=lambda *a: None, read_as_string=lambda p: Path(p).read_text(),
+                    copy_documentation_images=lambda *a: None, write_as_json=lambda p, d: Path(p).write_text(json.dumps(d)),
+                    generate_llms_examples=lambda: None, dedupe_examples_wasm=Dedupe(), normalize_example_path=lambda p: p)
+                exec(compile(ast.Module(body=[function], type_ignores=[]), 'update.py', 'exec'), namespace)
+                namespace['process_examples']()
+                self.assertEqual(Path('examples/sprite/demo/game.arcd').read_text(), 'bundle')
+                self.assertEqual(json.loads(Path('_data/examples_build.json').read_text())['examples_sha'], 'abc')
+            finally:
+                os.chdir(original)
+
+
+if __name__ == '__main__':
+    unittest.main()

update.py

@@ -990,24 +990,19 @@ def process_extension(extension_name, download = False):
 
 
 def process_examples(download = False, examples_ref = "master", changed_examples = None):
-    examples_sha1 = get_sha1(EXAMPLES_DEFOLD_CHANNEL)
+    snapshot_path = os.environ.get("EXAMPLES_SNAPSHOT", "examples-bundles.zip")
+    if not os.path.isfile(snapshot_path):
+        raise RuntimeError("Examples bundle snapshot is required")
+    with zipfile.ZipFile(snapshot_path) as snapshot:
+        manifest = json.loads(snapshot.read("manifest.json"))
+    examples_ref = manifest["examples_sha"]
 
     if download:
         if os.path.exists(EXAMPLES_ZIP):
             os.remove(EXAMPLES_ZIP)
         download_file("https://github.com/defold/examples/archive/{}.zip".format(examples_ref), ".", EXAMPLES_ZIP)
-        download_bob(examples_sha1)
-
     if not os.path.exists(EXAMPLES_ZIP):
-        print("File {} does not exist".format(EXAMPLES_ZIP))
-        sys.exit(1)
-
-    bob_jar = get_bob_filename(examples_sha1)
-    if not os.path.exists(bob_jar):
-        print("File {} does not exist".format(bob_jar))
-        sys.exit(1)
-
-    rebuild = True
+        raise RuntimeError("Examples source archive is missing")
 
     includes_dir = os.path.join("_includes", "examples")
     rmmkdir(includes_dir)
@@ -1021,7 +1016,15 @@ def process_examples(download = False, examples_ref = "master", changed_examples
             print("Unable to find unzipped examples directory")
             sys.exit(1)
 
+        with zipfile.ZipFile(snapshot_path) as snapshot:
+            for entry in snapshot.infolist():
+                path = Path(entry.filename)
+                if path.is_absolute() or ".." in path.parts:
+                    raise RuntimeError("Invalid bundle snapshot path")
+            snapshot.extractall(os.path.join(tmp_dir, "snapshot"))
+        bundles = Path(tmp_dir) / "snapshot" / "bundles"
         data_index_file = os.path.join("_data", "examplesindex.json")
+        old_examples = read_as_json(data_index_file) if os.path.exists(data_index_file) else []
         examplesindex = []
         category_dirs = os.listdir(unzipped_examples_dir)
         for category in category_dirs:
@@ -1030,9 +1033,6 @@ def process_examples(download = False, examples_ref = "master", changed_examples
             if os.path.isfile(category_src_dir) or category == ".github":
                 continue
 
-            if rebuild:
-                rmmkdir(category_dst_dir)
-
             for example in os.listdir(category_src_dir):
                 example_src_dir = os.path.join(category_src_dir, example)
                 example_dst_dir = os.path.join(category_dst_dir, example)
@@ -1051,29 +1051,13 @@ def process_examples(download = False, examples_ref = "master", changed_examples
                     sys.exit(1)
 
                 print("..processing %s" % example)
-                if rebuild:
-                    print("...building %s" % example)
-                    bob_out = os.path.join(example_src_dir, bob_jar)
-                    shutil.copyfile(bob_jar, bob_out)
-                    game_project = os.path.join(example_src_dir, "game.project")
-                    replace_in_file(game_project, r"title = .*", r"title = Defold-examples")
-                    subprocess.run([ "java", "-jar", bob_out, "--archive", "--platform", "wasm-web", "--architectures", "wasm-web", "--variant", "debug", "--build-server", EXAMPLES_BUILD_SERVER, "resolve", "build", "bundle" ], cwd=example_src_dir, check=True)
-                    os.remove(bob_out)
-
-                    print("...copying %s" % example)
-                    bundle_index_file = find_file(os.path.join(example_src_dir, "build", "default"), "index.html")
-                    if not bundle_index_file:
-                        print("File index.html not found")
-                        sys.exit(1)
-                    print(bundle_index_file, os.path.dirname(bundle_index_file))
-
-                    bundle_dir = os.path.dirname(bundle_index_file)
-                    rmtree(example_dst_dir)
-                    makedirs(category_dst_dir)
-                    shutil.copytree(bundle_dir, example_dst_dir)
-                    os.remove(os.path.join(example_dst_dir, "index.html"))
-                else:
-                    os.makedirs(example_dst_dir, exist_ok=True)
+                bundle_dir = bundles / category / example
+                if not (bundle_dir / "index.html").is_file():
+                    raise RuntimeError("Missing bundle for {}".format(example_path))
+                rmtree(example_dst_dir)
+                makedirs(category_dst_dir)
+                shutil.copytree(bundle_dir, example_dst_dir)
+                os.remove(os.path.join(example_dst_dir, "index.html"))
 
                 print("...parsing example.md")
                 fm["category"] = category
@@ -1111,6 +1095,13 @@ def process_examples(download = False, examples_ref = "master", changed_examples
                 print("...copying documentation images")
                 copy_documentation_images(example_src_dir, example_dst_dir, read_as_string(md_file))
 
+        current = {entry["path"] for entry in examplesindex}
+        for entry in old_examples:
+            path = normalize_example_path(entry.get("path", ""))
+            if path and path not in current:
+                rmtree(os.path.join("examples", path))
+        write_as_json(os.path.join("_data", "examples_build.json"), manifest)
+
         print("...generating index")
         if os.path.exists(data_index_file):
             os.remove(data_index_file)