diff --git a/.github/workflows/update_site.yml b/.github/workflows/update_site.yml index bc745f368..2f74f04b0 100644 --- a/.github/workflows/update_site.yml +++ b/.github/workflows/update_site.yml @@ -24,7 +24,8 @@ jobs: if: ${{ !contains(github.event.head_commit.message || '', 'skip-ci') }} env: ACTION: ${{ github.event.action || github.event.inputs.action }} - EXAMPLES_REF: ${{ github.event.client_payload.examples_sha || github.event.client_payload.after || github.event.client_payload.sha || 'master' }} + EXAMPLES_REF: ${{ github.event.client_payload.examples_sha || '' }} + EXAMPLES_RUN_ID: ${{ github.event.client_payload.examples_run_id || '' }} steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd @@ -50,7 +51,7 @@ jobs: architecture: x64 - name: Install Java - if: env.ACTION == 'examples' || env.ACTION == 'codepad' || env.ACTION == 'all' + if: env.ACTION == 'codepad' || env.ACTION == 'all' uses: actions/setup-java@f2beeb24e141e01a676f977032f5a29d81c9e27e with: java-version: '25' @@ -78,6 +79,32 @@ jobs: name: Update refdoc run: python update.py --download refdoc + - name: Download built example bundles + if: env.ACTION == 'examples' || env.ACTION == 'all' + env: + GH_TOKEN: ${{ secrets.SERVICES_GITHUB_TOKEN }} + run: | + set -euo pipefail + run_id="$EXAMPLES_RUN_ID" + if [ -z "$run_id" ]; then + run_id=$(gh run list --repo defold/examples --workflow trigger-site-rebuild.yml --branch master --status success --limit 50 --json databaseId --jq '.[].databaseId' | while read -r candidate; do + if gh api "repos/defold/examples/actions/runs/$candidate/artifacts" --jq '.artifacts[] | select(.name == "examples-bundles" and .expired == false) | .id' | grep -q .; then echo "$candidate"; break; fi + done) + fi + test -n "$run_id" + gh run download "$run_id" --repo defold/examples --name examples-bundles --dir . + snapshot_sha=$(python -c 'import json, zipfile; print(json.loads(zipfile.ZipFile("examples-bundles.zip").read("manifest.json"))["examples_sha"])') + if [ -n "$EXAMPLES_REF" ] && [ "$EXAMPLES_REF" != "$snapshot_sha" ]; then + echo "Snapshot SHA does not match the dispatch" >&2; exit 1 + fi + if [ -f _data/examples_build.json ]; then + previous_sha=$(python -c 'import json; print(json.load(open("_data/examples_build.json"))["examples_sha"])') + status=$(gh api "repos/defold/examples/compare/$previous_sha...$snapshot_sha" --jq .status) + if [ "$status" != ahead ] && [ "$status" != identical ]; then + echo "Refusing to publish an older or divergent examples snapshot" >&2; exit 1 + fi + fi + - if: env.ACTION == 'examples' name: Update examples run: python update.py --download --examples-ref "$EXAMPLES_REF" examples diff --git a/tests/test_example_bundle_import.py b/tests/test_example_bundle_import.py new file mode 100644 index 000000000..fbac236ad --- /dev/null +++ b/tests/test_example_bundle_import.py @@ -0,0 +1,72 @@ +import ast +import json +import os +import re +import shutil +import tempfile +import unittest +import zipfile +from contextlib import contextmanager +from pathlib import Path + + +class ExampleBundleImportTest(unittest.TestCase): + def test_imports_bundle_matching_source_snapshot(self): + source = (Path(__file__).resolve().parents[1] / 'update.py').read_text() + tree = ast.parse(source) + function = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == 'process_examples') + original = os.getcwd() + with tempfile.TemporaryDirectory() as temp: + try: + os.chdir(temp) + for path in ('_includes', '_data', 'examples'): + Path(path).mkdir() + Path('_data/examplesindex.json').write_text('[]') + with zipfile.ZipFile('examples-master.zip', 'w') as archive: + archive.writestr('examples-abc/sprite/demo/example.md', '---\ntitle: Demo\n---\n') + with zipfile.ZipFile('examples-bundles.zip', 'w') as archive: + archive.writestr('manifest.json', json.dumps({'examples_sha': 'abc'})) + archive.writestr('bundles/sprite/demo/index.html', 'index') + archive.writestr('bundles/sprite/demo/game.arcd', 'bundle') + + @contextmanager + def tmpdir(): + with tempfile.TemporaryDirectory() as path: + yield path + + def rmtree(path): + if os.path.isdir(path): + shutil.rmtree(path) + + def rmmkdir(path): + rmtree(path) + os.makedirs(path) + + def unzip(path, destination): + with zipfile.ZipFile(path) as archive: + archive.extractall(destination) + + class Dedupe: + @staticmethod + def run(): + pass + + namespace = dict(os=os, Path=Path, json=json, zipfile=zipfile, shutil=shutil, re=re, + EXAMPLES_ZIP='examples-master.zip', tmpdir=tmpdir, rmtree=rmtree, rmmkdir=rmmkdir, + unzip=unzip, find_unzipped_repo_dir=lambda root, prefix: next(str(p) for p in Path(root).iterdir() if p.is_dir() and p.name.startswith(prefix)), + read_as_json=lambda p: json.loads(Path(p).read_text()), load_frontmatter=lambda p: {'title': 'Demo'}, + find_example_scripts=lambda p: [], resolve_example_scripts=lambda a, b: [], + makedirs=lambda p: os.makedirs(p, exist_ok=True), copy_example_scripts=lambda *a: None, + replace_frontmatter=lambda *a: None, read_as_string=lambda p: Path(p).read_text(), + copy_documentation_images=lambda *a: None, write_as_json=lambda p, d: Path(p).write_text(json.dumps(d)), + generate_llms_examples=lambda: None, dedupe_examples_wasm=Dedupe(), normalize_example_path=lambda p: p) + exec(compile(ast.Module(body=[function], type_ignores=[]), 'update.py', 'exec'), namespace) + namespace['process_examples']() + self.assertEqual(Path('examples/sprite/demo/game.arcd').read_text(), 'bundle') + self.assertEqual(json.loads(Path('_data/examples_build.json').read_text())['examples_sha'], 'abc') + finally: + os.chdir(original) + + +if __name__ == '__main__': + unittest.main() diff --git a/update.py b/update.py index 352ff545c..75b56a9ef 100755 --- a/update.py +++ b/update.py @@ -990,24 +990,19 @@ def process_extension(extension_name, download = False): def process_examples(download = False, examples_ref = "master", changed_examples = None): - examples_sha1 = get_sha1(EXAMPLES_DEFOLD_CHANNEL) + snapshot_path = os.environ.get("EXAMPLES_SNAPSHOT", "examples-bundles.zip") + if not os.path.isfile(snapshot_path): + raise RuntimeError("Examples bundle snapshot is required") + with zipfile.ZipFile(snapshot_path) as snapshot: + manifest = json.loads(snapshot.read("manifest.json")) + examples_ref = manifest["examples_sha"] if download: if os.path.exists(EXAMPLES_ZIP): os.remove(EXAMPLES_ZIP) download_file("https://github.com/defold/examples/archive/{}.zip".format(examples_ref), ".", EXAMPLES_ZIP) - download_bob(examples_sha1) - if not os.path.exists(EXAMPLES_ZIP): - print("File {} does not exist".format(EXAMPLES_ZIP)) - sys.exit(1) - - bob_jar = get_bob_filename(examples_sha1) - if not os.path.exists(bob_jar): - print("File {} does not exist".format(bob_jar)) - sys.exit(1) - - rebuild = True + raise RuntimeError("Examples source archive is missing") includes_dir = os.path.join("_includes", "examples") rmmkdir(includes_dir) @@ -1021,7 +1016,15 @@ def process_examples(download = False, examples_ref = "master", changed_examples print("Unable to find unzipped examples directory") sys.exit(1) + with zipfile.ZipFile(snapshot_path) as snapshot: + for entry in snapshot.infolist(): + path = Path(entry.filename) + if path.is_absolute() or ".." in path.parts: + raise RuntimeError("Invalid bundle snapshot path") + snapshot.extractall(os.path.join(tmp_dir, "snapshot")) + bundles = Path(tmp_dir) / "snapshot" / "bundles" data_index_file = os.path.join("_data", "examplesindex.json") + old_examples = read_as_json(data_index_file) if os.path.exists(data_index_file) else [] examplesindex = [] category_dirs = os.listdir(unzipped_examples_dir) for category in category_dirs: @@ -1030,9 +1033,6 @@ def process_examples(download = False, examples_ref = "master", changed_examples if os.path.isfile(category_src_dir) or category == ".github": continue - if rebuild: - rmmkdir(category_dst_dir) - for example in os.listdir(category_src_dir): example_src_dir = os.path.join(category_src_dir, example) example_dst_dir = os.path.join(category_dst_dir, example) @@ -1051,29 +1051,13 @@ def process_examples(download = False, examples_ref = "master", changed_examples sys.exit(1) print("..processing %s" % example) - if rebuild: - print("...building %s" % example) - bob_out = os.path.join(example_src_dir, bob_jar) - shutil.copyfile(bob_jar, bob_out) - game_project = os.path.join(example_src_dir, "game.project") - replace_in_file(game_project, r"title = .*", r"title = Defold-examples") - subprocess.run([ "java", "-jar", bob_out, "--archive", "--platform", "wasm-web", "--architectures", "wasm-web", "--variant", "debug", "--build-server", EXAMPLES_BUILD_SERVER, "resolve", "build", "bundle" ], cwd=example_src_dir, check=True) - os.remove(bob_out) - - print("...copying %s" % example) - bundle_index_file = find_file(os.path.join(example_src_dir, "build", "default"), "index.html") - if not bundle_index_file: - print("File index.html not found") - sys.exit(1) - print(bundle_index_file, os.path.dirname(bundle_index_file)) - - bundle_dir = os.path.dirname(bundle_index_file) - rmtree(example_dst_dir) - makedirs(category_dst_dir) - shutil.copytree(bundle_dir, example_dst_dir) - os.remove(os.path.join(example_dst_dir, "index.html")) - else: - os.makedirs(example_dst_dir, exist_ok=True) + bundle_dir = bundles / category / example + if not (bundle_dir / "index.html").is_file(): + raise RuntimeError("Missing bundle for {}".format(example_path)) + rmtree(example_dst_dir) + makedirs(category_dst_dir) + shutil.copytree(bundle_dir, example_dst_dir) + os.remove(os.path.join(example_dst_dir, "index.html")) print("...parsing example.md") fm["category"] = category @@ -1111,6 +1095,13 @@ def process_examples(download = False, examples_ref = "master", changed_examples print("...copying documentation images") copy_documentation_images(example_src_dir, example_dst_dir, read_as_string(md_file)) + current = {entry["path"] for entry in examplesindex} + for entry in old_examples: + path = normalize_example_path(entry.get("path", "")) + if path and path not in current: + rmtree(os.path.join("examples", path)) + write_as_json(os.path.join("_data", "examples_build.json"), manifest) + print("...generating index") if os.path.exists(data_index_file): os.remove(data_index_file)