diff --git a/.github/workflows/update_site.yml b/.github/workflows/update_site.yml index bc745f368..2f74f04b0 100644 --- a/.github/workflows/update_site.yml +++ b/.github/workflows/update_site.yml @@ -24,7 +24,8 @@ jobs: if: ${{ !contains(github.event.head_commit.message || '', 'skip-ci') }} env: ACTION: ${{ github.event.action || github.event.inputs.action }} - EXAMPLES_REF: ${{ github.event.client_payload.examples_sha || github.event.client_payload.after || github.event.client_payload.sha || 'master' }} + EXAMPLES_REF: ${{ github.event.client_payload.examples_sha || '' }} + EXAMPLES_RUN_ID: ${{ github.event.client_payload.examples_run_id || '' }} steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd @@ -50,7 +51,7 @@ jobs: architecture: x64 - name: Install Java - if: env.ACTION == 'examples' || env.ACTION == 'codepad' || env.ACTION == 'all' + if: env.ACTION == 'codepad' || env.ACTION == 'all' uses: actions/setup-java@f2beeb24e141e01a676f977032f5a29d81c9e27e with: java-version: '25' @@ -78,6 +79,32 @@ jobs: name: Update refdoc run: python update.py --download refdoc + - name: Download built example bundles + if: env.ACTION == 'examples' || env.ACTION == 'all' + env: + GH_TOKEN: ${{ secrets.SERVICES_GITHUB_TOKEN }} + run: | + set -euo pipefail + run_id="$EXAMPLES_RUN_ID" + if [ -z "$run_id" ]; then + run_id=$(gh run list --repo defold/examples --workflow trigger-site-rebuild.yml --branch master --status success --limit 50 --json databaseId --jq '.[].databaseId' | while read -r candidate; do + if gh api "repos/defold/examples/actions/runs/$candidate/artifacts" --jq '.artifacts[] | select(.name == "examples-bundles" and .expired == false) | .id' | grep -q .; then echo "$candidate"; break; fi + done) + fi + test -n "$run_id" + gh run download "$run_id" --repo defold/examples --name examples-bundles --dir . + snapshot_sha=$(python -c 'import json, zipfile; print(json.loads(zipfile.ZipFile("examples-bundles.zip").read("manifest.json"))["examples_sha"])') + if [ -n "$EXAMPLES_REF" ] && [ "$EXAMPLES_REF" != "$snapshot_sha" ]; then + echo "Snapshot SHA does not match the dispatch" >&2; exit 1 + fi + if [ -f _data/examples_build.json ]; then + previous_sha=$(python -c 'import json; print(json.load(open("_data/examples_build.json"))["examples_sha"])') + status=$(gh api "repos/defold/examples/compare/$previous_sha...$snapshot_sha" --jq .status) + if [ "$status" != ahead ] && [ "$status" != identical ]; then + echo "Refusing to publish an older or divergent examples snapshot" >&2; exit 1 + fi + fi + - if: env.ACTION == 'examples' name: Update examples run: python update.py --download --examples-ref "$EXAMPLES_REF" examples diff --git a/_config.yml b/_config.yml index 249722132..7322d461f 100644 --- a/_config.yml +++ b/_config.yml @@ -30,6 +30,7 @@ exclude: # Downloaded inputs and tooling used to generate the site. - doc-master.zip - examples-master.zip + - examples-bundles.zip - codepad-master.zip - asset-portal-master.zip - games-showcase-master.zip diff --git a/scripts/example_bundle_validation.py b/scripts/example_bundle_validation.py new file mode 100644 index 000000000..32b7fa7af --- /dev/null +++ b/scripts/example_bundle_validation.py @@ -0,0 +1,73 @@ +"""Validate source revisions and complete Defold HTML5 bundle inputs.""" +import json +import re +import zipfile +from pathlib import Path, PurePosixPath + + +REQUIRED_ARCHIVES = {'game.projectc', 'game.arci', 'game.arcd', 'game.dmanifest'} +REQUIRED_BUNDLE_FILES = ('index.html', 'dmloader.js', 'Defoldexamples.wasm', 'Defoldexamples_wasm.js') + + +def validate_source_archive(archive_path, sha): + if not isinstance(sha, str) or not re.fullmatch(r'[0-9a-f]{40}', sha): + raise RuntimeError('Invalid examples revision in snapshot manifest') + expected_root = 'examples-' + sha + with zipfile.ZipFile(archive_path) as archive: + entries = archive.infolist() + if not entries: + raise RuntimeError('Examples source archive is empty') + for entry in entries: + path = PurePosixPath(entry.filename) + if path.is_absolute() or '..' in path.parts or not path.parts or path.parts[0] != expected_root: + raise RuntimeError('Examples source archive does not match snapshot revision {}'.format(sha)) + return expected_root + + +def validate_example_bundle(bundle_dir, example_path): + bundle_dir = Path(bundle_dir) + def fail(detail): + raise RuntimeError('Invalid bundle for {}: {}'.format(example_path, detail)) + for name in REQUIRED_BUNDLE_FILES: + file = bundle_dir / name + if not file.is_file() or file.stat().st_size == 0: + fail('missing or empty {}'.format(name)) + threaded = ('Defoldexamples_pthread.wasm', 'Defoldexamples_pthread_wasm.js') + if any((bundle_dir / name).exists() for name in threaded): + for name in threaded: + file = bundle_dir / name + if not file.is_file() or file.stat().st_size == 0: + fail('missing or empty {}'.format(name)) + archive_dir = bundle_dir / 'archive' + manifest_path = archive_dir / 'archive_files.json' + try: + manifest = json.loads(manifest_path.read_text(encoding='utf-8')) + content = manifest['content'] + if not isinstance(content, list): + raise ValueError('content must be an array') + names = set() + for item in content: + name, size, pieces = item['name'], item['size'], item['pieces'] + if not isinstance(name, str) or name in names or not isinstance(size, int) or isinstance(size, bool) or size < 0: + raise ValueError('invalid archive entry') + names.add(name) + if not isinstance(pieces, list) or not pieces: + raise ValueError('missing pieces for {}'.format(name)) + offset = 0 + for piece in pieces: + piece_name = piece['name'] + if not isinstance(piece_name, str): + raise ValueError('invalid piece name') + path = PurePosixPath(piece_name) + if path.is_absolute() or len(path.parts) != 1 or path.name in ('.', '..') or '\\' in piece_name: + raise ValueError('invalid archive piece path') + file = archive_dir / piece_name + if not file.is_file() or piece['offset'] != offset: + raise ValueError('missing or invalid piece {}'.format(piece_name)) + offset += file.stat().st_size + if offset != size: + raise ValueError('incorrect size for {}'.format(name)) + if not REQUIRED_ARCHIVES.issubset(names): + raise ValueError('missing required game archives: {}'.format(', '.join(sorted(REQUIRED_ARCHIVES - names)))) + except (OSError, ValueError, TypeError, KeyError) as error: + fail('invalid archive manifest or pieces ({})'.format(error)) diff --git a/tests/test_example_bundle_import.py b/tests/test_example_bundle_import.py new file mode 100644 index 000000000..47c299a11 --- /dev/null +++ b/tests/test_example_bundle_import.py @@ -0,0 +1,142 @@ +import ast +import json +import os +import re +import shutil +import tempfile +import unittest +import zipfile +from contextlib import contextmanager +from pathlib import Path + +from scripts.example_bundle_validation import validate_example_bundle, validate_source_archive + + +SHA_A = 'a' * 40 +SHA_B = 'b' * 40 + + +def bundle_files(): + entries = [] + result = {'index.html': b'index', 'dmloader.js': b'loader', + 'Defoldexamples.wasm': b'wasm', 'Defoldexamples_wasm.js': b'engine'} + for suffix in ('projectc', 'arci', 'arcd', 'dmanifest'): + data = ('data-' + suffix).encode() + name = 'game0.' + suffix + result['archive/' + name] = data + entries.append({'name': 'game.' + suffix, 'size': len(data), 'pieces': [{'name': name, 'offset': 0}]}) + result['archive/archive_files.json'] = json.dumps({'content': entries}).encode() + return result + + +class ExampleBundleImportTest(unittest.TestCase): + def setUp(self): + self.original = os.getcwd() + self.temp = tempfile.TemporaryDirectory() + os.chdir(self.temp.name) + for path in ('_includes/examples', '_data', 'examples/sprite/demo', 'examples/sprite/other'): + Path(path).mkdir(parents=True) + Path('_data/examplesindex.json').write_text('[]') + Path('_data/examples_build.json').write_text('{"examples_sha":"old"}') + Path('_includes/examples/keep.txt').write_text('unchanged') + for example in ('demo', 'other'): + Path('examples/sprite', example, 'keep.txt').write_text('unchanged') + self.make_source(SHA_A) + self.make_snapshot() + + def tearDown(self): + os.chdir(self.original) + self.temp.cleanup() + + def make_source(self, sha): + with zipfile.ZipFile('examples-master.zip', 'w') as archive: + for example in ('demo', 'other'): + archive.writestr('examples-{}/sprite/{}/example.md'.format(sha, example), '---\ntitle: Demo\n---\n') + + def make_snapshot(self, remove=None): + with zipfile.ZipFile('examples-bundles.zip', 'w') as archive: + archive.writestr('manifest.json', json.dumps({'examples_sha': SHA_A})) + for example in ('demo', 'other'): + for name, content in bundle_files().items(): + if (example, name) != remove: + archive.writestr('bundles/sprite/{}/{}'.format(example, name), content) + + def importer(self): + source = (Path(__file__).resolve().parents[1] / 'update.py').read_text() + tree = ast.parse(source) + function = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == 'process_examples') + @contextmanager + def tmpdir(): + with tempfile.TemporaryDirectory() as path: + yield path + def rmtree(path): + if os.path.isdir(path): shutil.rmtree(path) + def rmmkdir(path): + rmtree(path) + os.makedirs(path) + def unzip(path, destination): + with zipfile.ZipFile(path) as archive: archive.extractall(destination) + class Dedupe: + @staticmethod + def run(): pass + namespace = dict(os=os, Path=Path, json=json, zipfile=zipfile, shutil=shutil, re=re, + EXAMPLES_ZIP='examples-master.zip', tmpdir=tmpdir, rmtree=rmtree, rmmkdir=rmmkdir, + validate_source_archive=validate_source_archive, validate_example_bundle=validate_example_bundle, + unzip=unzip, read_as_json=lambda p: json.loads(Path(p).read_text()), + load_frontmatter=lambda p: {'title': 'Demo'}, find_example_scripts=lambda p: [], + resolve_example_scripts=lambda a, b: [], makedirs=lambda p: os.makedirs(p, exist_ok=True), + copy_example_scripts=lambda *a: None, replace_frontmatter=lambda *a: None, + read_as_string=lambda p: Path(p).read_text(), copy_documentation_images=lambda *a: None, + write_as_json=lambda p, d: Path(p).write_text(json.dumps(d)), + generate_llms_examples=lambda: None, dedupe_examples_wasm=Dedupe(), normalize_example_path=lambda p: p) + exec(compile(ast.Module(body=[function], type_ignores=[]), 'update.py', 'exec'), namespace) + return namespace['process_examples'] + + def assert_preserved(self): + self.assertEqual(Path('_includes/examples/keep.txt').read_text(), 'unchanged') + for example in ('demo', 'other'): + self.assertEqual(Path('examples/sprite', example, 'keep.txt').read_text(), 'unchanged') + self.assertEqual(json.loads(Path('_data/examples_build.json').read_text())['examples_sha'], 'old') + + def test_imports_bundle_matching_source_snapshot(self): + self.importer()() + self.assertEqual(Path('examples/sprite/demo/archive/game0.arcd').read_bytes(), b'data-arcd') + self.assertEqual(json.loads(Path('_data/examples_build.json').read_text())['examples_sha'], SHA_A) + + def test_mismatched_cached_source_does_not_change_existing_examples(self): + self.make_source(SHA_B) + with self.assertRaisesRegex(RuntimeError, 'does not match snapshot revision'): + self.importer()() + self.assert_preserved() + + def test_incomplete_bundle_does_not_change_any_examples(self): + for missing in ('dmloader.js', 'Defoldexamples.wasm', 'Defoldexamples_wasm.js', 'archive/game0.arcd', 'archive/archive_files.json'): + with self.subTest(missing=missing): + self.make_snapshot(remove=('other', missing)) + with self.assertRaisesRegex(RuntimeError, 'Invalid bundle for sprite/other'): + self.importer()() + self.assert_preserved() + + def test_split_archive_pieces_are_checked(self): + root = Path('split') + root.mkdir() + files = bundle_files() + manifest = json.loads(files['archive/archive_files.json']) + arcd = next(item for item in manifest['content'] if item['name'] == 'game.arcd') + data = files.pop('archive/game0.arcd') + files['archive/game0.arcd'] = data[:2] + files['archive/game1.arcd'] = data[2:] + arcd['pieces'] = [{'name': 'game0.arcd', 'offset': 0}, {'name': 'game1.arcd', 'offset': 2}] + files['archive/archive_files.json'] = json.dumps(manifest).encode() + for name, content in files.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + validate_example_bundle(root, 'sprite/demo') + (root / 'archive/game1.arcd').unlink() + with self.assertRaisesRegex(RuntimeError, 'game1.arcd'): + validate_example_bundle(root, 'sprite/demo') + + +if __name__ == '__main__': + unittest.main() diff --git a/update.py b/update.py index 352ff545c..414a29f9a 100755 --- a/update.py +++ b/update.py @@ -19,6 +19,7 @@ from llms import LLMS_DIR, generate_llms_manuals, generate_llms_apis, generate_llms_examples, path_to_manuals_anchor from utils import list_files, read_as_json, read_as_string, rmtree, write_as_string from scripts import dedupe_examples_wasm +from scripts.example_bundle_validation import validate_example_bundle, validate_source_archive from scripts.example_images import copy_documentation_images from scripts.frontpage_images import regenerate_game_images from argparse import ArgumentParser @@ -990,38 +991,50 @@ def process_extension(extension_name, download = False): def process_examples(download = False, examples_ref = "master", changed_examples = None): - examples_sha1 = get_sha1(EXAMPLES_DEFOLD_CHANNEL) + snapshot_path = os.environ.get("EXAMPLES_SNAPSHOT", "examples-bundles.zip") + if not os.path.isfile(snapshot_path): + raise RuntimeError("Examples bundle snapshot is required") + with zipfile.ZipFile(snapshot_path) as snapshot: + manifest = json.loads(snapshot.read("manifest.json")) + examples_ref = manifest["examples_sha"] if download: if os.path.exists(EXAMPLES_ZIP): os.remove(EXAMPLES_ZIP) download_file("https://github.com/defold/examples/archive/{}.zip".format(examples_ref), ".", EXAMPLES_ZIP) - download_bob(examples_sha1) - if not os.path.exists(EXAMPLES_ZIP): - print("File {} does not exist".format(EXAMPLES_ZIP)) - sys.exit(1) - - bob_jar = get_bob_filename(examples_sha1) - if not os.path.exists(bob_jar): - print("File {} does not exist".format(bob_jar)) - sys.exit(1) - - rebuild = True + raise RuntimeError("Examples source archive is missing") + source_root = validate_source_archive(EXAMPLES_ZIP, examples_ref) includes_dir = os.path.join("_includes", "examples") - rmmkdir(includes_dir) print("Processing examples") with tmpdir() as tmp_dir: shutil.copyfile(EXAMPLES_ZIP, os.path.join(tmp_dir, EXAMPLES_ZIP)) unzip(os.path.join(tmp_dir, EXAMPLES_ZIP), tmp_dir) - unzipped_examples_dir = find_unzipped_repo_dir(tmp_dir, "examples-") - if not unzipped_examples_dir: - print("Unable to find unzipped examples directory") - sys.exit(1) - + unzipped_examples_dir = os.path.join(tmp_dir, source_root) + if not os.path.isdir(unzipped_examples_dir): + raise RuntimeError("Examples source root is missing") + + with zipfile.ZipFile(snapshot_path) as snapshot: + for entry in snapshot.infolist(): + path = Path(entry.filename) + if path.is_absolute() or ".." in path.parts: + raise RuntimeError("Invalid bundle snapshot path") + snapshot.extractall(os.path.join(tmp_dir, "snapshot")) + bundles = Path(tmp_dir) / "snapshot" / "bundles" + # Reject an incomplete snapshot before modifying any existing examples. + for category in os.listdir(unzipped_examples_dir): + category_src_dir = os.path.join(unzipped_examples_dir, category) + if not os.path.isdir(category_src_dir) or category == ".github": + continue + for example in os.listdir(category_src_dir): + example_src_dir = os.path.join(category_src_dir, example) + if os.path.isdir(example_src_dir): + validate_example_bundle(bundles / category / example, "{}/{}".format(category, example)) + rmmkdir(includes_dir) data_index_file = os.path.join("_data", "examplesindex.json") + old_examples = read_as_json(data_index_file) if os.path.exists(data_index_file) else [] examplesindex = [] category_dirs = os.listdir(unzipped_examples_dir) for category in category_dirs: @@ -1030,9 +1043,6 @@ def process_examples(download = False, examples_ref = "master", changed_examples if os.path.isfile(category_src_dir) or category == ".github": continue - if rebuild: - rmmkdir(category_dst_dir) - for example in os.listdir(category_src_dir): example_src_dir = os.path.join(category_src_dir, example) example_dst_dir = os.path.join(category_dst_dir, example) @@ -1051,29 +1061,11 @@ def process_examples(download = False, examples_ref = "master", changed_examples sys.exit(1) print("..processing %s" % example) - if rebuild: - print("...building %s" % example) - bob_out = os.path.join(example_src_dir, bob_jar) - shutil.copyfile(bob_jar, bob_out) - game_project = os.path.join(example_src_dir, "game.project") - replace_in_file(game_project, r"title = .*", r"title = Defold-examples") - subprocess.run([ "java", "-jar", bob_out, "--archive", "--platform", "wasm-web", "--architectures", "wasm-web", "--variant", "debug", "--build-server", EXAMPLES_BUILD_SERVER, "resolve", "build", "bundle" ], cwd=example_src_dir, check=True) - os.remove(bob_out) - - print("...copying %s" % example) - bundle_index_file = find_file(os.path.join(example_src_dir, "build", "default"), "index.html") - if not bundle_index_file: - print("File index.html not found") - sys.exit(1) - print(bundle_index_file, os.path.dirname(bundle_index_file)) - - bundle_dir = os.path.dirname(bundle_index_file) - rmtree(example_dst_dir) - makedirs(category_dst_dir) - shutil.copytree(bundle_dir, example_dst_dir) - os.remove(os.path.join(example_dst_dir, "index.html")) - else: - os.makedirs(example_dst_dir, exist_ok=True) + bundle_dir = bundles / category / example + rmtree(example_dst_dir) + makedirs(category_dst_dir) + shutil.copytree(bundle_dir, example_dst_dir) + os.remove(os.path.join(example_dst_dir, "index.html")) print("...parsing example.md") fm["category"] = category @@ -1111,6 +1103,13 @@ def process_examples(download = False, examples_ref = "master", changed_examples print("...copying documentation images") copy_documentation_images(example_src_dir, example_dst_dir, read_as_string(md_file)) + current = {entry["path"] for entry in examplesindex} + for entry in old_examples: + path = normalize_example_path(entry.get("path", "")) + if path and path not in current: + rmtree(os.path.join("examples", path)) + write_as_json(os.path.join("_data", "examples_build.json"), manifest) + print("...generating index") if os.path.exists(data_index_file): os.remove(data_index_file)