diff --git a/.github/workflows/e2b-template.yml b/.github/workflows/e2b-template.yml index 692a11cbc..6537b838a 100644 --- a/.github/workflows/e2b-template.yml +++ b/.github/workflows/e2b-template.yml @@ -21,6 +21,13 @@ on: default: creative required: true + compute_profile: + description: Template CPU and memory allocation + type: choice + options: [2c2g, 2c8g, 4c8g] + default: 2c2g + required: true + resume_build: description: Existing unsubmitted templateID:build_UUID after an upload failure; requires ref type: string @@ -181,6 +188,7 @@ jobs: env: RESUME_BUILD: ${{ inputs.resume_build }} SOURCE_COMMIT: ${{ steps.source.outputs.revision }} + COMPUTE_PROFILE: ${{ inputs.compute_profile }} E2B_API_KEY: ${{ secrets.E2B_API_KEY }} E2B_API_URL: ${{ vars.E2B_API_URL }} run: | @@ -191,9 +199,16 @@ jobs: printf '%s' "$E2B_API_KEY" > "$key" unset E2B_API_KEY mkdir -p "$RUNNER_TEMP/e2b-result" + compute_args=() + case "$COMPUTE_PROFILE" in + 2c2g) ;; + 2c8g|4c8g) compute_args=(--compute-profile "$COMPUTE_PROFILE") ;; + *) echo '::error::Unknown compute profile.'; exit 1 ;; + esac "$RUNNER_TEMP/e2b-sdk/bin/python" - source/services/core/deploy/e2b/build-template.py \ --image "$(cat "$HOME/.oac/build/e2b-runtime/runtime-image.id")" \ --name "sandbase-oac-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" \ + "${compute_args[@]}" \ --api-key-file "$key" --output "$RUNNER_TEMP/e2b-result/template.json" <<'PY' import hashlib, json, os, pathlib, runpy, sys from types import SimpleNamespace @@ -252,7 +267,22 @@ jobs: result = original_request(client, **kwargs) receipt = {'template': result.template_id + ':' + result.build_id, 'source_commit': os.environ['SOURCE_COMMIT'], - 'api_url': os.environ['E2B_API_URL'], 'resumed': bool(selector)} + 'api_url': os.environ['E2B_API_URL'], 'resumed': bool(selector), + 'resources_verified': False} + (output / 'build-request.json').write_text(json.dumps(receipt, indent=2) + '\n') + response = client.get_httpx_client().get(f'/templates/{result.template_id}') + response.raise_for_status() + detail = response.json() + builds = [build for build in detail.get('builds', []) + if build.get('buildID') == result.build_id] + if (detail.get('templateID') != result.template_id or len(builds) != 1 + or type(builds[0].get('cpuCount')) is not int + or type(builds[0].get('memoryMB')) is not int + or builds[0]['cpuCount'] != kwargs['cpu_count'] + or builds[0]['memoryMB'] != kwargs['memory_mb']): + raise RuntimeError('Template build CPU/memory does not match the requested compute profile') + receipt.update(cpu_count=builds[0]['cpuCount'], memory_mb=builds[0]['memoryMB'], + resources_verified=True) (output / 'build-request.json').write_text(json.dumps(receipt, indent=2) + '\n') return result sdk.request_build = request @@ -271,12 +301,18 @@ jobs: import json, os, pathlib report = pathlib.Path(os.environ['RUNNER_TEMP']) / 'e2b-result/template.json' value = json.loads(report.read_text()) + receipt = json.loads(report.with_name('build-request.json').read_text()) + if (receipt.get('resources_verified') is not True + or receipt.get('template') != value['template']): + raise RuntimeError('Build report does not match a verified allocation receipt') + value.update(cpu_count=receipt['cpu_count'], memory_mb=receipt['memory_mb']) value.update(source_commit=os.environ['SOURCE_COMMIT'], source_branch=os.environ['SOURCE_BRANCH'], api_url=os.environ['E2B_API_URL'], harnesses=['codex', 'claude_sdk', 'mcode'], qualification='image identity and native package checks; no live Session/Turn') report.write_text(json.dumps(value, indent=2) + '\n') with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out: out.write(f"Source: `{value['source_commit']}`\n\nTemplate: `{value['template']}`\n\nImage: `{value['image']}`\n\n") + out.write(f"Compute: `{value['cpu_count']} vCPU / {value['memory_mb']} MiB`\n\n") out.write('One combined Runtime template built. No Core configuration or production deployment was changed.\n') PY - uses: actions/upload-artifact@v6 diff --git a/services/core/deploy/e2b/README.md b/services/core/deploy/e2b/README.md index 5e2e3c31c..99d05c3be 100644 --- a/services/core/deploy/e2b/README.md +++ b/services/core/deploy/e2b/README.md @@ -21,9 +21,11 @@ python -m venv "$HOME/.oac/build/e2b-sdk" --output "$HOME/.oac/build/e2b-template.json" ``` -[`build-template.py`](build-template.py) requires a `sha256:` image ID, a Linux amd64 image and the Runtime layout (`OAC_RUNTIME_WORKSPACE=/environment/workspace`). It copies the image's `/usr/local/bin`, `/opt` and, when present, `/usr/local/codex-resources` and `/etc/codex`, together with its `HOME` and `OAC_*` environment, onto a digest-pinned `node:22.23.1-bookworm-slim` base with `ca-certificates`, `bash`, `git`, `python3`, `python3-pip`, `ripgrep` and `util-linux`. It installs [`init.py`](init.py) and [`managed_init.py`](managed_init.py) read-only under `/opt/oac-e2b`, runs the daemon as UID/GID 1000 (`runtime`, home `/home/runtime`) and builds with 2 vCPUs and 2048 MiB. Only the `usr`, `usr/local` and `etc` archive ancestors it creates get traversable modes; Runtime file modes, private build contexts, key inputs and the output's umask stay unchanged. +[`build-template.py`](build-template.py) requires a `sha256:` image ID, a Linux amd64 image and the Runtime layout (`OAC_RUNTIME_WORKSPACE=/environment/workspace`). It copies the image's `/usr/local/bin`, `/opt` and, when present, `/usr/local/codex-resources` and `/etc/codex`, together with its `HOME` and `OAC_*` environment, onto a digest-pinned `node:22.23.1-bookworm-slim` base with `ca-certificates`, `bash`, `git`, `python3`, `python3-pip`, `ripgrep` and `util-linux`. It installs [`init.py`](init.py) and [`managed_init.py`](managed_init.py) read-only under `/opt/oac-e2b`, runs the daemon as UID/GID 1000 (`runtime`, home `/home/runtime`) and defaults to 2 vCPUs and 2048 MiB. Select `--compute-profile 2c8g` for 2 vCPUs and 8192 MiB or `--compute-profile 4c8g` for 4 vCPUs and 8192 MiB; only these profiles and the default `2c2g` are accepted. Only the `usr`, `usr/local` and `etc` archive ancestors it creates get traversable modes; Runtime file modes, private build contexts, key inputs and the output's umask stay unchanged. -The output file records `template` (the immutable `templateID:build_UUID`), the source `image`, the packaged `runtime_sha256` and the `base`. Use that exact `template` value. The template must qualify every Harness its image advertises. Install system dependencies into the image; the daemon runs as UID/GID 1000. No E2B account key, executor key or model credential belongs in a build, template environment, metadata, command argument or log. +The output file records `template` (the immutable `templateID:build_UUID`), the source `image`, the packaged `runtime_sha256`, the `base`, `compute_profile`, `cpu_count` and `memory_mb`. Use that exact `template` value. The template must qualify every Harness its image advertises. Install system dependencies into the image; the daemon runs as UID/GID 1000. No E2B account key, executor key or model credential belongs in a build, template environment, metadata, command argument or log. + +The `e2b-template-build` workflow exposes the same `compute_profile` choices and keeps `2c2g` as its default. The default omits the optional builder flag, allowing an exact historical source revision to retain its default allocation; 8 GiB selections require a source revision supporting `--compute-profile` and never fall back. It verifies the registered build’s CPU and memory before uploading, including when `resume_build` is supplied; missing or mismatched resource metadata stops the build. The build receipt records the verified allocation, and the final report takes its CPU and memory from that receipt after checking the immutable template identity. Building a separate 8 GiB template does not change the deployment’s default template. ## Launch an application-managed Runtime diff --git a/services/core/deploy/e2b/build-template.py b/services/core/deploy/e2b/build-template.py index 7d7be1af8..985c628d1 100644 --- a/services/core/deploy/e2b/build-template.py +++ b/services/core/deploy/e2b/build-template.py @@ -12,12 +12,15 @@ from e2b import Template BASE = 'node:22.23.1-bookworm-slim@sha256:8607a9064d4a571140998ae9e52a3b3fcf9cff361d04642d5971e6cd76d39e27' +COMPUTE_PROFILES = {'2c2g': (2, 2048), '2c8g': (2, 8192), '4c8g': (4, 8192)} parser = argparse.ArgumentParser() parser.add_argument('--image', required=True, help='Qualified linux/amd64 Runtime image digest') parser.add_argument('--name', required=True) parser.add_argument('--api-key-file', type=Path, required=True) parser.add_argument('--output', type=Path, required=True) +parser.add_argument('--compute-profile', choices=COMPUTE_PROFILES, default='2c2g') args = parser.parse_args() +cpu_count, memory_mb = COMPUTE_PROFILES[args.compute_profile] if not args.image.startswith('sha256:') or not args.output.is_absolute() or not args.api_key_file.is_absolute(): parser.error('Use an image digest and absolute private key/output paths') image = json.loads(subprocess.check_output(['docker', 'image', 'inspect', args.image]))[0] @@ -88,12 +91,13 @@ # Reinstall their system dependencies in E2B using the image's own installer. if creative_tools: template = template.run_cmd('sh /opt/oac-creative/install.sh --system-only', user='root') - result = Template.build(template, name=args.name, cpu_count=2, memory_mb=2048, + result = Template.build(template, name=args.name, cpu_count=cpu_count, memory_mb=memory_mb, on_build_logs=lambda entry: print(entry.message, flush=True), api_key=args.api_key_file.read_text().strip()) report = {'template': result.template_id + ':' + result.build_id, 'image': image['Id'], 'runtime_sha256': hashlib.sha256(bundle.read_bytes()).hexdigest(), 'base': BASE, - 'creative_tools': creative_tools} + 'creative_tools': creative_tools, 'compute_profile': args.compute_profile, + 'cpu_count': cpu_count, 'memory_mb': memory_mb} args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, indent=2) + '\n') print(json.dumps(report)) diff --git a/services/core/deploy/e2b/build_template_test.py b/services/core/deploy/e2b/build_template_test.py index 91e387419..5a3849e48 100644 --- a/services/core/deploy/e2b/build_template_test.py +++ b/services/core/deploy/e2b/build_template_test.py @@ -1,4 +1,6 @@ """Exercise real Runtime tar metadata without Docker or provider calls.""" +import argparse +import ast import contextlib import io import itertools @@ -7,9 +9,11 @@ from pathlib import Path import runpy import stat +import subprocess import sys import tarfile import tempfile +import textwrap from types import SimpleNamespace import unittest from unittest.mock import Mock, patch @@ -17,7 +21,8 @@ class BundlePermissionsTest(unittest.TestCase): def test_public_parents_and_runtime_modes_under_both_umasks(self): - for mask, creative in itertools.product((0o022, 0o077), (False, True)): + for mask, creative, profile in itertools.product((0o022, 0o077), (False, True), (None, '2c8g', '4c8g')): + expected_cpu, expected_memory = (4 if profile == '4c8g' else 2), (8192 if profile else 2048) with self.subTest(umask=oct(mask), creative=creative), tempfile.TemporaryDirectory() as temporary: root = Path(temporary) key = root / 'key' @@ -65,6 +70,7 @@ def run(argv, **kwargs): def build(instance, **kwargs): self.assertIs(instance, template) + self.assertEqual((kwargs['cpu_count'], kwargs['memory_mb']), (expected_cpu, expected_memory)) context = Path(factory.call_args.kwargs['file_context_path']) self.assertEqual(stat.S_IMODE(context.stat().st_mode), 0o700) with tarfile.open(context / 'runtime.tar.gz') as archive: @@ -93,7 +99,8 @@ def build(instance, **kwargs): with patch.dict(sys.modules, {'e2b': SimpleNamespace(Template=factory)}), \ patch.dict(os.environ, {'OAC_DEV_HOME': str(root / 'state')}), \ patch.object(sys, 'argv', ['build-template.py', '--image', 'sha256:fixture', - '--name', 'fixture', '--api-key-file', str(key), '--output', str(output)]), \ + '--name', 'fixture', '--api-key-file', str(key), '--output', str(output)] + + (['--compute-profile', profile] if profile else [])), \ patch('subprocess.check_output', side_effect=check_output), \ patch('subprocess.run', side_effect=run), contextlib.redirect_stdout(io.StringIO()): runpy.run_path(str(Path(__file__).with_name('build-template.py')), run_name='__main__') @@ -103,9 +110,108 @@ def build(instance, **kwargs): self.assertEqual(key.read_text(), 'fixture-only') self.assertEqual(json.loads(output.read_text())['creative_tools'], creative) factory.build.assert_called_once() + report = json.loads(output.read_text()) + self.assertEqual((report['cpu_count'], report['memory_mb']), (expected_cpu, expected_memory)) + self.assertEqual(report['compute_profile'], profile or '2c2g') finally: os.umask(original_mask) +class ComputeProfileTest(unittest.TestCase): + def test_invalid_profile_fails_before_docker_or_provider(self): + for profile in ('', '4c2g', '8c8g', '4c8g;command'): + factory = Mock() + with self.subTest(profile=profile), patch.dict(sys.modules, {'e2b': SimpleNamespace(Template=factory)}), \ + patch.object(sys, 'argv', ['build-template.py', '--image', 'sha256:fixture', + '--name', 'fixture', '--api-key-file', '/private/key', '--output', '/private/result', + '--compute-profile', profile]), patch('subprocess.check_output') as docker, \ + contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as error: + runpy.run_path(str(Path(__file__).with_name('build-template.py')), run_name='__main__') + self.assertEqual(error.exception.code, 2) + docker.assert_not_called() + factory.assert_not_called() + + def test_workflow_checks_fresh_and_resumed_build_resources(self): + workflow = Path(__file__).resolve().parents[4] / '.github/workflows/e2b-template.yml' + source = workflow.read_text() + start = source.index(' def request(client, **kwargs):') + end = source.index(' sdk.request_build = request', start) + function = compile(ast.parse(textwrap.dedent(source[start:end])), str(workflow), 'exec') + cases = ( + ('valid', [{'buildID': 'build', 'cpuCount': 4, 'memoryMB': 8192}]), + ('wrong-resource', [{'buildID': 'build', 'cpuCount': 2, 'memoryMB': 2048}]), + ('missing-memory', [{'buildID': 'build', 'cpuCount': 4}]), + ('non-integer', [{'buildID': 'build', 'cpuCount': True, 'memoryMB': 8192}]), + ('missing-build', []), + ('different-build', [{'buildID': 'other', 'cpuCount': 4, 'memoryMB': 8192}]), + ('duplicate', [{'buildID': 'build', 'cpuCount': 4, 'memoryMB': 8192}] * 2), + ('default', [{'buildID': 'build', 'cpuCount': 2, 'memoryMB': 2048}]), + ) + for resumed, (name, builds) in itertools.product((False, True), cases): + with self.subTest(resumed=resumed, case=name), tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + client = Mock() + status = {'templateID': 'template', 'buildID': 'build', 'status': 'waiting', 'logs': []} + detail = {'templateID': 'template', 'builds': builds} + responses = [Mock(json=Mock(return_value=detail))] + if resumed: + responses.insert(0, Mock(json=Mock(return_value=status))) + client.get_httpx_client.return_value.get.side_effect = responses + original = Mock(return_value=SimpleNamespace(template_id='template', build_id='build')) + namespace = dict(os=os, json=json, SimpleNamespace=SimpleNamespace, + original_request=original, output=output) + exec(function, namespace) + resources = (2, 2048) if name == 'default' else (4, 8192) + with patch.dict(os.environ, {'RESUME_BUILD': 'template:build' if resumed else '', + 'SOURCE_COMMIT': 'fixture', 'E2B_API_URL': 'https://fixture.invalid'}): + if name in ('valid', 'default'): + namespace['request'](client, cpu_count=resources[0], memory_mb=resources[1]) + receipt = json.loads((output / 'build-request.json').read_text()) + self.assertEqual((receipt['cpu_count'], receipt['memory_mb']), resources) + else: + with self.assertRaisesRegex(RuntimeError, 'CPU/memory'): + namespace['request'](client, cpu_count=4, memory_mb=8192) + self.assertFalse(json.loads((output / 'build-request.json').read_text())['resources_verified']) + self.assertEqual(original.call_count, 0 if resumed else 1) + + def test_default_omits_optional_flag_for_historical_source_parser(self): + workflow = Path(__file__).resolve().parents[4] / '.github/workflows/e2b-template.yml' + source = workflow.read_text() + start = source.index(' compute_args=()') + end = source.index(' esac', start) + len(' esac') + script = textwrap.dedent(source[start:end]) + '\nif (( ${#compute_args[@]} )); then printf "%s\\0" "${compute_args[@]}"; fi' + for profile in ('2c2g', '2c8g', '4c8g'): + result = subprocess.run(['bash', '-c', script], env={**os.environ, 'COMPUTE_PROFILE': profile}, + capture_output=True, check=True) + args = result.stdout.decode().split('\0')[:-1] + old_parser = argparse.ArgumentParser() + if profile == '2c2g': + self.assertEqual(vars(old_parser.parse_args(args)), {}) + else: + self.assertEqual(args, ['--compute-profile', profile]) + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + old_parser.parse_args(args) + + def test_report_uses_verified_receipt_for_old_builder_and_rejects_mismatch(self): + workflow = Path(__file__).resolve().parents[4] / '.github/workflows/e2b-template.yml' + source = workflow.read_text() + start = source.index(" value = json.loads(report.read_text())") + end = source.index(' value.update(source_commit=', start) + code = compile(textwrap.dedent(source[start:end]), str(workflow), 'exec') + for verified, template in itertools.product((True, False), ('template:build', 'template:other')): + with self.subTest(verified=verified, template=template), tempfile.TemporaryDirectory() as temporary: + report = Path(temporary) / 'template.json' + report.write_text(json.dumps({'template': 'template:build', 'image': 'fixture'})) + report.with_name('build-request.json').write_text(json.dumps({ + 'template': template, 'resources_verified': verified, 'cpu_count': 2, 'memory_mb': 2048})) + namespace = {'json': json, 'report': report} + if verified and template == 'template:build': + exec(code, namespace) + self.assertEqual((namespace['value']['cpu_count'], namespace['value']['memory_mb']), (2, 2048)) + else: + with self.assertRaisesRegex(RuntimeError, 'verified allocation receipt'): + exec(code, namespace) + + if __name__ == '__main__': unittest.main()