diff --git a/README.md b/README.md index be8bdeb..3831e75 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ uvx abench --help uvx abench experiments.yaml ``` -For a specific release use `uvx abench@0.1.1 experiments.yaml`; use +For a specific release use `uvx abench@0.1.2 experiments.yaml`; use `uvx abench@latest` to refresh to the latest release. Docker and model data must still be available locally. macOS and Linux hosts are supported. @@ -414,6 +414,12 @@ persistent host cache defaults to `~/.cache/abench/flows`. Change it with writes with `--no-reuse-flows` (`reuse_flows: false`). `--cache-from` remains an explicit seed option with its existing stricter dependency checks. +Containers set `NUMBA_CACHE_DIR` to `/results/cache/flows/.numba`, a writable +directory included in those snapshots. This covers installed-library helpers +(such as `sharrow.maths`) as well as generated flows without writing into +root-owned site-packages or relying on the container user's home directory. +Warmup may recompile entries from older snapshots that used Numba's default paths. + Compatibility uses the **installed** Sharrow, Numba, llvmlite, and NumPy versions, plus their source repository/commit identities when applicable, Python version, and container architecture/CPU features. ActivitySim and model settings are diff --git a/src/abench/__init__.py b/src/abench/__init__.py index 430437a..eec3576 100644 --- a/src/abench/__init__.py +++ b/src/abench/__init__.py @@ -1,3 +1,3 @@ """Reproducible ActivitySim experiments in Linux containers.""" -__version__ = "0.1.1" +__version__ = "0.1.2" diff --git a/src/abench/cli.py b/src/abench/cli.py index e68ed20..c2abda0 100644 --- a/src/abench/cli.py +++ b/src/abench/cli.py @@ -187,6 +187,12 @@ def container_phase(spec, output, data, image, phase_name): # so outputs and caches remain writable between attempts and experiments. "--user", f"{os.getuid()}:{os.getgid()}", + # Installed packages and the default home are not writable by this UID. + # Configure Numba before Python starts, including all spawned workers. + # Keep compiled artifacts in the shared flow snapshot so warmup work is + # reusable across measured attempts and compatible experiments. + "--env", + "NUMBA_CACHE_DIR=/results/cache/flows/.numba", "--env", f"BENCH_SPEC_PATH=/results/{phase_name}/{snapshot_name}", "--env", diff --git a/src/abench/runtime/worker.py b/src/abench/runtime/worker.py index 16acb5e..fb1d720 100644 --- a/src/abench/runtime/worker.py +++ b/src/abench/runtime/worker.py @@ -5,6 +5,7 @@ import os import subprocess import sys +import tempfile import time from pathlib import Path @@ -255,6 +256,14 @@ def supervise(spec, phase): (phase / "output").mkdir() for name in ("flows", "model"): Path(f"/results/cache/{name}").mkdir(parents=True, exist_ok=True) + # Fail before running models if the configured compilation cache is unusable. + numba_cache = Path(os.environ["NUMBA_CACHE_DIR"]) + try: + numba_cache.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryFile(dir=numba_cache): + pass + except OSError as error: + raise RuntimeError(f"Numba cache is not writable: {numba_cache}") from error env = dict(os.environ, BENCH_MODEL="1", BENCH_PHASE_DIR=str(phase)) env["BENCH_STRICT_CACHE"] = "0" env["BENCH_TRACK_CACHE"] = ( diff --git a/tests/fixtures/tiny/tiny_extension.py b/tests/fixtures/tiny/tiny_extension.py index c381b6a..6da19e5 100644 --- a/tests/fixtures/tiny/tiny_extension.py +++ b/tests/fixtures/tiny/tiny_extension.py @@ -28,6 +28,12 @@ def bench_initialize(state: workflow.State): @workflow.step def bench_compute(state: workflow.State, households: pd.DataFrame): """Exercise compiled overload reuse, including a measured-only test signature.""" + # Unlike generated modules, installed Sharrow helpers live in root-owned + # site-packages. Importing them must work as the unprivileged model user, + # including in spawned workers, and their compiled code must load on reuse. + from sharrow.maths import piece + + assert piece(3.0, 1.0, 4.0) == 2.0 sys.path.insert(0, state.settings.sharrow_cache_dir) twice = importlib.import_module("tiny_generated").twice households = households.copy() diff --git a/tests/test_docker.py b/tests/test_docker.py index 01cf5a5..e8cdcda 100644 --- a/tests/test_docker.py +++ b/tests/test_docker.py @@ -64,6 +64,7 @@ def test_tiny_model(tmp_path, multiprocess, sharrow, retry): assert cli.main(args) == 0 run = load_run(output) assert run["valid"] + assert list((output / "cache/flows/.numba").rglob("maths.piece-*.nbc")) attempts = run["spec"]["attempts"] assert len(attempts) == (2 if retry else 1) assert attempts[-1]["status"] == "accepted"