diff --git a/Makefile b/Makefile index 82e066b..0c822e9 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ export MIX_ENV := test export ORT_DYLIB_PATH ?= $(shell $(PYTHON) scripts/install_runtime.py path 2>/dev/null) .PHONY: help setup deps doctor test docs hooks fmt lint check +.PHONY: test.system test.package test.models test.tooling coverage bench fixtures package help: ## Show development commands @awk 'BEGIN {FS = ":.*## "} /^[a-zA-Z0-9_.-]+:.*## / {printf " %-22s %s\n", $$1, $$2}' $(MAKEFILE_LIST) @@ -43,3 +44,30 @@ lint: ## Check formatting, static analysis, and compiler warnings $(PYTHON) scripts/check.py --full check: lint test ## Run the ordinary local checks + +test.system: ## Run each system scenario in a fresh VM + $(PYTHON) scripts/scenario.py + +test.package: ## Test the assembled package in clean source and prebuilt consumers + $(PYTHON) scripts/package.py test --mode source + $(PYTHON) scripts/package.py test --mode prebuilt + +test.models: ## Check prepared model inputs against their reference outputs + $(PYTHON) scripts/fixtures.py + mix test test/models --include model + +test.tooling: ## Test the development harnesses + $(PYTHON) -m unittest discover -s test/tooling -p '*_test.py' -v + +coverage: ## Write scoped Elixir HTML and Markdown coverage reports + mix test --cover + mix run --no-start --no-compile scripts/coverage.exs + +bench: ## Measure the public inference workload (ad hoc) + mix run bench/inference.exs $(ARGS) + +fixtures: ## Regenerate model fixtures (requires the separate exporter environment) + $(PYTHON) fixtures/generate.py + +package: ## Assemble the local Hex archive + $(PYTHON) scripts/package.py build diff --git a/bench/inference.exs b/bench/inference.exs new file mode 100644 index 0000000..ca80268 --- /dev/null +++ b/bench/inference.exs @@ -0,0 +1,28 @@ +iterations = System.argv() |> List.first("2000") |> String.to_integer() +if iterations < 1, do: raise("iterations must be positive") +root = Path.expand("../test/fixtures", __DIR__) +cases = File.read!(Path.join(root, "xgboost.json")) |> :json.decode() +model = Ortex.load(Path.join(root, "xgboost.onnx"), [:cpu], 0) +input = Nx.tensor(cases["inputs"], type: :f32) +expected = List.flatten(cases["outputs"]) + +run = fn -> + {output} = Ortex.run(model, input) + actual = Nx.to_flat_list(output) + + unless length(actual) == length(expected) and + Enum.all?(Enum.zip(actual, expected), fn {a, b} -> abs(a - b) < 1.0e-6 end), + do: raise("benchmark reference mismatch") +end + +Enum.each(1..100, fn _ -> run.() end) +{elapsed, _} = :timer.tc(fn -> Enum.each(1..iterations, fn _ -> run.() end) end) + +IO.inspect(%{ + iterations: iterations, + calls_per_second: iterations * 1_000_000 / elapsed, + elixir: System.version(), + otp: System.otp_release(), + runtime: System.fetch_env!("ORT_DYLIB_PATH"), + workload: "synthetic XGBoost; public call including transfers; CPU; optimization disabled" +}) diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 0000000..455ed27 --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,46 @@ +# Testing + +`mix test` discovers library `*_test.exs` files normally under test/, including +doctests. Mix excludes system/, package/, support/, and fixtures/ from discovery. +Model ExUnit tests have `@moduletag :model` and are excluded by default; run +`make test.models` to select them explicitly. No selected suite silently skips +a missing fixture. + +`make test.system` invokes scripts/scenario.py. Each test/system/*.exs scenario +runs in its own BEAM after compilation. Nonzero exit (including a native shutdown +abort) fails. Timeout kills the process group on Unix and process tree on Windows. +The runner stores exit status and output in dist/reports/system.json. Additional +process-wide configuration scenarios should accompany the feature implementing +that configuration; the foundation tests main's existing behavior. + +`make test.package` assembles a Hex tarball, extracts its actual contents into a +temporary directory, and runs the separate test/package project against it. It +checks source compilation and an explicitly configured prebuilt library with +Cargo/rustc replaced by failing stubs. The runtime path comes from the caller's +ORT_DYLIB_PATH. This is a package installation check, not the feature stack's +automatic RustlerPrecompiled download path. That feature can reuse this harness. + +`make test.models` checks XGBoost, Iris, and MNIST. Prepared artifacts and reference +cases are recorded in test/fixtures/manifest.json. XGBoost and Iris results come +from the original estimators; MNIST uses Python ORT as a binding reference, with +synthetic pixels. None of these checks claim to validate ORT's operators or model +accuracy. Iris uses tensor probabilities on main; ZipMap needs the separate +container-output feature. Install fixtures/requirements.txt in a separate Python +3.12 environment before `make fixtures`. Generation validates exports with ONNX +and Python ORT; ordinary tests need neither Python exporters nor network access. + +`make test.tooling` checks the harnesses themselves, including partial staging, +filenames with shell characters, timeout handling, and archive path traversal. + +`make coverage` reports Elixir library lines using built-in OTP coverage. The +NIF interface is excluded; generated code should also be excluded when introduced. +The HTML and Markdown report are under cover/. Rust and fresh-VM scenarios are +outside that metric. Initially coverage is reported without a percentage gate. + +`make bench ARGS=5000` runs the ad hoc CPU XGBoost benchmark with reference checks +on every call. Timings include tensor transfer; they are observations for the +recorded environment, not pass/fail performance guarantees. + +The inherited ResNet, DistilBERT, and StableLM demos are manual examples. Their +export dependencies and weights are not part of normal checks. The original +tiny multi-input model remains for legacy API and dtype tests. diff --git a/examples/README.md b/examples/README.md index 8d75934..038c1a5 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,6 +1,10 @@ # Examples -Examples have an independent Mix project: from this directory, run `mix deps.get` +Small XGBoost, Iris, and MNIST examples run from the repository root via +`MIX_ENV=test mix run examples/run.exs NAME`, with ORT_DYLIB_PATH set. Their +prepared artifacts also power `make test.models`. + +The tokenizer examples have an independent Mix project: from this directory, run `mix deps.get` and `mix run distilbert/distilbert_classification.exs` (or another example). Set ORT_DYLIB_PATH to the separately installed runtime. Model paths are relative to this directory. The Python export dependencies are separate from the library. diff --git a/examples/iris/README.md b/examples/iris/README.md new file mode 100644 index 0000000..a8155a5 --- /dev/null +++ b/examples/iris/README.md @@ -0,0 +1,8 @@ +# Iris + +From the repository root, with ORT_DYLIB_PATH set: +`MIX_ENV=test mix run examples/run.exs iris`. + +StandardScaler and LogisticRegression produce class labels and a probability +matrix. `make test.models` compares both against scikit-learn. See +test/fixtures/NOTICE.md for attribution and dataset license information. diff --git a/examples/mnist/README.md b/examples/mnist/README.md new file mode 100644 index 0000000..0953217 --- /dev/null +++ b/examples/mnist/README.md @@ -0,0 +1,9 @@ +# MNIST + +From the repository root, with ORT_DYLIB_PATH set: +`MIX_ENV=test mix run examples/run.exs mnist`. + +A small trained convolutional model consumes deterministic synthetic pixels. +`make test.models` compares its numerical outputs against Python ORT; this checks +binding and tensor layout rather than classification accuracy. The exact model +revision, digest, and notices are under test/fixtures/. diff --git a/examples/resnet/README.md b/examples/resnet/README.md new file mode 100644 index 0000000..a0e576b --- /dev/null +++ b/examples/resnet/README.md @@ -0,0 +1,6 @@ +# ResNet (manual) + +This inherited exporter requires separately installed PyTorch and Torchvision, +downloads pretrained weights, and writes resnet50.onnx in its working directory. +It remains a manual demonstration pending pinned weights and export dependencies. +The ordinary model checks use the small prepared models documented in ../README.md. diff --git a/python/export_resnet.py b/examples/resnet/export.py similarity index 100% rename from python/export_resnet.py rename to examples/resnet/export.py diff --git a/examples/run.exs b/examples/run.exs new file mode 100644 index 0000000..09aecb5 --- /dev/null +++ b/examples/run.exs @@ -0,0 +1,6 @@ +name = List.first(System.argv(), "xgboost") +unless name in ~w(xgboost iris mnist), do: raise("choose xgboost, iris, or mnist") +root = Path.expand("../test/fixtures", __DIR__) +cases = File.read!(Path.join(root, name <> ".json")) |> :json.decode() +model = Ortex.load(Path.join(root, name <> ".onnx"), [:cpu], 0) +IO.inspect(Ortex.run(model, Nx.tensor(cases["inputs"], type: :f32)), label: name) diff --git a/examples/xgboost/README.md b/examples/xgboost/README.md new file mode 100644 index 0000000..c07658e --- /dev/null +++ b/examples/xgboost/README.md @@ -0,0 +1,8 @@ +# XGBoost + +From the repository root, with ORT_DYLIB_PATH set: +`MIX_ENV=test mix run examples/run.exs xgboost`. + +Eight shallow regression trees trained on deterministic synthetic data. Four +prepared rows are checked against the original estimator by `make test.models`. +See test/fixtures/manifest.json and fixtures/generate.py for provenance. diff --git a/fixtures/generate.py b/fixtures/generate.py new file mode 100644 index 0000000..eb831f6 --- /dev/null +++ b/fixtures/generate.py @@ -0,0 +1,135 @@ +"""Generate the small model suite and its reference cases with pinned exporters.""" + +import argparse +import hashlib +import json +import urllib.request +from pathlib import Path + +import numpy as np +import onnx +import onnxruntime as ort + +ROOT = Path(__file__).resolve().parents[1] +DIRECTORY = ROOT / "test/fixtures" +MNIST_REVISION = "1cd752f4a3d818fa215aa621c78b8a10a4a1a3a5" +MNIST_SHA = "5c688690f8bacf667d4c2074af5ad0646ca328d7ab03eccf944a65b320171bdd" + + +def xgboost(): + import onnxmltools + import xgboost as xgb + from onnxmltools.convert.common.data_types import FloatTensorType + + rng = np.random.default_rng(42) + x = rng.normal(size=(256, 3)).astype(np.float32) + y = (2 * (x[:, 0] > 0) - (x[:, 1] > 0.5) + x[:, 2] * 0.25).astype(np.float32) + model = xgb.XGBRegressor( + n_estimators=8, max_depth=3, learning_rate=0.25, n_jobs=1, random_state=42 + ) + model.fit(x, y) + converted = onnxmltools.convert_xgboost( + model, initial_types=[("features", FloatTensorType([None, 3]))], target_opset=15 + ) + rows = np.array([[0, 0, 0], [1, 1, 1], [-1, -1, -1], [0.25, 0.75, -0.5]], dtype=np.float32) + return ( + converted, + rows, + [model.predict(rows).reshape(-1, 1)], + { + "source": "deterministic synthetic training data; seed 42", + "license": "MIT", + "reference": "XGBoost predict, independently checked against Python ORT", + }, + ) + + +def iris(): + from skl2onnx import to_onnx + from sklearn.datasets import load_iris + from sklearn.linear_model import LogisticRegression + from sklearn.pipeline import make_pipeline + from sklearn.preprocessing import StandardScaler + + dataset = load_iris() + model = make_pipeline(StandardScaler(), LogisticRegression(random_state=42, max_iter=500)) + model.fit(dataset.data, dataset.target) + rows = dataset.data[[0, 50, 100, 149]].astype(np.float32) + converted = to_onnx(model, rows, options={id(model[-1]): {"zipmap": False}}, target_opset=15) + converted.ir_version = 8 + return ( + converted, + rows, + [model.predict(rows), model.predict_proba(rows)], + { + "source": "scikit-learn load_iris; StandardScaler + LogisticRegression", + "license": "CC-BY-4.0 (Iris dataset; see NOTICE.md)", + "reference": "scikit-learn predict and predict_proba", + }, + ) + + +def mnist(): + url = f"https://huggingface.co/onnxmodelzoo/mnist-12/resolve/{MNIST_REVISION}/mnist-12.onnx" + with urllib.request.urlopen(url, timeout=60) as response: + data = response.read() + if hashlib.sha256(data).hexdigest() != MNIST_SHA: + raise ValueError("MNIST download digest mismatch") + converted = onnx.load_model_from_string(data) + # A real trained network with synthetic pixels tests binding/layout/numerics, + # not classification accuracy or ORT operator correctness. + rows = np.random.default_rng(42).random((1, 1, 28, 28), dtype=np.float32) + return ( + converted, + rows, + None, + { + "source": url, + "license": "Apache-2.0 (model repository); upstream model documentation also records MIT", + "reference": "Python ONNX Runtime 1.24.4; synthetic pixels, seed 42", + }, + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("models", nargs="*", choices=["xgboost", "iris", "mnist"]) + args = parser.parse_args() + DIRECTORY.mkdir(parents=True, exist_ok=True) + manifest_path = DIRECTORY / "manifest.json" + manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else {} + for name in args.models or ["xgboost", "iris", "mnist"]: + model, inputs, expected, provenance = globals()[name]() + onnx.checker.check_model(model) + data = model.SerializeToString() + options = ort.SessionOptions() + options.intra_op_num_threads = 1 + session = ort.InferenceSession(data, options, providers=["CPUExecutionProvider"]) + actual = session.run(None, {session.get_inputs()[0].name: inputs}) + if expected is None: + expected = actual + for a, b in zip(actual, expected, strict=True): + np.testing.assert_allclose(a, b, atol=1e-5, rtol=1e-5) + (DIRECTORY / f"{name}.onnx").write_bytes(data) + cases = { + "inputs": inputs.tolist(), + "outputs": [output.tolist() for output in expected], + "types": [str(output.dtype) for output in actual], + } + case_data = (json.dumps(cases, indent=2) + "\n").encode() + (DIRECTORY / f"{name}.json").write_bytes(case_data) + manifest[name] = { + **provenance, + "sha256": hashlib.sha256(data).hexdigest(), + "cases_sha256": hashlib.sha256(case_data).hexdigest(), + "ir": model.ir_version, + "opsets": {x.domain: x.version for x in model.opset_import}, + "generator": "fixtures/generate.py", + "requirements": "fixtures/requirements.txt", + "atol": 1e-5, + } + manifest_path.write_text(json.dumps(manifest, sort_keys=True, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/python/multi_input.py b/fixtures/generate_tiny.py similarity index 100% rename from python/multi_input.py rename to fixtures/generate_tiny.py diff --git a/fixtures/requirements.txt b/fixtures/requirements.txt new file mode 100644 index 0000000..9221307 --- /dev/null +++ b/fixtures/requirements.txt @@ -0,0 +1,18 @@ +cloudpickle==3.1.2 +flatbuffers==25.12.19 +joblib==1.6.0 +ml_dtypes==0.6.0 +mpmath==1.3.0 +numpy==2.5.4 +onnx==1.20.1 +onnxmltools==1.16.0 +onnxruntime==1.24.4 +packaging==26.3 +protobuf==7.36.2 +scikit-learn==1.8.0 +scipy==1.18.1 +skl2onnx==1.20.0 +sympy==1.14.0 +threadpoolctl==3.7.0 +typing_extensions==4.16.0 +xgboost==3.2.0 diff --git a/mix.exs b/mix.exs index 32a9113..b86b807 100644 --- a/mix.exs +++ b/mix.exs @@ -7,6 +7,8 @@ defmodule Ortex.MixProject do version: "0.1.10", elixir: "~> 1.19", start_permanent: Mix.env() == :prod, + test_ignore_filters: [~r{test/(system|package|fixtures|support)/}], + test_coverage: [ignore_modules: [Ortex.Native], summary: [threshold: 0], export: "library"], deps: deps(), # Docs diff --git a/scripts/coverage.exs b/scripts/coverage.exs new file mode 100644 index 0000000..e553d97 --- /dev/null +++ b/scripts/coverage.exs @@ -0,0 +1,44 @@ +Mix.ensure_application!(:tools) +{:ok, _} = :cover.start() +:ok = :cover.import(~c"cover/library.coverdata") + +rows = + :cover.imported_modules() + |> Enum.reject(&(&1 == Ortex.Native)) + |> Enum.sort() + |> Enum.map(fn module -> + {:ok, lines} = :cover.analyse(module, :coverage, :line) + + # Match Mix's line metric: exclude generated line 0 and combine counters + # sharing a source line. Any covered counter makes that line covered. + unique = + Enum.reduce(lines, %{}, fn + {{_, 0}, _}, acc -> acc + {{_, line}, {hit, _}}, acc -> Map.update(acc, line, hit > 0, &(&1 or hit > 0)) + end) + + covered = Enum.count(unique, fn {_, hit} -> hit end) + missed = map_size(unique) - covered + {:ok, _} = :cover.analyse_to_file(module, ~c"cover/#{module}.html", [:html]) + {module, covered, missed} + end) + +if rows == [], do: raise("no library coverage data") + +percentage = fn hit, total -> + if total == 0, do: "n/a", else: :erlang.float_to_binary(hit / total * 100, decimals: 2) <> "%" +end + +covered = Enum.sum(for {_, hit, _} <- rows, do: hit) +total = Enum.sum(for {_, hit, missed} <- rows, do: hit + missed) + +report = + "# Elixir library coverage\n\nExcludes the NIF interface. Rust and fresh-VM scenarios are outside this report.\n\n" <> + "| Module | Covered | Executable | Coverage |\n|---|---:|---:|---:|\n" <> + Enum.map_join(rows, "\n", fn {module, hit, missed} -> + "| #{inspect(module)} | #{hit} | #{hit + missed} | #{percentage.(hit, hit + missed)} |" + end) <> + "\n| **Total** | #{covered} | #{total} | **#{percentage.(covered, total)}** |\n" + +File.write!("cover/summary.md", report) +IO.puts(report) diff --git a/scripts/fixtures.py b/scripts/fixtures.py new file mode 100644 index 0000000..ae0bd5f --- /dev/null +++ b/scripts/fixtures.py @@ -0,0 +1,22 @@ +"""Verify every prepared model and case file against its committed manifest.""" + +import hashlib +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + directory = ROOT / "test/fixtures" + manifest = json.loads((directory / "manifest.json").read_text()) + for name, metadata in manifest.items(): + for suffix, key in [("onnx", "sha256"), ("json", "cases_sha256")]: + path = directory / f"{name}.{suffix}" + if hashlib.sha256(path.read_bytes()).hexdigest() != metadata[key]: + raise SystemExit(f"Fixture digest mismatch: {path}") + print(f"Verified {len(manifest)} prepared model fixtures") + + +if __name__ == "__main__": + main() diff --git a/scripts/package.py b/scripts/package.py new file mode 100644 index 0000000..6512804 --- /dev/null +++ b/scripts/package.py @@ -0,0 +1,102 @@ +"""Assemble the Hex archive and test it in a clean consuming project.""" + +import argparse +import io +import os +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +from pathlib import Path + +from scenario import execute + +ROOT = Path(__file__).resolve().parents[1] + + +def version(): + return re.search(r'version: "([^"]+)"', (ROOT / "mix.exs").read_text())[1] + + +def build(): + destination = ROOT / "dist" / f"ortex-{version()}.tar" + destination.parent.mkdir(exist_ok=True) + subprocess.run(["mix", "hex.build", "--output", str(destination)], cwd=ROOT, check=True) + return destination + + +def unpack(archive, destination): + with tarfile.open(archive) as outer: + payload = outer.extractfile("contents.tar.gz").read() + with tarfile.open(fileobj=io.BytesIO(payload), mode="r:gz") as contents: + contents.extractall(destination, filter="data") + + +def test_package(archive, mode, nif, timeout): + with tempfile.TemporaryDirectory(prefix="ortex-package-") as temporary: + directory = Path(temporary) + package = directory / "ortex" + unpack(archive, package) + project = directory / "consumer" + shutil.copytree(ROOT / "test/package", project) + env = { + **os.environ, + "MIX_ENV": "prod", + "RUSTUP_TOOLCHAIN": "1.93.1", + "ORTEX_PACKAGE_PATH": str(package), + "ORTEX_PACKAGE_MODE": mode, + "ORTEX_FIXTURE_PATH": str(ROOT / "test/fixtures/xgboost.onnx"), + "ORTEX_CASES_PATH": str(ROOT / "test/fixtures/xgboost.json"), + } + if mode == "prebuilt": + binary = ( + Path(nif).resolve() + if nif + else ROOT / "priv/native" / ("libortex.dll" if os.name == "nt" else "libortex.so") + ) + native = package / "priv/native" + native.mkdir(parents=True, exist_ok=True) + shutil.copy2(binary, native / binary.name) + stubs = directory / "forbidden-compilers" + stubs.mkdir() + for name in ["cargo", "rustc"]: + stub = stubs / name + stub.write_text( + "#!/bin/sh\necho 'Compiler forbidden in prebuilt package check' >&2\nexit 99\n" + ) + stub.chmod(0o755) + (stubs / f"{name}.cmd").write_text( + "@echo Compiler forbidden in prebuilt package check\r\n@exit /b 99\r\n" + ) + env["PATH"] = str(stubs) + os.pathsep + env["PATH"] + for command in [["mix", "deps.get"], ["mix", "run", "smoke.exs"]]: + status, output = execute(command, timeout, cwd=project, env=env) + print(output, flush=True) + if status: + return status + return 0 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("build") + test = commands.add_parser("test") + test.add_argument("--archive", type=Path) + test.add_argument("--mode", choices=["source", "prebuilt"], default="source") + test.add_argument("--nif", help="prebuilt library; default: local priv/native library") + test.add_argument("--timeout", type=int, default=600) + args = parser.parse_args() + if args.command == "build": + print(build()) + return 0 + if args.timeout <= 0: + parser.error("timeout must be positive") + archive = args.archive or build() + return test_package(archive, args.mode, args.nif, args.timeout) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/scenario.py b/scripts/scenario.py new file mode 100644 index 0000000..2a87695 --- /dev/null +++ b/scripts/scenario.py @@ -0,0 +1,65 @@ +"""Run each system scenario in a fresh BEAM and require successful shutdown.""" + +import argparse +import json +import os +import signal +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def execute(command, timeout, cwd=ROOT, env=None): + process = subprocess.Popen( + command, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + start_new_session=os.name != "nt", + ) + try: + output, _ = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + if os.name == "nt": + subprocess.run(["taskkill", "/PID", str(process.pid), "/T", "/F"], capture_output=True) + else: + os.killpg(process.pid, signal.SIGKILL) + output, _ = process.communicate() + return 124, output + f"\nScenario exceeded {timeout}s\n" + return process.returncode, output + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("names", nargs="*", help="scenario stems; default: all") + parser.add_argument("--timeout", type=int, default=60) + args = parser.parse_args() + if args.timeout <= 0: + parser.error("timeout must be positive") + scenarios = {p.stem: p for p in sorted((ROOT / "test/system").glob("*.exs"))} + selected = args.names or list(scenarios) + if set(selected) - scenarios.keys(): + parser.error(f"unknown scenarios: {set(selected) - scenarios.keys()}") + env = {**os.environ, "MIX_ENV": "test"} + subprocess.run(["mix", "compile", "--warnings-as-errors"], cwd=ROOT, env=env, check=True) + results = [] + for name in selected: + status, output = execute( + ["mix", "run", "--no-compile", "--no-deps-check", "--no-start", str(scenarios[name])], + args.timeout, + env=env, + ) + print(f"{'PASS' if status == 0 else 'FAIL'} {name}\n{output}", flush=True) + results.append({"scenario": name, "exit_status": status, "output": output}) + directory = ROOT / "dist/reports" + directory.mkdir(parents=True, exist_ok=True) + (directory / "system.json").write_text(json.dumps(results, indent=2) + "\n") + return int(any(result["exit_status"] != 0 for result in results)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/fixtures/LICENSE-MNIST b/test/fixtures/LICENSE-MNIST new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/test/fixtures/LICENSE-MNIST @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/test/fixtures/NOTICE.md b/test/fixtures/NOTICE.md new file mode 100644 index 0000000..0bac09e --- /dev/null +++ b/test/fixtures/NOTICE.md @@ -0,0 +1,19 @@ +# Model fixture notices + +The XGBoost fixture is trained on synthetic data using fixtures/generate.py. +Its model and reference inputs are provided under this repository's MIT license. + +The Iris fixture adapts the dataset distributed with scikit-learn into a trained +StandardScaler / LogisticRegression pipeline. Attribution: Fisher, R. (1936). +Iris [Dataset]. UCI Machine Learning Repository. https://doi.org/10.24432/C56C76. +The UCI dataset is licensed under Creative Commons Attribution 4.0 International: +https://creativecommons.org/licenses/by/4.0/. Training, export, and selection of +reference rows are the modifications made here. + +The MNIST model is copied unchanged from onnxmodelzoo/mnist-12 at the revision +recorded in manifest.json. That model repository declares Apache-2.0; its license +text is in LICENSE-MNIST. The original ONNX Model Zoo MNIST documentation also +records MIT. We retain the Apache notice for the exact artifact source used here. +Reference pixels are synthetic; no MNIST image dataset is redistributed. + +These fixtures and notices are test inputs, excluded from the Hex package. diff --git a/test/fixtures/iris.json b/test/fixtures/iris.json new file mode 100644 index 0000000..6af9d86 --- /dev/null +++ b/test/fixtures/iris.json @@ -0,0 +1,62 @@ +{ + "inputs": [ + [ + 5.099999904632568, + 3.5, + 1.399999976158142, + 0.20000000298023224 + ], + [ + 7.0, + 3.200000047683716, + 4.699999809265137, + 1.399999976158142 + ], + [ + 6.300000190734863, + 3.299999952316284, + 6.0, + 2.5 + ], + [ + 5.900000095367432, + 3.0, + 5.099999904632568, + 1.7999999523162842 + ] + ], + "outputs": [ + [ + 0, + 1, + 2, + 2 + ], + [ + [ + 0.9847031709514064, + 0.015296767010838049, + 6.2037755696216e-08 + ], + [ + 0.0047141481533477005, + 0.8650593933406213, + 0.13022645850603093 + ], + [ + 1.4917856588872743e-05, + 0.006202664103992609, + 0.9937824180394186 + ], + [ + 0.003053286819346972, + 0.29364978735805086, + 0.7032969258226021 + ] + ] + ], + "types": [ + "int64", + "float32" + ] +} diff --git a/test/fixtures/iris.onnx b/test/fixtures/iris.onnx new file mode 100644 index 0000000..13a6a61 Binary files /dev/null and b/test/fixtures/iris.onnx differ diff --git a/test/fixtures/manifest.json b/test/fixtures/manifest.json new file mode 100644 index 0000000..a21b0bf --- /dev/null +++ b/test/fixtures/manifest.json @@ -0,0 +1,45 @@ +{ + "iris": { + "atol": 1e-05, + "cases_sha256": "f87eb73635c7a3a75898d3e8fd78693f42f38c026e0bad65422ae37745bce618", + "generator": "fixtures/generate.py", + "ir": 8, + "license": "CC-BY-4.0 (Iris dataset; see NOTICE.md)", + "opsets": { + "": 15, + "ai.onnx.ml": 1 + }, + "reference": "scikit-learn predict and predict_proba", + "requirements": "fixtures/requirements.txt", + "sha256": "6128c916198679d0ac1ad16dcb72325c2586893bbf7309baec7c0c8ce04db4fb", + "source": "scikit-learn load_iris; StandardScaler + LogisticRegression" + }, + "mnist": { + "atol": 1e-05, + "cases_sha256": "a3f8ef2472a13494a7cf022ff44cd3f1399d6607eccefffaff6efdda33bf70ba", + "generator": "fixtures/generate.py", + "ir": 7, + "license": "Apache-2.0 (model repository); upstream model documentation also records MIT", + "opsets": { + "": 12 + }, + "reference": "Python ONNX Runtime 1.24.4; synthetic pixels, seed 42", + "requirements": "fixtures/requirements.txt", + "sha256": "5c688690f8bacf667d4c2074af5ad0646ca328d7ab03eccf944a65b320171bdd", + "source": "https://huggingface.co/onnxmodelzoo/mnist-12/resolve/1cd752f4a3d818fa215aa621c78b8a10a4a1a3a5/mnist-12.onnx" + }, + "xgboost": { + "atol": 1e-05, + "cases_sha256": "f16d557b5816d9fdb253a0a0db7e7a9d4d6abf67ceb247dbe584a60f57e44f4a", + "generator": "fixtures/generate.py", + "ir": 8, + "license": "MIT", + "opsets": { + "ai.onnx.ml": 1 + }, + "reference": "XGBoost predict, independently checked against Python ORT", + "requirements": "fixtures/requirements.txt", + "sha256": "ab9d76d2fbc49faf4396cf1187978c33597ce680c077911e6cd0347c0d1a5ebf", + "source": "deterministic synthetic training data; seed 42" + } +} diff --git a/test/fixtures/mnist.json b/test/fixtures/mnist.json new file mode 100644 index 0000000..189082a --- /dev/null +++ b/test/fixtures/mnist.json @@ -0,0 +1,867 @@ +{ + "inputs": [ + [ + [ + [ + 0.08925092220306396, + 0.7739560008049011, + 0.6545714735984802, + 0.4388784170150757, + 0.43301522731781006, + 0.8585978746414185, + 0.08594560623168945, + 0.6973680257797241, + 0.20146948099136353, + 0.09417730569839478, + 0.5264789462089539, + 0.975622296333313, + 0.7357523441314697, + 0.7611396908760071, + 0.7174772620201111, + 0.7860642671585083, + 0.5132265686988831, + 0.12811362743377686, + 0.839748203754425, + 0.4503859281539917, + 0.5003519058227539, + 0.3707979917526245, + 0.1825495958328247, + 0.926764965057373, + 0.7815674543380737, + 0.6438651084899902, + 0.40241432189941406, + 0.822761595249176 + ], + [ + 0.5454291105270386, + 0.4434141516685486, + 0.4504595398902893, + 0.2272387146949768, + 0.09213590621948242, + 0.5545847415924072, + 0.8878898024559021, + 0.06381720304489136, + 0.8582912683486938, + 0.8276311159133911, + 0.27675968408584595, + 0.6316643953323364, + 0.16522902250289917, + 0.758087694644928, + 0.7005229592323303, + 0.35452592372894287, + 0.06791996955871582, + 0.9706979990005493, + 0.445687472820282, + 0.893121063709259, + 0.6779189705848694, + 0.7783834934234619, + 0.7598994374275208, + 0.1946386694908142, + 0.36390602588653564, + 0.4667209982872009, + 0.4977915287017822, + 0.04380375146865845 + ], + [ + 0.5465694665908813, + 0.15428948402404785, + 0.7433758974075317, + 0.6830489039421082, + 0.9225277900695801, + 0.744762122631073, + 0.36664265394210815, + 0.9675096869468689, + 0.4108503460884094, + 0.3258253335952759, + 0.9055357575416565, + 0.3704596757888794, + 0.07634317874908447, + 0.4695557951927185, + 0.7956799268722534, + 0.1894713044166565, + 0.462940514087677, + 0.12992149591445923, + 0.6864781975746155, + 0.4757049083709717, + 0.3301963210105896, + 0.226909339427948, + 0.5644998550415039, + 0.6698139905929565, + 0.9403060674667358, + 0.4371519088745117, + 0.16068816184997559, + 0.8326781392097473 + ], + [ + 0.6297642588615417, + 0.7002650499343872, + 0.09722661972045898, + 0.3123666048049927, + 0.7678439021110535, + 0.8322597742080688, + 0.43521666526794434, + 0.8047643303871155, + 0.8416918516159058, + 0.3874783515930176, + 0.8980876207351685, + 0.28832805156707764, + 0.23955774307250977, + 0.6824954748153687, + 0.6367617845535278, + 0.13975244760513306, + 0.8327367305755615, + 0.19990819692611694, + 0.8048990964889526, + 0.007362246513366699, + 0.7968425750732422, + 0.7869243621826172, + 0.7803642749786377, + 0.6648508310317993, + 0.4715670943260193, + 0.7051653265953064, + 0.2771216034889221, + 0.7807289958000183 + ], + [ + 0.5556469559669495, + 0.4589157700538635, + 0.5056062340736389, + 0.5687411427497864, + 0.03726226091384888, + 0.13979697227478027, + 0.24548113346099854, + 0.11453002691268921, + 0.4396135210990906, + 0.668402910232544, + 0.6544288396835327, + 0.4710961580276489, + 0.8550799489021301, + 0.5652360916137695, + 0.07919734716415405, + 0.7649988532066345, + 0.5746005773544312, + 0.6347182989120483, + 0.5653733015060425, + 0.5535793900489807, + 0.09056639671325684, + 0.5592071413993835, + 0.794912576675415, + 0.30395007133483887, + 0.6028382182121277, + 0.030817806720733643, + 0.34749919176101685, + 0.43671733140945435 + ], + [ + 0.9827383756637573, + 0.21458464860916138, + 0.2765483260154724, + 0.4085286259651184, + 0.9926655888557434, + 0.8534030318260193, + 0.03469294309616089, + 0.23393946886062622, + 0.8216744065284729, + 0.05830270051956177, + 0.8556708097457886, + 0.2813838720321655, + 0.9176521301269531, + 0.29359370470046997, + 0.4343167543411255, + 0.6619164943695068, + 0.12628072500228882, + 0.5570321083068848, + 0.50501549243927, + 0.7838981747627258, + 0.9961068630218506, + 0.6643134951591492, + 0.4091864228248596, + 0.4063868522644043, + 0.41795873641967773, + 0.8140203356742859, + 0.3212156891822815, + 0.16697287559509277 + ], + [ + 0.33429139852523804, + 0.02271205186843872, + 0.10614258050918579, + 0.09004783630371094, + 0.7715523838996887, + 0.722359299659729, + 0.6967707872390747, + 0.4618772268295288, + 0.7170584797859192, + 0.16127175092697144, + 0.9006216526031494, + 0.501044750213623, + 0.9380513429641724, + 0.15231209993362427, + 0.49733006954193115, + 0.6963203549385071, + 0.4956258535385132, + 0.44615626335144043, + 0.16621404886245728, + 0.3810212016105652, + 0.23876625299453735, + 0.30151206254959106, + 0.6837006211280823, + 0.6302825808525085, + 0.6071963906288147, + 0.3618125915527344, + 0.9596925377845764, + 0.0876498818397522 + ], + [ + 0.3425907492637634, + 0.11800587177276611, + 0.3389437794685364, + 0.961897611618042, + 0.3662635087966919, + 0.9085806608200073, + 0.4950656294822693, + 0.6997070908546448, + 0.45741742849349976, + 0.2658699154853821, + 0.764229416847229, + 0.9691763520240784, + 0.2635248303413391, + 0.7787508964538574, + 0.26113998889923096, + 0.7168901562690735, + 0.7890065312385559, + 0.4493614435195923, + 0.7369195818901062, + 0.2722415328025818, + 0.07925349473953247, + 0.09639090299606323, + 0.4470236897468567, + 0.9026023745536804, + 0.1275768280029297, + 0.45577627420425415, + 0.7040320038795471, + 0.20236331224441528 + ], + [ + 0.7216163873672485, + 0.3059566020965576, + 0.8086607456207275, + 0.5792195200920105, + 0.5469072461128235, + 0.17677277326583862, + 0.46320587396621704, + 0.8566142320632935, + 0.019523978233337402, + 0.7585194706916809, + 0.49276238679885864, + 0.7194629311561584, + 0.6632980108261108, + 0.4320930242538452, + 0.3086671829223633, + 0.6273087859153748, + 0.14022421836853027, + 0.5840979218482971, + 0.09469467401504517, + 0.6498465538024902, + 0.6287452578544617, + 0.08444428443908691, + 0.7574878931045532, + 0.4158073663711548, + 0.78688645362854, + 0.0416141152381897, + 0.17945647239685059, + 0.49399077892303467 + ], + [ + 0.1882251501083374, + 0.3298611640930176, + 0.6605989933013916, + 0.14452415704727173, + 0.6777029037475586, + 0.10340291261672974, + 0.17518514394760132, + 0.5876445174217224, + 0.7934831976890564, + 0.17059296369552612, + 0.29999274015426636, + 0.9251201152801514, + 0.9896836876869202, + 0.5810611248016357, + 0.44703811407089233, + 0.3468697667121887, + 0.6124519109725952, + 0.5909154415130615, + 0.2947530150413513, + 0.022803843021392822, + 0.15149825811386108, + 0.9585591554641724, + 0.41815489530563354, + 0.4823033809661865, + 0.46710753440856934, + 0.7827351689338684, + 0.06495654582977295, + 0.0827299952507019 + ], + [ + 0.2758471965789795, + 0.4866582751274109, + 0.6711239814758301, + 0.4907069802284241, + 0.44116806983947754, + 0.9378264546394348, + 0.16101890802383423, + 0.5717280507087708, + 0.5422446131706238, + 0.47348934412002563, + 0.8075973987579346, + 0.26697564125061035, + 0.9537728428840637, + 0.33156895637512207, + 0.5450044274330139, + 0.5206723809242249, + 0.84977787733078, + 0.43891143798828125, + 0.2845589518547058, + 0.021612048149108887, + 0.3636782169342041, + 0.8262919187545776, + 0.009916186332702637, + 0.8961607217788696, + 0.5428270697593689, + 0.1402490735054016, + 0.3021792769432068, + 0.5540361404418945 + ], + [ + 0.31569212675094604, + 0.10857570171356201, + 0.6659888625144958, + 0.6722400784492493, + 0.11887425184249878, + 0.2812337279319763, + 0.2438732385635376, + 0.6594225764274597, + 0.7302955389022827, + 0.7269945740699768, + 0.9387612342834473, + 0.7686474323272705, + 0.3206794261932373, + 0.10774093866348267, + 0.6146427989006042, + 0.9160118103027344, + 0.9751552939414978, + 0.2302139401435852, + 0.23645538091659546, + 0.03741252422332764, + 0.14126211404800415, + 0.5548524260520935, + 0.5468032360076904, + 0.3709222674369812, + 0.3099554181098938, + 0.8297896981239319, + 0.4600338935852051, + 0.8082514405250549 + ], + [ + 0.9632118344306946, + 0.3171388506889343, + 0.20273828506469727, + 0.9528993368148804, + 0.6348310708999634, + 0.2909178137779236, + 0.6218599677085876, + 0.5150570869445801, + 0.90553218126297, + 0.25596505403518677, + 0.9913827776908875, + 0.9360435605049133, + 0.18070197105407715, + 0.16460776329040527, + 0.5331568121910095, + 0.04491060972213745, + 0.2987979054450989, + 0.4350970387458801, + 0.9693732857704163, + 0.9923755526542664, + 0.15153992176055908, + 0.8916772603988647, + 0.09092879295349121, + 0.7486079931259155, + 0.8456426858901978, + 0.8907924890518188, + 0.22341793775558472, + 0.8934466242790222 + ], + [ + 0.11598086357116699, + 0.5188583135604858, + 0.46954524517059326, + 0.31592899560928345, + 0.022114157676696777, + 0.7720124125480652, + 0.3355836272239685, + 0.6616612076759338, + 0.4102199077606201, + 0.3736577033996582, + 0.8862959146499634, + 0.09446662664413452, + 0.7366173267364502, + 0.7467895746231079, + 0.37999141216278076, + 0.26246047019958496, + 0.005307018756866455, + 0.9368131160736084, + 0.9477314352989197, + 0.24097055196762085, + 0.03191101551055908, + 0.1227579116821289, + 0.06764280796051025, + 0.8311126232147217, + 0.8919114470481873, + 0.15328431129455566, + 0.7467394471168518, + 0.17926830053329468 + ], + [ + 0.36720895767211914, + 0.599382758140564, + 0.7199267745018005, + 0.8745620250701904, + 0.0447847843170166, + 0.1964346170425415, + 0.2805085778236389, + 0.31032365560531616, + 0.9980278015136719, + 0.77740478515625, + 0.32041722536087036, + 0.9718263745307922, + 0.49930697679519653, + 0.500741183757782, + 0.433698832988739, + 0.14389747381210327, + 0.9343724250793457, + 0.013936281204223633, + 0.17427849769592285, + 0.22965598106384277, + 0.27929747104644775, + 0.13182216882705688, + 0.01796138286590576, + 0.6776586174964905, + 0.9693098664283752, + 0.12183248996734619, + 0.2965351939201355, + 0.5063298940658569 + ], + [ + 0.9275023341178894, + 0.6942623853683472, + 0.7661880850791931, + 0.5811165571212769, + 0.4170357584953308, + 0.19977563619613647, + 0.8643143773078918, + 0.8041244745254517, + 0.025101304054260254, + 0.7154070734977722, + 0.08438807725906372, + 0.7389839887619019, + 0.8259538412094116, + 0.1310577392578125, + 0.43383586406707764, + 0.12375378608703613, + 0.5672821998596191, + 0.9275625348091125, + 0.08086949586868286, + 0.3975781798362732, + 0.9731205105781555, + 0.3009486794471741, + 0.1322166919708252, + 0.488584041595459, + 0.5083725452423096, + 0.6628642082214355, + 0.31373751163482666, + 0.955623209476471 + ], + [ + 0.5210227370262146, + 0.286446213722229, + 0.960284411907196, + 0.9248083829879761, + 0.49916476011276245, + 0.024859488010406494, + 0.5969361066818237, + 0.5551980137825012, + 0.1482083797454834, + 0.633975088596344, + 0.8407935500144958, + 0.10589736700057983, + 0.7233226299285889, + 0.14033955335617065, + 0.748826265335083, + 0.41911429166793823, + 0.00651097297668457, + 0.9662318825721741, + 0.1659703254699707, + 0.5960425138473511, + 0.1814681887626648, + 0.93302321434021, + 0.7383255362510681, + 0.8043608665466309, + 0.43207985162734985, + 0.4673815965652466, + 0.6343011260032654, + 0.7847633957862854 + ], + [ + 0.8469794988632202, + 0.01783674955368042, + 0.6635613441467285, + 0.10914397239685059, + 0.28793174028396606, + 0.8294286131858826, + 0.25051313638687134, + 0.7968170642852783, + 0.46300774812698364, + 0.23264068365097046, + 0.3342441916465759, + 0.5307695865631104, + 0.7651471495628357, + 0.6060158014297485, + 0.9332238435745239, + 0.8677389025688171, + 0.753353476524353, + 0.6031071543693542, + 0.2898557782173157, + 0.4125715494155884, + 0.11889427900314331, + 0.3741840124130249, + 0.9264721870422363, + 0.4258820414543152, + 0.11372882127761841, + 0.6519309878349304, + 0.9626148343086243, + 0.8674905896186829 + ], + [ + 0.2740764021873474, + 0.4538968801498413, + 0.53101646900177, + 0.24783951044082642, + 0.2686919569969177, + 0.23666232824325562, + 0.9945812821388245, + 0.7460142374038696, + 0.8408563137054443, + 0.8165687322616577, + 0.9591434001922607, + 0.10527807474136353, + 0.881632387638092, + 0.066558837890625, + 0.4754210114479065, + 0.594433605670929, + 0.3578610420227051, + 0.14617323875427246, + 0.19230127334594727, + 0.8246641755104065, + 0.3187238574028015, + 0.31033462285995483, + 0.9233618378639221, + 0.14387190341949463, + 0.9152668714523315, + 0.9209704399108887, + 0.5985596179962158, + 0.1655316948890686 + ], + [ + 0.15586084127426147, + 0.2847200632095337, + 0.5774156451225281, + 0.1536133885383606, + 0.4556578993797302, + 0.11549001932144165, + 0.5402840971946716, + 0.021147966384887695, + 0.7090361714363098, + 0.05539536476135254, + 0.19482183456420898, + 0.1746414303779602, + 0.8309940695762634, + 0.053381919860839844, + 0.42925965785980225, + 0.591143786907196, + 0.5159955024719238, + 0.68071448802948, + 0.8031013011932373, + 0.3936304450035095, + 0.8282178640365601, + 0.31799107789993286, + 0.9844226837158203, + 0.5045261979103088, + 0.7360132932662964, + 0.8750048875808716, + 0.7926512360572815, + 0.8511316180229187 + ], + [ + 0.5365135669708252, + 0.04347503185272217, + 0.49308353662490845, + 0.18149840831756592, + 0.8891165256500244, + 0.23674482107162476, + 0.22806978225708008, + 0.24938756227493286, + 0.6425662040710449, + 0.5712326169013977, + 0.948840856552124, + 0.4162623882293701, + 0.2822781205177307, + 0.04925411939620972, + 0.007075905799865723, + 0.3736141324043274, + 0.9258173704147339, + 0.5237529277801514, + 0.8935475945472717, + 0.10167187452316284, + 0.09332096576690674, + 0.8334585428237915, + 0.1067420244216919, + 0.05196183919906616, + 0.4040926694869995, + 0.9248418211936951, + 0.689383327960968, + 0.0991131067276001 + ], + [ + 0.8371310234069824, + 0.8435749411582947, + 0.8006964921951294, + 0.9026530981063843, + 0.23987799882888794, + 0.9795706272125244, + 0.32069629430770874, + 0.8020258545875549, + 0.579364538192749, + 0.7794775366783142, + 0.6278194189071655, + 0.6424832344055176, + 0.6162840723991394, + 0.7789963483810425, + 0.7390429377555847, + 0.13455218076705933, + 0.8260548710823059, + 0.5360680222511292, + 0.32308340072631836, + 0.5142228603363037, + 0.966983437538147, + 0.8575721383094788, + 0.8336984515190125, + 0.4627993106842041, + 0.7841377854347229, + 0.3850894570350647, + 0.686165452003479, + 0.639563262462616 + ], + [ + 0.24979865550994873, + 0.2664632797241211, + 0.06113308668136597, + 0.13976836204528809, + 0.43361860513687134, + 0.47787725925445557, + 0.16512072086334229, + 0.4168893098831177, + 0.6041774749755859, + 0.23256993293762207, + 0.33823686838150024, + 0.3675118088722229, + 0.5050538778305054, + 0.36639243364334106, + 0.7369446158409119, + 0.3274955153465271, + 0.4338967204093933, + 0.3794640302658081, + 0.18291574716567993, + 0.6857433319091797, + 0.23544615507125854, + 0.2968764305114746, + 0.7927519083023071, + 0.9488579034805298, + 0.6708431243896484, + 0.9163479804992676, + 0.8541905879974365, + 0.48091042041778564 + ], + [ + 0.7702901363372803, + 0.3283611536026001, + 0.46049702167510986, + 0.5354347825050354, + 0.9604929685592651, + 0.8485604524612427, + 0.4892529249191284, + 0.6525872945785522, + 0.2023564577102661, + 0.8043918013572693, + 0.5530379414558411, + 0.5327222347259521, + 0.7662835121154785, + 0.632917582988739, + 0.07246702909469604, + 0.28815555572509766, + 0.17671895027160645, + 0.7348931431770325, + 0.035147130489349365, + 0.20240455865859985, + 0.597445011138916, + 0.6947981119155884, + 0.5033048987388611, + 0.8607190251350403, + 0.9189400672912598, + 0.13210278749465942, + 0.9525110721588135, + 0.6143797039985657 + ], + [ + 0.6362191438674927, + 0.095095694065094, + 0.621871829032898, + 0.7257155776023865, + 0.5807911157608032, + 0.08449316024780273, + 0.5232273936271667, + 0.9359397888183594, + 0.7234165072441101, + 0.13740789890289307, + 0.9599384665489197, + 0.9588802456855774, + 0.16939550638198853, + 0.8008841276168823, + 0.40671706199645996, + 0.5936819911003113, + 0.4451862573623657, + 0.7826240658760071, + 0.9966598153114319, + 0.7951148152351379, + 0.9313713908195496, + 0.9460270404815674, + 0.09010541439056396, + 0.2533833384513855, + 0.7218565344810486, + 0.5900758504867554, + 0.05014616250991821, + 0.09504914283752441 + ], + [ + 0.5205473899841309, + 0.6161656975746155, + 0.167719304561615, + 0.1712912917137146, + 0.12934017181396484, + 0.5649505853652954, + 0.00785362720489502, + 0.5724304914474487, + 0.4759286046028137, + 0.46598511934280396, + 0.6792762279510498, + 0.5226317644119263, + 0.7277487516403198, + 0.7639233469963074, + 0.9201177358627319, + 0.7992447018623352, + 0.5177061557769775, + 0.4921531677246094, + 0.23570853471755981, + 0.5995934009552002, + 0.1391354203224182, + 0.931236207485199, + 0.8450442552566528, + 0.11973357200622559, + 0.7859558463096619, + 0.11710351705551147, + 0.7212251424789429, + 0.08770900964736938 + ], + [ + 0.4881933927536011, + 0.6578632593154907, + 0.7288107872009277, + 0.41860824823379517, + 0.9216234087944031, + 0.7743213772773743, + 0.20826470851898193, + 0.6712313890457153, + 0.730534553527832, + 0.33363771438598633, + 0.7382313013076782, + 0.8983665108680725, + 0.8171142339706421, + 0.762532114982605, + 0.6344974040985107, + 0.2705349326133728, + 0.7952229380607605, + 0.36419200897216797, + 0.5321968197822571, + 0.31443995237350464, + 0.3130170702934265, + 0.15761160850524902, + 0.9612976312637329, + 0.1477833390235901, + 0.08625876903533936, + 0.9361274242401123, + 0.555013120174408, + 0.4379040002822876 + ], + [ + 0.7374913096427917, + 0.38331979513168335, + 0.12792158126831055, + 0.7296856641769409, + 0.1759994626045227, + 0.5529930591583252, + 0.9490661025047302, + 0.9361399412155151, + 0.3607829809188843, + 0.7803014516830444, + 0.6126757860183716, + 0.47936952114105225, + 0.13992810249328613, + 0.3763594627380371, + 0.6519180536270142, + 0.9866315126419067, + 0.004890561103820801, + 0.7177602052688599, + 0.846336841583252, + 0.9511946439743042, + 0.4238036870956421, + 0.11847853660583496, + 0.27096712589263916, + 0.850533664226532, + 0.8779348731040955, + 0.6370738744735718, + 0.12274044752120972, + 0.12192165851593018 + ] + ] + ] + ], + "outputs": [ + [ + [ + -0.34393584728240967, + -0.5412918925285339, + -1.5185871124267578, + 0.6088137626647949, + -1.0386784076690674, + 1.61068856716156, + 0.7106994986534119, + -1.7952651977539062, + 1.2416664361953735, + -0.340225487947464 + ] + ] + ], + "types": [ + "float32" + ] +} diff --git a/test/fixtures/mnist.onnx b/test/fixtures/mnist.onnx new file mode 100644 index 0000000..6661bfe Binary files /dev/null and b/test/fixtures/mnist.onnx differ diff --git a/test/fixtures/xgboost.json b/test/fixtures/xgboost.json new file mode 100644 index 0000000..1817807 --- /dev/null +++ b/test/fixtures/xgboost.json @@ -0,0 +1,43 @@ +{ + "inputs": [ + [ + 0.0, + 0.0, + 0.0 + ], + [ + 1.0, + 1.0, + 1.0 + ], + [ + -1.0, + -1.0, + -1.0 + ], + [ + 0.25, + 0.75, + -0.5 + ] + ], + "outputs": [ + [ + [ + 0.02070039138197899 + ], + [ + 1.1609605550765991 + ], + [ + -0.14878709614276886 + ], + [ + 0.8968343138694763 + ] + ] + ], + "types": [ + "float32" + ] +} diff --git a/test/fixtures/xgboost.onnx b/test/fixtures/xgboost.onnx new file mode 100644 index 0000000..a1f0ece Binary files /dev/null and b/test/fixtures/xgboost.onnx differ diff --git a/test/models/models_test.exs b/test/models/models_test.exs new file mode 100644 index 0000000..2fff08a --- /dev/null +++ b/test/models/models_test.exs @@ -0,0 +1,23 @@ +defmodule Ortex.ModelsTest do + use ExUnit.Case, async: false + @moduletag :model + + for name <- ~w(xgboost iris mnist) do + test "#{name} matches its prepared reference results" do + name = unquote(name) + root = Path.expand("../fixtures", __DIR__) + cases = File.read!(Path.join(root, name <> ".json")) |> :json.decode() + model = Ortex.load(Path.join(root, name <> ".onnx"), [:cpu], 0) + outputs = Ortex.run(model, Nx.tensor(cases["inputs"], type: :f32)) |> Tuple.to_list() + assert length(outputs) == length(cases["outputs"]) + + Enum.zip(outputs, cases["outputs"]) + |> Enum.each(fn {tensor, expected} -> + actual = Nx.to_flat_list(tensor) + expected = List.flatten(expected) + assert length(actual) == length(expected) + Enum.zip(actual, expected) |> Enum.each(fn {a, b} -> assert_in_delta a, b, 1.0e-5 end) + end) + end + end +end diff --git a/test/ortex_test.exs b/test/ortex_test.exs index bb41f8b..1f550db 100644 --- a/test/ortex_test.exs +++ b/test/ortex_test.exs @@ -2,27 +2,6 @@ defmodule OrtexTest do use ExUnit.Case doctest Ortex - @tag :resnet50 - test "resnet50" do - model = Ortex.load("./models/resnet50.onnx") - - input = Nx.broadcast(0.0, {1, 3, 224, 224}) - {output} = Ortex.run(model, {input}) - argmax = output |> Nx.backend_transfer() |> Nx.argmax(axis: 1) - - assert argmax == Nx.tensor([499]) - end - - @tag :resnet50 - test "Nx.Serving with resnet50" do - model = Ortex.load("./models/resnet50.onnx") - - serving = Nx.Serving.new(Ortex.Serving, model) - batch = Nx.Batch.stack([{Nx.broadcast(0.0, {3, 224, 224})}]) - {result} = Nx.Serving.run(serving, batch) - assert result |> Nx.backend_transfer() |> Nx.argmax(axis: 1) == Nx.tensor([499]) - end - test "Nx.Serving with tinymodel" do model = Ortex.load("./models/tinymodel.onnx") diff --git a/test/package/config/config.exs b/test/package/config/config.exs new file mode 100644 index 0000000..5e62539 --- /dev/null +++ b/test/package/config/config.exs @@ -0,0 +1,5 @@ +import Config + +config :ortex, Ortex.Native, + features: ["ort/load-dynamic"], + skip_compilation?: System.fetch_env!("ORTEX_PACKAGE_MODE") == "prebuilt" diff --git a/test/package/mix.exs b/test/package/mix.exs new file mode 100644 index 0000000..b74d265 --- /dev/null +++ b/test/package/mix.exs @@ -0,0 +1,12 @@ +defmodule Ortex.PackageCheck.MixProject do + use Mix.Project + + def project do + [ + app: :ortex_package_check, + version: "0.1.0", + elixir: "~> 1.19", + deps: [{:ortex, path: System.fetch_env!("ORTEX_PACKAGE_PATH")}] + ] + end +end diff --git a/test/package/smoke.exs b/test/package/smoke.exs new file mode 100644 index 0000000..4d36dfb --- /dev/null +++ b/test/package/smoke.exs @@ -0,0 +1,11 @@ +model = Ortex.load(System.fetch_env!("ORTEX_FIXTURE_PATH"), [:cpu], 0) +cases = File.read!(System.fetch_env!("ORTEX_CASES_PATH")) |> :json.decode() +{result} = Ortex.run(model, Nx.tensor(cases["inputs"], type: :f32)) +actual = Nx.to_flat_list(result) +expected = List.flatten(cases["outputs"]) + +unless length(actual) == length(expected) and + Enum.all?(Enum.zip(actual, expected), fn {a, b} -> abs(a - b) < 1.0e-6 end), + do: raise("installed package produced incorrect reference scores") + +IO.puts("Installed package passed #{System.fetch_env!("ORTEX_PACKAGE_MODE")} inference") diff --git a/test/support/scenario.exs b/test/support/scenario.exs new file mode 100644 index 0000000..890e4f9 --- /dev/null +++ b/test/support/scenario.exs @@ -0,0 +1,10 @@ +defmodule Ortex.Scenario do + def fixture, do: Path.expand("../../models/tinymodel.onnx", __DIR__) + + def score(model) do + inputs = {Nx.broadcast(0, {1, 100}) |> Nx.as_type(:s32), Nx.broadcast(0.0, {1, 100})} + Ortex.run(model, inputs) |> Tuple.to_list() |> Enum.map(&Nx.to_flat_list/1) + end +end + +{:ok, _} = Application.ensure_all_started(:ortex) diff --git a/test/system/owner_exit.exs b/test/system/owner_exit.exs new file mode 100644 index 0000000..b52df26 --- /dev/null +++ b/test/system/owner_exit.exs @@ -0,0 +1,10 @@ +Code.require_file(Path.expand("../support/scenario.exs", __DIR__)) +model = Task.async(fn -> Ortex.load(Ortex.Scenario.fixture(), [:cpu]) end) |> Task.await(30_000) +:erlang.garbage_collect() +expected = Ortex.Scenario.score(model) + +1..16 +|> Task.async_stream(fn _ -> Ortex.Scenario.score(model) end, max_concurrency: 4) +|> Enum.each(fn result -> + unless result == {:ok, expected}, do: raise("result changed after model owner exited") +end) diff --git a/test/system/restart.exs b/test/system/restart.exs new file mode 100644 index 0000000..f5f37d4 --- /dev/null +++ b/test/system/restart.exs @@ -0,0 +1,8 @@ +Code.require_file(Path.expand("../support/scenario.exs", __DIR__)) +model = Ortex.load(Ortex.Scenario.fixture(), [:cpu]) +before = Ortex.Scenario.score(model) +:ok = Application.stop(:ortex) +{:ok, _} = Application.ensure_all_started(:ortex) + +unless Ortex.Scenario.score(model) == before, + do: raise("model changed across application restart") diff --git a/test/system/runtime_override.exs b/test/system/runtime_override.exs new file mode 100644 index 0000000..0d3a6d3 --- /dev/null +++ b/test/system/runtime_override.exs @@ -0,0 +1,9 @@ +import ExUnit.Assertions + +# A separately supplied runtime must load before the first inference in this VM. +path = System.fetch_env!("ORT_DYLIB_PATH") +assert File.regular?(path) +model = Ortex.load(Path.expand("../../models/tinymodel.onnx", __DIR__), [:cpu]) +{inputs, outputs} = Ortex.Native.show_session(model.reference) +assert length(inputs) == 2 +assert length(outputs) == 3 diff --git a/test/test_helper.exs b/test/test_helper.exs index b8accf6..3fcc82c 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,16 +1 @@ -exclude = - if File.exists?("models/resnet50.onnx") do - [] - else - IO.warn( - """ - skipping resnet50 tests because model is not available. - Run python/export_resnet.py before for a complete test suite\ - """, - [] - ) - - [:resnet50] - end - -ExUnit.start(exclude: exclude) +ExUnit.start(exclude: [model: true]) diff --git a/test/tooling/harness_test.py b/test/tooling/harness_test.py new file mode 100644 index 0000000..2d6d86e --- /dev/null +++ b/test/tooling/harness_test.py @@ -0,0 +1,85 @@ +"""Exercise staging isolation, scenario timeouts, and package extraction boundaries.""" + +import contextlib +import importlib.util +import io +import subprocess +import sys +import tarfile +import tempfile +import time +import unittest +from pathlib import Path +from unittest.mock import patch + +SCRIPTS = Path(__file__).resolve().parents[2] / "scripts" +sys.path.insert(0, str(SCRIPTS)) + + +def load(name): + spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class HarnessTest(unittest.TestCase): + def test_staged_check_sees_index_and_preserves_unstaged_contents(self): + checker = load("check") + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + subprocess.run(["git", "init", "--quiet", str(root)], check=True) + source = root / "space $(echo not-executed).py" + source.write_text("import os\n") # invalid staged content + subprocess.run(["git", "-C", str(root), "add", "--", source.name], check=True) + source.write_text('print("valid working tree")\n') + with ( + patch.object(checker, "ROOT", root), + patch.object(sys, "argv", ["check", "--staged"]), + contextlib.redirect_stdout(io.StringIO()) as output, + ): + self.assertEqual(checker.main(), 1) + self.assertIn("F401", output.getvalue()) + self.assertEqual(source.read_text(), 'print("valid working tree")\n') + staged = subprocess.check_output( + ["git", "-C", str(root), "show", f":{source.name}"], text=True + ) + self.assertEqual(staged, "import os\n") + + def test_scenario_success_and_failure_are_preserved(self): + runner = load("scenario") + status, output = runner.execute( + [sys.executable, "-c", 'print("done"); raise SystemExit(7)'], 10 + ) + self.assertEqual(status, 7) + self.assertIn("done", output) + + def test_scenario_timeout_terminates_the_process(self): + runner = load("scenario") + before = time.monotonic() + status, output = runner.execute([sys.executable, "-c", "import time; time.sleep(30)"], 0.2) + self.assertEqual(status, 124) + self.assertLess(time.monotonic() - before, 5) + self.assertIn("exceeded", output) + + def test_hex_contents_cannot_escape_the_unpack_directory(self): + package = load("package") + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + payload = io.BytesIO() + with tarfile.open(fileobj=payload, mode="w:gz") as inner: + entry = tarfile.TarInfo("../escaped") + entry.size = 4 + inner.addfile(entry, io.BytesIO(b"oops")) + archive = root / "bad.tar" + with tarfile.open(archive, "w") as outer: + entry = tarfile.TarInfo("contents.tar.gz") + entry.size = len(payload.getvalue()) + outer.addfile(entry, io.BytesIO(payload.getvalue())) + with self.assertRaises(tarfile.FilterError): + package.unpack(archive, root / "unpacked") + self.assertFalse((root / "escaped").exists()) + + +if __name__ == "__main__": + unittest.main()