From 523cce67448b23a6cfe49a72d33d6667ee5aa0b4 Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Thu, 29 Oct 2026 17:16:03 -0700 Subject: [PATCH 1/4] test: isolate system scenarios and package harness discovery --- Makefile | 28 +++++++++++ docs/testing.md | 46 +++++++++++++++++ mix.exs | 2 + scripts/scenario.py | 65 ++++++++++++++++++++++++ test/support/scenario.exs | 10 ++++ test/system/owner_exit.exs | 10 ++++ test/system/restart.exs | 8 +++ test/system/runtime_override.exs | 9 ++++ test/test_helper.exs | 17 +------ test/tooling/harness_test.py | 85 ++++++++++++++++++++++++++++++++ 10 files changed, 264 insertions(+), 16 deletions(-) create mode 100644 docs/testing.md create mode 100644 scripts/scenario.py create mode 100644 test/support/scenario.exs create mode 100644 test/system/owner_exit.exs create mode 100644 test/system/restart.exs create mode 100644 test/system/runtime_override.exs create mode 100644 test/tooling/harness_test.py diff --git a/Makefile b/Makefile index 82e066b..0c822e9 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ export MIX_ENV := test export ORT_DYLIB_PATH ?= $(shell $(PYTHON) scripts/install_runtime.py path 2>/dev/null) .PHONY: help setup deps doctor test docs hooks fmt lint check +.PHONY: test.system test.package test.models test.tooling coverage bench fixtures package help: ## Show development commands @awk 'BEGIN {FS = ":.*## "} /^[a-zA-Z0-9_.-]+:.*## / {printf " %-22s %s\n", $$1, $$2}' $(MAKEFILE_LIST) @@ -43,3 +44,30 @@ lint: ## Check formatting, static analysis, and compiler warnings $(PYTHON) scripts/check.py --full check: lint test ## Run the ordinary local checks + +test.system: ## Run each system scenario in a fresh VM + $(PYTHON) scripts/scenario.py + +test.package: ## Test the assembled package in clean source and prebuilt consumers + $(PYTHON) scripts/package.py test --mode source + $(PYTHON) scripts/package.py test --mode prebuilt + +test.models: ## Check prepared model inputs against their reference outputs + $(PYTHON) scripts/fixtures.py + mix test test/models --include model + +test.tooling: ## Test the development harnesses + $(PYTHON) -m unittest discover -s test/tooling -p '*_test.py' -v + +coverage: ## Write scoped Elixir HTML and Markdown coverage reports + mix test --cover + mix run --no-start --no-compile scripts/coverage.exs + +bench: ## Measure the public inference workload (ad hoc) + mix run bench/inference.exs $(ARGS) + +fixtures: ## Regenerate model fixtures (requires the separate exporter environment) + $(PYTHON) fixtures/generate.py + +package: ## Assemble the local Hex archive + $(PYTHON) scripts/package.py build diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 0000000..455ed27 --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,46 @@ +# Testing + +`mix test` discovers library `*_test.exs` files normally under test/, including +doctests. Mix excludes system/, package/, support/, and fixtures/ from discovery. +Model ExUnit tests have `@moduletag :model` and are excluded by default; run +`make test.models` to select them explicitly. No selected suite silently skips +a missing fixture. + +`make test.system` invokes scripts/scenario.py. Each test/system/*.exs scenario +runs in its own BEAM after compilation. Nonzero exit (including a native shutdown +abort) fails. Timeout kills the process group on Unix and process tree on Windows. +The runner stores exit status and output in dist/reports/system.json. Additional +process-wide configuration scenarios should accompany the feature implementing +that configuration; the foundation tests main's existing behavior. + +`make test.package` assembles a Hex tarball, extracts its actual contents into a +temporary directory, and runs the separate test/package project against it. It +checks source compilation and an explicitly configured prebuilt library with +Cargo/rustc replaced by failing stubs. The runtime path comes from the caller's +ORT_DYLIB_PATH. This is a package installation check, not the feature stack's +automatic RustlerPrecompiled download path. That feature can reuse this harness. + +`make test.models` checks XGBoost, Iris, and MNIST. Prepared artifacts and reference +cases are recorded in test/fixtures/manifest.json. XGBoost and Iris results come +from the original estimators; MNIST uses Python ORT as a binding reference, with +synthetic pixels. None of these checks claim to validate ORT's operators or model +accuracy. Iris uses tensor probabilities on main; ZipMap needs the separate +container-output feature. Install fixtures/requirements.txt in a separate Python +3.12 environment before `make fixtures`. Generation validates exports with ONNX +and Python ORT; ordinary tests need neither Python exporters nor network access. + +`make test.tooling` checks the harnesses themselves, including partial staging, +filenames with shell characters, timeout handling, and archive path traversal. + +`make coverage` reports Elixir library lines using built-in OTP coverage. The +NIF interface is excluded; generated code should also be excluded when introduced. +The HTML and Markdown report are under cover/. Rust and fresh-VM scenarios are +outside that metric. Initially coverage is reported without a percentage gate. + +`make bench ARGS=5000` runs the ad hoc CPU XGBoost benchmark with reference checks +on every call. Timings include tensor transfer; they are observations for the +recorded environment, not pass/fail performance guarantees. + +The inherited ResNet, DistilBERT, and StableLM demos are manual examples. Their +export dependencies and weights are not part of normal checks. The original +tiny multi-input model remains for legacy API and dtype tests. diff --git a/mix.exs b/mix.exs index 32a9113..b86b807 100644 --- a/mix.exs +++ b/mix.exs @@ -7,6 +7,8 @@ defmodule Ortex.MixProject do version: "0.1.10", elixir: "~> 1.19", start_permanent: Mix.env() == :prod, + test_ignore_filters: [~r{test/(system|package|fixtures|support)/}], + test_coverage: [ignore_modules: [Ortex.Native], summary: [threshold: 0], export: "library"], deps: deps(), # Docs diff --git a/scripts/scenario.py b/scripts/scenario.py new file mode 100644 index 0000000..2a87695 --- /dev/null +++ b/scripts/scenario.py @@ -0,0 +1,65 @@ +"""Run each system scenario in a fresh BEAM and require successful shutdown.""" + +import argparse +import json +import os +import signal +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def execute(command, timeout, cwd=ROOT, env=None): + process = subprocess.Popen( + command, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + start_new_session=os.name != "nt", + ) + try: + output, _ = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + if os.name == "nt": + subprocess.run(["taskkill", "/PID", str(process.pid), "/T", "/F"], capture_output=True) + else: + os.killpg(process.pid, signal.SIGKILL) + output, _ = process.communicate() + return 124, output + f"\nScenario exceeded {timeout}s\n" + return process.returncode, output + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("names", nargs="*", help="scenario stems; default: all") + parser.add_argument("--timeout", type=int, default=60) + args = parser.parse_args() + if args.timeout <= 0: + parser.error("timeout must be positive") + scenarios = {p.stem: p for p in sorted((ROOT / "test/system").glob("*.exs"))} + selected = args.names or list(scenarios) + if set(selected) - scenarios.keys(): + parser.error(f"unknown scenarios: {set(selected) - scenarios.keys()}") + env = {**os.environ, "MIX_ENV": "test"} + subprocess.run(["mix", "compile", "--warnings-as-errors"], cwd=ROOT, env=env, check=True) + results = [] + for name in selected: + status, output = execute( + ["mix", "run", "--no-compile", "--no-deps-check", "--no-start", str(scenarios[name])], + args.timeout, + env=env, + ) + print(f"{'PASS' if status == 0 else 'FAIL'} {name}\n{output}", flush=True) + results.append({"scenario": name, "exit_status": status, "output": output}) + directory = ROOT / "dist/reports" + directory.mkdir(parents=True, exist_ok=True) + (directory / "system.json").write_text(json.dumps(results, indent=2) + "\n") + return int(any(result["exit_status"] != 0 for result in results)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/support/scenario.exs b/test/support/scenario.exs new file mode 100644 index 0000000..890e4f9 --- /dev/null +++ b/test/support/scenario.exs @@ -0,0 +1,10 @@ +defmodule Ortex.Scenario do + def fixture, do: Path.expand("../../models/tinymodel.onnx", __DIR__) + + def score(model) do + inputs = {Nx.broadcast(0, {1, 100}) |> Nx.as_type(:s32), Nx.broadcast(0.0, {1, 100})} + Ortex.run(model, inputs) |> Tuple.to_list() |> Enum.map(&Nx.to_flat_list/1) + end +end + +{:ok, _} = Application.ensure_all_started(:ortex) diff --git a/test/system/owner_exit.exs b/test/system/owner_exit.exs new file mode 100644 index 0000000..b52df26 --- /dev/null +++ b/test/system/owner_exit.exs @@ -0,0 +1,10 @@ +Code.require_file(Path.expand("../support/scenario.exs", __DIR__)) +model = Task.async(fn -> Ortex.load(Ortex.Scenario.fixture(), [:cpu]) end) |> Task.await(30_000) +:erlang.garbage_collect() +expected = Ortex.Scenario.score(model) + +1..16 +|> Task.async_stream(fn _ -> Ortex.Scenario.score(model) end, max_concurrency: 4) +|> Enum.each(fn result -> + unless result == {:ok, expected}, do: raise("result changed after model owner exited") +end) diff --git a/test/system/restart.exs b/test/system/restart.exs new file mode 100644 index 0000000..f5f37d4 --- /dev/null +++ b/test/system/restart.exs @@ -0,0 +1,8 @@ +Code.require_file(Path.expand("../support/scenario.exs", __DIR__)) +model = Ortex.load(Ortex.Scenario.fixture(), [:cpu]) +before = Ortex.Scenario.score(model) +:ok = Application.stop(:ortex) +{:ok, _} = Application.ensure_all_started(:ortex) + +unless Ortex.Scenario.score(model) == before, + do: raise("model changed across application restart") diff --git a/test/system/runtime_override.exs b/test/system/runtime_override.exs new file mode 100644 index 0000000..0d3a6d3 --- /dev/null +++ b/test/system/runtime_override.exs @@ -0,0 +1,9 @@ +import ExUnit.Assertions + +# A separately supplied runtime must load before the first inference in this VM. +path = System.fetch_env!("ORT_DYLIB_PATH") +assert File.regular?(path) +model = Ortex.load(Path.expand("../../models/tinymodel.onnx", __DIR__), [:cpu]) +{inputs, outputs} = Ortex.Native.show_session(model.reference) +assert length(inputs) == 2 +assert length(outputs) == 3 diff --git a/test/test_helper.exs b/test/test_helper.exs index b8accf6..3fcc82c 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,16 +1 @@ -exclude = - if File.exists?("models/resnet50.onnx") do - [] - else - IO.warn( - """ - skipping resnet50 tests because model is not available. - Run python/export_resnet.py before for a complete test suite\ - """, - [] - ) - - [:resnet50] - end - -ExUnit.start(exclude: exclude) +ExUnit.start(exclude: [model: true]) diff --git a/test/tooling/harness_test.py b/test/tooling/harness_test.py new file mode 100644 index 0000000..2d6d86e --- /dev/null +++ b/test/tooling/harness_test.py @@ -0,0 +1,85 @@ +"""Exercise staging isolation, scenario timeouts, and package extraction boundaries.""" + +import contextlib +import importlib.util +import io +import subprocess +import sys +import tarfile +import tempfile +import time +import unittest +from pathlib import Path +from unittest.mock import patch + +SCRIPTS = Path(__file__).resolve().parents[2] / "scripts" +sys.path.insert(0, str(SCRIPTS)) + + +def load(name): + spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class HarnessTest(unittest.TestCase): + def test_staged_check_sees_index_and_preserves_unstaged_contents(self): + checker = load("check") + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + subprocess.run(["git", "init", "--quiet", str(root)], check=True) + source = root / "space $(echo not-executed).py" + source.write_text("import os\n") # invalid staged content + subprocess.run(["git", "-C", str(root), "add", "--", source.name], check=True) + source.write_text('print("valid working tree")\n') + with ( + patch.object(checker, "ROOT", root), + patch.object(sys, "argv", ["check", "--staged"]), + contextlib.redirect_stdout(io.StringIO()) as output, + ): + self.assertEqual(checker.main(), 1) + self.assertIn("F401", output.getvalue()) + self.assertEqual(source.read_text(), 'print("valid working tree")\n') + staged = subprocess.check_output( + ["git", "-C", str(root), "show", f":{source.name}"], text=True + ) + self.assertEqual(staged, "import os\n") + + def test_scenario_success_and_failure_are_preserved(self): + runner = load("scenario") + status, output = runner.execute( + [sys.executable, "-c", 'print("done"); raise SystemExit(7)'], 10 + ) + self.assertEqual(status, 7) + self.assertIn("done", output) + + def test_scenario_timeout_terminates_the_process(self): + runner = load("scenario") + before = time.monotonic() + status, output = runner.execute([sys.executable, "-c", "import time; time.sleep(30)"], 0.2) + self.assertEqual(status, 124) + self.assertLess(time.monotonic() - before, 5) + self.assertIn("exceeded", output) + + def test_hex_contents_cannot_escape_the_unpack_directory(self): + package = load("package") + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + payload = io.BytesIO() + with tarfile.open(fileobj=payload, mode="w:gz") as inner: + entry = tarfile.TarInfo("../escaped") + entry.size = 4 + inner.addfile(entry, io.BytesIO(b"oops")) + archive = root / "bad.tar" + with tarfile.open(archive, "w") as outer: + entry = tarfile.TarInfo("contents.tar.gz") + entry.size = len(payload.getvalue()) + outer.addfile(entry, io.BytesIO(payload.getvalue())) + with self.assertRaises(tarfile.FilterError): + package.unpack(archive, root / "unpacked") + self.assertFalse((root / "escaped").exists()) + + +if __name__ == "__main__": + unittest.main() From 7b5d2ff84926340501ac54641fc15cbc6f0565e1 Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Thu, 29 Oct 2026 17:16:03 -0700 Subject: [PATCH 2/4] test: prepare representative model references and consolidate examples --- bench/inference.exs | 28 + examples/README.md | 6 +- examples/iris/README.md | 8 + examples/mnist/README.md | 9 + examples/resnet/README.md | 6 + .../resnet/export.py | 0 examples/run.exs | 6 + examples/xgboost/README.md | 8 + fixtures/generate.py | 135 +++ .../generate_tiny.py | 0 fixtures/requirements.txt | 18 + scripts/fixtures.py | 22 + test/fixtures/LICENSE-MNIST | 202 ++++ test/fixtures/NOTICE.md | 19 + test/fixtures/iris.json | 62 ++ test/fixtures/iris.onnx | Bin 0 -> 623 bytes test/fixtures/manifest.json | 45 + test/fixtures/mnist.json | 867 ++++++++++++++++++ test/fixtures/mnist.onnx | Bin 0 -> 26143 bytes test/fixtures/xgboost.json | 43 + test/fixtures/xgboost.onnx | Bin 0 -> 4250 bytes test/models/models_test.exs | 23 + test/ortex_test.exs | 21 - 23 files changed, 1506 insertions(+), 22 deletions(-) create mode 100644 bench/inference.exs create mode 100644 examples/iris/README.md create mode 100644 examples/mnist/README.md create mode 100644 examples/resnet/README.md rename python/export_resnet.py => examples/resnet/export.py (100%) create mode 100644 examples/run.exs create mode 100644 examples/xgboost/README.md create mode 100644 fixtures/generate.py rename python/multi_input.py => fixtures/generate_tiny.py (100%) create mode 100644 fixtures/requirements.txt create mode 100644 scripts/fixtures.py create mode 100644 test/fixtures/LICENSE-MNIST create mode 100644 test/fixtures/NOTICE.md create mode 100644 test/fixtures/iris.json create mode 100644 test/fixtures/iris.onnx create mode 100644 test/fixtures/manifest.json create mode 100644 test/fixtures/mnist.json create mode 100644 test/fixtures/mnist.onnx create mode 100644 test/fixtures/xgboost.json create mode 100644 test/fixtures/xgboost.onnx create mode 100644 test/models/models_test.exs diff --git a/bench/inference.exs b/bench/inference.exs new file mode 100644 index 0000000..ca80268 --- /dev/null +++ b/bench/inference.exs @@ -0,0 +1,28 @@ +iterations = System.argv() |> List.first("2000") |> String.to_integer() +if iterations < 1, do: raise("iterations must be positive") +root = Path.expand("../test/fixtures", __DIR__) +cases = File.read!(Path.join(root, "xgboost.json")) |> :json.decode() +model = Ortex.load(Path.join(root, "xgboost.onnx"), [:cpu], 0) +input = Nx.tensor(cases["inputs"], type: :f32) +expected = List.flatten(cases["outputs"]) + +run = fn -> + {output} = Ortex.run(model, input) + actual = Nx.to_flat_list(output) + + unless length(actual) == length(expected) and + Enum.all?(Enum.zip(actual, expected), fn {a, b} -> abs(a - b) < 1.0e-6 end), + do: raise("benchmark reference mismatch") +end + +Enum.each(1..100, fn _ -> run.() end) +{elapsed, _} = :timer.tc(fn -> Enum.each(1..iterations, fn _ -> run.() end) end) + +IO.inspect(%{ + iterations: iterations, + calls_per_second: iterations * 1_000_000 / elapsed, + elixir: System.version(), + otp: System.otp_release(), + runtime: System.fetch_env!("ORT_DYLIB_PATH"), + workload: "synthetic XGBoost; public call including transfers; CPU; optimization disabled" +}) diff --git a/examples/README.md b/examples/README.md index 8d75934..038c1a5 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,6 +1,10 @@ # Examples -Examples have an independent Mix project: from this directory, run `mix deps.get` +Small XGBoost, Iris, and MNIST examples run from the repository root via +`MIX_ENV=test mix run examples/run.exs NAME`, with ORT_DYLIB_PATH set. Their +prepared artifacts also power `make test.models`. + +The tokenizer examples have an independent Mix project: from this directory, run `mix deps.get` and `mix run distilbert/distilbert_classification.exs` (or another example). Set ORT_DYLIB_PATH to the separately installed runtime. Model paths are relative to this directory. The Python export dependencies are separate from the library. diff --git a/examples/iris/README.md b/examples/iris/README.md new file mode 100644 index 0000000..a8155a5 --- /dev/null +++ b/examples/iris/README.md @@ -0,0 +1,8 @@ +# Iris + +From the repository root, with ORT_DYLIB_PATH set: +`MIX_ENV=test mix run examples/run.exs iris`. + +StandardScaler and LogisticRegression produce class labels and a probability +matrix. `make test.models` compares both against scikit-learn. See +test/fixtures/NOTICE.md for attribution and dataset license information. diff --git a/examples/mnist/README.md b/examples/mnist/README.md new file mode 100644 index 0000000..0953217 --- /dev/null +++ b/examples/mnist/README.md @@ -0,0 +1,9 @@ +# MNIST + +From the repository root, with ORT_DYLIB_PATH set: +`MIX_ENV=test mix run examples/run.exs mnist`. + +A small trained convolutional model consumes deterministic synthetic pixels. +`make test.models` compares its numerical outputs against Python ORT; this checks +binding and tensor layout rather than classification accuracy. The exact model +revision, digest, and notices are under test/fixtures/. diff --git a/examples/resnet/README.md b/examples/resnet/README.md new file mode 100644 index 0000000..a0e576b --- /dev/null +++ b/examples/resnet/README.md @@ -0,0 +1,6 @@ +# ResNet (manual) + +This inherited exporter requires separately installed PyTorch and Torchvision, +downloads pretrained weights, and writes resnet50.onnx in its working directory. +It remains a manual demonstration pending pinned weights and export dependencies. +The ordinary model checks use the small prepared models documented in ../README.md. diff --git a/python/export_resnet.py b/examples/resnet/export.py similarity index 100% rename from python/export_resnet.py rename to examples/resnet/export.py diff --git a/examples/run.exs b/examples/run.exs new file mode 100644 index 0000000..09aecb5 --- /dev/null +++ b/examples/run.exs @@ -0,0 +1,6 @@ +name = List.first(System.argv(), "xgboost") +unless name in ~w(xgboost iris mnist), do: raise("choose xgboost, iris, or mnist") +root = Path.expand("../test/fixtures", __DIR__) +cases = File.read!(Path.join(root, name <> ".json")) |> :json.decode() +model = Ortex.load(Path.join(root, name <> ".onnx"), [:cpu], 0) +IO.inspect(Ortex.run(model, Nx.tensor(cases["inputs"], type: :f32)), label: name) diff --git a/examples/xgboost/README.md b/examples/xgboost/README.md new file mode 100644 index 0000000..c07658e --- /dev/null +++ b/examples/xgboost/README.md @@ -0,0 +1,8 @@ +# XGBoost + +From the repository root, with ORT_DYLIB_PATH set: +`MIX_ENV=test mix run examples/run.exs xgboost`. + +Eight shallow regression trees trained on deterministic synthetic data. Four +prepared rows are checked against the original estimator by `make test.models`. +See test/fixtures/manifest.json and fixtures/generate.py for provenance. diff --git a/fixtures/generate.py b/fixtures/generate.py new file mode 100644 index 0000000..eb831f6 --- /dev/null +++ b/fixtures/generate.py @@ -0,0 +1,135 @@ +"""Generate the small model suite and its reference cases with pinned exporters.""" + +import argparse +import hashlib +import json +import urllib.request +from pathlib import Path + +import numpy as np +import onnx +import onnxruntime as ort + +ROOT = Path(__file__).resolve().parents[1] +DIRECTORY = ROOT / "test/fixtures" +MNIST_REVISION = "1cd752f4a3d818fa215aa621c78b8a10a4a1a3a5" +MNIST_SHA = "5c688690f8bacf667d4c2074af5ad0646ca328d7ab03eccf944a65b320171bdd" + + +def xgboost(): + import onnxmltools + import xgboost as xgb + from onnxmltools.convert.common.data_types import FloatTensorType + + rng = np.random.default_rng(42) + x = rng.normal(size=(256, 3)).astype(np.float32) + y = (2 * (x[:, 0] > 0) - (x[:, 1] > 0.5) + x[:, 2] * 0.25).astype(np.float32) + model = xgb.XGBRegressor( + n_estimators=8, max_depth=3, learning_rate=0.25, n_jobs=1, random_state=42 + ) + model.fit(x, y) + converted = onnxmltools.convert_xgboost( + model, initial_types=[("features", FloatTensorType([None, 3]))], target_opset=15 + ) + rows = np.array([[0, 0, 0], [1, 1, 1], [-1, -1, -1], [0.25, 0.75, -0.5]], dtype=np.float32) + return ( + converted, + rows, + [model.predict(rows).reshape(-1, 1)], + { + "source": "deterministic synthetic training data; seed 42", + "license": "MIT", + "reference": "XGBoost predict, independently checked against Python ORT", + }, + ) + + +def iris(): + from skl2onnx import to_onnx + from sklearn.datasets import load_iris + from sklearn.linear_model import LogisticRegression + from sklearn.pipeline import make_pipeline + from sklearn.preprocessing import StandardScaler + + dataset = load_iris() + model = make_pipeline(StandardScaler(), LogisticRegression(random_state=42, max_iter=500)) + model.fit(dataset.data, dataset.target) + rows = dataset.data[[0, 50, 100, 149]].astype(np.float32) + converted = to_onnx(model, rows, options={id(model[-1]): {"zipmap": False}}, target_opset=15) + converted.ir_version = 8 + return ( + converted, + rows, + [model.predict(rows), model.predict_proba(rows)], + { + "source": "scikit-learn load_iris; StandardScaler + LogisticRegression", + "license": "CC-BY-4.0 (Iris dataset; see NOTICE.md)", + "reference": "scikit-learn predict and predict_proba", + }, + ) + + +def mnist(): + url = f"https://huggingface.co/onnxmodelzoo/mnist-12/resolve/{MNIST_REVISION}/mnist-12.onnx" + with urllib.request.urlopen(url, timeout=60) as response: + data = response.read() + if hashlib.sha256(data).hexdigest() != MNIST_SHA: + raise ValueError("MNIST download digest mismatch") + converted = onnx.load_model_from_string(data) + # A real trained network with synthetic pixels tests binding/layout/numerics, + # not classification accuracy or ORT operator correctness. + rows = np.random.default_rng(42).random((1, 1, 28, 28), dtype=np.float32) + return ( + converted, + rows, + None, + { + "source": url, + "license": "Apache-2.0 (model repository); upstream model documentation also records MIT", + "reference": "Python ONNX Runtime 1.24.4; synthetic pixels, seed 42", + }, + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("models", nargs="*", choices=["xgboost", "iris", "mnist"]) + args = parser.parse_args() + DIRECTORY.mkdir(parents=True, exist_ok=True) + manifest_path = DIRECTORY / "manifest.json" + manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else {} + for name in args.models or ["xgboost", "iris", "mnist"]: + model, inputs, expected, provenance = globals()[name]() + onnx.checker.check_model(model) + data = model.SerializeToString() + options = ort.SessionOptions() + options.intra_op_num_threads = 1 + session = ort.InferenceSession(data, options, providers=["CPUExecutionProvider"]) + actual = session.run(None, {session.get_inputs()[0].name: inputs}) + if expected is None: + expected = actual + for a, b in zip(actual, expected, strict=True): + np.testing.assert_allclose(a, b, atol=1e-5, rtol=1e-5) + (DIRECTORY / f"{name}.onnx").write_bytes(data) + cases = { + "inputs": inputs.tolist(), + "outputs": [output.tolist() for output in expected], + "types": [str(output.dtype) for output in actual], + } + case_data = (json.dumps(cases, indent=2) + "\n").encode() + (DIRECTORY / f"{name}.json").write_bytes(case_data) + manifest[name] = { + **provenance, + "sha256": hashlib.sha256(data).hexdigest(), + "cases_sha256": hashlib.sha256(case_data).hexdigest(), + "ir": model.ir_version, + "opsets": {x.domain: x.version for x in model.opset_import}, + "generator": "fixtures/generate.py", + "requirements": "fixtures/requirements.txt", + "atol": 1e-5, + } + manifest_path.write_text(json.dumps(manifest, sort_keys=True, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/python/multi_input.py b/fixtures/generate_tiny.py similarity index 100% rename from python/multi_input.py rename to fixtures/generate_tiny.py diff --git a/fixtures/requirements.txt b/fixtures/requirements.txt new file mode 100644 index 0000000..9221307 --- /dev/null +++ b/fixtures/requirements.txt @@ -0,0 +1,18 @@ +cloudpickle==3.1.2 +flatbuffers==25.12.19 +joblib==1.6.0 +ml_dtypes==0.6.0 +mpmath==1.3.0 +numpy==2.5.4 +onnx==1.20.1 +onnxmltools==1.16.0 +onnxruntime==1.24.4 +packaging==26.3 +protobuf==7.36.2 +scikit-learn==1.8.0 +scipy==1.18.1 +skl2onnx==1.20.0 +sympy==1.14.0 +threadpoolctl==3.7.0 +typing_extensions==4.16.0 +xgboost==3.2.0 diff --git a/scripts/fixtures.py b/scripts/fixtures.py new file mode 100644 index 0000000..ae0bd5f --- /dev/null +++ b/scripts/fixtures.py @@ -0,0 +1,22 @@ +"""Verify every prepared model and case file against its committed manifest.""" + +import hashlib +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + directory = ROOT / "test/fixtures" + manifest = json.loads((directory / "manifest.json").read_text()) + for name, metadata in manifest.items(): + for suffix, key in [("onnx", "sha256"), ("json", "cases_sha256")]: + path = directory / f"{name}.{suffix}" + if hashlib.sha256(path.read_bytes()).hexdigest() != metadata[key]: + raise SystemExit(f"Fixture digest mismatch: {path}") + print(f"Verified {len(manifest)} prepared model fixtures") + + +if __name__ == "__main__": + main() diff --git a/test/fixtures/LICENSE-MNIST b/test/fixtures/LICENSE-MNIST new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/test/fixtures/LICENSE-MNIST @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/test/fixtures/NOTICE.md b/test/fixtures/NOTICE.md new file mode 100644 index 0000000..0bac09e --- /dev/null +++ b/test/fixtures/NOTICE.md @@ -0,0 +1,19 @@ +# Model fixture notices + +The XGBoost fixture is trained on synthetic data using fixtures/generate.py. +Its model and reference inputs are provided under this repository's MIT license. + +The Iris fixture adapts the dataset distributed with scikit-learn into a trained +StandardScaler / LogisticRegression pipeline. Attribution: Fisher, R. (1936). +Iris [Dataset]. UCI Machine Learning Repository. https://doi.org/10.24432/C56C76. +The UCI dataset is licensed under Creative Commons Attribution 4.0 International: +https://creativecommons.org/licenses/by/4.0/. Training, export, and selection of +reference rows are the modifications made here. + +The MNIST model is copied unchanged from onnxmodelzoo/mnist-12 at the revision +recorded in manifest.json. That model repository declares Apache-2.0; its license +text is in LICENSE-MNIST. The original ONNX Model Zoo MNIST documentation also +records MIT. We retain the Apache notice for the exact artifact source used here. +Reference pixels are synthetic; no MNIST image dataset is redistributed. + +These fixtures and notices are test inputs, excluded from the Hex package. diff --git a/test/fixtures/iris.json b/test/fixtures/iris.json new file mode 100644 index 0000000..6af9d86 --- /dev/null +++ b/test/fixtures/iris.json @@ -0,0 +1,62 @@ +{ + "inputs": [ + [ + 5.099999904632568, + 3.5, + 1.399999976158142, + 0.20000000298023224 + ], + [ + 7.0, + 3.200000047683716, + 4.699999809265137, + 1.399999976158142 + ], + [ + 6.300000190734863, + 3.299999952316284, + 6.0, + 2.5 + ], + [ + 5.900000095367432, + 3.0, + 5.099999904632568, + 1.7999999523162842 + ] + ], + "outputs": [ + [ + 0, + 1, + 2, + 2 + ], + [ + [ + 0.9847031709514064, + 0.015296767010838049, + 6.2037755696216e-08 + ], + [ + 0.0047141481533477005, + 0.8650593933406213, + 0.13022645850603093 + ], + [ + 1.4917856588872743e-05, + 0.006202664103992609, + 0.9937824180394186 + ], + [ + 0.003053286819346972, + 0.29364978735805086, + 0.7032969258226021 + ] + ] + ], + "types": [ + "int64", + "float32" + ] +} diff --git a/test/fixtures/iris.onnx b/test/fixtures/iris.onnx new file mode 100644 index 0000000000000000000000000000000000000000..13a6a61f379909d8d0ab943734c92f88acfa4862 GIT binary patch literal 623 zcmZvZ&ui2`6vvY^n+?x)-5n2>^%v}E4YIU*EIu=~pdj0=vQoBQmLyDM>}(P;QB)`h z#e*PPR0Q#|N1>oM!6F+xh^PJ$f*_u}iHZl4cClUcGBdm{@AG~0eWpoFxhk4r5ZtU9 zvyJ9VW2RPgcq1>=m8LTHN{4Hpc8PhziMitnRy8`FBUoG;-PQ>(Lf@Ax!Kc5rD6VWS zP$cUS#dqt^?15@n$Dtte7Q8V1+{UPLh~m#X6&p{ly~y=B82L5&0v>Dd$B7h#<1#@= z6o;0{Zy@UlajH~4q5111#2uDip^K6`dD%KGwd%3adq>}5FfqV8e;Ielj6fuKhEIQN-M>+ zw}j%SFLk>6N67dr_E?m`KHk4H#Ethl#k(tC(_ANTpnqK?ytgyav^r2V>nN0>P$s=3 zb^_^#alcmVES3Gw zrFOepzsMsdvN%s(spOlU1Ta-%fC8GSyA>F1!34uB5)Bmh7>qlQbxh2cVBbQ^>bwq0 I`QYyP8(B2N!2kdN literal 0 HcmV?d00001 diff --git a/test/fixtures/manifest.json b/test/fixtures/manifest.json new file mode 100644 index 0000000..a21b0bf --- /dev/null +++ b/test/fixtures/manifest.json @@ -0,0 +1,45 @@ +{ + "iris": { + "atol": 1e-05, + "cases_sha256": "f87eb73635c7a3a75898d3e8fd78693f42f38c026e0bad65422ae37745bce618", + "generator": "fixtures/generate.py", + "ir": 8, + "license": "CC-BY-4.0 (Iris dataset; see NOTICE.md)", + "opsets": { + "": 15, + "ai.onnx.ml": 1 + }, + "reference": "scikit-learn predict and predict_proba", + "requirements": "fixtures/requirements.txt", + "sha256": "6128c916198679d0ac1ad16dcb72325c2586893bbf7309baec7c0c8ce04db4fb", + "source": "scikit-learn load_iris; StandardScaler + LogisticRegression" + }, + "mnist": { + "atol": 1e-05, + "cases_sha256": "a3f8ef2472a13494a7cf022ff44cd3f1399d6607eccefffaff6efdda33bf70ba", + "generator": "fixtures/generate.py", + "ir": 7, + "license": "Apache-2.0 (model repository); upstream model documentation also records MIT", + "opsets": { + "": 12 + }, + "reference": "Python ONNX Runtime 1.24.4; synthetic pixels, seed 42", + "requirements": "fixtures/requirements.txt", + "sha256": "5c688690f8bacf667d4c2074af5ad0646ca328d7ab03eccf944a65b320171bdd", + "source": "https://huggingface.co/onnxmodelzoo/mnist-12/resolve/1cd752f4a3d818fa215aa621c78b8a10a4a1a3a5/mnist-12.onnx" + }, + "xgboost": { + "atol": 1e-05, + "cases_sha256": "f16d557b5816d9fdb253a0a0db7e7a9d4d6abf67ceb247dbe584a60f57e44f4a", + "generator": "fixtures/generate.py", + "ir": 8, + "license": "MIT", + "opsets": { + "ai.onnx.ml": 1 + }, + "reference": "XGBoost predict, independently checked against Python ORT", + "requirements": "fixtures/requirements.txt", + "sha256": "ab9d76d2fbc49faf4396cf1187978c33597ce680c077911e6cd0347c0d1a5ebf", + "source": "deterministic synthetic training data; seed 42" + } +} diff --git a/test/fixtures/mnist.json b/test/fixtures/mnist.json new file mode 100644 index 0000000..189082a --- /dev/null +++ b/test/fixtures/mnist.json @@ -0,0 +1,867 @@ +{ + "inputs": [ + [ + [ + [ + 0.08925092220306396, + 0.7739560008049011, + 0.6545714735984802, + 0.4388784170150757, + 0.43301522731781006, + 0.8585978746414185, + 0.08594560623168945, + 0.6973680257797241, + 0.20146948099136353, + 0.09417730569839478, + 0.5264789462089539, + 0.975622296333313, + 0.7357523441314697, + 0.7611396908760071, + 0.7174772620201111, + 0.7860642671585083, + 0.5132265686988831, + 0.12811362743377686, + 0.839748203754425, + 0.4503859281539917, + 0.5003519058227539, + 0.3707979917526245, + 0.1825495958328247, + 0.926764965057373, + 0.7815674543380737, + 0.6438651084899902, + 0.40241432189941406, + 0.822761595249176 + ], + [ + 0.5454291105270386, + 0.4434141516685486, + 0.4504595398902893, + 0.2272387146949768, + 0.09213590621948242, + 0.5545847415924072, + 0.8878898024559021, + 0.06381720304489136, + 0.8582912683486938, + 0.8276311159133911, + 0.27675968408584595, + 0.6316643953323364, + 0.16522902250289917, + 0.758087694644928, + 0.7005229592323303, + 0.35452592372894287, + 0.06791996955871582, + 0.9706979990005493, + 0.445687472820282, + 0.893121063709259, + 0.6779189705848694, + 0.7783834934234619, + 0.7598994374275208, + 0.1946386694908142, + 0.36390602588653564, + 0.4667209982872009, + 0.4977915287017822, + 0.04380375146865845 + ], + [ + 0.5465694665908813, + 0.15428948402404785, + 0.7433758974075317, + 0.6830489039421082, + 0.9225277900695801, + 0.744762122631073, + 0.36664265394210815, + 0.9675096869468689, + 0.4108503460884094, + 0.3258253335952759, + 0.9055357575416565, + 0.3704596757888794, + 0.07634317874908447, + 0.4695557951927185, + 0.7956799268722534, + 0.1894713044166565, + 0.462940514087677, + 0.12992149591445923, + 0.6864781975746155, + 0.4757049083709717, + 0.3301963210105896, + 0.226909339427948, + 0.5644998550415039, + 0.6698139905929565, + 0.9403060674667358, + 0.4371519088745117, + 0.16068816184997559, + 0.8326781392097473 + ], + [ + 0.6297642588615417, + 0.7002650499343872, + 0.09722661972045898, + 0.3123666048049927, + 0.7678439021110535, + 0.8322597742080688, + 0.43521666526794434, + 0.8047643303871155, + 0.8416918516159058, + 0.3874783515930176, + 0.8980876207351685, + 0.28832805156707764, + 0.23955774307250977, + 0.6824954748153687, + 0.6367617845535278, + 0.13975244760513306, + 0.8327367305755615, + 0.19990819692611694, + 0.8048990964889526, + 0.007362246513366699, + 0.7968425750732422, + 0.7869243621826172, + 0.7803642749786377, + 0.6648508310317993, + 0.4715670943260193, + 0.7051653265953064, + 0.2771216034889221, + 0.7807289958000183 + ], + [ + 0.5556469559669495, + 0.4589157700538635, + 0.5056062340736389, + 0.5687411427497864, + 0.03726226091384888, + 0.13979697227478027, + 0.24548113346099854, + 0.11453002691268921, + 0.4396135210990906, + 0.668402910232544, + 0.6544288396835327, + 0.4710961580276489, + 0.8550799489021301, + 0.5652360916137695, + 0.07919734716415405, + 0.7649988532066345, + 0.5746005773544312, + 0.6347182989120483, + 0.5653733015060425, + 0.5535793900489807, + 0.09056639671325684, + 0.5592071413993835, + 0.794912576675415, + 0.30395007133483887, + 0.6028382182121277, + 0.030817806720733643, + 0.34749919176101685, + 0.43671733140945435 + ], + [ + 0.9827383756637573, + 0.21458464860916138, + 0.2765483260154724, + 0.4085286259651184, + 0.9926655888557434, + 0.8534030318260193, + 0.03469294309616089, + 0.23393946886062622, + 0.8216744065284729, + 0.05830270051956177, + 0.8556708097457886, + 0.2813838720321655, + 0.9176521301269531, + 0.29359370470046997, + 0.4343167543411255, + 0.6619164943695068, + 0.12628072500228882, + 0.5570321083068848, + 0.50501549243927, + 0.7838981747627258, + 0.9961068630218506, + 0.6643134951591492, + 0.4091864228248596, + 0.4063868522644043, + 0.41795873641967773, + 0.8140203356742859, + 0.3212156891822815, + 0.16697287559509277 + ], + [ + 0.33429139852523804, + 0.02271205186843872, + 0.10614258050918579, + 0.09004783630371094, + 0.7715523838996887, + 0.722359299659729, + 0.6967707872390747, + 0.4618772268295288, + 0.7170584797859192, + 0.16127175092697144, + 0.9006216526031494, + 0.501044750213623, + 0.9380513429641724, + 0.15231209993362427, + 0.49733006954193115, + 0.6963203549385071, + 0.4956258535385132, + 0.44615626335144043, + 0.16621404886245728, + 0.3810212016105652, + 0.23876625299453735, + 0.30151206254959106, + 0.6837006211280823, + 0.6302825808525085, + 0.6071963906288147, + 0.3618125915527344, + 0.9596925377845764, + 0.0876498818397522 + ], + [ + 0.3425907492637634, + 0.11800587177276611, + 0.3389437794685364, + 0.961897611618042, + 0.3662635087966919, + 0.9085806608200073, + 0.4950656294822693, + 0.6997070908546448, + 0.45741742849349976, + 0.2658699154853821, + 0.764229416847229, + 0.9691763520240784, + 0.2635248303413391, + 0.7787508964538574, + 0.26113998889923096, + 0.7168901562690735, + 0.7890065312385559, + 0.4493614435195923, + 0.7369195818901062, + 0.2722415328025818, + 0.07925349473953247, + 0.09639090299606323, + 0.4470236897468567, + 0.9026023745536804, + 0.1275768280029297, + 0.45577627420425415, + 0.7040320038795471, + 0.20236331224441528 + ], + [ + 0.7216163873672485, + 0.3059566020965576, + 0.8086607456207275, + 0.5792195200920105, + 0.5469072461128235, + 0.17677277326583862, + 0.46320587396621704, + 0.8566142320632935, + 0.019523978233337402, + 0.7585194706916809, + 0.49276238679885864, + 0.7194629311561584, + 0.6632980108261108, + 0.4320930242538452, + 0.3086671829223633, + 0.6273087859153748, + 0.14022421836853027, + 0.5840979218482971, + 0.09469467401504517, + 0.6498465538024902, + 0.6287452578544617, + 0.08444428443908691, + 0.7574878931045532, + 0.4158073663711548, + 0.78688645362854, + 0.0416141152381897, + 0.17945647239685059, + 0.49399077892303467 + ], + [ + 0.1882251501083374, + 0.3298611640930176, + 0.6605989933013916, + 0.14452415704727173, + 0.6777029037475586, + 0.10340291261672974, + 0.17518514394760132, + 0.5876445174217224, + 0.7934831976890564, + 0.17059296369552612, + 0.29999274015426636, + 0.9251201152801514, + 0.9896836876869202, + 0.5810611248016357, + 0.44703811407089233, + 0.3468697667121887, + 0.6124519109725952, + 0.5909154415130615, + 0.2947530150413513, + 0.022803843021392822, + 0.15149825811386108, + 0.9585591554641724, + 0.41815489530563354, + 0.4823033809661865, + 0.46710753440856934, + 0.7827351689338684, + 0.06495654582977295, + 0.0827299952507019 + ], + [ + 0.2758471965789795, + 0.4866582751274109, + 0.6711239814758301, + 0.4907069802284241, + 0.44116806983947754, + 0.9378264546394348, + 0.16101890802383423, + 0.5717280507087708, + 0.5422446131706238, + 0.47348934412002563, + 0.8075973987579346, + 0.26697564125061035, + 0.9537728428840637, + 0.33156895637512207, + 0.5450044274330139, + 0.5206723809242249, + 0.84977787733078, + 0.43891143798828125, + 0.2845589518547058, + 0.021612048149108887, + 0.3636782169342041, + 0.8262919187545776, + 0.009916186332702637, + 0.8961607217788696, + 0.5428270697593689, + 0.1402490735054016, + 0.3021792769432068, + 0.5540361404418945 + ], + [ + 0.31569212675094604, + 0.10857570171356201, + 0.6659888625144958, + 0.6722400784492493, + 0.11887425184249878, + 0.2812337279319763, + 0.2438732385635376, + 0.6594225764274597, + 0.7302955389022827, + 0.7269945740699768, + 0.9387612342834473, + 0.7686474323272705, + 0.3206794261932373, + 0.10774093866348267, + 0.6146427989006042, + 0.9160118103027344, + 0.9751552939414978, + 0.2302139401435852, + 0.23645538091659546, + 0.03741252422332764, + 0.14126211404800415, + 0.5548524260520935, + 0.5468032360076904, + 0.3709222674369812, + 0.3099554181098938, + 0.8297896981239319, + 0.4600338935852051, + 0.8082514405250549 + ], + [ + 0.9632118344306946, + 0.3171388506889343, + 0.20273828506469727, + 0.9528993368148804, + 0.6348310708999634, + 0.2909178137779236, + 0.6218599677085876, + 0.5150570869445801, + 0.90553218126297, + 0.25596505403518677, + 0.9913827776908875, + 0.9360435605049133, + 0.18070197105407715, + 0.16460776329040527, + 0.5331568121910095, + 0.04491060972213745, + 0.2987979054450989, + 0.4350970387458801, + 0.9693732857704163, + 0.9923755526542664, + 0.15153992176055908, + 0.8916772603988647, + 0.09092879295349121, + 0.7486079931259155, + 0.8456426858901978, + 0.8907924890518188, + 0.22341793775558472, + 0.8934466242790222 + ], + [ + 0.11598086357116699, + 0.5188583135604858, + 0.46954524517059326, + 0.31592899560928345, + 0.022114157676696777, + 0.7720124125480652, + 0.3355836272239685, + 0.6616612076759338, + 0.4102199077606201, + 0.3736577033996582, + 0.8862959146499634, + 0.09446662664413452, + 0.7366173267364502, + 0.7467895746231079, + 0.37999141216278076, + 0.26246047019958496, + 0.005307018756866455, + 0.9368131160736084, + 0.9477314352989197, + 0.24097055196762085, + 0.03191101551055908, + 0.1227579116821289, + 0.06764280796051025, + 0.8311126232147217, + 0.8919114470481873, + 0.15328431129455566, + 0.7467394471168518, + 0.17926830053329468 + ], + [ + 0.36720895767211914, + 0.599382758140564, + 0.7199267745018005, + 0.8745620250701904, + 0.0447847843170166, + 0.1964346170425415, + 0.2805085778236389, + 0.31032365560531616, + 0.9980278015136719, + 0.77740478515625, + 0.32041722536087036, + 0.9718263745307922, + 0.49930697679519653, + 0.500741183757782, + 0.433698832988739, + 0.14389747381210327, + 0.9343724250793457, + 0.013936281204223633, + 0.17427849769592285, + 0.22965598106384277, + 0.27929747104644775, + 0.13182216882705688, + 0.01796138286590576, + 0.6776586174964905, + 0.9693098664283752, + 0.12183248996734619, + 0.2965351939201355, + 0.5063298940658569 + ], + [ + 0.9275023341178894, + 0.6942623853683472, + 0.7661880850791931, + 0.5811165571212769, + 0.4170357584953308, + 0.19977563619613647, + 0.8643143773078918, + 0.8041244745254517, + 0.025101304054260254, + 0.7154070734977722, + 0.08438807725906372, + 0.7389839887619019, + 0.8259538412094116, + 0.1310577392578125, + 0.43383586406707764, + 0.12375378608703613, + 0.5672821998596191, + 0.9275625348091125, + 0.08086949586868286, + 0.3975781798362732, + 0.9731205105781555, + 0.3009486794471741, + 0.1322166919708252, + 0.488584041595459, + 0.5083725452423096, + 0.6628642082214355, + 0.31373751163482666, + 0.955623209476471 + ], + [ + 0.5210227370262146, + 0.286446213722229, + 0.960284411907196, + 0.9248083829879761, + 0.49916476011276245, + 0.024859488010406494, + 0.5969361066818237, + 0.5551980137825012, + 0.1482083797454834, + 0.633975088596344, + 0.8407935500144958, + 0.10589736700057983, + 0.7233226299285889, + 0.14033955335617065, + 0.748826265335083, + 0.41911429166793823, + 0.00651097297668457, + 0.9662318825721741, + 0.1659703254699707, + 0.5960425138473511, + 0.1814681887626648, + 0.93302321434021, + 0.7383255362510681, + 0.8043608665466309, + 0.43207985162734985, + 0.4673815965652466, + 0.6343011260032654, + 0.7847633957862854 + ], + [ + 0.8469794988632202, + 0.01783674955368042, + 0.6635613441467285, + 0.10914397239685059, + 0.28793174028396606, + 0.8294286131858826, + 0.25051313638687134, + 0.7968170642852783, + 0.46300774812698364, + 0.23264068365097046, + 0.3342441916465759, + 0.5307695865631104, + 0.7651471495628357, + 0.6060158014297485, + 0.9332238435745239, + 0.8677389025688171, + 0.753353476524353, + 0.6031071543693542, + 0.2898557782173157, + 0.4125715494155884, + 0.11889427900314331, + 0.3741840124130249, + 0.9264721870422363, + 0.4258820414543152, + 0.11372882127761841, + 0.6519309878349304, + 0.9626148343086243, + 0.8674905896186829 + ], + [ + 0.2740764021873474, + 0.4538968801498413, + 0.53101646900177, + 0.24783951044082642, + 0.2686919569969177, + 0.23666232824325562, + 0.9945812821388245, + 0.7460142374038696, + 0.8408563137054443, + 0.8165687322616577, + 0.9591434001922607, + 0.10527807474136353, + 0.881632387638092, + 0.066558837890625, + 0.4754210114479065, + 0.594433605670929, + 0.3578610420227051, + 0.14617323875427246, + 0.19230127334594727, + 0.8246641755104065, + 0.3187238574028015, + 0.31033462285995483, + 0.9233618378639221, + 0.14387190341949463, + 0.9152668714523315, + 0.9209704399108887, + 0.5985596179962158, + 0.1655316948890686 + ], + [ + 0.15586084127426147, + 0.2847200632095337, + 0.5774156451225281, + 0.1536133885383606, + 0.4556578993797302, + 0.11549001932144165, + 0.5402840971946716, + 0.021147966384887695, + 0.7090361714363098, + 0.05539536476135254, + 0.19482183456420898, + 0.1746414303779602, + 0.8309940695762634, + 0.053381919860839844, + 0.42925965785980225, + 0.591143786907196, + 0.5159955024719238, + 0.68071448802948, + 0.8031013011932373, + 0.3936304450035095, + 0.8282178640365601, + 0.31799107789993286, + 0.9844226837158203, + 0.5045261979103088, + 0.7360132932662964, + 0.8750048875808716, + 0.7926512360572815, + 0.8511316180229187 + ], + [ + 0.5365135669708252, + 0.04347503185272217, + 0.49308353662490845, + 0.18149840831756592, + 0.8891165256500244, + 0.23674482107162476, + 0.22806978225708008, + 0.24938756227493286, + 0.6425662040710449, + 0.5712326169013977, + 0.948840856552124, + 0.4162623882293701, + 0.2822781205177307, + 0.04925411939620972, + 0.007075905799865723, + 0.3736141324043274, + 0.9258173704147339, + 0.5237529277801514, + 0.8935475945472717, + 0.10167187452316284, + 0.09332096576690674, + 0.8334585428237915, + 0.1067420244216919, + 0.05196183919906616, + 0.4040926694869995, + 0.9248418211936951, + 0.689383327960968, + 0.0991131067276001 + ], + [ + 0.8371310234069824, + 0.8435749411582947, + 0.8006964921951294, + 0.9026530981063843, + 0.23987799882888794, + 0.9795706272125244, + 0.32069629430770874, + 0.8020258545875549, + 0.579364538192749, + 0.7794775366783142, + 0.6278194189071655, + 0.6424832344055176, + 0.6162840723991394, + 0.7789963483810425, + 0.7390429377555847, + 0.13455218076705933, + 0.8260548710823059, + 0.5360680222511292, + 0.32308340072631836, + 0.5142228603363037, + 0.966983437538147, + 0.8575721383094788, + 0.8336984515190125, + 0.4627993106842041, + 0.7841377854347229, + 0.3850894570350647, + 0.686165452003479, + 0.639563262462616 + ], + [ + 0.24979865550994873, + 0.2664632797241211, + 0.06113308668136597, + 0.13976836204528809, + 0.43361860513687134, + 0.47787725925445557, + 0.16512072086334229, + 0.4168893098831177, + 0.6041774749755859, + 0.23256993293762207, + 0.33823686838150024, + 0.3675118088722229, + 0.5050538778305054, + 0.36639243364334106, + 0.7369446158409119, + 0.3274955153465271, + 0.4338967204093933, + 0.3794640302658081, + 0.18291574716567993, + 0.6857433319091797, + 0.23544615507125854, + 0.2968764305114746, + 0.7927519083023071, + 0.9488579034805298, + 0.6708431243896484, + 0.9163479804992676, + 0.8541905879974365, + 0.48091042041778564 + ], + [ + 0.7702901363372803, + 0.3283611536026001, + 0.46049702167510986, + 0.5354347825050354, + 0.9604929685592651, + 0.8485604524612427, + 0.4892529249191284, + 0.6525872945785522, + 0.2023564577102661, + 0.8043918013572693, + 0.5530379414558411, + 0.5327222347259521, + 0.7662835121154785, + 0.632917582988739, + 0.07246702909469604, + 0.28815555572509766, + 0.17671895027160645, + 0.7348931431770325, + 0.035147130489349365, + 0.20240455865859985, + 0.597445011138916, + 0.6947981119155884, + 0.5033048987388611, + 0.8607190251350403, + 0.9189400672912598, + 0.13210278749465942, + 0.9525110721588135, + 0.6143797039985657 + ], + [ + 0.6362191438674927, + 0.095095694065094, + 0.621871829032898, + 0.7257155776023865, + 0.5807911157608032, + 0.08449316024780273, + 0.5232273936271667, + 0.9359397888183594, + 0.7234165072441101, + 0.13740789890289307, + 0.9599384665489197, + 0.9588802456855774, + 0.16939550638198853, + 0.8008841276168823, + 0.40671706199645996, + 0.5936819911003113, + 0.4451862573623657, + 0.7826240658760071, + 0.9966598153114319, + 0.7951148152351379, + 0.9313713908195496, + 0.9460270404815674, + 0.09010541439056396, + 0.2533833384513855, + 0.7218565344810486, + 0.5900758504867554, + 0.05014616250991821, + 0.09504914283752441 + ], + [ + 0.5205473899841309, + 0.6161656975746155, + 0.167719304561615, + 0.1712912917137146, + 0.12934017181396484, + 0.5649505853652954, + 0.00785362720489502, + 0.5724304914474487, + 0.4759286046028137, + 0.46598511934280396, + 0.6792762279510498, + 0.5226317644119263, + 0.7277487516403198, + 0.7639233469963074, + 0.9201177358627319, + 0.7992447018623352, + 0.5177061557769775, + 0.4921531677246094, + 0.23570853471755981, + 0.5995934009552002, + 0.1391354203224182, + 0.931236207485199, + 0.8450442552566528, + 0.11973357200622559, + 0.7859558463096619, + 0.11710351705551147, + 0.7212251424789429, + 0.08770900964736938 + ], + [ + 0.4881933927536011, + 0.6578632593154907, + 0.7288107872009277, + 0.41860824823379517, + 0.9216234087944031, + 0.7743213772773743, + 0.20826470851898193, + 0.6712313890457153, + 0.730534553527832, + 0.33363771438598633, + 0.7382313013076782, + 0.8983665108680725, + 0.8171142339706421, + 0.762532114982605, + 0.6344974040985107, + 0.2705349326133728, + 0.7952229380607605, + 0.36419200897216797, + 0.5321968197822571, + 0.31443995237350464, + 0.3130170702934265, + 0.15761160850524902, + 0.9612976312637329, + 0.1477833390235901, + 0.08625876903533936, + 0.9361274242401123, + 0.555013120174408, + 0.4379040002822876 + ], + [ + 0.7374913096427917, + 0.38331979513168335, + 0.12792158126831055, + 0.7296856641769409, + 0.1759994626045227, + 0.5529930591583252, + 0.9490661025047302, + 0.9361399412155151, + 0.3607829809188843, + 0.7803014516830444, + 0.6126757860183716, + 0.47936952114105225, + 0.13992810249328613, + 0.3763594627380371, + 0.6519180536270142, + 0.9866315126419067, + 0.004890561103820801, + 0.7177602052688599, + 0.846336841583252, + 0.9511946439743042, + 0.4238036870956421, + 0.11847853660583496, + 0.27096712589263916, + 0.850533664226532, + 0.8779348731040955, + 0.6370738744735718, + 0.12274044752120972, + 0.12192165851593018 + ] + ] + ] + ], + "outputs": [ + [ + [ + -0.34393584728240967, + -0.5412918925285339, + -1.5185871124267578, + 0.6088137626647949, + -1.0386784076690674, + 1.61068856716156, + 0.7106994986534119, + -1.7952651977539062, + 1.2416664361953735, + -0.340225487947464 + ] + ] + ], + "types": [ + "float32" + ] +} diff --git a/test/fixtures/mnist.onnx b/test/fixtures/mnist.onnx new file mode 100644 index 0000000000000000000000000000000000000000..6661bfe3c5c1771594fe50f180df38f4416f2c5f GIT binary patch literal 26143 zcmce-cU+I*_dnjAR2tGyG-yx~sb2Rvg~~{ShAk^(rj$*4DN$LGj7UWYQQhZcBvK-K zM@DvJ&%UqsXTCq<`+Yoq|NI{JU-!Asb=~JW=Ne}`FBN5Ng`vKI-iC_(?FQO8m?=*S zvzxJS$vi7r*I)IrY9+F2N+TAAM=fzwQ}dq|F>OKUlF*2O+Fge(S{S`(e$ohMkVqxg~kjOdH!b9B^-4%DqD%)tODMv1e2%8xi z=`QOo`wxnNn##1OC5uABr_D4|3mEL<84~30?>T0dth`MpHO1Kxi=x82Ng3sAwAEB+ zhRvV$kH7zWCT+90d;-?_B4{TD5oAGUC|lhc2;RQ(T{nX=EcW&e2l zA3BnAm-}xz`Zr1!H3g}9BHd;F_4x;r-an~L$=5G1z%x*)vVRe&QdaGMkw8uTKTPR7 zNL&A(Z06wbPioQluOI$@x8_K-|C?z4JOF2Ht$&!$!Qr25qx^4_|Fq~Jwg17Tt@RHJ zIyemYPs=F(8|9yjnk}pLKlrqD{>_38PXDAXwf`jk-|Z^zF8|-``frr~Z+2a(_P>ds zX7-;V{Kud|B0~RV+4cL^)u(Oz|Kc(15*W51G_t=`ng8C*OsbTB>EK@`)O58@|1ozQ zTpZPm{`u|S+c^B&uk`-ckN@)@(p~4D|1m}F|0$!||K6?ke~SJe+mXt@kH$>NXW9~< zsQLdg22ZvB$@*W0HQ-;TRR8}nHU2TBq?G@dQrfEj*fxhnObef5v)x}sOGQEYsAYw2dpKfu==p?yVy`4CupI0k^?(ZGYYze}M8ny~9q= zzX_Gr<{TC>3YWDtQpA8i{G$2*nw=krjk~Vl{P5rK$3BA%`8|KsXKwyoA|BF~XR}r> z5W~*l<#h|#_|{c?mscT}Mdw5RjkeWI*(>PpdNAxeW!I?x zukFGY&L(;rL$w`8;aO88=$>@O!nOwxc4|1sj+Gbo?u~_^i4j7KkWCIP_h8+-6)f{? zxgeKU!dpivQSnnV*4Uuuc6HrmTCS|%wqMQ?4>BbcP(&-yJR9J$G+6^&y z{5JG_G=<$=_QSRnDd?Y-!<*Ko;OV!1d?E8MZCrj%GPybw=QgQh;kM`E<{OzD*jg#v z8k-2O*XglmuO{fVt`lFc4}}*!_ETV|SCWo>0bG2u6{bcxv#aYVoDn#Wf7SezJU%m? zueCMN=?#^z`+XXYP15D5g=Mh0K9huQ>UieT2Y4{zLRF2730>1Y0uB9#z$cxq^cxF7 zcf~=N@KY5Py2et!Qzs;2ym!Lix*GA$kS(-!)iPAm_rbH9p0fk(@bkMI z>UFBQLL*<3zSzd%4W(1upg|Z;O1sx+MizD>_vQkHYAyCGUfB)8!rG~1X_bJKQeExLDVdW#~I zmFM$@1?7-2Hizvm?k0m#Yq0KJZ?~ylIUEyk6jj>p@d%w5vdgjtBgeD!Sx=iM1na@% z_e#9d*hR3pYD?o!UJ>Qe26A=cFu3CVN}}>plP-7rM%&&PgVX#GJoAweb_`E}r9$uU>(E9pH+S%|MFAu zFutxj3xghHuur8Yj{dTYSG&zb>o3l@(6^NHW9o!HnqTS4g<_m~B9cqy{}TE)6$|US z&8IJ8y+wQ-%ih}#^0QrkarcKt9GY@UApKD8K0!_r60nzz%03GtavZUJ<3zMNn}UD5 zAH$Fj8`$Q2AnHz_~6z))tv_3L!_Us(m-&2l) zoVvRm{+y15?_AjAT0bGceLcDl9|oV&J+Ps;3$MI>8sv9%;Tt=LQhiKkOp<#ARs~(E z=6>=LhUm-l`PQ*Gp`{L*KYpmXYFyc~jR)d@6G<uyc)(mW&TpytTZ^h=LeZB41w65xvcY2vvSu?d;DbQ>9+G&8teHk z2Gvj6yu)=hV|zZwJ=Bc@g{<27})lZJ0eM*(F!0g_Mu)fuQ&> zvA}x}bSkMKjqpTrf3#KL{db__@pCvi%>f4vOQN8+^5C~R8P2v{qk+w?RQ0-omUKM~ z&L7k1){O)BKDaaMe2byWz5AhW%{=lc*Ib7=)zr(mV$~oLRX+ss+3D}0)>9yV@jFP4e1y@@%W;S5U{-vt%oYPCqkP>)y8I%V zY8TEGx4gIr8-gBE^wB>;Nb3dKQqv%mUXJG7ogWL^HPs~j{60fdQZ2{??Zrj?Y&h_4 zZ?ycqhC>c0fZ3k2(6}ZJwv4XBy15M;>^y{hM=$5g^^A93ALrAdrIP#IoJl8b5gd0* z;(L+^Qt>F~tM3D_>%=tNc3=`J$(-a5I@)5UPXjxYPZd0#-4V35cEJVH{CI28dX(#V zze2;rlP6T0VdW?idvAInJScn*pK`OrL33u4f9qZvZM221_P#-t7wkac%@Ld^SI$Nb z32>qdlf2e@u*jZ>Yo}!6rGQi1>&-6S<8&NW9BSl_<`K|mTq2%maHSZZQu?BO9`#AozUk;JleaoP=}#5?@T-n50vNQX=6)j;BBOFXg9mmmWzcA zI-HlBLm@V1ylz5IR(zdInxjtOckL&V(N_&2A~}SxFQ$W5j4$@HJr5I0 zKGLvdU*Y9~46$INAEXyFL&$hL(tr0tyf|Y9jto#lRqb(9!|uN$S?xjw>mOJSv(FC0zCHFr{6c;1obSu_9lJdGPru1C^nkS3M{Ps>kq4f~v>)6nzZjKxR%DCqELUEq?NBS6*4!17a(OK&* z7&I>q!i_z-*5f73^YRjJuD=i3nYp-lcLzmRb)`tv%_uV>iMt+{NO`KIbg%h4v}eeO z%4u8pyGlO? zyPN{HsIS1BowhvdRyoX^rVV%BmXp1|P*piXmA9;z&q}{d!Dy&0e|_nM&mGTDdDAwr zG@ujab`>w4)DE)*pYFjlDZy0gOM75asTz)obr(l@ z#X{H6Ct@$(-GbT39n^PJZ@AN_N@mfSl;Jgs+P-w<+>5?&@%d6wZ~a3k>{Aajo7FkE z-I9tWmDGLwV$A)Vz;CV&q+a%Kg|6~}aQNvOiB(AtT(kHJgoFmNQqE}_kr;;yBu2b5 z=M2B;X@D0FT!VMxLRj)BwDO|HdwQu`2}ZqC+$IbGoIcqE<;wCXRdYLRKNf}Cpp};k zN|Lmj_1vvNkz30fL=TS)_&C{yJXa=x>-SjtcG3+}N`1r~-QH30rG@on!$h93W*fd3`WT&7EP>DjSux809LtWG%HvyOaLq#p zJX+eHch556Hd_~bWOEXeW;@azdq%lqt^B9f4c7`|#ftgbJkwvNx_6)gkB@Rdqqpa9 zm2!8yGRmA6e9{M9z4c=BjLvjP*^%v`8G3Bp1}0KJ>AIXX88@cEIyT|M%X(wpm@r)T z%nfc#_$(U#xdA8);cH*j(EZ{mGHFP{#o|WJd}ZnOa;`f2`6R&I^shqhy$tL!b_QF# zw7^B1qPS;$D2JAYi9K~*p{f2{@lovvHXWMIT~`i7)mPbY`tWBe8zPNu{F}jhsv|dy zFDJNWk7}PIX?x-;*lKT$+Z2y;51}(@hJT>H6IFT67jxEkO~#CV&CqUfl6K5<;DPI= zko|fITp3mfXZ%Vb$EyHhGnTU4Rz@YUk%AWY;NmoYh|d~@A8S@{vxXW9b()fafgR$^ zwD0ipy)38o9Kst9+Th;&AA)U-30{;NN1lUj3)^1LVWVxT{3l@t_L;||=K!3Z-GIMi zr}I+pepq|HU9@+80xLeAqTjXCs>0;E;pWi+pkQo|Zt1z2?FVp3?+dee! zOgt_eGzfkA-Q=y|X9bP3{n#d7t~$QqH%#2UgXetf20X+Q73JD-^zldZ;OhlR%Sfc@ zlYelGwIw9aKLmpk<=u)sr9Sp+16-P6K^_Ap^SBEu@Y4`AJ|1*S8h1Iny>*o1jkd}B zV!|(ovYI9N>}guva48Oan?49#G!9~W+DEb1@Mbu5Ng9_q`tkJZ%P=Oa8FE+b81#-ruAU=L{n&+cbDHO$fMiXqd0wi7I#{bia!np z9M7v*7f0_ zbSYSLnVbRFPt-^e@&z>>W?HNCGUkU3suCHdI?Tc@DmhTA}Qovt3<`i5}gVIz^zD`R`_-0 z(0;a5HR*z|%)*jx&Ce$7ur(C*>=G$`*h%XRns)LW^p5UsOFj9!#lwXF{(l*{fn&NoedB zjm?EN{AgiscJg#a{exXZ=Uc5}pDK9{Z8nBOs`BCog_~$GK7bb-k>^{vs<>NTscKW@ zdhRqhhd;wFy35*pa@beUR4jcN`;HjQL4UHZ0gvigTvKi2FWm7uHUH zLjJXytnPULOqcA&AMf|z$xgae6>7@hRJ)m0uUNqAs0e6K5h|Q&qm92h^QC1UpvU8O zGPu7SFGbi2J3sg0r1W|kFW*E_N4t>O4R2mI2~h5et?SsA1H?}ONRRcUbJ!?||7zHa z`RU}*V3~u&PeoB)M?heGSzXX0Y%M35rbU@?PyJDrk56s{LLb3G% zICv|bJeRH&$MxxkpJv>m`%7iHmysn~q&*k*TG_G3_MXTEUt#=XE1p(efyOZ@oYkcZ z%X*yyNpURfe|MJ`zGv!Pkp^A&-2lr;Cs2QawcuO0kEV_EBemB)ELN5Z*X9ejVP;?c zyQW+yZGKGG7Hy?*PCC^EJx<^;t#7b+&KMkUM4q;7RmR0u5q#d5Y5xWVF7EaS-p00b zk8F2R@4u24etJvWzDd~P`W83Qe=KiW)ros{JwzYuuHui=eQ@f^S77}1H<-LS!QW?j zV0tw0ala(c)bc^&_(Iwf?7@2#57Cj6rPSHKysBTnV6GT)7~GGHk%YH*g@D_J5c%ah z&HZ~5F7-SuG);U$Q>TSXa|SbLc@N(zf4Mr!ww(>dql+ow<3`Fwh1ksxPC-xTFf4FKnX0^B&OKTjQ};_7s%dDuwFkr5y8YjW97% z4?DtRu{%^!!#+bS8SF$*q>8eQa@?zEI1h*^#33Ju>kpjb?>Bp+Qq@RqZ=c9NCk5cK zXa!E|TNJwQj_qz=0eT zos9nV1EqVLA?%oLPfN!x;9e0CaP*QXKhYX4Oo%uMZ|er|i}TZfepL5NOL?pcvQ^; z*c{p+aq%mINeP)eI6#}Pj+C!1xibp~jFH8?2J0{h?CHrAx9XbFUSPA>9{+|W@cl+l zH}l>NY=7D0t=Q~N--gZpFn^!EL9$6gNGa+cGc)`hrL>9t_sb%w@Ic|#fd zRH56Ep?L3Q29?ekK}mk;^lMlk4NMroGYpV}j80I&b2F@6s1J9QilAtt6SOS9El#8j zB*r`heN!c2xo$tMEV97Qm-djETQDp=9EP>?cW^-C0~pvk6&|`?L%q7uLeH7NZn2GE z_3e|eJ9;J;?6^lK4rOs`Lw`xrp>1MO$TMNSL8~YX9f&Trn<|q-`||U4SBMxa%X1rR zY2~|nv}0);EZI1M5=n)X3*JsC`OEP$JIl>({lYNXi}8r z?B5G{+1E`pC?L+YLVdaTZHPA)R1D?`QR;YS|2QnEJT7+0PZejjtKqB6%TRLlC0^Ut zn++$-Xw6$r=^|i{Sx1F)zxL8E)pf%9;RDHP z)OOfg-5Is27vZSMd&TUQR9Jj-J!P%%g+KR=V4&kq64#eeWNaM0zS2Onx5!Tu0V(r)+G}AJdW!8)%vCDkCniI-C9*;=9XfU1e?u@?uV|m$@?zqFC zklxoh;iYYhaYlkOce^=VnVv_=a~A1@(&xgMfH#T|OmtC4gaw}5(RFKk}i z3l$r4VBUumel;i+QiF!$Wb^)bM!A?@t{l%l)Q<>_rIysMvi0#WY+krR3{zqlV3X2tO_*Euw(aTAz_MA5_qk#63e#k;Ib_=e`TN}B^I zn5-|0!tVl9TB*Pj`dWc)x-s9F)rJ|~F}#1lJ`y{3;~3v9aHDoV>y1sL2}!>tv5hgJ zl2HWSHhPDBYS(l4oo>9+DUWJKFUBP{0pyx3A2a#(9L1en%i0YQgr~{qc8Qf%;beR1~|@0fsfZ+h4>@oq}{TYrkXvG ze3`7vM}&5IDsvRB7yKsG*B5YL@(9V<3U8XFF_o+8jB%TRDlS<07FMp8r?ZB)>Fdd0 zoM!b_EZ#kvU-YYm0cWOh>Ahc|8?M2Vl>5Q3qDz#0BoUq}eh^09Jq(L9NAm8>i}3K% zLveL@H*VN?MVxwX365VmS@g7Npkrl~6n89MSokRehmE>THVbNKby8o5xA&vsbK}{` z^dl?|84rpp^x53!zNqa|4ZrSRpnlpy|8jlPky3s znF3_f;pf2;l2Kj*T4{-_HAuquCz$ZVis|SvZ!7vQ?SeMF9tyoR2eV)BO?uViIgEAk z#J|noAT4x_L>6tyzP~&#KYc~=GWIVSxJ;7zYmxLoRD=FK)6p{dVbwd?Itr2fB~FRF z0*Nu|ouKj~$-=Ub zL{bg!gw~Q+etB~#zDNwjI!#BKG9v*S4-G`4XR7!jbuV`r+93A!Tg|;+hH+<~&8RNT zgJ$*5sq&4@hW^J2ut6gWs@HX`s_S!~-bwwyq_(KaU(3>@YxF)j|4iix!>dWr@dVrG z=W=4uaWuJ=Of5Nk_@QGMpRk=y8IM2V=#jhe6SYZhWS!@su8lORvjO(1`zx3{D}}k= zPJ+SeE8^Wl7pbv05j$NUf#LY8%I(*0^hlQaqa9BHpPr&2MYh=E!9rG6n##+HM)Hdu zKj5>%IvCMY7n>(cW7FwKO_@dF<=us@iD_MMa#RdkE36i-h(|Ch_$dGGS0dDY%)=nZ zaFDr*;Cy_#GzR-iC(ZMv=fgJI+IpW%6Sh+R9z$BI8V^}+FJbSsPHeZt249R*go=d^ zAjGRTP3dA*)nb%~`ZM})?fi}W;!8Qc>tl*?e_ip^5j||%>&)&?mb3QXRJ!s`4;A%f zsHNi!FBU6tZt-!5*_=cAHJPL_;jaX8?z1%6M13+YV&CL^4qp6*v`wS%rKvR6^GbwO zQwr$Ymo)mmwSfO*jKk-7D*Wt3I8RC*#5cz+huFjRls#bz77YsEVJ$v5Zu&k@TpNlD z&2+05$#|0W%n0@>If(IE+E}c%6_5N)Wtl&Iu&QsASo%bbD`v+C%O=kfdi$xlEj@pi zT`7S(hAn{Jf0HmRw*ch!E#oNpZ_r|N3apcV;-tty_By{~*uM=a)J;=N2V*%gD!Uu1VNV@>NOGeKMZ>YWOohje3?b34lbCXFG%tNV zn!Q##@a}mkAbTR5*68I|)p`%%D5pVI^mqW7Wc26zu^Vx3>~|iJznMqOOG1sb z<8)}r1TH?`f@9PgNw2LAeYoROWGe<^5OoC zkQX|cj?Eg&f6sgdYvm;D)>)m$e~=Th_Eh4Cb2;pE*AT7k{s?-*Dq;GMx19TR3ZHZ; ztn%uvhe{hJ;)5S!ux6(gD}CyTW|uX@>Ngij_Us-wEU(E9_g2Ay`KhqlFHLNieF;WC zok>eJt`r`%s-cE~3_G4zpi!CYK(>1TyjXmaobMe`2$CQYbkByeLn)?-kEWqPisZ> z%cEh9xh5K%yG_T(>GOkzJ}lA6qcPGLP2+|~)#9N}ylK*KAt!Mjxf-|7m-W{~$Lws# zcqxyuQy$Q}lZI5hMVkNnI9Kef`GPD?GiLt@;w6O|_~7XyIuaIuw^^N4PZWsu<0rur zcmwlPBk9TMEIy|;l1(ZTaDuZl#p@`brn@pN=yVfiy}LzGoh!KD##Q#6tKv4kdkar@ zX~)Br^KpRWAfMN{g1PT{@~xg1Xm9Oo8h+*=8Te<>*fvuf`P2m`+uHKq;xp7b;~O>j z=L+A_dg1Z5G%|g+2ts`N(dIW@&_lKt2X{kWLQBB9M=ClWdny_nRpN{wM_d(P$U?Cq z=9k*Ba&A6cQOV+5kDu`J7}3GlPWa@$AM3fylI%`#$MPAk1ivMQV*k$-!gMd``f7FN zvgK`fada>4=+_AIOm~TKu#u+4^`V3P+9hJ2Z z_NEQizWhRw?gMatyd#@^jmL}DeQ=XYp7`sB2OIe6b8i)WJU2UnuWx<`xnAR7@RwL# z^{5V4Mt-W2@%Tmo3L5;i{Q}7-3mb^I_xaDL8dYA`YIK0z7EHZ19&)Hqxbmrq`g_Xa7z zV~uLIw(}KESmm&OP%>728OnDg@v!?wGtN#pEv*}Uq(fiQ@Mq>OII{FEC2X1tKe8^1 zPQi*4Al>|+DzU!uRcN!ggzvgeW51zAy!TE5>ps3FX4<=R zSgr*WB`CPX4ppF~o0Q?lm1J7(L>Sk01Z4B3@Whjaykv+!re)ed&5dqkJS-mL4kWUC z@Ky|GWoccz442((;Lv&JM9U0g{1A7H6*peSUal2YJAcKo?w4PpOTu@0b3==d>qbEM zwlZ?htblkT74ddn zXMG$pWlRO3Z7!P|#nZB?GGX5^Z`z;I7Zp4fmAYV2gX4jc;vx6$XjoS_Wo8Ci2h8k~I=%9AvK+d?@ zg@0G9qz%UoQPi)&JooK1u0J=KyCol`f_DjU=xhS7yMKdJR~B>r6IHg&m!)N8t-`x` zCqbp~5y^aTV_CaUzBBqd%>Fxs40P{`g6D?WhMx4vjKHkAz4 zOoP8Fj;PQli5863;LCD5`J3V`Qd>@7ma`3S%~}DOLqqxQw<#=aIZE-%I>WBGA8_}9 zHXp0h;;AR@@C^AbM3=_HoZhv<=E8WvXr)-MsV;%;-;l6=0B zphLh1N7Ma}{?EsfRJ| zhe=P)I?7s|#`=>DxN`7evfm^tNlBRuD^|F{nU5BH?>GzNHzvZm8!M$d$ubCc9V2-E zI78OQ?@-;Hx!5*fG+sA4Mb;}<^QI~t^t%=a2JSUt$mIq|`5s0*Zaa+`HIIZ1qv7)y zDf&@&Ef^0p;q6Z~dCAUwf?-&TU}1ihVy3(iZs^K$PQ)Bul$XlB;VRhCk`KRb{3e5^ zx~#3?zzt#BV94W%SaWZyXgChVfQ+7eqrC@deVr>F=&4C6S39BH^B5f3+Yyq~>-cd- zK80o*;IN(nINx83(=T{(e69mUUOWZEhEG9F)q~Wm>&H7QlsG=Ef$WAosr=YmDywz);i6*7f1g07%u+by!2S>%;BYN_1xJ1`-6ZJ9~5s790O4)0Ko$I_Nfom+8my87ttY-m|E6+c{_) zcY_@7_vP*8JsIx4cXji+C~42GscO;)#sv@KKzEWg4&Rf3vsTZeP^U7;_R>eFR7dmS z#$tVo9NxXML2OCRr?Ed3>7a)ZXB}2Y1Fx^b*6CTaRN5zI?xkGw3&O4KwblhK#wU^h5RnMm2v60bOUK+-#OFD_>Plxk`XXWH_(uudd^ya9Dot$*6 zg=*)1CdC>Xur(Bf=gAe6ILVTe)>!iLGu3p{;|jd&HXqHkO7LK+8Hag27jM2d!m1m} zpz+NgmyLe_JDlY(A?=Rf`vcfz(j-pSHezd)VLVv4N;daW=>8`ce%L&VOCGJE@X)dB zvbc;^+)?67A3jlttpWc%)Bsm+e1=JH!?4Qkwb)@X5Zp)g#QA+ikZGNRCHu<6ZzDX= z_04A3G;26K*w%wKR=Sf+*jx@$(WR17YZ&!oC$%0;u(rEDzWt_b)*q9%s^v}q127Qz)DpY6F*U?Nm z8Juo>mx`>%Rzk*3D$IKU_gXJPvX>c|^_0g)cnr2INy8QCemFBG5I-xJ!-EIiSm8i5 zMLu~zbEXgD1^d(3KfDKD)4L~*I#mmsCuoS%#w-=DwGU!r&2XA8zlTzOOyfkU{^LF5 zs+Szx$&t6Z@Ezru!t~)Pl(1n1{fY=<$1i<&Sm*~DG@+6{C?&!{uf^Qplqw#a+9u3C z5sZBz?b+=3E-aLy3U3-jvdSYnPInFA$Ai65{Z}9K?Yj!}Cfd{KuvE@!NQT`}oA6qL z0c;;{i!-k5jJiV!Fb9y5Zdp6*}thMT+&VUC-Ze}HxJ{n2{K6!bb#CvI3B zNLNb=ME{Pv@H6@?X?$n`yRLBr+uu`XQJT?xlSAW-cR`8bZozKVT{s?P!^A=tK-!U3T!(*oowym@OtJ22->`e?&WvlSMjm@ z`d%{Lf9?XWzpnxFhl5aSsUtf1o`BQiNVrIO^y9tMYpp<%Nk2#v#wL(oiVS}>J4o)Q z<_N1ay1=t7R(vmJ1xw6{FWhwIPgNS2c;*^Fbpx%^H|6T}m-(n}0nGM)508EigHA00 zxM9sLh}7I#6>GCpVl`nf4_R6TaeGgYsx&Gs$g|{YzTK(VaTw{O1k#e<(kfQNF=5qS zIUX>=iw_M?<0+%gK=HXbcxTi(P`})ZdhU&6w|SZz=4t`oKTgD#_Upm!_jYWamqwe- zmy*Za8DdW1Oq@~l6)wxi&?Sd_TGx~*p?~-VD!FW^rNPEd(>bKSB}a^3$4xUgaz*1L z2DL5JxPLgyJYUBSbCNi?2AS{ALW_h$!pOIQt|p#?g!INF2>78&y3qlwyKW3CEDq!$ z(R;*;k9|QSEmu;kD@EC~EXG}*;z?e9B24F0eE#(a{+Kx$7tjsv*?ASKcibf1MV+6I@Fzn*bDlP9KUlRd0>i_z$wG=S)(~Z+=VW&*e5)k|^B0QoMb>;ivWT~D z4P|SMO+x;RFcxLk!`Ac;sGsl`4({}!pziEC=kx`rf0Br^zxD;=cU3gdDFxjFJp~8l zIQTJn6VAI6%vSgJNzrXae4{E&NL9Wqbob9@#VMxzVaz+ix7OUGH%y8UtD;w<4|B}> z19a|{9&f9bMygryRC4q-SSKh#X|x61n`=v_-{~fz)#Vyu!W{|U z(^7=wdsAUx)NF_dsDNafO;p~|3%)CF6waKNg7)8*!W^y5;N!OiP9%@Owk|5T)p$QG zju*)|bV6v9#Ywn7BzgEz( zw;M!-R}(l^*^ZrF`%=;p3!Jgy7UhWok6&0woAb{LcVF$s?r#r>3+nZ0tPm#_K3EU2 zudbq3lLwx!G~%-F1*EuU2$tNB!@CD6BuOrD)MM9aaSy!`ntMGY`LtbB(b^&jjQ%2y zit$G^+3V06vJ!+hcc@V}5Bk1JaUFEc16{@}f}MSX`CwfO4fi#|amL-~Z0lt5j-N%Y zlD@b>#Q~r6%_g;U6OQn_0Iv=mA;SVg@GSpMTlNm%D^m=p$@V*``y^6T8;B1gkSCva z#H5rk9I@a9D9M+={HtH-<|ip4WbXu&bDIiRg5vSVb7$JqBac?ee-tfM&S4sUfZVEB zK__Prdz)<#muuXHSN^S%BO{||-}bDkG4+~!qCSPH?1%A8KM(3sF-r(eA5W$^7a*k4 z6)#1tL#LbiSa07T=P9c6K*PULcwA#eEfjUbxvi~VVU$};v+%HWHYYou?&@F zzl8kSv#`o9r>Z_C1002Rq1zWFnmN9NZguTJ-aT|-mRmIW|2YhU*JRM)(}vKsq!pU_ zY^svU3nbYt$y9o~JIz$f61QKT%)`@?IqJ$Helo2ywJy5|^L!-us+$E4aa@TpLmc6& z{z&Y9cQXvz6pGmoqR9RJI`NEKEvfD}M^;_VOPum#_>D^~T$*Enj&Wz{Z(%aqKlSIT zsA+7sU{oco_wia7B#xvrw6Snq-@#^zCS!g!#5nH#gAn8Qmr8yE_^0A92x}o zZWRk!_yV4eks`zT7{GPkWJo*xNHBJoE-3bSN@xGf=e}np@HFKB{JJ-b@5@}HZRX7+ zH|ru5$Ndub?Hi0{j|Re-b-AKniW)n*+w&>~7Yqu12)!HQM474TJg0jIN8M|uAXBLf z(pbZ^DT*8mn5u4A@xiqj6!>Qr_{AK6%2ru^{W}i|T?^4j)g1GRE$MWN43kR|R)5}$ zy(8N|p0{B7jce$6J&wZgJouiyN|{k)S zKZn587TCXJD9T??6SK4{sa#1zNc1{RwsWWBjoW+h!>7(1cz7vn*JZke~mRDC|%g-~|cVR4t z2BgyaHhFfv_+0RkW_LQ37l|v@946nMHrQ209nYEsfiy47=Nlbaz2|6jmZGGOKQrMI zqhonPxf9&-x5oI*`KXiND(o8^4FyY#*jMhc@Y@b3AiAEwu>$l?{G@>oXOXwpB^tZ@ zg}9}mmI5}usZw!rmZEp!$v63t71bq{ZX1j`kALnqiBvu?g zq%XEk>y8=EW594g4{87YLTH{QoLqig99nr^6t}yG2R~cFl#&jK-Hp5Oyv2rf9vMVPEB+pjeUxyE@)e^qfR)_k2b>T1wDa@fLWStagkO^1(ePW)?hsIc>QH}UknMh-QuWJkF=Va)zE zI=Ep3FWk9|x5#ec!HMT6;?Gw8Wf#oX`X$iD)M^@7qsZCac9N;_8qs9B3I~3<3`u(h zj(3&n#aRRGt(AD@@?znA+EC%(xfJ2rs^2s&*?}!@+=B)P;Jbr+R}J$iptYfK;tcl? z{B~d@51neuwdV1pFi?&MPEy8eP6q7PdWX(Rk-YWiYvAXQC%8W1E#3SulrOEc<}rN~ zIrdxxS(%UF-q*IV%VSMm(>zA_nrn%>LubPa|H&+vN@GVqFOt_wgo#V*#noGa`OJ4^ zj&cdYw?$iVRmmk@G=I0N<0=dMUEd9tY4_vk^`U(J$WK}?h31+4Dy7VqQM|ZsAa{jU z>gtnERtdj&)rv%Z8GRZK4_``FI{jElI7^quEJpVapM{n$D!ig^8}*2cgTWbI_{wXO zuzv0f`m^X1C7oVUHMf%ug^Mjhk$)$(X}KCE$h%bhT2$X?1h;RC;pZ>2rE zbYbiHRQMq767H3~sG8orXVv>|T`_O-Df;^{U7YY}NY#r?y`U;u1r^*q`T6PY5SqIk z3fG#_v334fwK;@Zekt?J2X_3emlWZIIuK%_jE~!9gX}7OZYmr@6SW%zr`WS#Qg94n z+XT)zP{<3H_LHK+)?&xcr?7eYPwEzz!UdOS(0Q*zI7X(Q=ypksUF@qx`A{?7VyJ?= zMY>kIc7>y|m3Y_hP}Tdt<&<{cmX%K3KsBv{;PCQTRp!fneDLc)emx+LM?`q=A?-=L z@A7vcwRawGRMh7$dcFD0z6$YTg&7~-bA*RqN)~(@;&4vWEu1Phkkudd;s^}`X`#~> z+|Kp|=vv5y!XEMD8;NK!!VQR^%t!5^&pDJv{hmrtq$>3r}_ZMRT7`M$Zk3 zxa-AguD!mKj1S*|jZH_Sw*!svSh+Lbblm}4>MiL_&Td>daxqVo?$MII#iHeDdF)u& zAsMbYlZ+1aBA;P7!oy?Rq5OkBWz6;BeD`#=b`FQdyY@&Dc!piaEGX1|F(rlLI%$&{1NmX^%k@X{o=#xMV(H z9t7KxlNZdGHS6EQN^221N1%^+>sLbBcbj2uXEg3sxJBmby#qh9f`175H*}jC!frgTso6AUa@K2~8-Ak%v!YMqgqC3y)ux~3Cppf(nP!Wt_XJa*3 zsV+t9w?`U^H(v(vtq1Vo>?qu#a1bx2x?=Iz-FW6xES3vBf=?l1@pFe5t{*v#t2$~i zw)zRXK-3E8F9}*VEuK_5^^%{N49yRAr)?hN(QAb?HkT5}OXe7H#D!Q)w!&_i?ck6W zguV%{z;#0sW+s+HN81cM(V4>K|EHr}%{uUS8I3oWRHKl1Ic_~Dh7R89c=RC;ULCZ= zJiP+;t2C0x(4$PzTQ4-8#b;)@Nq{Pck^a6T6~eg+)Ynw%aXC&PUS>zl!-X1NRc^;dEoC%X zA`VYac90oyg_vHfj&cGEc~_5*p%A?i$L))P+_50f!`9ZFU^inRO8BBQxMkR2*vFw#J7So)I=h z7t>allj9s3W`fow607aXP9J^=JKe(Jd}=P!AA1SNW<*pSAWH(YXvHB)B_1xsfZREl zvU464(d;B?T+Z8?mxO6~JQP2Z3y^Rh*p?hzT(kq1?bXDg(@W4RdmN)$<3K9!X|TFK z#$Z972|m5-4*0!?xhp%C{rt-m4?CQP4QFlu96H7d8pJ^9Wk3FDIR|tS`pUZ^w1^aL zwxv16O;l={D6Q9CPeapFXpr1YTK4q_{Qh0aMt8T87u!1dhdGsnn^GAxNmd2&gjUfh z&DQk&d2LKqizgrLAAm|M#d*nDQ08L<#|kQl$jReaAKgMuCyC(FFVjfLO*y<-u@suV zCO~qL2}tTNY&H8FUT2rEFa8{9SSo*=br;Nm%asLGwWXEaHg-Bq)SgNcO#;XsmyI+j zb^%c*CGaq%ke_m+9YnePJ}*`TCz@Vo@KkxcBj7i{4jIHgF*F>FmWy0 zlXe2SmdnEz-HYHFvI<5SoF=;Ba_Fqp2KNHSQmU#)ms;G10@G^f7IDSL}u-6yRUN2t;#~amenv#9%1E}vaS-Mo&mS^DI0j#Sq?Ta*H6h3UB#=7Mgviv-_pH{|k#sOHd zFPkai>K-xKeEd|}j!qt*Ax<%C<*jgx13kz z;6XS1P-dPh-X-Fh>FkNPR;XLJ7m__rL)_31S6QA#700x(LwB=T=W`rVW^5&tdVFQI z8Ap_#I09~0>+#E-9w_#`2Wk=s`=_YUQPvu$?YRaAbXD-HrW*Q%-z2@evKX}*{K=FW@o_We1q-y#u=dGW~4&><^jEwDl85=3=h28-*l zFil`SsXlU?R29l$I30tMveW zZFr2d*!2-;U>2OY<;RLRUS$0D%TR}9X9;ho7ynhxQFJ~x7hie`VA!f2;-1pS44eDF zDydXFzQGBGbZbE){~>ff%Yvxmp4e2RhR(Nam_idVEaS`Kvp!k;ZS{%BpB`lUPg~-i z@%P|QaSS`!1w3PepVjrKX9B zY_L7)2+^4rfHPG7KyHXB?NYZy#V8TTI(r;1ID4a|i2;g<#q;dbx05*i0`gRB40sf* zW}{WQVBLx*L~THga+O~^VR{KVBlg2u5hv)-6vR-&>rCn8r;K0309#e!g`L$MjM5DW zI_uI$(!u3ho$jcTF5N_GZg-b_)EGz0lFDh2-Vx?m=05OiJqun(xIK5JD?EcF;>ICV ze)^Q~U%!rGlHAR}X+;}3(7hh_e7geYG!MbibD<>9%983WC}EF8EFvp~^GKv)d|?i@9)H0Ws6T)RBQ5ONQ2>#nmO#E` z8eS}Oz7{rX zTn(`^`pgJjZ6p%EPC(}D%S_d2Usx)h4u6coz&>Rgt}&9vsy9*iGNTP*1M7*n;d6fb zhz%|=2}HZ!d>DQn%ukng!&g&kD4ZPL)e)JjfGD>bx7_}Q8n@++m8cMEM&V&~0d{#7UD%yqD^G<&{ z49j&5aYbY;jEEWF&ety>c=A}7U~fn#{aH*8kKaHRuap6Yl2(}b@G)8Wi>sT9&4h#J z9q_1MFUWp#z#&r+{NSLAd)K7kn62GVcTR=ha#aGZ#u&1Ke!95*tpxZTb!I#x1&Kt? zJ95bH5bV>HgUZV<+1tWW^h04Zzvk3;=(#tSr#Z%z_%TvoydwsDhHtfYY#%WTxNRwGIu&mhD4?_bS-9`j2s(lF zpx2ZG9S4s=a?d5UH{u<;ZCXBacfA9Ro90e$t;n`c*!BggriV~d&p`Slu!iip8-?k1 z#Zc+L1w%(wjWLlHZbKvns|lG=#S7m8Ma)#<;{ShW2)=QHyh$cu~2OzpCQ_ zWHkzqZVw^!%!^@`4&8*V*^|hxcPGh-)lv98_%L%+V-vq8r;J^wst)<=IZ~?m12zvP z!|l{u=oNX)Yx4O+?zjuUqO4SG`&vtGjcO!1nF~pMi#t8^@&NTu*C#tmrlR+yI_68W zG(2CDPqK`DFizikVTXnYp8hUNG!8|O*uGJ8wQn0d$_zuTKcb9m#%8jeIMa>=o0+2K z1@QHU7_+DB5)(5)6F;`D!55EA@U!a~$Zjuzs9U!f?VGMRRW2VN`N!iP%SAZjpaAaq z*~DfZ7>`YS6?}aD9#lu%B^!*iN$VedYMK@Z6aBQ{#LicwW93Hj!tgrXAg@d&MBil; z7K+kkm+w*E*S&Q0;CKF@@-yaw_hsg|RwrAtM-X!_cJUmGFOu{h+evfV891%@hFPZe z3~~dG6M+$5#*NF@-xb(J_mniAo6%hfp9j3bc0&ZzS~ie`9#!UY+EkLVRtGO^G-PL3 zctYRF>9lHx8;MZ74f8bTu%peK$)q>q@Z^f+#B%dBqBKDaqqH9|ktU_gj^S*w(`1;u z{HzCyrVNvN$I}>XrzZF$YfRp(Q6~##KZcHTn^=jG9_Cl}(FT(PqmfJdW3#6$l?gro zXRT+_1xprFle};IO_OVg&tqBQqLjjTA6mm$&C@3vOG82Iupe6|a1;6;a}RKyFO=W( zB2}CUQ*_1<^lJ&?_5lH${roA}Tvt!tkJ$t|d0d{zwH=O&cvIoqi|Nbq6r!ybjaNVF z;A`GKCSk;g^qlJ=vUl`wAb1bT>^Nq9-~%5vEKP@^s5>yk_r@D8>&W!;%b5#(UJd0B zL@~F(4|5hgW(qcFgVQEYEKd3be?;#xPu#{5|I@w1rsg{=8BoZ8+AO3qcWm$QJE_g ztJ0PAMi}jX0fO$h@s2*RC6hO+lJz^!LXCvQ|DH;x3sJpO<;0Ek26>XoUr18MvPE0_4cd(F!>G z$TfcSwwFYxNSg*&)WB;&SKMgl3Zecj4R3d7g4z2CP_`nCIq0&8H&4z6OK!zd&%htN zjHO{5&V@T}kuy2y(M@#3)$pW11~zdBy57`>tmJ}K=wUB|p#}@FcBu>|uP6pp2Yal0 zCV>w|D`Vl>)$p`P1qYVSBzGc$No7SPJGmf+i3q2()oKZuWShh+4}1cmEn|5(js~bJ z`ihNQkpXZ~87)jom6Q#Jqu~Kqo@)-NV@E?( zh&d+OCh$T&M?t62MKV!pGbqJ#i1Z#I@JQaz?iv@sOxtKrUin&pbhZTM$r^z}s~DUQ z(T8PTz08~Rtu$ukTnrA0<@dI0(&a~IV7a0#Uw+CKcK@o`)GAws?9qyc8M>b2es?6t zo*Pd^C%gu0cMfO5<+sZ#k|6P0Ab$2f1dZ}P8H>Jo=<;Pbo;&gpzScj3kQo*;Bi+x+GcS$ zh_GE`!jg+%_|p~_AN>RuR*l8xOWZRl@El}iWHLHWuCl*tZo|wfPmGjrB*FIM!DqDx z+8Q`<`%p79HRuJ=PrCGZg#`}kE=R9bLF6+YfLEPuJY^wwnw~CavvI*3x;H@#=VfNn z$)6LcS6Km_SNoYX{c0oPP)K%I$I(m$I36(=!a=v=i@%}TVOK3oNP?W!o6lE@cAn#T$^%~Tn=4Bo`laLn|>Hz zqqiZ}m&V{$`&wpk&3Tf2@G^-Kt0$7a!l-;YiPxo~1O|JO7@npPDys_NXfG|^S|K6S zdAk>XORmM?(7CW~u_HNA=mcS{;#4iDgzVj(0r&3$ncB6OEWUk@bbp%8qHtGiOU=!p-w|Y=wuQM zFPu$YFYh6px3h^#l{4|QE(fbUJ>c0tpKTZz0$fxJ4zDKD0et~rr!m-=jr&m z*BPykDpCIfU%+#3?n6qa>9Ch7*y*~$=Se5o-PC7q&irok2R_|$r8crb$qH;j-ah2My& zgBIO3dVr}rRtt|GPop;@Cen;}0c4+ep#BOs`0d6#?=8(g$76>GXOb$Fp;cEIgb7ulL#%GZ|@e~Hlp1}c)Xx^8SljK!|A(7OwrACc& zaGAU+TDo6jGp~B#1MV6x2?`)ub+2II2QxaDa2T9l%p$cP6)n{H6b-}u*NF4Ce&*4{E7mSs;;5qDDsp3NFgAo7<>~%0 z!TcZ2)FddG99|n|J=$Xr`Q|bL#)XQgQ>}^*gBVohl;zr2+Mxe_6BJvvm-qgh45pa+ z!ze#K(|kDq_9vP#J9vLUzfl*GReb1F=WAqXZw0v}8_Df)9>Sv$Yip%9nbdo2Bf0a3 z0r}aLjDJ%x#}aR(5>q^h$qGZ%i7FvwU*3~j$24e2=O&twVMEuaNl^##2~0i-pvMv{(KeMYvuS0l3J;*e=;FavXP$T&f4qYfAFFMqT zUi%JA|2`gT+vGv^%U*O+YiFkziIbQ+$|zqwo&S7`o zwAGld?w(Gcl|E+2)NO{+5Ouo2$DCeuT@8x*j<`PgEoi>GOD1^55DYoTsCqg=e|{o7 zv(Z6I!5m7%n#jYw9vt3b0ZNZfg3{O~Y89VOcFjKzz31ydg;V_LoSjd%kJSb>jv<|1 zRttKLrKH)#oQ~KZU=NSPVY0s-SWTCqK5rvIK6xW5m~gci&3KqoRR%el3z>0ielaDJ zEYWGy9;Ri68`*KG3tG-uV+G>^(#QG<|7AE@b84U^_hQgWR1ovzpOc4NO{7zphfZ&P zg1*8}=zQ&f6$MAIG?))t7p({FFcC)9upiRzl;Yr4JMuj+iZn*JbL*1?zOyZdR%U?M z-@i+mPi{v2msjDFtpL!ZTju-Afg(PDHOo)Q{^KQhBfL&8ViWAH{3 zxyA5)niwhUoljJ~&w_Wu5xBQk0p&R5!f8GeiB+8t8t**HKJcuCI8zZat@sUO#4Ka> z&micn5rjX+6{7LFB3wNA8f10NWL_++hO>|M;1rAd?CVA?bXAzd^dF97vTRI&DrmLVp!w;SwSEr3ywVAZ8Od;#O z=Ha&d2<&?t%(M@Q(YIfm>Ft0nbWl$vAA{rIOo$v_8Ek@S3iqJIxtTAx?@dGV$3FoIqq#KfKj#U2K#?jUpJ^1<)|S7|4j}l5LM}xXS)uDqPD$Y zOt6rrP03>#a~FR@CoLC_LO9c=o0I2Qcl8%~-dRq9H$>a03dqm$7lp z7-O^eeKz_GX*3R2Ym(fM)zs=%0N!2Dk3K$GSm~<`g&hHOlJ_4P{h@ij*hvG)PA9SW5KQ#oY^%U^P|^NCI52T z{=FG%hJ)}_>0Yo;3uga3OQ$Jf&Fovz2V_HM6di9}Pd~ggwwV@QL5CyTDI?xOHFS@E0P0CYLr+qvYG0)N zLoi=ZgFi)oH9Csx;!Xdv#QcprWz{$!w)|=?87AA9aDFSO>$K9rB^%hqlfPl_O1Z|K zijVLxN}isr-bZbAO(SQl2rJz+j#(hGi%!>{U=!Fg&1MH1LO*Z`Ea)!B?N=SJYl1Hs z9+abL_B<*#PpZ-FJg@Q0A(cj<`b#J=U(zPk;xx@Oj6%^3r%~^x7f#-|jGj8FZ}a<7 z15FJVrnvMu#8q!4`(<h94QXzopX(gyPrpJs(OBNb ztMV&^pK_wbRhR0iPNDi@0rxyGC1r#0Iz4c;M1&8jUjvma1FFa^|y~P;d(l% zeXvLSe{!KPnDJk?jsVB1noz+lYIWnE;swK*voB$ei4Vd*wVPq! z#qqHO#Q$n%s}NcWhkiVTQ|sl>>Y6)xZnVG`xl7PQSOP`XJE7j15CM$hmeBQ~fpBWJ3rL=>BrPig;BN#pGoK0n-UpdcqHC@%2d9}VtXBXHtB>=FK1@m@hjQi_We|L-`-ApQR&ye}&$%>_zG3W;-P zA}B5yWc;6OIR54MfA#%e#}AtP{~B)meW-s6{fG0$|8+R{kG&fHJM)Ww6D`AyLndhY zf3%zUmnMH*zJD|^_&2T+e;@bXV*k}|;y?B~_21e%{F`uLZYaV*M*q8Eg2Mky&|kCu W*CYjL{~HH{f1Pi8VaZW)QvU}WMo+{5 literal 0 HcmV?d00001 diff --git a/test/fixtures/xgboost.json b/test/fixtures/xgboost.json new file mode 100644 index 0000000..1817807 --- /dev/null +++ b/test/fixtures/xgboost.json @@ -0,0 +1,43 @@ +{ + "inputs": [ + [ + 0.0, + 0.0, + 0.0 + ], + [ + 1.0, + 1.0, + 1.0 + ], + [ + -1.0, + -1.0, + -1.0 + ], + [ + 0.25, + 0.75, + -0.5 + ] + ], + "outputs": [ + [ + [ + 0.02070039138197899 + ], + [ + 1.1609605550765991 + ], + [ + -0.14878709614276886 + ], + [ + 0.8968343138694763 + ] + ] + ], + "types": [ + "float32" + ] +} diff --git a/test/fixtures/xgboost.onnx b/test/fixtures/xgboost.onnx new file mode 100644 index 0000000000000000000000000000000000000000..a1f0ecefa74b2939f1cc9c670a17fb5fd0ca1412 GIT binary patch literal 4250 zcmeHLeQXp(6u;eG@2+$G@){JY#?l%rN7D2I`gw1=EmUiB7HDcTBzj(Zd*$l&4sNd$ z^)DnLR4BFl@4WZDE2i@)Aa8S;-l$DfKmXsD+!kF0;EH;&!YR?y-y#>hT zm%P#5h$K5~+r5#1w<9R!FKdrTQd3BldN9={bz!L-jufV5JC`AQhgX(7+r7bFNro4G zsIDL5EY4h%5%NU6kuE7JFXzTM-nolMj!@Vq$sWHqC`)7*@X4Y@Oc%|fNoz>V(4LuE zZ@VF}#<_OwnY~n{W1Q8wlSjF66m;YX1}2dcd2Lt5C^enX`?sY$_L#&h(d+&syh=zD zTwP0}2nK^Zx?9JfCm_p#P?vsqJkf}^^9dP0dZjoR`my+r_XLs)R%DMuLas$+x-yPYvKyPF#BYoYtpQy3Zy_9EM0tioXbzxd73X28T?Q|K`?7;I`HHDo9=*o!f7 zF>Em0W3Z_esUbs|!Twtl$2gmFfkRpPO((sP{nKz2OvlyGD%x-rM50|JRgmQSRFW)_ zRKX=DC8^E;Ad25JbO2Zxp2J&YHy(q-ZfO`eb~9tR*)1K1g54}QOzdXGA!IfJz%x9$ zMLz+g(j`uT9!nhvat6pqf8Q}A7L7|z2a#rJ{SuTC5WX~{OdW*yi(|!0buO5bGV9@F z+uHyp{G+K1-2?fyft^1WI7<9;&BE%~ZMYp+fP#dUuc}4O8W# z=jbIBbOE8m8`;ziHD59D@~y)`UH z$y;Se_J<=qh33``txaQGnsb0dNdIr*`Rn|{%1evLCzoM}L)meE66rus`lJ+)LO>`| z8c10nssA6QlYB~@r<3z?awbmBy~)`&Ilm@n)TA6rd^*kFz^P9PbZv>skhk8i!tqZz z6}Fu`tAg3HRefp#1=>p+1X$V8FM#v+ zN*7dGCjhQ|?NDKK+lvZp5}sBe|Hw`S_VV8Wtg2lvz-iN|7!-Wo0`U6U83h_%o{qs! zTOLZ0UYcK`TU9eh9H2iR4Qb4=g)QUwOC-l;;zwFw2D=iUIwa@z&C z=YS~&_r8|{FnG|dz=}@U1s7+qsRu4hDzNt0ybJpIDF9zE-35VzI{@J5umZL2-xSE3 zoQr`kYY*Pha52Dbn@0dP1}?|ot;2_0aHwsy0#n7u6v+A2u0YkrZ)?C|%>&r?QGXq5 zntT|bVc|I7J5r0hffDjfprj}0C@7UWeHCTZ-fCZ`e~quKyt=Z+Us1WH#^>v-kgCdj jo0n6c3o;S1aSjVIA>NkOfhCF02M$`oHD)7fja=?e(g3E8 literal 0 HcmV?d00001 diff --git a/test/models/models_test.exs b/test/models/models_test.exs new file mode 100644 index 0000000..2fff08a --- /dev/null +++ b/test/models/models_test.exs @@ -0,0 +1,23 @@ +defmodule Ortex.ModelsTest do + use ExUnit.Case, async: false + @moduletag :model + + for name <- ~w(xgboost iris mnist) do + test "#{name} matches its prepared reference results" do + name = unquote(name) + root = Path.expand("../fixtures", __DIR__) + cases = File.read!(Path.join(root, name <> ".json")) |> :json.decode() + model = Ortex.load(Path.join(root, name <> ".onnx"), [:cpu], 0) + outputs = Ortex.run(model, Nx.tensor(cases["inputs"], type: :f32)) |> Tuple.to_list() + assert length(outputs) == length(cases["outputs"]) + + Enum.zip(outputs, cases["outputs"]) + |> Enum.each(fn {tensor, expected} -> + actual = Nx.to_flat_list(tensor) + expected = List.flatten(expected) + assert length(actual) == length(expected) + Enum.zip(actual, expected) |> Enum.each(fn {a, b} -> assert_in_delta a, b, 1.0e-5 end) + end) + end + end +end diff --git a/test/ortex_test.exs b/test/ortex_test.exs index bb41f8b..1f550db 100644 --- a/test/ortex_test.exs +++ b/test/ortex_test.exs @@ -2,27 +2,6 @@ defmodule OrtexTest do use ExUnit.Case doctest Ortex - @tag :resnet50 - test "resnet50" do - model = Ortex.load("./models/resnet50.onnx") - - input = Nx.broadcast(0.0, {1, 3, 224, 224}) - {output} = Ortex.run(model, {input}) - argmax = output |> Nx.backend_transfer() |> Nx.argmax(axis: 1) - - assert argmax == Nx.tensor([499]) - end - - @tag :resnet50 - test "Nx.Serving with resnet50" do - model = Ortex.load("./models/resnet50.onnx") - - serving = Nx.Serving.new(Ortex.Serving, model) - batch = Nx.Batch.stack([{Nx.broadcast(0.0, {3, 224, 224})}]) - {result} = Nx.Serving.run(serving, batch) - assert result |> Nx.backend_transfer() |> Nx.argmax(axis: 1) == Nx.tensor([499]) - end - test "Nx.Serving with tinymodel" do model = Ortex.load("./models/tinymodel.onnx") From e6f6fb1f03be693a4aaf32ef4da8054bfda848c9 Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Thu, 29 Oct 2026 17:16:04 -0700 Subject: [PATCH 3/4] test: install assembled packages in clean source and prebuilt consumers --- scripts/package.py | 102 +++++++++++++++++++++++++++++++++ test/package/config/config.exs | 5 ++ test/package/mix.exs | 12 ++++ test/package/smoke.exs | 11 ++++ 4 files changed, 130 insertions(+) create mode 100644 scripts/package.py create mode 100644 test/package/config/config.exs create mode 100644 test/package/mix.exs create mode 100644 test/package/smoke.exs diff --git a/scripts/package.py b/scripts/package.py new file mode 100644 index 0000000..6512804 --- /dev/null +++ b/scripts/package.py @@ -0,0 +1,102 @@ +"""Assemble the Hex archive and test it in a clean consuming project.""" + +import argparse +import io +import os +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +from pathlib import Path + +from scenario import execute + +ROOT = Path(__file__).resolve().parents[1] + + +def version(): + return re.search(r'version: "([^"]+)"', (ROOT / "mix.exs").read_text())[1] + + +def build(): + destination = ROOT / "dist" / f"ortex-{version()}.tar" + destination.parent.mkdir(exist_ok=True) + subprocess.run(["mix", "hex.build", "--output", str(destination)], cwd=ROOT, check=True) + return destination + + +def unpack(archive, destination): + with tarfile.open(archive) as outer: + payload = outer.extractfile("contents.tar.gz").read() + with tarfile.open(fileobj=io.BytesIO(payload), mode="r:gz") as contents: + contents.extractall(destination, filter="data") + + +def test_package(archive, mode, nif, timeout): + with tempfile.TemporaryDirectory(prefix="ortex-package-") as temporary: + directory = Path(temporary) + package = directory / "ortex" + unpack(archive, package) + project = directory / "consumer" + shutil.copytree(ROOT / "test/package", project) + env = { + **os.environ, + "MIX_ENV": "prod", + "RUSTUP_TOOLCHAIN": "1.93.1", + "ORTEX_PACKAGE_PATH": str(package), + "ORTEX_PACKAGE_MODE": mode, + "ORTEX_FIXTURE_PATH": str(ROOT / "test/fixtures/xgboost.onnx"), + "ORTEX_CASES_PATH": str(ROOT / "test/fixtures/xgboost.json"), + } + if mode == "prebuilt": + binary = ( + Path(nif).resolve() + if nif + else ROOT / "priv/native" / ("libortex.dll" if os.name == "nt" else "libortex.so") + ) + native = package / "priv/native" + native.mkdir(parents=True, exist_ok=True) + shutil.copy2(binary, native / binary.name) + stubs = directory / "forbidden-compilers" + stubs.mkdir() + for name in ["cargo", "rustc"]: + stub = stubs / name + stub.write_text( + "#!/bin/sh\necho 'Compiler forbidden in prebuilt package check' >&2\nexit 99\n" + ) + stub.chmod(0o755) + (stubs / f"{name}.cmd").write_text( + "@echo Compiler forbidden in prebuilt package check\r\n@exit /b 99\r\n" + ) + env["PATH"] = str(stubs) + os.pathsep + env["PATH"] + for command in [["mix", "deps.get"], ["mix", "run", "smoke.exs"]]: + status, output = execute(command, timeout, cwd=project, env=env) + print(output, flush=True) + if status: + return status + return 0 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("build") + test = commands.add_parser("test") + test.add_argument("--archive", type=Path) + test.add_argument("--mode", choices=["source", "prebuilt"], default="source") + test.add_argument("--nif", help="prebuilt library; default: local priv/native library") + test.add_argument("--timeout", type=int, default=600) + args = parser.parse_args() + if args.command == "build": + print(build()) + return 0 + if args.timeout <= 0: + parser.error("timeout must be positive") + archive = args.archive or build() + return test_package(archive, args.mode, args.nif, args.timeout) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/package/config/config.exs b/test/package/config/config.exs new file mode 100644 index 0000000..5e62539 --- /dev/null +++ b/test/package/config/config.exs @@ -0,0 +1,5 @@ +import Config + +config :ortex, Ortex.Native, + features: ["ort/load-dynamic"], + skip_compilation?: System.fetch_env!("ORTEX_PACKAGE_MODE") == "prebuilt" diff --git a/test/package/mix.exs b/test/package/mix.exs new file mode 100644 index 0000000..b74d265 --- /dev/null +++ b/test/package/mix.exs @@ -0,0 +1,12 @@ +defmodule Ortex.PackageCheck.MixProject do + use Mix.Project + + def project do + [ + app: :ortex_package_check, + version: "0.1.0", + elixir: "~> 1.19", + deps: [{:ortex, path: System.fetch_env!("ORTEX_PACKAGE_PATH")}] + ] + end +end diff --git a/test/package/smoke.exs b/test/package/smoke.exs new file mode 100644 index 0000000..4d36dfb --- /dev/null +++ b/test/package/smoke.exs @@ -0,0 +1,11 @@ +model = Ortex.load(System.fetch_env!("ORTEX_FIXTURE_PATH"), [:cpu], 0) +cases = File.read!(System.fetch_env!("ORTEX_CASES_PATH")) |> :json.decode() +{result} = Ortex.run(model, Nx.tensor(cases["inputs"], type: :f32)) +actual = Nx.to_flat_list(result) +expected = List.flatten(cases["outputs"]) + +unless length(actual) == length(expected) and + Enum.all?(Enum.zip(actual, expected), fn {a, b} -> abs(a - b) < 1.0e-6 end), + do: raise("installed package produced incorrect reference scores") + +IO.puts("Installed package passed #{System.fetch_env!("ORTEX_PACKAGE_MODE")} inference") From ab3963d00474bee002a2ca0dc22a286ce35ff561 Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Thu, 29 Oct 2026 17:16:04 -0700 Subject: [PATCH 4/4] test: render scoped line coverage as HTML and Markdown --- scripts/coverage.exs | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 scripts/coverage.exs diff --git a/scripts/coverage.exs b/scripts/coverage.exs new file mode 100644 index 0000000..e553d97 --- /dev/null +++ b/scripts/coverage.exs @@ -0,0 +1,44 @@ +Mix.ensure_application!(:tools) +{:ok, _} = :cover.start() +:ok = :cover.import(~c"cover/library.coverdata") + +rows = + :cover.imported_modules() + |> Enum.reject(&(&1 == Ortex.Native)) + |> Enum.sort() + |> Enum.map(fn module -> + {:ok, lines} = :cover.analyse(module, :coverage, :line) + + # Match Mix's line metric: exclude generated line 0 and combine counters + # sharing a source line. Any covered counter makes that line covered. + unique = + Enum.reduce(lines, %{}, fn + {{_, 0}, _}, acc -> acc + {{_, line}, {hit, _}}, acc -> Map.update(acc, line, hit > 0, &(&1 or hit > 0)) + end) + + covered = Enum.count(unique, fn {_, hit} -> hit end) + missed = map_size(unique) - covered + {:ok, _} = :cover.analyse_to_file(module, ~c"cover/#{module}.html", [:html]) + {module, covered, missed} + end) + +if rows == [], do: raise("no library coverage data") + +percentage = fn hit, total -> + if total == 0, do: "n/a", else: :erlang.float_to_binary(hit / total * 100, decimals: 2) <> "%" +end + +covered = Enum.sum(for {_, hit, _} <- rows, do: hit) +total = Enum.sum(for {_, hit, missed} <- rows, do: hit + missed) + +report = + "# Elixir library coverage\n\nExcludes the NIF interface. Rust and fresh-VM scenarios are outside this report.\n\n" <> + "| Module | Covered | Executable | Coverage |\n|---|---:|---:|---:|\n" <> + Enum.map_join(rows, "\n", fn {module, hit, missed} -> + "| #{inspect(module)} | #{hit} | #{hit + missed} | #{percentage.(hit, hit + missed)} |" + end) <> + "\n| **Total** | #{covered} | #{total} | **#{percentage.(covered, total)}** |\n" + +File.write!("cover/summary.md", report) +IO.puts(report)