test: establish library, system, package, and model harnesses

Makefile

@@ -5,6 +5,7 @@ export MIX_ENV := test
 export ORT_DYLIB_PATH ?= $(shell $(PYTHON) scripts/install_runtime.py path 2>/dev/null)
 
 .PHONY: help setup deps doctor test docs hooks fmt lint check
+.PHONY: test.system test.package test.models test.tooling coverage bench fixtures package
 
 help: ## Show development commands
 	@awk 'BEGIN {FS = ":.*## "} /^[a-zA-Z0-9_.-]+:.*## / {printf "  %-22s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -43,3 +44,30 @@ lint: ## Check formatting, static analysis, and compiler warnings
 	$(PYTHON) scripts/check.py --full
 
 check: lint test ## Run the ordinary local checks
+
+test.system: ## Run each system scenario in a fresh VM
+	$(PYTHON) scripts/scenario.py
+
+test.package: ## Test the assembled package in clean source and prebuilt consumers
+	$(PYTHON) scripts/package.py test --mode source
+	$(PYTHON) scripts/package.py test --mode prebuilt
+
+test.models: ## Check prepared model inputs against their reference outputs
+	$(PYTHON) scripts/fixtures.py
+	mix test test/models --include model
+
+test.tooling: ## Test the development harnesses
+	$(PYTHON) -m unittest discover -s test/tooling -p '*_test.py' -v
+
+coverage: ## Write scoped Elixir HTML and Markdown coverage reports
+	mix test --cover
+	mix run --no-start --no-compile scripts/coverage.exs
+
+bench: ## Measure the public inference workload (ad hoc)
+	mix run bench/inference.exs $(ARGS)
+
+fixtures: ## Regenerate model fixtures (requires the separate exporter environment)
+	$(PYTHON) fixtures/generate.py
+
+package: ## Assemble the local Hex archive
+	$(PYTHON) scripts/package.py build

bench/inference.exs

@@ -0,0 +1,28 @@
+iterations = System.argv() |> List.first("2000") |> String.to_integer()
+if iterations < 1, do: raise("iterations must be positive")
+root = Path.expand("../test/fixtures", __DIR__)
+cases = File.read!(Path.join(root, "xgboost.json")) |> :json.decode()
+model = Ortex.load(Path.join(root, "xgboost.onnx"), [:cpu], 0)
+input = Nx.tensor(cases["inputs"], type: :f32)
+expected = List.flatten(cases["outputs"])
+
+run = fn ->
+  {output} = Ortex.run(model, input)
+  actual = Nx.to_flat_list(output)
+
+  unless length(actual) == length(expected) and
+           Enum.all?(Enum.zip(actual, expected), fn {a, b} -> abs(a - b) < 1.0e-6 end),
+         do: raise("benchmark reference mismatch")
+end
+
+Enum.each(1..100, fn _ -> run.() end)
+{elapsed, _} = :timer.tc(fn -> Enum.each(1..iterations, fn _ -> run.() end) end)
+
+IO.inspect(%{
+  iterations: iterations,
+  calls_per_second: iterations * 1_000_000 / elapsed,
+  elixir: System.version(),
+  otp: System.otp_release(),
+  runtime: System.fetch_env!("ORT_DYLIB_PATH"),
+  workload: "synthetic XGBoost; public call including transfers; CPU; optimization disabled"
+})

docs/testing.md

@@ -0,0 +1,46 @@
+# Testing
+
+`mix test` discovers library `*_test.exs` files normally under test/, including
+doctests. Mix excludes system/, package/, support/, and fixtures/ from discovery.
+Model ExUnit tests have `@moduletag :model` and are excluded by default; run
+`make test.models` to select them explicitly. No selected suite silently skips
+a missing fixture.
+
+`make test.system` invokes scripts/scenario.py. Each test/system/*.exs scenario
+runs in its own BEAM after compilation. Nonzero exit (including a native shutdown
+abort) fails. Timeout kills the process group on Unix and process tree on Windows.
+The runner stores exit status and output in dist/reports/system.json. Additional
+process-wide configuration scenarios should accompany the feature implementing
+that configuration; the foundation tests main's existing behavior.
+
+`make test.package` assembles a Hex tarball, extracts its actual contents into a
+temporary directory, and runs the separate test/package project against it. It
+checks source compilation and an explicitly configured prebuilt library with
+Cargo/rustc replaced by failing stubs. The runtime path comes from the caller's
+ORT_DYLIB_PATH. This is a package installation check, not the feature stack's
+automatic RustlerPrecompiled download path. That feature can reuse this harness.
+
+`make test.models` checks XGBoost, Iris, and MNIST. Prepared artifacts and reference
+cases are recorded in test/fixtures/manifest.json. XGBoost and Iris results come
+from the original estimators; MNIST uses Python ORT as a binding reference, with
+synthetic pixels. None of these checks claim to validate ORT's operators or model
+accuracy. Iris uses tensor probabilities on main; ZipMap needs the separate
+container-output feature. Install fixtures/requirements.txt in a separate Python
+3.12 environment before `make fixtures`. Generation validates exports with ONNX
+and Python ORT; ordinary tests need neither Python exporters nor network access.
+
+`make test.tooling` checks the harnesses themselves, including partial staging,
+filenames with shell characters, timeout handling, and archive path traversal.
+
+`make coverage` reports Elixir library lines using built-in OTP coverage. The
+NIF interface is excluded; generated code should also be excluded when introduced.
+The HTML and Markdown report are under cover/. Rust and fresh-VM scenarios are
+outside that metric. Initially coverage is reported without a percentage gate.
+
+`make bench ARGS=5000` runs the ad hoc CPU XGBoost benchmark with reference checks
+on every call. Timings include tensor transfer; they are observations for the
+recorded environment, not pass/fail performance guarantees.
+
+The inherited ResNet, DistilBERT, and StableLM demos are manual examples. Their
+export dependencies and weights are not part of normal checks. The original
+tiny multi-input model remains for legacy API and dtype tests.

examples/README.md

@@ -1,6 +1,10 @@
 # Examples
 
-Examples have an independent Mix project: from this directory, run `mix deps.get`
+Small XGBoost, Iris, and MNIST examples run from the repository root via
+`MIX_ENV=test mix run examples/run.exs NAME`, with ORT_DYLIB_PATH set. Their
+prepared artifacts also power `make test.models`.
+
+The tokenizer examples have an independent Mix project: from this directory, run `mix deps.get`
 and `mix run distilbert/distilbert_classification.exs` (or another example).
 Set ORT_DYLIB_PATH to the separately installed runtime. Model paths are relative
 to this directory. The Python export dependencies are separate from the library.

examples/iris/README.md

@@ -0,0 +1,8 @@
+# Iris
+
+From the repository root, with ORT_DYLIB_PATH set:
+`MIX_ENV=test mix run examples/run.exs iris`.
+
+StandardScaler and LogisticRegression produce class labels and a probability
+matrix. `make test.models` compares both against scikit-learn. See
+test/fixtures/NOTICE.md for attribution and dataset license information.

examples/mnist/README.md

@@ -0,0 +1,9 @@
+# MNIST
+
+From the repository root, with ORT_DYLIB_PATH set:
+`MIX_ENV=test mix run examples/run.exs mnist`.
+
+A small trained convolutional model consumes deterministic synthetic pixels.
+`make test.models` compares its numerical outputs against Python ORT; this checks
+binding and tensor layout rather than classification accuracy. The exact model
+revision, digest, and notices are under test/fixtures/.

examples/resnet/README.md

@@ -0,0 +1,6 @@
+# ResNet (manual)
+
+This inherited exporter requires separately installed PyTorch and Torchvision,
+downloads pretrained weights, and writes resnet50.onnx in its working directory.
+It remains a manual demonstration pending pinned weights and export dependencies.
+The ordinary model checks use the small prepared models documented in ../README.md.

examples/run.exs

@@ -0,0 +1,6 @@
+name = List.first(System.argv(), "xgboost")
+unless name in ~w(xgboost iris mnist), do: raise("choose xgboost, iris, or mnist")
+root = Path.expand("../test/fixtures", __DIR__)
+cases = File.read!(Path.join(root, name <> ".json")) |> :json.decode()
+model = Ortex.load(Path.join(root, name <> ".onnx"), [:cpu], 0)
+IO.inspect(Ortex.run(model, Nx.tensor(cases["inputs"], type: :f32)), label: name)

examples/xgboost/README.md

@@ -0,0 +1,8 @@
+# XGBoost
+
+From the repository root, with ORT_DYLIB_PATH set:
+`MIX_ENV=test mix run examples/run.exs xgboost`.
+
+Eight shallow regression trees trained on deterministic synthetic data. Four
+prepared rows are checked against the original estimator by `make test.models`.
+See test/fixtures/manifest.json and fixtures/generate.py for provenance.

fixtures/generate.py

@@ -0,0 +1,135 @@
+"""Generate the small model suite and its reference cases with pinned exporters."""
+
+import argparse
+import hashlib
+import json
+import urllib.request
+from pathlib import Path
+
+import numpy as np
+import onnx
+import onnxruntime as ort
+
+ROOT = Path(__file__).resolve().parents[1]
+DIRECTORY = ROOT / "test/fixtures"
+MNIST_REVISION = "1cd752f4a3d818fa215aa621c78b8a10a4a1a3a5"
+MNIST_SHA = "5c688690f8bacf667d4c2074af5ad0646ca328d7ab03eccf944a65b320171bdd"
+
+
+def xgboost():
+    import onnxmltools
+    import xgboost as xgb
+    from onnxmltools.convert.common.data_types import FloatTensorType
+
+    rng = np.random.default_rng(42)
+    x = rng.normal(size=(256, 3)).astype(np.float32)
+    y = (2 * (x[:, 0] > 0) - (x[:, 1] > 0.5) + x[:, 2] * 0.25).astype(np.float32)
+    model = xgb.XGBRegressor(
+        n_estimators=8, max_depth=3, learning_rate=0.25, n_jobs=1, random_state=42
+    )
+    model.fit(x, y)
+    converted = onnxmltools.convert_xgboost(
+        model, initial_types=[("features", FloatTensorType([None, 3]))], target_opset=15
+    )
+    rows = np.array([[0, 0, 0], [1, 1, 1], [-1, -1, -1], [0.25, 0.75, -0.5]], dtype=np.float32)
+    return (
+        converted,
+        rows,
+        [model.predict(rows).reshape(-1, 1)],
+        {
+            "source": "deterministic synthetic training data; seed 42",
+            "license": "MIT",
+            "reference": "XGBoost predict, independently checked against Python ORT",
+        },
+    )
+
+
+def iris():
+    from skl2onnx import to_onnx
+    from sklearn.datasets import load_iris
+    from sklearn.linear_model import LogisticRegression
+    from sklearn.pipeline import make_pipeline
+    from sklearn.preprocessing import StandardScaler
+
+    dataset = load_iris()
+    model = make_pipeline(StandardScaler(), LogisticRegression(random_state=42, max_iter=500))
+    model.fit(dataset.data, dataset.target)
+    rows = dataset.data[[0, 50, 100, 149]].astype(np.float32)
+    converted = to_onnx(model, rows, options={id(model[-1]): {"zipmap": False}}, target_opset=15)
+    converted.ir_version = 8
+    return (
+        converted,
+        rows,
+        [model.predict(rows), model.predict_proba(rows)],
+        {
+            "source": "scikit-learn load_iris; StandardScaler + LogisticRegression",
+            "license": "CC-BY-4.0 (Iris dataset; see NOTICE.md)",
+            "reference": "scikit-learn predict and predict_proba",
+        },
+    )
+
+
+def mnist():
+    url = f"https://huggingface.co/onnxmodelzoo/mnist-12/resolve/{MNIST_REVISION}/mnist-12.onnx"
+    with urllib.request.urlopen(url, timeout=60) as response:
+        data = response.read()
+    if hashlib.sha256(data).hexdigest() != MNIST_SHA:
+        raise ValueError("MNIST download digest mismatch")
+    converted = onnx.load_model_from_string(data)
+    # A real trained network with synthetic pixels tests binding/layout/numerics,
+    # not classification accuracy or ORT operator correctness.
+    rows = np.random.default_rng(42).random((1, 1, 28, 28), dtype=np.float32)
+    return (
+        converted,
+        rows,
+        None,
+        {
+            "source": url,
+            "license": "Apache-2.0 (model repository); upstream model documentation also records MIT",
+            "reference": "Python ONNX Runtime 1.24.4; synthetic pixels, seed 42",
+        },
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("models", nargs="*", choices=["xgboost", "iris", "mnist"])
+    args = parser.parse_args()
+    DIRECTORY.mkdir(parents=True, exist_ok=True)
+    manifest_path = DIRECTORY / "manifest.json"
+    manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else {}
+    for name in args.models or ["xgboost", "iris", "mnist"]:
+        model, inputs, expected, provenance = globals()[name]()
+        onnx.checker.check_model(model)
+        data = model.SerializeToString()
+        options = ort.SessionOptions()
+        options.intra_op_num_threads = 1
+        session = ort.InferenceSession(data, options, providers=["CPUExecutionProvider"])
+        actual = session.run(None, {session.get_inputs()[0].name: inputs})
+        if expected is None:
+            expected = actual
+        for a, b in zip(actual, expected, strict=True):
+            np.testing.assert_allclose(a, b, atol=1e-5, rtol=1e-5)
+        (DIRECTORY / f"{name}.onnx").write_bytes(data)
+        cases = {
+            "inputs": inputs.tolist(),
+            "outputs": [output.tolist() for output in expected],
+            "types": [str(output.dtype) for output in actual],
+        }
+        case_data = (json.dumps(cases, indent=2) + "\n").encode()
+        (DIRECTORY / f"{name}.json").write_bytes(case_data)
+        manifest[name] = {
+            **provenance,
+            "sha256": hashlib.sha256(data).hexdigest(),
+            "cases_sha256": hashlib.sha256(case_data).hexdigest(),
+            "ir": model.ir_version,
+            "opsets": {x.domain: x.version for x in model.opset_import},
+            "generator": "fixtures/generate.py",
+            "requirements": "fixtures/requirements.txt",
+            "atol": 1e-5,
+        }
+    manifest_path.write_text(json.dumps(manifest, sort_keys=True, indent=2) + "\n")
+
+
+if __name__ == "__main__":
+    main()

fixtures/requirements.txt

@@ -0,0 +1,18 @@
+cloudpickle==3.1.2
+flatbuffers==25.12.19
+joblib==1.6.0
+ml_dtypes==0.6.0
+mpmath==1.3.0
+numpy==2.5.4
+onnx==1.20.1
+onnxmltools==1.16.0
+onnxruntime==1.24.4
+packaging==26.3
+protobuf==7.36.2
+scikit-learn==1.8.0
+scipy==1.18.1
+skl2onnx==1.20.0
+sympy==1.14.0
+threadpoolctl==3.7.0
+typing_extensions==4.16.0
+xgboost==3.2.0

mix.exs

@@ -7,6 +7,8 @@ defmodule Ortex.MixProject do
       version: "0.1.10",
       elixir: "~> 1.19",
       start_permanent: Mix.env() == :prod,
+      test_ignore_filters: [~r{test/(system|package|fixtures|support)/}],
+      test_coverage: [ignore_modules: [Ortex.Native], summary: [threshold: 0], export: "library"],
       deps: deps(),
 
       # Docs

scripts/coverage.exs

@@ -0,0 +1,44 @@
+Mix.ensure_application!(:tools)
+{:ok, _} = :cover.start()
+:ok = :cover.import(~c"cover/library.coverdata")
+
+rows =
+  :cover.imported_modules()
+  |> Enum.reject(&(&1 == Ortex.Native))
+  |> Enum.sort()
+  |> Enum.map(fn module ->
+    {:ok, lines} = :cover.analyse(module, :coverage, :line)
+
+    # Match Mix's line metric: exclude generated line 0 and combine counters
+    # sharing a source line. Any covered counter makes that line covered.
+    unique =
+      Enum.reduce(lines, %{}, fn
+        {{_, 0}, _}, acc -> acc
+        {{_, line}, {hit, _}}, acc -> Map.update(acc, line, hit > 0, &(&1 or hit > 0))
+      end)
+
+    covered = Enum.count(unique, fn {_, hit} -> hit end)
+    missed = map_size(unique) - covered
+    {:ok, _} = :cover.analyse_to_file(module, ~c"cover/#{module}.html", [:html])
+    {module, covered, missed}
+  end)
+
+if rows == [], do: raise("no library coverage data")
+
+percentage = fn hit, total ->
+  if total == 0, do: "n/a", else: :erlang.float_to_binary(hit / total * 100, decimals: 2) <> "%"
+end
+
+covered = Enum.sum(for {_, hit, _} <- rows, do: hit)
+total = Enum.sum(for {_, hit, missed} <- rows, do: hit + missed)
+
+report =
+  "# Elixir library coverage\n\nExcludes the NIF interface. Rust and fresh-VM scenarios are outside this report.\n\n" <>
+    "| Module | Covered | Executable | Coverage |\n|---|---:|---:|---:|\n" <>
+    Enum.map_join(rows, "\n", fn {module, hit, missed} ->
+      "| #{inspect(module)} | #{hit} | #{hit + missed} | #{percentage.(hit, hit + missed)} |"
+    end) <>
+    "\n| **Total** | #{covered} | #{total} | **#{percentage.(covered, total)}** |\n"
+
+File.write!("cover/summary.md", report)
+IO.puts(report)