diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..427136a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,15 @@ +.git +deps +_build +priv +native/ortex/target +.venv +.cache +.ruff_cache +runtime +dist +cover +doc +**/__pycache__ +examples/deps +examples/_build diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..7bea8c9 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + groups: + actions: + patterns: ['*'] + - package-ecosystem: pip + directory: /scripts + schedule: + interval: weekly diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml new file mode 100644 index 0000000..3709217 --- /dev/null +++ b/.github/workflows/audit.yml @@ -0,0 +1,41 @@ +name: Dependency audit +on: + schedule: + - cron: '41 7 * * *' + workflow_dispatch: + workflow_call: +permissions: + contents: read +concurrency: + group: audit-${{ github.ref }} + cancel-in-progress: false +jobs: + audit: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 + with: + elixir-version: '1.20.4' + otp-version: '29.1.1' + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.14.10' + - run: make deps setup.audit + - run: make audit + - name: Attach audit summary + if: always() + run: | + if test -f dist/reports/audit.md; then + cat dist/reports/audit.md >> "$GITHUB_STEP_SUMMARY" + fi + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: always() + with: + name: dependency-audit + path: dist/reports/ + retention-days: 14 + if-no-files-found: ignore diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 0000000..865bd14 --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,200 @@ +name: Checks +on: + pull_request: + types: [opened, synchronize, reopened, labeled, unlabeled] + push: + branches: [main] + workflow_dispatch: + workflow_call: + inputs: + scope: + type: string + default: pr +permissions: + contents: read +concurrency: + group: checks-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} +jobs: + policy: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + matrix: ${{ steps.matrix.outputs.value }} + scope: ${{ steps.matrix.outputs.scope }} + alpine: ${{ steps.matrix.outputs.alpine }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.14.10' + - id: matrix + env: + SCOPE: ${{ inputs.scope || (github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ci:full') && 'pr' || 'full') }} + run: | + python scripts/ci.py policy + { + echo "value=$(python scripts/ci.py matrix --scope "$SCOPE")" + echo "scope=$SCOPE" + echo "alpine=$(python scripts/ci.py matrix --scope alpine)" + } >> "$GITHUB_OUTPUT" + hygiene: + runs-on: ubuntu-24.04 + timeout-minutes: 25 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 + with: + elixir-version: '1.20.4' + otp-version: '29.1.1' + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.14.10' + - uses: jdx/mise-action@6be3c7ed4c21d4f76b208d351d95aac273ebb76d # v5.2.4 + with: + version: '2026.10.23' + install_args: actionlint shellcheck shfmt + cache: false + add_shims_to_path: false + - name: Prepare checkers + run: | + make setup.rust + python -m venv .venv + .venv/bin/python -m pip install -r scripts/requirements.txt + make deps + - run: make lint test.tooling docs + source: + needs: policy + name: ${{ matrix.id }} + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.policy.outputs.matrix) }} + defaults: + run: + shell: bash + env: + MIX_ENV: test + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 + with: + elixir-version: ${{ matrix.elixir }} + otp-version: ${{ matrix.otp }} + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.14.10' + - name: Select runtime and Nx boundary + env: + RUNTIME: ${{ matrix.runtime }} + NX_MODE: ${{ matrix.nx }} + run: | + python scripts/install_runtime.py install --version "$RUNTIME" + python scripts/ci.py nx "$NX_MODE" + if [[ "$NX_MODE" == locked ]]; then + mix deps.get --check-locked + else + mix deps.update nx + mix deps.get + fi + - name: Validate library, model, system, and package suites + run: | + mix compile --warnings-as-errors + make test test.models test.system test.package + - name: Render library coverage + if: matrix.id == 'linux-current' + run: | + make coverage + cat cover/summary.md >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: always() + with: + name: reports-${{ matrix.id }} + path: | + cover/ + dist/reports/ + retention-days: 14 + if-no-files-found: ignore + - name: Assemble native target archive + env: + TARGET: ${{ matrix.target }} + run: python scripts/package.py nif --target "$TARGET" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: needs.policy.outputs.scope == 'full' && matrix.nx == 'locked' + with: + name: nif-${{ matrix.target }} + path: dist/*-${{ matrix.target }}.tar.gz + retention-days: 14 + if-no-files-found: error + msrv: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Check the declared minimum Rust version + working-directory: native/ortex + run: | + rustup toolchain install 1.92.0 --profile minimal + cargo +1.92.0 check --locked --features ort/load-dynamic + alpine: + needs: policy + if: needs.policy.outputs.scope == 'full' + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.policy.outputs.alpine) }} + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Validate Alpine source and package consumers + env: + TARGET: ${{ matrix.target }} + run: | + docker build -f ci/containers/alpine.Dockerfile -t ortex-alpine . + docker run --name ortex-alpine -e "TARGET=$TARGET" ortex-alpine + docker cp ortex-alpine:/app/dist ./dist + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: nif-${{ matrix.target }} + path: dist/*-${{ matrix.target }}.tar.gz + retention-days: 14 + if-no-files-found: error + validated: + if: always() + needs: [policy, hygiene, source, msrv, alpine] + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Require every selected validation job + env: + RESULTS: ${{ toJSON(needs) }} + run: | + python3 - <<'PY' + import json, os + jobs = json.loads(os.environ['RESULTS']) + scope = jobs['policy']['outputs'].get('scope') + for name, job in jobs.items(): + allowed = {'success', 'skipped'} if name == 'alpine' and scope == 'pr' else {'success'} + if job['result'] not in allowed: + raise SystemExit(f"{name}: {job['result']}") + PY + - name: Record full-validation marker + if: needs.policy.outputs.scope == 'full' && github.ref == 'refs/heads/main' + run: echo validated > validation.txt + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: needs.policy.outputs.scope == 'full' && github.ref == 'refs/heads/main' + with: + name: compatibility-validated + path: validation.txt + retention-days: 14 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 345fa56..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Baseline validation -on: - push: - branches: [main] - pull_request: -permissions: - contents: read -concurrency: - group: baseline-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true -jobs: - baseline: - strategy: - fail-fast: false - matrix: - os: [ubuntu-24.04, macos-15] - runs-on: ${{ matrix.os }} - timeout-minutes: 25 - env: - MIX_ENV: test - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 - with: - elixir-version: '1.20.4' - otp-version: '29.1.1' - - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 - with: - python-version: '3.14.10' - - name: Supply the separately installed runtime - run: | - runtime_path=$(python scripts/install_runtime.py install) - echo "ORT_DYLIB_PATH=$runtime_path" >> "$GITHUB_ENV" - - run: mix deps.get --check-locked - - run: mix compile --warnings-as-errors - - run: mix format --check-formatted - - run: mix test diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 0000000..70be405 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,35 @@ +name: Nightly compatibility +on: + schedule: + - cron: '23 8 * * *' + workflow_dispatch: +permissions: + contents: read + actions: read +concurrency: + group: nightly-main + cancel-in-progress: false +jobs: + changed: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + run: ${{ steps.changed.outputs.run }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.14.10' + - id: changed + env: + GITHUB_TOKEN: ${{ github.token }} + run: echo "run=$(python scripts/ci.py changed "$GITHUB_SHA")" >> "$GITHUB_OUTPUT" + full: + needs: changed + if: needs.changed.outputs.run == 'true' || github.event_name == 'workflow_dispatch' + # actionlint 1.7.12 does not yet parse GitHub's $/ reusable-workflow syntax. + uses: ./.github/workflows/checks.yml # zizmor: ignore[self-repository] + with: + scope: full diff --git a/Makefile b/Makefile index 0c822e9..d8dc4c9 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,26 @@ .DEFAULT_GOAL := help PYTHON ?= python3 -export PATH := $(CURDIR)/.venv/bin:$(CURDIR)/.venv/Scripts:$(PATH) +VENV_PYTHON = $(if $(filter Windows_NT,$(OS)),.venv/Scripts/python.exe,.venv/bin/python) +export PATH := $(CURDIR)/.venv/bin:$(CURDIR)/.venv/Scripts:$(CURDIR)/.cache/audit/bin:$(PATH) export MIX_ENV := test export ORT_DYLIB_PATH ?= $(shell $(PYTHON) scripts/install_runtime.py path 2>/dev/null) .PHONY: help setup deps doctor test docs hooks fmt lint check .PHONY: test.system test.package test.models test.tooling coverage bench fixtures package +.PHONY: test.alpine +.PHONY: setup.audit audit setup.rust help: ## Show development commands @awk 'BEGIN {FS = ":.*## "} /^[a-zA-Z0-9_.-]+:.*## / {printf " %-22s %s\n", $$1, $$2}' $(MAKEFILE_LIST) setup: ## Fetch Mix dependencies and the development runtime + $(MAKE) setup.rust mix local.hex --force mix local.rebar --force $(MAKE) deps $(PYTHON) scripts/install_runtime.py install $(PYTHON) -m venv .venv - .venv/bin/python -m pip install -r scripts/requirements.txt + $(VENV_PYTHON) -m pip install -r scripts/requirements.txt deps: ## Fetch the locked test and documentation dependencies mix deps.get --check-locked @@ -71,3 +75,16 @@ fixtures: ## Regenerate model fixtures (requires the separate exporter environme package: ## Assemble the local Hex archive $(PYTHON) scripts/package.py build + +test.alpine: ## Run the same suites in the pinned Alpine environment + docker build -f ci/containers/alpine.Dockerfile -t ortex-foundation-alpine . + docker run --rm ortex-foundation-alpine + +setup.audit: ## Install the pinned advisory checker into the local cache + cargo install cargo-audit --version 0.22.2 --locked --root .cache/audit + +setup.rust: ## Install the pinned Rust compiler, formatter, and Clippy + $(PYTHON) scripts/toolchain.py + +audit: ## Report Rust advisories and retired Hex dependencies + $(PYTHON) scripts/audit.py diff --git a/ci/compatibility.json b/ci/compatibility.json new file mode 100644 index 0000000..a3b1297 --- /dev/null +++ b/ci/compatibility.json @@ -0,0 +1,17 @@ +{ + "tools": {"elixir": "1.20.4", "otp": "29.1.1", "rust": "1.93.1", "msrv": "1.92.0", "python": "3.14.10"}, + "pr": [ + {"id": "linux-current", "os": "ubuntu-24.04", "target": "x86_64-unknown-linux-gnu", "elixir": "1.20.4", "otp": "29.1.1", "runtime": "1.24.4", "nx": "locked"}, + {"id": "linux-minimum", "os": "ubuntu-24.04", "target": "x86_64-unknown-linux-gnu", "elixir": "1.19.7", "otp": "28.5.0.8", "runtime": "1.19.2", "nx": "minimum"}, + {"id": "mac-arm", "os": "macos-15", "target": "aarch64-apple-darwin", "elixir": "1.20.4", "otp": "29.1.1", "runtime": "1.24.4", "nx": "locked"} + ], + "platforms": [ + {"id": "linux-arm", "os": "ubuntu-24.04-arm", "target": "aarch64-unknown-linux-gnu", "runtime": "1.24.4"}, + {"id": "mac-intel", "os": "macos-15-intel", "target": "x86_64-apple-darwin", "runtime": "1.23.2"}, + {"id": "windows", "os": "windows-2022", "target": "x86_64-pc-windows-msvc", "runtime": "1.24.4"} + ], + "alpine": [ + {"os": "ubuntu-24.04", "target": "x86_64-unknown-linux-musl"}, + {"os": "ubuntu-24.04-arm", "target": "aarch64-unknown-linux-musl"} + ] +} diff --git a/ci/containers/alpine.Dockerfile b/ci/containers/alpine.Dockerfile new file mode 100644 index 0000000..fe0ef07 --- /dev/null +++ b/ci/containers/alpine.Dockerfile @@ -0,0 +1,14 @@ +FROM hexpm/elixir:1.20.4-erlang-29.1.1-alpine-3.24.2@sha256:ad851f40ce103dcb4ad56f23877d99473ef5c013e09b9e57921ffe877ac6d6a9 +RUN apk add --no-cache build-base git bash python3 rust=1.96.1-r0 cargo=1.96.1-r0 ca-certificates && \ + apk add --no-cache \ + --repository=https://dl-cdn.alpinelinux.org/alpine/edge/main \ + --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community \ + --repository=https://dl-cdn.alpinelinux.org/alpine/edge/testing \ + onnxruntime=1.24.4-r1 +WORKDIR /app +ENV MIX_ENV=test ORT_DYLIB_PATH=/usr/lib/libonnxruntime.so.1.24.4 RUSTUP_TOOLCHAIN=1.96.1 +COPY mix.exs mix.lock ./ +RUN mix local.hex --force && mix local.rebar --force && mix deps.get --check-locked +COPY . . +RUN mix compile --warnings-as-errors +CMD ["sh", "-ec", "mix test && python3 scripts/fixtures.py && mix test test/models --include model && python3 scripts/scenario.py && python3 scripts/package.py test --mode source && python3 scripts/package.py test --mode prebuilt && if test -n \"${TARGET:-}\"; then python3 scripts/package.py nif --target \"$TARGET\"; fi"] diff --git a/ci/runtimes.json b/ci/runtimes.json new file mode 100644 index 0000000..74b7216 --- /dev/null +++ b/ci/runtimes.json @@ -0,0 +1,26 @@ +{ + "1.19.2/linux-x64": { + "sha256": "eb00c64e0041f719913c4080e0fed7d9963dc3aa9b54664df6036d8308dbcd33", + "url": "https://github.com/microsoft/onnxruntime/releases/download/v1.19.2/onnxruntime-linux-x64-1.19.2.tgz" + }, + "1.23.2/osx-x86_64": { + "sha256": "d10359e16347b57d9959f7e80a225a5b4a66ed7d7e007274a15cae86836485a6", + "url": "https://github.com/microsoft/onnxruntime/releases/download/v1.23.2/onnxruntime-osx-x86_64-1.23.2.tgz" + }, + "1.24.4/linux-aarch64": { + "sha256": "866109a9248d057671a039b9d725be4bd86888e3754140e6701ec621be9d4d7e", + "url": "https://github.com/microsoft/onnxruntime/releases/download/v1.24.4/onnxruntime-linux-aarch64-1.24.4.tgz" + }, + "1.24.4/linux-x64": { + "sha256": "3a211fbea252c1e66290658f1b735b772056149f28321e71c308942cdb54b747", + "url": "https://github.com/microsoft/onnxruntime/releases/download/v1.24.4/onnxruntime-linux-x64-1.24.4.tgz" + }, + "1.24.4/osx-arm64": { + "sha256": "93787795f47e1eee369182e43ed51b9e5da0878ab0346aecf4258979b8bba989", + "url": "https://github.com/microsoft/onnxruntime/releases/download/v1.24.4/onnxruntime-osx-arm64-1.24.4.tgz" + }, + "1.24.4/win-x64": { + "sha256": "d2319fddfb6ea4db99ccc4b60c85c517bcd855721f5daa6a06d40d7cb2ee2357", + "url": "https://github.com/microsoft/onnxruntime/releases/download/v1.24.4/onnxruntime-win-x64-1.24.4.zip" + } +} diff --git a/docs/ci.md b/docs/ci.md new file mode 100644 index 0000000..b197929 --- /dev/null +++ b/docs/ci.md @@ -0,0 +1,48 @@ +# CI policy + +Checks runs all hygiene checks and the library, model, system, and package suites +on three PR combinations. Main and the nightly use seven native combinations +(pairwise BEAM/Nx/runtime boundaries and five host platforms), plus both Alpine +architectures. This is an explicit selection, not a Cartesian product. +The `ci:full` PR label opts into the complete matrix before merge. +ci/compatibility.json owns the combinations and artifact targets. Toolchain +consistency is checked against mise.toml and rust-toolchain.toml. The current +native implementation requires ORT API 19; API 17 support belongs to the runtime +migration. The declared Nx floor is exercised separately from the locked version. + +Nightly checks look for an unexpired full-validation artifact on the exact main +SHA, verify that its run succeeded, and reuse it for at most seven days. A skipped +nightly creates no marker. Manual dispatch forces validation. API errors fail +the decision job instead of incorrectly claiming a successful validation. + +PR jobs use ephemeral GitHub-hosted runners, read-only permissions, no publishing +credentials, no persisted checkout credentials, and no shared build caches. All +actions use full commit SHAs with release comments. Release privileges belong in +separate jobs. Job timeouts and artifact retention are explicit. Markdown coverage +is included in the job summary, with HTML and scenario reports attached as artifacts. + +## Trusted workflow enforcement after merge + +The ordinary pull_request workflow is unprivileged but can be changed by the PR. +To require the definition from main, configure an organization ruleset targeting +only RiverFinancial/ortex's main branch, requiring `.github/workflows/checks.yml` +from RiverFinancial/ortex at refs/heads/main. Enable it after the workflow has +landed. Require the `validated` aggregate and review changes to workflows, tool +configuration, and hooks. Organization ruleset availability and installation are +administrative configuration; repository YAML alone does not enforce this policy. +This stack does not change organization settings or bypass lists. Do not substitute +a privileged pull_request_target checkout of PR code for the required workflow. + +Advisories are checked independently every day even when source compatibility +validation is skipped. `make setup.audit audit` runs the same checks locally. +Findings produce a report and fail the audit; they are not suppressed to make the +inherited dependency graph appear clean. Releases must satisfy that gate. + +## Alpine reproduction + +`make test.alpine` uses the same Dockerfile as CI. The base image is pinned by +digest; Rust/Cargo and ORT package versions are explicit. ORT is obtained from +Alpine edge repositories, whose transitive package selection is mutable. These +images establish compatibility, not bit-for-bit reproducibility. glibc libraries +are tested separately from musl. Published native artifacts include only the NIF; +the consumer supplies ONNX Runtime. diff --git a/native/ortex/.cargo/config.toml b/native/ortex/.cargo/config.toml index 61fee89..eb4cf68 100644 --- a/native/ortex/.cargo/config.toml +++ b/native/ortex/.cargo/config.toml @@ -15,3 +15,6 @@ rustflags = [ rustflags = [ "-Clink-args=-Wl,-rpath,$ORIGIN" ] [target.aarch64-unknown-linux-gnu] rustflags = [ "-Clink-args=-Wl,-rpath,$ORIGIN" ] + +[target.'cfg(target_env = "musl")'] +rustflags = [ "-C", "target-feature=-crt-static" ] diff --git a/native/ortex/Cargo.lock b/native/ortex/Cargo.lock index cd74a84..705e174 100644 --- a/native/ortex/Cargo.lock +++ b/native/ortex/Cargo.lock @@ -1,12 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 - -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +version = 4 [[package]] name = "aho-corasick" @@ -23,39 +17,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "cc" version = "1.0.83" @@ -71,101 +32,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "cpufeatures" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" -dependencies = [ - "libc", -] - -[[package]] -name = "crc32fast" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" -dependencies = [ - "cfg-if", -] - [[package]] name = "crunchy" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", -] - -[[package]] -name = "errno" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" -dependencies = [ - "libc", - "windows-sys", -] - -[[package]] -name = "filetime" -version = "0.2.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "windows-sys", -] - -[[package]] -name = "flate2" -version = "1.0.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" -dependencies = [ - "crc32fast", - "miniz_oxide", -] - -[[package]] -name = "form_urlencoded" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getrandom" version = "0.2.12" @@ -193,16 +65,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -[[package]] -name = "idna" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -216,10 +78,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] -name = "linux-raw-sys" -version = "0.4.13" +name = "libloading" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] [[package]] name = "log" @@ -254,15 +120,6 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" -[[package]] -name = "miniz_oxide" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" -dependencies = [ - "adler", -] - [[package]] name = "ndarray" version = "0.16.1" @@ -329,6 +186,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11826e6118cc42fea0cb2b102f7d006c1bb339cb167f8badb5fb568616438234" dependencies = [ "half", + "libloading", "ndarray", "ort-sys", "tracing", @@ -340,11 +198,7 @@ version = "2.0.0-rc.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4780a8b8681e653b2bed85c7f0e2c6e8547224c3e983e5ad27bf0457e012407" dependencies = [ - "flate2", "pkg-config", - "sha2", - "tar", - "ureq", ] [[package]] @@ -366,12 +220,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" -[[package]] -name = "percent-encoding" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" - [[package]] name = "pin-project-lite" version = "0.2.9" @@ -423,15 +271,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" -[[package]] -name = "redox_syscall" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "regex" version = "1.7.1" @@ -473,19 +312,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "rustix" -version = "0.38.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" -dependencies = [ - "bitflags 2.4.2", - "errno", - "libc", - "linux-raw-sys", - "windows-sys", -] - [[package]] name = "rustler" version = "0.29.1" @@ -550,17 +376,6 @@ dependencies = [ "untrusted", ] -[[package]] -name = "sha2" -version = "0.10.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sharded-slab" version = "0.1.7" @@ -576,17 +391,6 @@ version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" -[[package]] -name = "socks" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" -dependencies = [ - "byteorder", - "libc", - "winapi", -] - [[package]] name = "spin" version = "0.9.8" @@ -610,17 +414,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "tar" -version = "0.4.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb" -dependencies = [ - "filetime", - "libc", - "xattr", -] - [[package]] name = "thread_local" version = "1.1.7" @@ -631,21 +424,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "tinyvec" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - [[package]] name = "tracing" version = "0.1.37" @@ -696,33 +474,12 @@ dependencies = [ "tracing-log", ] -[[package]] -name = "typenum" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" - -[[package]] -name = "unicode-bidi" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" - [[package]] name = "unicode-ident" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" -[[package]] -name = "unicode-normalization" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" -dependencies = [ - "tinyvec", -] - [[package]] name = "unreachable" version = "1.0.0" @@ -738,46 +495,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -[[package]] -name = "ureq" -version = "2.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11f214ce18d8b2cbe84ed3aa6486ed3f5b285cf8d8fbdbce9f3f767a724adc35" -dependencies = [ - "base64", - "log", - "once_cell", - "rustls", - "rustls-pki-types", - "rustls-webpki", - "socks", - "url", - "webpki-roots", -] - -[[package]] -name = "url" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", -] - [[package]] name = "valuable" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - [[package]] name = "void" version = "1.0.2" @@ -790,15 +513,6 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" -[[package]] -name = "webpki-roots" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "winapi" version = "0.3.9" @@ -821,6 +535,12 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + [[package]] name = "windows-sys" version = "0.52.0" @@ -887,17 +607,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" -[[package]] -name = "xattr" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" -dependencies = [ - "libc", - "linux-raw-sys", - "rustix", -] - [[package]] name = "zeroize" version = "1.7.0" diff --git a/native/ortex/Cargo.toml b/native/ortex/Cargo.toml index be152a2..10f6db3 100644 --- a/native/ortex/Cargo.toml +++ b/native/ortex/Cargo.toml @@ -12,7 +12,7 @@ crate-type = ["cdylib"] [dependencies] rustler = "0.29.0" -ort = { version = "2.0.0-rc.8" } +ort = { version = "2.0.0-rc.8", default-features = false, features = ["ndarray", "half"] } ndarray = "0.16.1" half = "2.2.1" tracing-subscriber = { version = "0.3", features = [ "env-filter", "fmt" ] } @@ -20,6 +20,7 @@ num-traits = "0.2.15" rustls = "0.22.4" [features] +default = ["ort/load-dynamic"] # ONNXRuntime Execution providers directml = ["ort/directml"] coreml = ["ort/coreml"] diff --git a/scripts/audit.py b/scripts/audit.py new file mode 100644 index 0000000..36e9b82 --- /dev/null +++ b/scripts/audit.py @@ -0,0 +1,55 @@ +"""Report Rust advisories and retired Hex dependencies without hiding failures.""" + +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + destination = ROOT / "dist/reports" + destination.mkdir(parents=True, exist_ok=True) + sections = [] + failed = False + for name, command in [ + ( + "cargo-audit", + [ + "cargo", + "audit", + "--file", + "native/ortex/Cargo.lock", + "--db", + ".cache/advisory-db", + "--json", + ], + ), + ("hex-audit", ["mix", "hex.audit"]), + ]: + try: + result = subprocess.run( + command, + cwd=ROOT, + env={**os.environ, "MIX_ENV": "test"}, + capture_output=True, + text=True, + timeout=300, + ) + status, output = result.returncode, result.stdout + result.stderr + except (OSError, subprocess.TimeoutExpired) as error: + status, output = 1, str(error) + (destination / f"{name}.txt").write_text(output) + sections.append( + f"## {name}: {'failed' if status else 'passed'}\n\n```text\n{output.rstrip()}\n```\n" + ) + failed |= status != 0 + report = "# Dependency audit\n\n" + "\n".join(sections) + (destination / "audit.md").write_text(report) + print(report) + return int(failed) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check.py b/scripts/check.py index c7202c0..deb8005 100644 --- a/scripts/check.py +++ b/scripts/check.py @@ -3,6 +3,7 @@ import argparse import concurrent.futures import os +import shutil import subprocess import sys import tempfile @@ -17,6 +18,7 @@ def git(*arguments): def run_check(item, directory): name, command = item + command = [shutil.which(command[0]) or command[0], *command[1:]] try: result = subprocess.run( command, diff --git a/scripts/ci.py b/scripts/ci.py new file mode 100644 index 0000000..5da0b33 --- /dev/null +++ b/scripts/ci.py @@ -0,0 +1,99 @@ +"""Emit explicit CI combinations and inspect successful full-validation markers.""" + +import argparse +import datetime +import json +import os +import re +import tomllib +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +POLICY = ROOT / "ci/compatibility.json" + + +def api(path): + repository = os.environ.get("GITHUB_REPOSITORY", "RiverFinancial/ortex") + if not re.fullmatch(r"[\w.-]+/[\w.-]+", repository): + raise ValueError("invalid repository") + headers = {"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"} + if token := os.environ.get("GITHUB_TOKEN"): + headers["Authorization"] = f"Bearer {token}" + request = urllib.request.Request( + f"https://api.github.com/repos/{repository}/{path}", headers=headers + ) + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) + + +def should_run(sha): + now = datetime.datetime.now(datetime.timezone.utc) + artifacts = api("actions/artifacts?name=compatibility-validated&per_page=100")["artifacts"] + for artifact in artifacts: + run = artifact.get("workflow_run", {}) + if artifact["expired"] or run.get("head_branch") != "main" or run.get("head_sha") != sha: + continue + created = datetime.datetime.fromisoformat(artifact["created_at"].replace("Z", "+00:00")) + if now - created > datetime.timedelta(days=7): + continue + details = api(f"actions/runs/{run['id']}") + if details["conclusion"] == "success" and details["event"] in { + "push", + "schedule", + "workflow_dispatch", + }: + return False + return True + + +def matrix(scope): + policy = json.loads(POLICY.read_text()) + if scope == "alpine": + return {"include": policy["alpine"]} + rows = list(policy["pr"]) + if scope == "full": + defaults = {k: policy["tools"][k] for k in ["elixir", "otp"]} + rows += [{**defaults, "nx": "locked", **row} for row in policy["platforms"]] + for runtime, nx in [("1.24.4", "minimum"), ("1.19.2", "latest")]: + rows.append({**rows[0], "id": f"linux-{runtime}-{nx}", "runtime": runtime, "nx": nx}) + return {"include": rows} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + matrices = commands.add_parser("matrix") + matrices.add_argument("--scope", choices=["pr", "full", "alpine"], default="pr") + nightly = commands.add_parser("changed") + nightly.add_argument("sha") + commands.add_parser("policy") + nx = commands.add_parser("nx") + nx.add_argument("mode", choices=["locked", "minimum", "latest"]) + args = parser.parse_args() + if args.command == "matrix": + print(json.dumps(matrix(args.scope), separators=(",", ":"))) + elif args.command == "changed": + if not re.fullmatch(r"[0-9a-f]{40}", args.sha): + parser.error("expected a full commit SHA") + print(str(should_run(args.sha)).lower()) + elif args.command == "nx": + if args.mode == "minimum": + project = ROOT / "mix.exs" + text, count = re.subn(r'\{:nx, "[^"]+"\}', '{:nx, "== 0.6.2"}', project.read_text()) + if count != 1: + raise ValueError("Nx dependency declaration changed") + project.write_text(text) + else: + tools = json.loads(POLICY.read_text())["tools"] + mise = tomllib.loads((ROOT / "mise.toml").read_text())["tools"] + rust = tomllib.loads((ROOT / "rust-toolchain.toml").read_text())["toolchain"] + assert mise["elixir"] == tools["elixir"] + "-otp-" + tools["otp"].split(".")[0] + assert mise["erlang"] == tools["otp"] + assert mise["python"] == tools["python"] + assert rust["channel"] == tools["rust"] + print("Toolchain declarations agree with CI policy") + + +if __name__ == "__main__": + main() diff --git a/scripts/doctor.py b/scripts/doctor.py index 5cc6ae5..7560201 100644 --- a/scripts/doctor.py +++ b/scripts/doctor.py @@ -17,7 +17,9 @@ def main(): print(f"MISSING: {executable}; run mise install / install rustup") failed = True continue - result = subprocess.run([executable, *arguments], capture_output=True, text=True) + result = subprocess.run( + [shutil.which(executable), *arguments], capture_output=True, text=True + ) print(result.stdout.strip() or result.stderr.strip()) failed |= result.returncode != 0 runtime = os.environ.get("ORT_DYLIB_PATH") diff --git a/scripts/install_runtime.py b/scripts/install_runtime.py index d76a947..5f2cff8 100644 --- a/scripts/install_runtime.py +++ b/scripts/install_runtime.py @@ -1,8 +1,11 @@ """Install the separately supplied ONNX Runtime; print its path for Make.""" import argparse +import hashlib import json +import os import platform +import re import tarfile import urllib.request import zipfile @@ -32,14 +35,18 @@ def main(): "Windows": "win-arm64" if arm else "win-x64", }[system] version = args.version or ("1.23.2" if target == "osx-x86_64" else "1.24.4") + if not re.fullmatch(r"\d+\.\d+\.\d+", version): + parser.error("expected a stable runtime version") extension = "zip" if system == "Windows" else "tgz" name = f"onnxruntime-{target}-{version}" DESTINATION.mkdir(exist_ok=True) archive = DESTINATION / f"{name}.{extension}" - urllib.request.urlretrieve( - f"https://github.com/microsoft/onnxruntime/releases/download/v{version}/{archive.name}", - archive, - ) + records = json.loads((ROOT / "ci/runtimes.json").read_text()) + record = records[f"{version}/{target}"] + urllib.request.urlretrieve(record["url"], archive) + if hashlib.sha256(archive.read_bytes()).hexdigest() != record["sha256"]: + archive.unlink() + raise SystemExit("runtime archive digest mismatch") if extension == "zip": with zipfile.ZipFile(archive) as source: source.extractall(DESTINATION) @@ -56,6 +63,9 @@ def main(): json.dumps({"version": version, "target": target, "library": str(library)}) + "\n" ) print(library) + if output := os.environ.get("GITHUB_ENV"): + with open(output, "a", encoding="utf-8") as destination: + destination.write(f"ORT_DYLIB_PATH={library}\n") if __name__ == "__main__": diff --git a/scripts/package.py b/scripts/package.py index 6512804..a9ef151 100644 --- a/scripts/package.py +++ b/scripts/package.py @@ -1,6 +1,7 @@ """Assemble the Hex archive and test it in a clean consuming project.""" import argparse +import gzip import io import os import re @@ -23,7 +24,11 @@ def version(): def build(): destination = ROOT / "dist" / f"ortex-{version()}.tar" destination.parent.mkdir(exist_ok=True) - subprocess.run(["mix", "hex.build", "--output", str(destination)], cwd=ROOT, check=True) + subprocess.run( + [shutil.which("mix") or "mix", "hex.build", "--output", str(destination)], + cwd=ROOT, + check=True, + ) return destination @@ -34,9 +39,30 @@ def unpack(archive, destination): contents.extractall(destination, filter="data") +def nif_archive(target): + policy = __import__("json").loads((ROOT / "ci/compatibility.json").read_text()) + targets = {row["target"] for key in ["pr", "platforms", "alpine"] for row in policy[key]} + if target not in targets: + raise ValueError("target is not in the artifact contract") + source = ROOT / "priv/native" / ("libortex.dll" if "windows" in target else "libortex.so") + destination = ROOT / "dist" / f"ortex-{version()}-{target}.tar.gz" + destination.parent.mkdir(exist_ok=True) + with ( + destination.open("wb") as stream, + gzip.GzipFile(fileobj=stream, mode="wb", filename="", mtime=0) as compressed, + tarfile.open(fileobj=compressed, mode="w") as archive, + ): + entry = tarfile.TarInfo(source.name) + data = source.read_bytes() + entry.size, entry.mode = len(data), 0o644 + archive.addfile(entry, io.BytesIO(data)) + return destination + + def test_package(archive, mode, nif, timeout): with tempfile.TemporaryDirectory(prefix="ortex-package-") as temporary: - directory = Path(temporary) + # Expand Windows 8.3 names before Mix compares priv-directory symlink targets. + directory = Path(temporary).resolve() package = directory / "ortex" unpack(archive, package) project = directory / "consumer" @@ -83,6 +109,8 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest="command", required=True) commands.add_parser("build") + nif = commands.add_parser("nif") + nif.add_argument("--target", required=True) test = commands.add_parser("test") test.add_argument("--archive", type=Path) test.add_argument("--mode", choices=["source", "prebuilt"], default="source") @@ -92,6 +120,9 @@ def main(): if args.command == "build": print(build()) return 0 + if args.command == "nif": + print(nif_archive(args.target)) + return 0 if args.timeout <= 0: parser.error("timeout must be positive") archive = args.archive or build() diff --git a/scripts/scenario.py b/scripts/scenario.py index 2a87695..585410c 100644 --- a/scripts/scenario.py +++ b/scripts/scenario.py @@ -3,6 +3,7 @@ import argparse import json import os +import shutil import signal import subprocess import sys @@ -12,6 +13,8 @@ def execute(command, timeout, cwd=ROOT, env=None): + search = (os.environ if env is None else env).get("PATH", os.defpath) + command = [shutil.which(command[0], path=search) or command[0], *command[1:]] process = subprocess.Popen( command, cwd=cwd, @@ -45,7 +48,12 @@ def main(): if set(selected) - scenarios.keys(): parser.error(f"unknown scenarios: {set(selected) - scenarios.keys()}") env = {**os.environ, "MIX_ENV": "test"} - subprocess.run(["mix", "compile", "--warnings-as-errors"], cwd=ROOT, env=env, check=True) + subprocess.run( + [shutil.which("mix") or "mix", "compile", "--warnings-as-errors"], + cwd=ROOT, + env=env, + check=True, + ) results = [] for name in selected: status, output = execute( diff --git a/scripts/toolchain.py b/scripts/toolchain.py new file mode 100644 index 0000000..dde70f5 --- /dev/null +++ b/scripts/toolchain.py @@ -0,0 +1,19 @@ +"""Install the declared Rust toolchain once, before checks run concurrently.""" + +import subprocess +import tomllib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + spec = tomllib.loads((ROOT / "rust-toolchain.toml").read_text())["toolchain"] + command = ["rustup", "toolchain", "install", spec["channel"], "--profile", spec["profile"]] + for component in spec["components"]: + command += ["--component", component] + subprocess.run(command, check=True) + + +if __name__ == "__main__": + main() diff --git a/test/tooling/ci_test.py b/test/tooling/ci_test.py new file mode 100644 index 0000000..a5abf83 --- /dev/null +++ b/test/tooling/ci_test.py @@ -0,0 +1,57 @@ +"""Check matrix boundaries and evidence-based nightly skipping.""" + +import datetime +import unittest +from unittest.mock import patch + +from harness_test import load + + +class CiTest(unittest.TestCase): + def test_full_matrix_covers_seven_distinct_native_targets(self): + ci = load("ci") + targets = { + row["target"] for row in ci.matrix("full")["include"] + ci.matrix("alpine")["include"] + } + self.assertEqual(len(targets), 7) + self.assertLess(len(ci.matrix("pr")["include"]), len(ci.matrix("full")["include"])) + + def marker(self, age=0): + created = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=age) + return { + "expired": False, + "created_at": created.isoformat(), + "workflow_run": {"id": 42, "head_branch": "main", "head_sha": "a" * 40}, + } + + def test_recent_success_on_same_sha_skips(self): + ci = load("ci") + with patch.object( + ci, + "api", + side_effect=[ + {"artifacts": [self.marker()]}, + {"conclusion": "success", "event": "push"}, + ], + ): + self.assertFalse(ci.should_run("a" * 40)) + + def test_failed_old_or_different_sha_cannot_skip(self): + ci = load("ci") + with patch.object( + ci, + "api", + side_effect=[ + {"artifacts": [self.marker()]}, + {"conclusion": "failure", "event": "push"}, + ], + ): + self.assertTrue(ci.should_run("a" * 40)) + with patch.object(ci, "api", return_value={"artifacts": [self.marker(age=8)]}): + self.assertTrue(ci.should_run("a" * 40)) + with patch.object(ci, "api", return_value={"artifacts": [self.marker()]}): + self.assertTrue(ci.should_run("b" * 40)) + + +if __name__ == "__main__": + unittest.main()