diff --git a/.github/workflows/terminal-bench.yml b/.github/workflows/terminal-bench.yml deleted file mode 100644 index 218496dcc..000000000 --- a/.github/workflows/terminal-bench.yml +++ /dev/null @@ -1,160 +0,0 @@ -name: Terminal-Bench integration - -on: - pull_request: - paths: - - evals/terminal-bench/** - - .github/workflows/terminal-bench.yml - - packages/coding-agent/src/modes/print-mode.ts - - AGENTS_release.md - push: - branches: [main, 'feat/terminal-bench-*'] - paths: - - evals/terminal-bench/** - - .github/workflows/terminal-bench.yml - - packages/coding-agent/src/modes/print-mode.ts - - AGENTS_release.md - workflow_dispatch: - inputs: - through: - description: Run Daytona stages through this gate (uses credits) - type: choice - default: oracle - options: [oracle, trial, integration, pilot, reliability] - required: true - -permissions: - contents: read - -concurrency: - group: terminal-bench-${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} - cancel-in-progress: false - -jobs: - deterministic: - runs-on: ubuntu-24.04 - timeout-minutes: 30 - defaults: - run: - working-directory: evals/terminal-bench - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: 22.23.2 - - name: Install source fixture dependencies - working-directory: . - run: | - npm ci --ignore-scripts - npm run hydrate:model-data - - uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7 - with: - version: 0.11.2 - python-version: '3.13' - enable-cache: false - - name: Install locked benchmark dependencies - run: uv sync --locked - - name: Adapter, converter, and release gate regressions - run: uv run --locked python -m unittest discover -s tests -v - - name: Verify pinned dataset - run: uv run --locked python -m diffex_terminal_bench.benchmark --prepare - - name: Linux execution and trajectory smoke - run: uv run --locked python scripts/smoke_execution.py --platform linux/amd64 - - name: Linux failure and interruption smoke - run: uv run --locked python scripts/smoke_outcomes.py --platform linux/amd64 - - name: Retain smoke evidence - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: terminal-bench-deterministic-${{ github.run_id }}-${{ github.run_attempt }} - path: | - evals/terminal-bench/results/ - !evals/terminal-bench/results/**/dataset-cache/ - include-hidden-files: true - if-no-files-found: error - retention-days: 30 - - production-contents: - runs-on: ubuntu-24.04 - timeout-minutes: 20 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: ./.github/actions/setup - - uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7 - with: - version: 0.11.2 - python-version: '3.13' - enable-cache: false - - name: Inspect production tarballs - working-directory: evals/terminal-bench - run: | - uv sync --locked - uv run --locked python scripts/check_production_contents.py - - name: Retain production exclusion evidence - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: terminal-bench-production-contents-${{ github.run_id }}-${{ github.run_attempt }} - path: evals/terminal-bench/results/production-contents.json - if-no-files-found: error - retention-days: 30 - - daytona: - if: github.event_name == 'workflow_dispatch' - needs: [deterministic, production-contents] - runs-on: ubuntu-24.04 - timeout-minutes: 360 - environment: terminal-bench - defaults: - run: - working-directory: evals/terminal-bench - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7 - with: - version: 0.11.2 - python-version: '3.13' - enable-cache: false - - run: uv sync --locked - - name: Run ordered Daytona qualification - env: - THROUGH: ${{ inputs.through }} - DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - # Leave time for cleanup and artifact upload before the job deadline. - run: timeout --signal=TERM --kill-after=180s 330m uv run --locked python -m diffex_terminal_bench.benchmark --through "$THROUGH" - - name: Recover sandboxes after a failed or interrupted stage - if: failure() || cancelled() - env: - DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }} - run: | - if test -d results/release-gate; then - uv run --locked python -m diffex_terminal_bench.recover results/release-gate - fi - - name: Retain complete benchmark evidence - if: always() - id: artifacts - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: terminal-bench-daytona-${{ github.run_id }}-${{ github.run_attempt }} - path: | - evals/terminal-bench/results/release-gate/ - !evals/terminal-bench/results/release-gate/dataset-cache/ - include-hidden-files: true - if-no-files-found: error - retention-days: 90 - - name: Record artifact receipt - if: always() && steps.artifacts.outcome == 'success' - env: - ARTIFACT_URL: ${{ steps.artifacts.outputs.artifact-url }} - ARTIFACT_DIGEST: ${{ steps.artifacts.outputs.artifact-digest }} - run: | - printf 'Evidence: %s\n\nArchive SHA-256: %s\n' "$ARTIFACT_URL" "$ARTIFACT_DIGEST" >> "$GITHUB_STEP_SUMMARY" - - name: Download the uploaded evidence for verification - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 - with: - name: terminal-bench-daytona-${{ github.run_id }}-${{ github.run_attempt }} - path: evals/terminal-bench/results/downloaded-evidence - - name: Revalidate uploaded trajectories and usage - env: - THROUGH: ${{ inputs.through }} - run: uv run --locked python -m diffex_terminal_bench.benchmark --verify-through "$THROUGH" --output results/downloaded-evidence diff --git a/.gitignore b/.gitignore index 8698cc536..df9650f91 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ node_modules/ dist/ .artifacts/ +/evals/ *.log .DS_Store *.tsbuildinfo diff --git a/docs/dependency-maintenance.md b/docs/dependency-maintenance.md index 576381e8d..1adfb893e 100644 --- a/docs/dependency-maintenance.md +++ b/docs/dependency-maintenance.md @@ -55,12 +55,6 @@ Google automatic continuation is explicitly disabled for Gemini and Vertex reque Anthropic adopts the SDK's API version **2026-10-21** and no longer sends the removed sampling parameters. A model-context-window limit is reported as `length`. The Google GenAI install-script allowlist was reviewed and updated: its published `preinstall` is still a no-op, and installs continue to skip scripts. -## Python evaluation dependencies - -The shared Terminal-Bench/DeepSWE project pins Harbor **0.24.0**, Daytona **0.226.1**, E2B **2.59.0**, Pydantic **2.14.1**, PyYAML **6.0.3**, and build backend Hatchling **1.32.4**. Runtime guards and live setup guides match the checked-in lock; DeepSWE no longer resolves separate `uv run --with` overlays. - -The refreshed Python lock patches the reported certifi, aiohttp, OpenTelemetry API, PyJWT, multidict, and urllib3 advisories. Stable direct dependencies still require beta-only OpenTelemetry instrumentation transitives; `prerelease = "if-necessary"` records that resolver policy. Dependency validation uses published wheels without build hooks and mocked tests, not paid sandbox/model benchmark runs. Linux Docker qualification and live service qualification remain separate release checks. - ## Security review The initial audit reported seven affected packages, all present in the baseline lockfile. Updating Vitest/coverage to 4.1.12 and refreshing compatible Vite and source-map-js transitives removes five findings. The remaining sandbox-runtime/Zod dependency chain requires further review; do not force a Zod major-version override across consumers without checking their APIs. diff --git a/docs/identity-handoff.md b/docs/identity-handoff.md index 0fdc9e7ee..a62094872 100644 --- a/docs/identity-handoff.md +++ b/docs/identity-handoff.md @@ -1,7 +1,7 @@ # Product identity handoff -The release candidate uses the identities below and the approved backend policy in [the candidate handoff](release/candidate-handoff.md). -That handoff supersedes the earlier identity migration's security/staff exceptions and absent-API-hostname statements. +The current Diffex identities and package map are recorded below. +The [candidate handoff](release/candidate-handoff.md) preserves historical acceptance evidence; its service and attribution approvals are superseded by current policy. The canonical source remains private. No package, image, release asset, or source archive is published by this change. @@ -21,7 +21,9 @@ No package, image, release asset, or source archive is published by this change. Diffex and Luntraa, Inc. are unrelated to Earendil. The former staff-team authorization, security mailbox reference, announcement, and product affiliation wording have been removed. -The owner explicitly retained OpenAI Codex and xAI OAuth and provider attribution. +Provider attribution now identifies Diffex; see [provider validation](release/provider-branding-validation.md). +ChatGPT now registers Diffex and uses public model and inference APIs; existing inherited registrations require a new sign-in as described in [provider setup](../packages/coding-agent/docs/providers.md#chatgpt). +The xAI subscription integration remains available with Diffex attribution. Radius is removed; the older release audit does not authorize its use. See [migration and validation](release/radius-removal.md). The catalog authority is now Diffex; see [the catalog contract](model-catalog.md). @@ -29,23 +31,24 @@ The exact residual inventory is [identity-residual-allowlist.md](../identity-res | Workspace | Final package | Publication | | --------------------------------------- | ---------------------------------------------- | ---------------------------- | -| `packages/agent` | `@diffexai/diffex-agent-core` | Public; publication deferred | -| `packages/ai` | `@diffexai/diffex-ai` | Public; publication deferred | -| `packages/client` | `@diffexai/diffex-client` | Public; publication deferred | -| `packages/coding-agent` | `@diffexai/diffex` | Public; publication deferred | -| `packages/coding-agent-server` | `@diffexai/diffex-coding-agent-server` | Public; publication deferred | -| `packages/harness-state` | `@diffexai/diffex-harness-state` | Public; publication deferred | -| `packages/protocol` | `@diffexai/diffex-protocol` | Public; publication deferred | -| `packages/server` | `@diffexai/diffex-server` | Public; publication deferred | -| `packages/session-backends/sqlite-node` | `@diffexai/diffex-session-backend-sqlite-node` | Public; publication deferred | -| `packages/telemetry` | `@diffexai/diffex-telemetry` | Public; publication deferred | -| `packages/tui` | `@diffexai/diffex-tui` | Public; publication deferred | +| `packages/agent` | `@diffexai/diffex-agent-core` | Private internal workspace | +| `packages/ai` | `@diffexai/diffex-ai` | Private internal workspace | +| `packages/client` | `@diffexai/diffex-client` | Private internal workspace | +| `packages/coding-agent` | `@diffexai/diffex` | Public CLI | +| `packages/coding-agent-server` | `@diffexai/diffex-coding-agent-server` | Private internal workspace | +| `packages/harness-state` | `@diffexai/diffex-harness-state` | Private internal workspace | +| `packages/protocol` | `@diffexai/diffex-protocol` | Private internal workspace | +| `packages/server` | `@diffexai/diffex-server` | Private internal workspace | +| `packages/session-backends/sqlite-node` | `@diffexai/diffex-session-backend-sqlite-node` | Private internal workspace | +| `packages/telemetry` | `@diffexai/diffex-telemetry` | Private internal workspace | +| `packages/tui` | `@diffexai/diffex-tui` | Private internal workspace | | `packages/evals` | `@diffexai/diffex-evals` | Private | | `packages/evolution` | `@diffexai/diffex-evolution` | Private | | `packages/harness-trace` | `@diffexai/diffex-harness-trace` | Private | | `packages/coding-agent/install-lock` | `@diffexai/diffex-coding-agent-install` | Generated private installer | The public package inventory is derived from `scripts/release-packages.mjs`. +Only `@diffexai/diffex` is public; required internal runtime modules are bundled into its tarball under the [distribution policy](release/distribution-policy.json). The generated installer package remains private and is not an additional publication target. `@diffexai/diffex-coding-agent` is not a supported package identity. diff --git a/docs/release/candidate-handoff.md b/docs/release/candidate-handoff.md index 9f5561340..65c5666a7 100644 --- a/docs/release/candidate-handoff.md +++ b/docs/release/candidate-handoff.md @@ -6,16 +6,19 @@ The accepted implementation is [`33704b32c5d17b849c013834fd1c56d65de843fd`](http The follow-up evidence commit changes only documents; [the artifact manifest](candidate-artifacts.json) pins the tested implementation and bytes. The companion cloud change is stacked on PR #147. -Current policy supersedes this historical candidate's Radius approval. +Current policy supersedes this historical candidate's service, OAuth registration, and application-attribution approvals. Radius is removed and must not be used in development, testing, evaluation, or production. -The pinned JSON manifests and identity reports below describe historical bytes containing Radius and must not be reused as release candidates. +The pinned JSON manifests and identity reports below preserve historical bytes, including former URLs and headers, and must not be reused as release candidates. +Their approval labels are historical evidence, not current runtime exceptions. +Current catalog ownership is defined by [the catalog contract](../model-catalog.md), and current ChatGPT registration and inference by [provider setup](../../packages/coding-agent/docs/providers.md#chatgpt). See [Radius removal](radius-removal.md) for current migration and validation. ## Reconciled decisions The canonical private source, package map, executable, hosted image identity, and Luntraa, Inc. copyright are recorded in [the identity handoff](../identity-handoff.md). The historical release audit approved external catalogs and provider attribution. -The owner additionally instructed this implementation to retain OpenAI Codex and xAI OAuth. +The owner additionally instructed that implementation to retain OpenAI Codex and xAI OAuth. +Current provider defaults identify Diffex, and ChatGPT now uses a Diffex registration and public APIs instead of the inherited client registration. Radius gateway/OAuth approval has been revoked; no replacement Radius service is planned. No Diffex ownership of those external services or registrations is asserted. @@ -34,18 +37,20 @@ Install reporting remains removed. The updater still accepts only canonical stable versions for `@diffexai/diffex`, pins installation to the verified version, and treats the no-release response as no update. The [exact residual inventory](../../identity-residual-allowlist.md) replaces the obsolete exceptions and is enforced by `npm run check:identities`. -## Endpoint ownership and allowed egress +## Current endpoint ownership and allowed egress + +This table reflects current source behavior, not the endpoints captured in the historical JSON evidence. | Operation | Endpoint or authority | Policy | | ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | | Release discovery | `https://api.diffex.ai/api/latest-version` | Diffex Cloud public Worker; anonymous; fixed canonical package identity; no redirects | | Diffex account login and product APIs | `https://api.diffex.ai`, existing Render/WorkOS routes | Separate account credentials | -| Built-in catalog refresh | `https://pi.dev/api/models/providers/` | Explicitly retained external catalogs for configured providers | -| OpenAI Codex | `https://auth.openai.com`, `https://chatgpt.com/backend-api`; client `app_EMoamEEZ73f0CkXaXp7hrann` | Explicit owner retention of OAuth and provider attribution | -| xAI subscription | `https://auth.x.ai/oauth2/device/code`, `https://auth.x.ai/oauth2/token`; client `b1a00492-073a-47ea-816f-4c329264a828` | Explicit owner retention of OAuth and provider attribution | +| Built-in catalog refresh | `https://api.diffex.ai/api/models/providers/{providerId}` | Diffex-owned catalogs; cache provenance and compatibility follow the catalog contract | +| ChatGPT (`openai-codex`) | `https://auth.openai.com`, `https://api.openai.com/v1` | Diffex account registration; public account model catalog and Responses API; inherited registrations require sign-in again | +| xAI subscription | `https://auth.x.ai/oauth2/device/code`, `https://auth.x.ai/oauth2/token`; client `b1a00492-073a-47ea-816f-4c329264a828` | Existing subscription OAuth with Diffex attribution | | Other model providers | Selected built-in provider endpoints or explicitly configured custom endpoints | Provider-specific credentials and model traffic | -| Optional attribution | OpenRouter, NVIDIA NIM, Cloudflare | Existing optional provider headers; default off; separate from removed install reporting | -| OpenCode attribution | Selected OpenCode endpoint | Existing client/session headers retained independently of optional attribution | +| Optional attribution | OpenRouter, NVIDIA NIM, Cloudflare | Diffex application headers; default off; separate from removed install reporting | +| OpenCode attribution | Selected OpenCode endpoint | Diffex client identity and session header, independent of optional attribution | Radius discovery, authentication, and inference are no longer supported or allowed egress. Custom gateways remain explicit user configuration. @@ -61,7 +66,9 @@ The static provider IDs in that historical manifest describe the candidate's cat Dynamic `radius` is no longer supported by the source adapters. The installed artifact manifest lists all 40 provider IDs and display names; listing a provider does not claim live authentication or inference acceptance. -Configured built-in providers retain the external overlay and its four-hour refresh interval. +Configured built-in providers use the Diffex-owned overlay and its four-hour refresh interval. +Only caches matching the exact catalog URL and CLI version can restore models or validators; former-host caches trigger an unconditional refresh when online. +ChatGPT uses its signed-in account's public model catalog and an account-specific cache instead. Newer remote models override same-ID built-ins; cached bodies support ETag `304` responses. Older or undated overlays do not override the frozen built-in generation timestamp. Unavailable `404`/`501` overlays and transient errors preserve the documented built-in/cache fallback. @@ -108,7 +115,8 @@ All eleven local tarballs use version `0.2.0` and are fixture candidates, not pu Their SHA-256 values, the standalone executable/archive hashes, provider inventory, exact commands, exit codes, and smoke results are retained in [candidate-artifacts.json](candidate-artifacts.json). The [artifact identity review](candidate-identity-review.json) binds every match to an unpacked file hash, byte offset and context, or decoded source-map line. -It records 49 raw approved-integration/legal matches and 28 approved matches from decoded source maps. +It records 49 raw approved-integration/legal matches and 28 approved matches from decoded source maps under the policy at that revision. +Those historical approvals do not authorize the former catalogs, registrations, attribution headers, or Radius service in current source. Five additional lexical hits are identified separately: an XML processing-instruction error, three syntax-highlighting mathematical constants, and Jiti's bundled detection of an external Pi agent from PATH. The detector neither names Diffex as Pi nor uses a Pi package, service, credential, or organization; it is third-party interoperability vocabulary rather than an inherited Diffex identity. These contexts do not expand the runtime identity allowlist or authorize any new egress. diff --git a/docs/release/candidate-rehearsal.md b/docs/release/candidate-rehearsal.md index 6a9b1574b..87895c2c3 100644 --- a/docs/release/candidate-rehearsal.md +++ b/docs/release/candidate-rehearsal.md @@ -93,9 +93,11 @@ Lifecycle tests use loopback discovery/registry fixtures, with third-party depen These application-level guards and event logs are not a whole-host packet capture or proof of live provider egress. Catalog preparation contacts its approved catalog inputs; the local refresh encountered a transport failure, so the existing validated local snapshot was used for source gates while the native build retained its freshly hydrated snapshot. -The source identity gate accepts exactly seventeen reviewed occurrences across 915 files. +The historical source identity gate accepted seventeen reviewed occurrences across 915 files. Inspection of that historical tarball found sixteen identity matches in nine files, then approved as Radius, catalog, OAuth, and provider-attribution entries. Radius approval has since been revoked for every environment. +Catalogs, ChatGPT registration, and application attribution have also migrated to the current Diffex defaults described in [the residual inventory](../../identity-residual-allowlist.md). +The JSON evidence preserves its original URLs, headers, and review labels; it does not define current service permissions. The final payload gate verifies required notices, runtime-file hashes, the documentation allowlist, and absence of prohibited source and embedded maps. Image inspection accepts 1,414 project files across twelve runtime workspaces and thirty approved documentation assets. All isolated installation prefixes, caches, homes, mock sessions, fixture servers, and runtime containers are removed by their harnesses. diff --git a/docs/release/client-canaries.md b/docs/release/client-canaries.md index f216a6b2f..b0f0f7266 100644 --- a/docs/release/client-canaries.md +++ b/docs/release/client-canaries.md @@ -69,6 +69,7 @@ No package-manager process executed, no uninstall was attempted, and no installe The preload intercepted the separate `https://pi.dev/api/report-install` call during non-offline startup and returned a controlled failure without forwarding it. There was no Pi version-check traffic or any Pi socket connection. At the time of this historical run, install telemetry, model catalogs, and Radius remained separate migration work. +Install telemetry has since been [removed](install-telemetry-removal.md), and catalogs now use the [Diffex authority](../model-catalog.md). Radius is now removed and prohibited in all environments; see [Radius removal](radius-removal.md). ## Regression and artifact checks diff --git a/docs/release/provider-branding-validation.md b/docs/release/provider-branding-validation.md index a6b8c27e9..059bda876 100644 --- a/docs/release/provider-branding-validation.md +++ b/docs/release/provider-branding-validation.md @@ -1,5 +1,10 @@ # Provider branding validation +This record describes the provider presentation and attribution checks performed before the ChatGPT authentication migration. +Its OpenAI Codex registration, endpoint, device-flow, and transport descriptions are historical. +Current ChatGPT setup uses Diffex registration and the public model and Responses APIs; see [provider setup](../../packages/coding-agent/docs/providers.md#chatgpt). +The historical mock results below do not establish acceptance of the current ChatGPT flow. + Diffex controls the local callback presentation and default application attribution. Registered OAuth client IDs, provider consent pages, authentication protocols, stored credentials, inference endpoints, and payloads remain unchanged. The separate Diffex account feature is excluded. diff --git a/docs/release/publication.md b/docs/release/publication.md index 7aa6f34ce..bf30510ae 100644 --- a/docs/release/publication.md +++ b/docs/release/publication.md @@ -35,9 +35,9 @@ Missing credentials fail before npm verification or storage access. Use an AWS CLI that supports `put-object --if-match` and `--if-none-match`. The original `diffex-model-upload` environment had no secrets or protection rules when inspected on 2026-09-10. -Release publication now has a dedicated environment so configuring release credentials cannot redirect the model publisher. -The model-catalog workflow retains its existing environment and former account endpoint; model publication and any old-account data migration require their own setup. -Do not copy release credentials into that environment. +That historical setup has been superseded by the dedicated model-upload environments and Diffex-owned storage described in [the catalog contract](../model-catalog.md). +Release publication and model publication use separate environment credentials. +Do not copy release credentials into model-upload environments. Bucket-scoped Object Read & Write credentials can mutate other objects within their own bucket, including any future model objects in the production bucket. They do not enforce a `releases/v1/` prefix boundary and must never be distributed to clients. @@ -172,6 +172,7 @@ Canaries cover missing releases, newer/equal/older releases, normal and forced u Malformed JSON, wrong schema, prerelease versions, missing or duplicate canonical packages, package-version mismatches, and foreign identities must produce an unavailable response and no install even with force. Every successful install attempt must match the complete expected npm argument list, pinned to `@diffexai/diffex@` without uninstall. The JSON evidence captures requested and actual URLs, response status and public body, socket destination, intercepted install command, and verified local cleanup. -Install telemetry is intercepted separately without network traffic; its Pi dependency remains separate migration work. +Install telemetry is removed, and the current canary rejects every unexpected startup request, including the former install-report route. +See [install telemetry removal](install-telemetry-removal.md) for the regression boundary; the original acceptance results remain historical evidence. The source canary does not establish published-package installation or binary acceptance. See [the client canary acceptance record](client-canaries.md) for executed results and remaining limits. diff --git a/evals/deepswe-full-steps-itemized.md b/evals/deepswe-full-steps-itemized.md deleted file mode 100644 index 6fd9e1df6..000000000 --- a/evals/deepswe-full-steps-itemized.md +++ /dev/null @@ -1,124 +0,0 @@ -# DeepSWE 1.1: complete 113-task evaluation - -This workflow schedules all 113 tasks once, for 113 independent trials in one Harbor job. -Concurrency is 8, using E2B. -This preserves the launcher's one-attempt configuration; it does not reproduce four leaderboard passes. - -| Setting | Value | -| --- | --- | -| Dataset | [datacurve/deep-swe-1-1](https://hub.harborframework.com/datasets/datacurve/deep-swe-1-1) | -| Revision | `sha256:5affcd534fd90ac85d202d4c63f8b35ddc942140afdd5d60014a21365440a2f5` | -| Coverage | 113 distinct tasks, 1 attempt each, 113 trials | -| Diffex | Published 0.3.2 | -| Model | `openai/gpt-6-luna` | -| Thinking | `max` | -| Subagents | Disabled | -| Self evolution | Disabled by the adapter's fresh per-run settings | -| Environment | `deepswe_e2b:ResourceCheckedE2B` | -| Dependencies | Harbor 0.24.0, E2B 2.59.0, shared Terminal-Bench lock | -| Concurrent trials | 8 | -| Automatic Harbor trial retries | 0 | - -The official [leaderboard data](https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json) records four whole-benchmark passes. -The official leaderboard uses mini-swe-agent on Modal; this run uses Diffex on E2B and must be identified accordingly. -Diffex 0.3.2 retains its published request retry behavior; the proposed 5, 15, 45-second change is not part of this pinned release. - -## Preparation - -1. Synchronize the shared locked evaluation project; no `--with` dependency overlays are needed: - - ```zsh - cd /Users/jialeyu/Projects/diffex-new/evals/terminal-bench - uv --cache-dir /tmp/diffex-e2b-uv-cache sync --locked - ``` - -2. Download and validate every task without starting sandboxes or model calls: - - ```zsh - zsh /Users/jialeyu/Projects/diffex-new/evals/deepswe/run-full.zsh --prepare - ``` - - Preparation saves `job.yaml`, `manifest.json`, and `task-cache/` in `.artifacts/deepswe/prepared-113x1-c8-runtime-v3/`. - It verifies the release pins, all 113 task identities, separate verification, agent-only model network access, and offline verifiers. - Every task is included explicitly; no random subset or seed is used. - The manifest records each task's original resource requirements and timeouts. - The configuration preserves these limits. - -3. Inspect the resolved configuration: - - ```zsh - zsh /Users/jialeyu/Projects/diffex-new/evals/deepswe/run-full.zsh --print-config - ``` - - Confirm one attempt, concurrency eight, the pinned dataset, all 113 task names, Luna at max effort, and disabled subagents. - Evolution is disabled inside the adapter, not by a Harbor YAML setting. - The upstream artifact overlap warning about `/logs/artifacts/model.patch` is expected; inspect retained patches after execution. - -## Execution and evidence - -The launcher revalidates preparation and prompts silently for missing `E2B_API_KEY` and `OPENAI_API_KEY` values. -It creates a fresh `evals/terminal-bench/results/deepswe-1-1-luna-max-113x1--/` directory. -The native Harbor job lives in its `full-113x1/` subdirectory, whose absolute path is printed at launch. -The parent directory retains the manifest, configuration, launcher, backend, adapter snapshot, Git revision, and adapter diff. - -The adapter installs Diffex 0.3.2, uses its evaluation system prompt, disables automatic context files, and starts with fresh state for every attempt. -The selected tools contain no subagent tools. -Agents commit their changes, and Harbor extracts the resulting patch for a separate pristine verifier sandbox as required by DeepSWE 1.1. -Only `api.openai.com` is added to the agent's network allowlist; the verifier remains offline. -Disclose this model-endpoint exception with results. - -Eight active trials can require up to sixteen agent and verifier sandboxes. -The backend checks actual CPU, memory, and disk capacity for each sandbox. -Harbor collects local artifacts and deletes sandboxes through its normal lifecycle. -Keep the terminal and computer running until verification and cleanup finish. - -## After completion - -1. Set `DS_JOB` to the exact absolute job path printed by the launcher and inspect `result.json`. -2. Require 113 terminal trial results covering 113 distinct tasks exactly once each. - Inspect errors separately from failed solutions and retain all attempts. -3. Inspect verifier rewards, `verifier/test-stdout.txt`, collected `model.patch`, `agent/trajectory.json`, and native `agent/run-*/sessions/`. - Check requested and effective settings for the model, max thinking, disabled subagents and evolution, delivered instructions, and confirmed shutdown. -4. Validate existing trajectories: - - ```zsh - ( - set -e - cd /Users/jialeyu/Projects/diffex-new/evals/terminal-bench - DS_TRACES=("$DS_JOB"/*/agent/trajectory.json(N)) - [[ ${#DS_TRACES} -gt 0 ]] - for DS_TRACE in "${DS_TRACES[@]}"; do - uv --cache-dir /tmp/diffex-e2b-uv-cache run --locked \ - python -m harbor.utils.trajectory_validator "$DS_TRACE" - done - ) - ``` - - Separately check missing trajectories and incomplete token accounting. - Official pass@1 is successful scored attempts divided by scored attempts; pass@4 is tasks with at least one successful attempt divided by tasks attempted across four attempts. - This one-attempt run cannot measure pass@4. - Official scoring excludes provider, verifier, and network errors but counts agent timeouts and context-window failures as failures. - Report coverage and exclusions alongside scores; Harbor's raw mean may use a different denominator. - -5. To upload the native job privately under Diffex after verification: - - ```zsh - cd /Users/jialeyu/Projects/diffex-new/evals/terminal-bench - uv --cache-dir /tmp/diffex-e2b-uv-cache run --locked \ - harbor upload "$DS_JOB" --org diffex --private - ``` - - If authentication is required, run `uv run --locked harbor auth login` first. - Verify ownership, private visibility, all 113 trials, and available traces at the printed Hub link. - Retry a partial upload using the same job directory. - -If interrupted, preserve the entire output directory, inspect job and trial logs, and check E2B for remaining sandboxes after Harbor cleanup. -Each launcher invocation starts a fresh job; it does not resume an interrupted job or replace failed attempts. - -## Final command - -Run this command to start all 113 trials: - -```zsh -zsh /Users/jialeyu/Projects/diffex-new/evals/deepswe/run-full.zsh -``` diff --git a/evals/deepswe-steps-itemized.md b/evals/deepswe-steps-itemized.md deleted file mode 100644 index 3d1091714..000000000 --- a/evals/deepswe-steps-itemized.md +++ /dev/null @@ -1,174 +0,0 @@ -# DeepSWE 1.1: five random tasks, two attempts each - -This workflow schedules 10 independent trials: five distinct randomly selected tasks, each attempted twice, with concurrency 5. -Both attempts use the same task selection; Harbor schedules them in one job rather than enforcing a barrier between two passes. -Preparation downloads task definitions and validates configuration without provisioning E2B sandboxes or calling the model. -The final command starts the evaluation. - -| Setting | Value | -| --- | --- | -| Dataset | [datacurve/deep-swe-1-1](https://hub.harborframework.com/datasets/datacurve/deep-swe-1-1), 113 tasks | -| Dataset revision | `sha256:5affcd534fd90ac85d202d4c63f8b35ddc942140afdd5d60014a21365440a2f5` | -| Diffex | Published `0.3.2` | -| Model | `openai/gpt-6-luna` | -| Thinking | `max` | -| Subagents | Disabled | -| Self evolution | Disabled by the adapter's fresh `evolution.enabled=false` settings | -| Environment | E2B, through `deepswe_e2b:ResourceCheckedE2B` | -| Harness dependencies | Harbor `0.24.0`, E2B `2.59.0`, shared Terminal-Bench lock | -| Sample | Five tasks selected without replacement using a saved random seed | -| Attempts per task | 2 | -| Concurrent trials | 5 | -| Automatic Harbor retries | 0 | - -The published leaderboard uses four whole-dataset passes, or 452 trials per configuration, and reports pass@1 across scored attempts. -This 5-by-2 run is a test subset, not a full leaderboard evaluation. -See the official [leaderboard data](https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json). - -## Prepare the environment - -1. Open zsh and synchronize the existing evaluation project: - - ```zsh - cd /Users/jialeyu/Projects/diffex-new/evals/terminal-bench - uv --cache-dir /tmp/diffex-e2b-uv-cache sync --locked - ``` - -2. Using the shared locked environment without dependency overlays, select five random tasks and download and validate their definitions: - - ```zsh - zsh /Users/jialeyu/Projects/diffex-new/evals/deepswe/run-sample.zsh --prepare - ``` - - Preparation saves `seed.json`, `manifest.json`, `job.yaml`, and a task cache in `.artifacts/deepswe/prepared-5x2-runtime-v3/` under the repository root. - It confirms Diffex 0.3.2, the 113-task dataset digest, five selected task identities, two attempts, concurrency five, and separate verification. - Repeating preparation retains the existing seed and selected tasks. - Harbor's `n_tasks` takes the first entries, so this launcher explicitly samples a sorted task list with the recorded seed and writes `task_names` into the prepared configuration. - Harbor warns that the original explicit `model.patch` artifact overlaps its standard `/logs/artifacts` collection. - The upstream task definitions are preserved; check that the patch is retained after execution. - -3. Inspect the resolved configuration and selection: - - ```zsh - zsh /Users/jialeyu/Projects/diffex-new/evals/deepswe/run-sample.zsh --print-config - python3 -m json.tool /Users/jialeyu/Projects/diffex-new/.artifacts/deepswe/prepared-5x2-runtime-v3/manifest.json - ``` - - Confirm `n_attempts: 2`, `n_concurrent_trials: 5`, the five task names, the pinned dataset, `thinking: max`, and disabled subagents. - The backend import must be `deepswe_e2b:ResourceCheckedE2B`. - No trial, agent, verifier, CPU, memory, or storage limit overrides are configured. - The manifest records the original resource requirements and deadlines for each task. - -The initial preparation selected these tasks with seed `5414959033685670815`: - -- `datacurve/go-critic-doc-link-checker` -- `datacurve/koota-composite-trait-aspects` -- `datacurve/superjson-error-stack-serialization` -- `datacurve/anko-typed-variable-bindings` -- `datacurve/bandit-structured-nosec-directives` - -Each selected task declares 2 vCPU, 8192 MiB RAM, 20480 MiB disk capacity, a 5400-second agent deadline, and an 1800-second verifier deadline. -The saved manifest is authoritative if a separate preparation directory is used later. - -## Execution and retained evidence - -The adapter installs the published Diffex 0.3.2 release inside each agent sandbox. -It uses the checked-in evaluation system prompt, disables automatic context files, and includes the existing anti-cheating instruction. -The fresh Diffex settings disable self evolution, and the selected tools contain no subagent tools. - -DeepSWE 1.1 requires the agent to commit its work. -Harbor collects the commit-derived `model.patch`, applies it in a separate pristine verifier sandbox, and runs the held-out tests there. -The original task instruction, commit collection, verifier, and resource limits remain intact. - -The original tasks forbid execution-time network access. -This configuration adds the existing adapter's agent-only `api.openai.com` exception so Diffex can call its model inside the sandbox. -The verifier remains offline, and other agent destinations remain blocked during execution. -Installation occurs before the restricted execution phase. -Disclose this network exception and the Diffex harness with results. - -E2B capacity checks require the task's declared CPU and memory and at least its declared disk capacity in both sandboxes. -Disk capacity is not a free-space measurement. -Concurrency five means five active trials; each trial uses distinct agent and verifier sandboxes, so allow E2B quota for up to ten simultaneous sandboxes. -Actual E2B provisioning, API access, and runtime behavior are validated only when the evaluation starts. - -The launcher creates a fresh `evals/terminal-bench/results/deepswe-1-1-luna-max-5x2--/` directory each time. -It saves the seed, manifest, job configuration, launcher, E2B backend, adapter source snapshot, Git revision, and adapter working-tree diff. -The native Harbor job is its `sample-5x2/` subdirectory. -No Daytona volumes or volume-cleanup plugins are used. -Harbor collects local logs and deletes sandboxes through its normal lifecycle. - -## Inspect results after completion - -1. Use the exact absolute job path printed by the launcher: - - ```zsh - DS_JOB='/Users/jialeyu/Projects/diffex-new/evals/terminal-bench/results/ACTUAL-OUTPUT/sample-5x2' - python3 -m json.tool "$DS_JOB/result.json" - open "$DS_JOB" - ``` - -2. Require 10 terminal trial results covering exactly the selected five tasks twice each. - Inspect job and trial `config.json`, `lock.json`, `result.json`, and any execution exceptions. - A finished job does not imply that the solutions passed. - -3. Inspect every trial's verifier reward, `verifier/test-stdout.txt`, `model.patch`, `agent/trajectory.json`, and native `agent/run-*/sessions/`. - Check `requested.json` for `thinking: max`, `subagents: disabled`, and `evolution: {enabled: false}`. - Check `effective.json` for the expected model and thinking setting, disabled context files, and delivered instructions. - Check confirmed shutdown and complete usage accounting before treating execution evidence as complete. - Successful trials should retain two distinct verified records in `artifacts/e2b-resources/`. - -4. Validate retained trajectories with the pinned environment: - - ```zsh - ( - set -e - cd /Users/jialeyu/Projects/diffex-new/evals/terminal-bench - DS_TRACES=("$DS_JOB"/*/agent/trajectory.json(N)) - [[ ${#DS_TRACES} -gt 0 ]] - for DS_TRACE in "${DS_TRACES[@]}"; do - uv --cache-dir /tmp/diffex-e2b-uv-cache run --locked \ - python -m harbor.utils.trajectory_validator "$DS_TRACE" - done - ) - ``` - - This validates existing trajectories; separately check that every successfully executed trial has one. - Retain incomplete traces and errors rather than silently dropping attempts. - Report successful attempts, failed solutions, infrastructure errors, and completed coverage separately. - Official DeepSWE scoring excludes provider, verifier, and network errors while counting agent timeouts and context-window failures as failures. - Harbor's raw job mean may use different error handling, so disclose the denominator when reporting a score. - -## Optional private upload - -Native Harbor jobs can be uploaded without the Terminal-Bench staged export command: - -```zsh -cd /Users/jialeyu/Projects/diffex-new/evals/terminal-bench -uv --cache-dir /tmp/diffex-e2b-uv-cache run --locked \ - harbor auth login -uv --cache-dir /tmp/diffex-e2b-uv-cache run --locked \ - harbor upload "$DS_JOB" --org diffex --private -``` - -`--org diffex` assigns ownership to Diffex. -Save the printed link and verify visibility, ownership, ten trials, and available trajectories. -Retry a partial upload using the same job directory. - -## Failure or interruption - -Preserve the complete output directory and inspect `job.log`, trial exceptions, and retained logs. -Wait for Harbor cleanup and inspect E2B for sandboxes left by this job before restarting. -Do not reuse an existing job directory or selectively replace failed attempts. -The launcher starts a new output directory while keeping the same saved five-task sample. -To intentionally choose another random sample, set `DS_PREPARED` to a fresh absolute directory before preparation and launch. -Keep each sample's seed and results separate. - -## Final command: start the 10-trial test - -Run this in your terminal after preparation. -If `E2B_API_KEY` or `OPENAI_API_KEY` is missing, the launcher prompts for it without echoing or saving it in the configuration. -Keep the terminal running until Harbor finishes verification and cleanup. - -```zsh -zsh /Users/jialeyu/Projects/diffex-new/evals/deepswe/run-sample.zsh -``` diff --git a/evals/deepswe/REPAIR_NOTES.md b/evals/deepswe/REPAIR_NOTES.md deleted file mode 100644 index ffa3c766e..000000000 --- a/evals/deepswe/REPAIR_NOTES.md +++ /dev/null @@ -1,104 +0,0 @@ -# DeepSWE runtime repairs - -These changes apply to new evaluations using the working-tree adapter and DeepSWE runners. -No Diffex package release is required; the runtime remains pinned to Diffex 0.3.2. -Existing trial results and the pinned dataset have not been modified. - -## Implemented - -- DeepSWE explicitly selects `task_environment: preserve`. - The launcher preserves task environment variables and HOME-dependent dependency caches while keeping Diffex settings, sessions, and mutable state in a fresh run directory. - Inherited Diffex controls and unselected provider credentials are excluded. -- Both E2B sandboxes receive the published task image's environment and working directory explicitly. - Image metadata comes from public ECR without downloading filesystem layers, and configuration and manifest digests are checked. - Explicit task and command environment overrides keep precedence. - Startup verifies propagation and saves evidence under `artifacts/e2b-runtime/` before model execution or grading. -- Verifier shutdown attempts a bounded download before sandbox deletion, including after a timeout. - Partial logs and recovery status are saved under `artifacts/verifier-recovery/`, separately from completed grading output. - Download failures remain explicit and do not prevent sandbox deletion. -- Full, sample, and smoke runners use new preparation directories ending in `runtime-v3`, avoiding reuse of previously prepared launcher settings. - Smoke runs retain the complete adapter source, and smoke qualification requires environment checks for both sandboxes. -- Two adapter tests with obsolete model/reasoning expectations now validate the selected configuration. - -## Validation - -The real launcher was reproduced in an offline Linux container with a fixture CLI. -Before the patch, a child Python process failed with `ModuleNotFoundError` despite a supplied PYTHONPATH. -After the patch, imports, explicit cache paths, HOME-dependent caches, credential filtering, and fresh Diffex settings passed. -The existing launcher isolation regression also passed. - -Public ECR metadata confirmed `DENO_DIR=/deno-cache` for Cliffy and `PYTHONPATH=/app/src` for PSD Tools. -Fifteen offline E2B tests cover image configuration integrity, platform selection, agent/verifier environment propagation through actual child processes, override precedence, fail-fast behavior, verifier log recovery, registry retries, and prepared catalogs. -The adapter Python suite and `npm run check` passed. -The full configuration selects all 113 tasks with one attempt and concurrency eight. - -No live E2B sandbox or model call was started during this repair. -E2B credentials were not available in the session, so Cliffy and PSD Tools still need live verifier confirmation before treating their environment failures as resolved end to end. -Cliffy's cache contents have not been independently checked; preserving DENO_DIR cannot repair an incomplete published cache. - -## Remaining issues - -| Task | Remaining issue | -| --- | --- | -| Cliffy | Runtime propagation is patched; offline cache completeness and successful live verification remain unconfirmed. | -| PSD Tools | Runtime propagation is patched; live verification and solution correctness remain unconfirmed. | -| Pwntools | The cause of the historical 1,800-second timeout is unknown because the old verifier logs were not retained; future shutdowns now attempt to preserve them. | -| Obsidian TOC grading | The whitelist uses `Auto Table of Contents`, while the retained reporter output uses `Auto TOC`; its new-suite report contains 37 passes and four failures. | -| Aiomonitor | Reported solution failure: snapshot endpoints return 404. | -| Bandit | Reported solution failure: suppression regions and skipped-test metrics. | -| Clack | Reported solution failure: caching, cancellation, retry timing, and async UI behavior. | -| Etree | Reported solution failure: whitespace and patch parent paths. | -| FastAPI | Reported solution failure: non-deprecated route tracking. | -| Go Critic | Reported solution failure: documentation-link checker assertions. | -| Koota predicates | Reported solution failure: Changed and Removed predicates. | -| Obsidian TOC solution | Four formatting failures remain even with the test-ID mapping corrected. | -| Onedump | Reported solution failure: encryption key loading and derivation. | -| Pest | Reported solution failure: 19 character-class optimization tests. | -| Prometheus reload | Reported solution failure: fractional timings where tests expect integers. | -| Python Statemachine | Reported solution failure: state-data defaults and SCXML. | -| Vulture | Reported solution failure: cache backup/metadata-file assertion. | - -The reported solution failures do not establish a general Diffex implementation defect. -Task-specific solutions or hidden-test hints were not added to the eval harness. -Obsidian's scoring correction requires a separately versioned dataset correction or an explicitly identified local grading variant; the pinned benchmark's test IDs and rewards remain unchanged. -Preparation also reports an upstream overlapping-artifact warning for `/logs/artifacts` and `/logs/artifacts/model.patch`; the pinned task definitions remain unchanged. - -## Next run - -From the repository root, use `zsh evals/deepswe/run-full.zsh --print-config` to inspect the newly prepared job. -Use a fresh preparation directory if overriding `DS_PREPARED`; old prepared configurations deliberately fail comparison rather than silently changing an evaluation. -Run live smoke/verifier qualification with the normal credentials before starting the next full evaluation. - -## Login-shell startup regression - -The first 113-task, one-attempt run failed during sandbox startup because the new environment check rejected PATH. -E2B 2.51 constructs commands as `/bin/bash -l -c`, allowing login profiles to replace PATH after it is supplied through the SDK environment. -The original local environment regression used a non-login shell and missed this behavior. -The backend now restores the fully merged PATH inside the login shell before executing each command, preserving image, per-command, and scoped override precedence. -Provider credentials remain in the SDK environment parameter rather than shell command text. -The strict environment check remains enabled. - -A regression now exercises the pinned E2B SDK's actual command construction and executes the resulting shell process locally with a profile that replaces PATH. -It failed before the fix with `task-only-tool: command not found` and passes after the fix, including paths containing spaces, quotes, and dollar signs. -The login-shell regression passes alongside the other E2B regression tests. -Live E2B validation of this follow-up still requires credentials in the launching terminal. -The next full run remains Diffex 0.3.2, GPT-6 Luna at max thinking, subagents disabled, evolution disabled, 113 tasks with one attempt each, and concurrency eight. - -## Public registry throttling regression - -The subsequent run failed before creating sandboxes because concurrent image metadata requests received HTTP 429 from public ECR. -Harbor's subsequent log download reported `Sandbox not found` because startup had never created one. -The overlapping-artifact warnings are independent of this failure. - -Image runtime metadata is now collected serially during preparation with paced requests, bounded retries for transient failures, and support for `Retry-After`. -Successful reads are checkpointed atomically so interrupted preparation resumes without repeating them. -Concurrent preparers for the same catalog share a file lock. -Preparation writes a catalog with integrity checks and pins its SHA-256 in the job configuration. -Both agent and verifier startup load that catalog locally, eliminating these public ECR metadata requests from concurrent trial startup. -Each runner archives the catalog with the run's configuration and source snapshots. - -Regression tests reproduce a throttled manifest response, validate bounded retries, reject modified catalogs, resume partial preparation, and exercise eight concurrent starts through Harbor's real environment factory without public registry requests. -Live public ECR preparation successfully cached all 113 image configurations in `prepared-113x1-c8-runtime-v3`. -An offline check then loaded all 113 agent and 113 verifier configurations through Harbor's factory and reused the complete catalog with zero network requests. -All 15 E2B regressions, runner syntax checks, Harbor configuration rendering, and `npm run check` passed. -E2B still performs its own image provisioning, which this host-side metadata fix does not validate. diff --git a/evals/deepswe/__pycache__/deepswe_e2b.cpython-313.pyc b/evals/deepswe/__pycache__/deepswe_e2b.cpython-313.pyc deleted file mode 100644 index d6e722b05..000000000 Binary files a/evals/deepswe/__pycache__/deepswe_e2b.cpython-313.pyc and /dev/null differ diff --git a/evals/deepswe/__pycache__/prepare-full.cpython-314.pyc b/evals/deepswe/__pycache__/prepare-full.cpython-314.pyc deleted file mode 100644 index 0d757f760..000000000 Binary files a/evals/deepswe/__pycache__/prepare-full.cpython-314.pyc and /dev/null differ diff --git a/evals/deepswe/__pycache__/test_deepswe_e2b.cpython-313.pyc b/evals/deepswe/__pycache__/test_deepswe_e2b.cpython-313.pyc deleted file mode 100644 index d4ec3ea68..000000000 Binary files a/evals/deepswe/__pycache__/test_deepswe_e2b.cpython-313.pyc and /dev/null differ diff --git a/evals/deepswe/deepswe_e2b.py b/evals/deepswe/deepswe_e2b.py deleted file mode 100644 index 95576ca62..000000000 --- a/evals/deepswe/deepswe_e2b.py +++ /dev/null @@ -1,266 +0,0 @@ -"""Preserve DeepSWE image runtime settings, capacity checks, and verifier evidence.""" - -import asyncio -from asyncio import sleep as registry_sleep -from datetime import datetime, timezone -from email.utils import parsedate_to_datetime -import fcntl -import hashlib -import json -import math -import os -from pathlib import Path -import re -import shlex -import tempfile - -import httpx -from harbor.environments.e2b import E2BEnvironment - - -async def registry_get(client: httpx.AsyncClient, url: str, **kwargs) -> httpx.Response: - """Pace anonymous ECR reads and retry only transient, read-only failures.""" - delay = 1.1 - for attempt in range(6): - await registry_sleep(delay) - try: - response = await client.get(url, **kwargs) - except httpx.TransportError: - if attempt == 5: - raise - delay = 2 ** (attempt + 1) - continue - if response.status_code not in {429, 500, 502, 503, 504} or attempt == 5: - response.raise_for_status() - return response - delay = 2 ** (attempt + 1) - retry_after = response.headers.get("Retry-After") - if retry_after: - try: - wait = float(retry_after) - except ValueError: - try: - wait = (parsedate_to_datetime(retry_after) - datetime.now(timezone.utc)).total_seconds() - except (TypeError, ValueError, OverflowError): - wait = 0 - if math.isfinite(wait): - delay = max(delay, wait) - # Do not retry earlier than Retry-After or wait indefinitely on a - # registry outage. Preparation fails without starting any eval trials. - if delay > 60: - response.raise_for_status() - raise AssertionError("Unreachable registry retry state") - - -async def image_runtime(client: httpx.AsyncClient, image: str) -> dict: - """Read public ECR image configuration without downloading filesystem layers.""" - match = re.fullmatch(r"public\.ecr\.aws/([a-z0-9_./-]+)(?::([\w.-]+)|@(sha256:[a-f0-9]{64}))", image) - if match is None: - raise ValueError(f"Expected a versioned public ECR task image: {image}") - repository, tag, digest = match.groups() - response = await registry_get(client, "https://public.ecr.aws/token/", params={ - "service": "public.ecr.aws", "scope": f"repository:{repository}:pull", - }) - response.raise_for_status() - headers = {"Authorization": f"Bearer {response.json()['token']}", "Accept": ", ".join(( - "application/vnd.oci.image.index.v1+json", "application/vnd.oci.image.manifest.v1+json", - "application/vnd.docker.distribution.manifest.list.v2+json", - "application/vnd.docker.distribution.manifest.v2+json", - ))} - base = f"https://public.ecr.aws/v2/{repository}" - reference = digest or tag - for _ in range(2): - response = await registry_get(client, f"{base}/manifests/{reference}", headers=headers) - response.raise_for_status() - actual_digest = "sha256:" + hashlib.sha256(response.content).hexdigest() - if reference.startswith("sha256:") and reference != actual_digest: - raise ValueError("Image manifest digest mismatch") - manifest = response.json() - if "manifests" not in manifest: - break - candidates = [entry for entry in manifest["manifests"] - if entry.get("platform", {}).get("os") == "linux" - and entry.get("platform", {}).get("architecture") == "amd64"] - if len(candidates) != 1: - raise ValueError("DeepSWE requires exactly one linux/amd64 image manifest") - reference = candidates[0]["digest"] - else: - raise ValueError("Nested image indexes are not supported") - config_digest = manifest["config"]["digest"] - response = await registry_get(client, f"{base}/blobs/{config_digest}", headers=headers) - response.raise_for_status() - if config_digest != "sha256:" + hashlib.sha256(response.content).hexdigest(): - raise ValueError("Image configuration digest mismatch") - config = response.json() - if (config["os"], config["architecture"]) != ("linux", "amd64"): - raise ValueError("DeepSWE requires linux/amd64 image configuration") - runtime = config["config"] - return {"image": image, "manifest_digest": actual_digest, "config_digest": config_digest, - "env": dict(value.split("=", 1) for value in runtime.get("Env") or []), - "workdir": runtime.get("WorkingDir") or "/"} - - -def read_runtime_catalog(data: bytes) -> dict: - catalog = json.loads(data) - images = catalog["images"] - digest = hashlib.sha256(json.dumps(images, sort_keys=True).encode()).hexdigest() - if catalog.get("schema_version") != 1 or catalog.get("images_sha256") != digest: - raise ValueError("Image runtime catalog integrity check failed") - for image, runtime in images.items(): - if runtime.get("image") != image or not isinstance(runtime.get("env"), dict) or not isinstance(runtime.get("workdir"), str): - raise ValueError("Invalid image runtime catalog entry") - return images - - -async def prepare_image_runtimes(images: list[str], path: Path) -> str: - """Resolve metadata serially once, checkpointing successful reads for reuse.""" - path.parent.mkdir(parents=True, exist_ok=True) - # Separate preparers for the same job must share both cached results and - # pacing. Never hold a blocking flock on the asyncio event-loop thread. - with path.with_suffix(".lock").open("a") as lock: - async with asyncio.timeout(1200): - while True: - try: - fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) - break - except BlockingIOError: - await asyncio.sleep(0.1) - try: - records = read_runtime_catalog(path.read_bytes()) if path.exists() else {} - async with httpx.AsyncClient(timeout=30, follow_redirects=True) as client: - for index, image in enumerate(sorted(set(images)), 1): - if image in records: - continue - print(f"Preparing image runtime {index}/{len(set(images))}: {image}", flush=True) - records[image] = await image_runtime(client, image) - catalog = {"schema_version": 1, "images": records, "images_sha256": - hashlib.sha256(json.dumps(records, sort_keys=True).encode()).hexdigest()} - with tempfile.NamedTemporaryFile(dir=path.parent, mode="w", delete=False) as stream: - temporary = Path(stream.name) - try: - stream.write(json.dumps(catalog, indent=2, sort_keys=True) + "\n") - stream.close() - temporary.replace(path) - finally: - temporary.unlink(missing_ok=True) - return hashlib.sha256(path.read_bytes()).hexdigest() - finally: - fcntl.flock(lock, fcntl.LOCK_UN) - - -class ResourceCheckedE2B(E2BEnvironment): - def __init__(self, *args, runtime_catalog: str | None = None, - runtime_catalog_sha256: str | None = None, **kwargs): - self._runtime_catalog = runtime_catalog - self._runtime_catalog_sha256 = runtime_catalog_sha256 - super().__init__(*args, **kwargs) - - def _load_runtime(self) -> dict: - path = self._runtime_catalog - expected = self._runtime_catalog_sha256 - if not path or not expected: - raise RuntimeError("Prepare the DeepSWE runtime catalog before starting trials") - data = Path(path).read_bytes() - if hashlib.sha256(data).hexdigest() != expected: - raise ValueError("Prepared image runtime catalog changed") - return read_runtime_catalog(data)[self._runtime_image()] - - async def _dispatch_command( - self, command: str, *, cwd: str | None, env: dict[str, str] | None, - timeout_sec: int | None, user: str, - ): - # E2B executes /bin/bash -l -c: login profiles can replace the PATH - # supplied through envs. Restore the fully merged PATH inside that shell - # so startup probes, agent tools, and verifier commands use task binaries. - # Keep credentials in the env parameter, never in shell command text. - if env is not None and "PATH" in env: - command = f"export PATH={shlex.quote(env['PATH'])}; {command}" - return await super()._dispatch_command( - command, cwd=cwd, env=env, timeout_sec=timeout_sec, user=user, - ) - - def _runtime_image(self) -> str: - if self.task_env_config.docker_image: - return self.task_env_config.docker_image - # The pinned dataset's verifier Dockerfiles add only files and chmod to - # the task image. Reject future runtime overrides rather than ignore them. - lines = self._environment_definition_path.read_text().splitlines() - instructions = [shlex.split(line, comments=True) for line in lines if line.strip()] - instructions = [parts for parts in instructions if parts] - bases = [parts for parts in instructions if parts[0].upper() == "FROM"] - if len(bases) != 1 or len(bases[0]) != 2 or any( - parts[0].upper() not in {"FROM", "COPY", "RUN"} for parts in instructions - ): - raise ValueError("Unsupported DeepSWE verifier Dockerfile runtime configuration") - return bases[0][1] - - async def start(self, force_build: bool) -> None: - runtime = self._load_runtime() - # Explicit task/trial settings still take precedence over image defaults. - self._persistent_env = {**runtime["env"], **self._startup_env()} - self._workdir = runtime["workdir"] - await super().start(force_build) - if self._sandbox is None: - raise RuntimeError("E2B sandbox was not created.") - probe = await self.exec("env -0", timeout_sec=30) - actual = dict(value.split("=", 1) for value in (probe.stdout or "").split("\0") if "=" in value) - expected = self._merge_env(None) or {} - mismatches = sorted(key for key, value in expected.items() if actual.get(key) != value) - runtime_record = {key: value for key, value in runtime.items() if key != "env"} - runtime_record.update(env_names=sorted(runtime["env"]), mismatches=mismatches, - verified=probe.return_code == 0 and not mismatches) - directory = self.trial_paths.trial_dir / "artifacts" / "e2b-runtime" - directory.mkdir(parents=True, exist_ok=True) - (directory / f"{self._sandbox.sandbox_id}.json").write_text(json.dumps(runtime_record, indent=2) + "\n") - if not runtime_record["verified"]: - raise RuntimeError(f"E2B did not preserve image environment variables: {mismatches}") - async with httpx.AsyncClient(timeout=30) as client: - response = await client.get( - f"https://api.e2b.dev/sandboxes/{self._sandbox.sandbox_id}", - headers={"X-API-Key": os.environ["E2B_API_KEY"]}, - ) - response.raise_for_status() - data = response.json() - resources = { - "sandbox_id": self._sandbox.sandbox_id, - "session_id": self.session_id, - "cpu_count": data["cpuCount"], - "memory_mb": data["memoryMB"], - "disk_size_mb": data["diskSizeMB"], - "expected_cpu_count": self.task_env_config.cpus, - "expected_memory_mb": self.task_env_config.memory_mb, - "minimum_disk_mb": self.task_env_config.storage_mb, - } - resources["verified"] = ( - resources["cpu_count"] == resources["expected_cpu_count"] - and resources["memory_mb"] == resources["expected_memory_mb"] - and resources["disk_size_mb"] >= resources["minimum_disk_mb"] - ) - directory = self.trial_paths.trial_dir / "artifacts" / "e2b-resources" - directory.mkdir(parents=True, exist_ok=True) - (directory / f"{self._sandbox.sandbox_id}.json").write_text( - json.dumps(resources, indent=2) + "\n" - ) - if not resources["verified"]: - raise RuntimeError(f"E2B capacity differs from the task requirements: {resources}") - - async def stop(self, delete: bool) -> None: - # Recover verifier logs before teardown, including when verify() fails. - # Retain a separate recovery snapshot on timeout/transport failure too, - # without turning an incomplete reward file into a completed grade. - try: - if self._sandbox is not None and not self.task_env_config.docker_image: - directory = self.trial_paths.trial_dir / "artifacts" / "verifier-recovery" / self._sandbox.sandbox_id - directory.mkdir(parents=True, exist_ok=True) - record = {"complete": False} - try: - async with asyncio.timeout(60): - await self.download_dir("/logs/verifier", directory / "logs") - record["complete"] = True - except Exception as exc: - record["error"] = type(exc).__name__ - self.logger.warning("Verifier recovery download failed: %s", type(exc).__name__) - finally: - (directory / "recovery.json").write_text(json.dumps(record, indent=2) + "\n") - finally: - await super().stop(delete) diff --git a/evals/deepswe/full.yaml b/evals/deepswe/full.yaml deleted file mode 100644 index 4d73b5fc0..000000000 --- a/evals/deepswe/full.yaml +++ /dev/null @@ -1,25 +0,0 @@ -n_attempts: 1 -n_concurrent_trials: 8 -retry: - max_retries: 0 -environment: - import_path: deepswe_e2b:ResourceCheckedE2B - force_build: true - delete: true -agents: - - import_path: diffex_terminal_bench.adapter:DiffexAgent - model_name: openai/gpt-6-luna - extra_allowed_hosts: - - api.openai.com - kwargs: - thinking: max - tools: read,bash,edit,write,update_plan - subagents: disabled - project_resources: context-only - background_process_lifetime: environment - task_environment: preserve - env: - OPENAI_API_KEY: ${OPENAI_API_KEY} -datasets: - - name: datacurve/deep-swe-1-1 - ref: sha256:5affcd534fd90ac85d202d4c63f8b35ddc942140afdd5d60014a21365440a2f5 diff --git a/evals/deepswe/prepare-full.py b/evals/deepswe/prepare-full.py deleted file mode 100644 index 290e70de4..000000000 --- a/evals/deepswe/prepare-full.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Prepare a reproducible complete 113-task evaluation without starting paid execution.""" - -import argparse -import asyncio -from importlib.metadata import version -import json -from pathlib import Path - -import yaml -from harbor.models.job.config import JobConfig -from harbor.models.task.config import TaskConfig -from harbor.models.task.verifier_mode import resolve_effective_verifier_env_config -from harbor.registry.client.package import PackageDatasetClient -from harbor.tasks.client import TaskClient -from harbor.trial.network_policy import resolve_trial_network_plan - -from diffex_terminal_bench.config import PINS -from diffex_terminal_bench.execution import ExecutionOptions -from deepswe_e2b import prepare_image_runtimes - - -async def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - source = Path(__file__).resolve().parent - output = args.output.resolve() - output.mkdir(parents=True, exist_ok=True) - config = JobConfig.model_validate(yaml.safe_load((source / "full.yaml").read_text())) - if (PINS["diffex_version"], version("harbor"), version("e2b")) != ("0.3.2", "0.24.0", "2.59.0"): - raise RuntimeError("Expected Diffex 0.3.2, Harbor 0.24.0, E2B 2.59.0.") - if (config.n_attempts, config.n_concurrent_trials, config.retry.max_retries) != (1, 8, 0): - raise RuntimeError("Expected one attempt, eight concurrent trials, and zero retries.") - if len(config.agents) != 1 or len(config.datasets) != 1 or config.tasks: - raise RuntimeError("Expected one agent and one dataset only.") - agent = config.agents[0] - policy = ExecutionOptions.model_validate(agent.kwargs) - if policy.task_environment != "preserve": - raise RuntimeError("DeepSWE requires the task image's runtime environment.") - if (agent.model_name, policy.thinking, policy.subagents) != ("openai/gpt-6-luna", "max", "disabled"): - raise RuntimeError("Unexpected model or execution policy.") - if config.environment.import_path != "deepswe_e2b:ResourceCheckedE2B": - raise RuntimeError("Expected the resource-checked E2B backend.") - if agent.extra_allowed_hosts != ["api.openai.com"] or config.environment.extra_allowed_hosts: - raise RuntimeError("Only the model endpoint may be allowlisted.") - dataset = config.datasets[0] - if dataset.task_names or dataset.exclude_task_names or dataset.n_tasks: - raise RuntimeError("The template must include the entire dataset.") - metadata = await PackageDatasetClient().get_dataset_metadata(f"{dataset.name}@{dataset.ref}") - if metadata.version != dataset.ref or len(metadata.task_ids) != 113: - raise RuntimeError("Expected the pinned 113-task dataset.") - selected = sorted(metadata.task_ids, key=lambda task: task.get_name()) - dataset.task_names = [task.get_name() for task in selected] - dataset.download_dir = output / "task-cache" - downloads = await TaskClient().download_tasks(selected, output_dir=dataset.download_dir) - records = [] - images = [] - for task_id, path in zip(selected, downloads.paths, strict=True): - task = TaskConfig.model_validate_toml((path / "task.toml").read_text()) - if not task.environment.docker_image: - raise RuntimeError("DeepSWE requires a published task image.") - images.append(task.environment.docker_image) - if task.verifier.environment_mode.value != "separate" or not task.verifier.collect: - raise RuntimeError(f"{task_id.get_name()} requires separate commit-based verification.") - verifier_env = resolve_effective_verifier_env_config(task, None) - plan = resolve_trial_network_plan( - task, agent, config.environment, None, - verifier_mode=task.verifier.environment_mode, env_config=verifier_env, - ) - if plan.agent_phase.network_mode.value != "allowlist" or plan.agent_phase.allowed_hosts != ["api.openai.com"]: - raise RuntimeError("Unexpected agent network policy.") - if plan.verifier_phase.network_mode.value != "no-network": - raise RuntimeError("The verifier must remain offline.") - records.append({ - "name": task_id.get_name(), "ref": task_id.ref, - "agent_timeout_sec": task.agent.timeout_sec, - "verifier_timeout_sec": task.verifier.timeout_sec, - "cpus": task.environment.cpus, "memory_mb": task.environment.memory_mb, - "minimum_storage_mb": task.environment.storage_mb, - "agent_network": plan.agent_phase.model_dump(mode="json"), - "verifier_network": plan.verifier_phase.model_dump(mode="json"), - }) - resolved_tasks = await dataset.get_task_configs() - if {task.name for task in resolved_tasks} != set(dataset.task_names) or len(resolved_tasks) != 113: - raise RuntimeError("Harbor did not resolve exactly the complete 113 tasks.") - catalog = output / "runtime-images.json" - config.environment.kwargs.update(runtime_catalog=str(catalog), - runtime_catalog_sha256=await prepare_image_runtimes(images, catalog)) - config_path = output / "job.yaml" - # Keep preparation idempotent despite Harbor's timestamp-based job-name default. - config.job_name = "full-113x1" - serialized = yaml.safe_dump(config.model_dump(mode="json", exclude_none=True), sort_keys=False) - if config_path.exists(): - previous = JobConfig.model_validate(yaml.safe_load(config_path.read_text())) - if previous != config: - raise RuntimeError("Prepared configuration changed; use a fresh preparation directory.") - else: - config_path.write_text(serialized) - manifest = { - "dataset": metadata.name, "dataset_ref": metadata.version, - "dataset_task_count": 113, "tasks": records, "n_attempts": 1, - "n_trials": 113, "n_concurrent_trials": 8, "diffex_version": PINS["diffex_version"], - "model": agent.model_name, "thinking": policy.thinking, - "subagents": policy.subagents, "evolution_enabled": False, - "task_environment": policy.task_environment, - "live_validation": "not_started", - } - (output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") - print("Prepared 113 tasks x 1 attempt = 113 trials; concurrency 8.") - for record in records: - print(f" {record['name']}: {record['cpus']} CPU, {record['memory_mb']} MiB RAM, {record['minimum_storage_mb']} MiB disk") - print(f"Configuration: {config_path}\nNo sandboxes or model calls have started.") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/evals/deepswe/prepare-sample.py b/evals/deepswe/prepare-sample.py deleted file mode 100644 index 641edeb69..000000000 --- a/evals/deepswe/prepare-sample.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Prepare a reproducible five-task sample without starting paid execution.""" - -import argparse -import asyncio -from importlib.metadata import version -import json -from pathlib import Path -import random -import secrets - -import yaml -from harbor.models.job.config import JobConfig -from harbor.models.task.config import TaskConfig -from harbor.models.task.verifier_mode import resolve_effective_verifier_env_config -from harbor.registry.client.package import PackageDatasetClient -from harbor.tasks.client import TaskClient -from harbor.trial.network_policy import resolve_trial_network_plan - -from diffex_terminal_bench.config import PINS -from diffex_terminal_bench.execution import ExecutionOptions -from deepswe_e2b import prepare_image_runtimes - - -async def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--seed", type=int) - args = parser.parse_args() - source = Path(__file__).resolve().parent - output = args.output.resolve() - output.mkdir(parents=True, exist_ok=True) - seed_path = output / "seed.json" - if seed_path.exists(): - seed = json.loads(seed_path.read_text())["seed"] - if args.seed is not None and args.seed != seed: - raise RuntimeError("Use a new output directory for a different seed.") - else: - seed = args.seed if args.seed is not None else secrets.randbits(64) - with seed_path.open("x") as stream: - json.dump({"seed": seed}, stream) - stream.write("\n") - config = JobConfig.model_validate(yaml.safe_load((source / "sample.yaml").read_text())) - if (PINS["diffex_version"], version("harbor"), version("e2b")) != ("0.3.2", "0.24.0", "2.59.0"): - raise RuntimeError("Expected Diffex 0.3.2, Harbor 0.24.0, E2B 2.59.0.") - if (config.n_attempts, config.n_concurrent_trials, config.retry.max_retries) != (2, 5, 0): - raise RuntimeError("Expected two attempts, five concurrent trials, and zero retries.") - if len(config.agents) != 1 or len(config.datasets) != 1 or config.tasks: - raise RuntimeError("Expected one agent and one dataset only.") - agent = config.agents[0] - policy = ExecutionOptions.model_validate(agent.kwargs) - if policy.task_environment != "preserve": - raise RuntimeError("DeepSWE requires the task image's runtime environment.") - if (agent.model_name, policy.thinking, policy.subagents) != ("openai/gpt-6-luna", "max", "disabled"): - raise RuntimeError("Unexpected model or execution policy.") - if config.environment.import_path != "deepswe_e2b:ResourceCheckedE2B": - raise RuntimeError("Expected the resource-checked E2B backend.") - if agent.extra_allowed_hosts != ["api.openai.com"] or config.environment.extra_allowed_hosts: - raise RuntimeError("Only the model endpoint may be allowlisted.") - dataset = config.datasets[0] - if dataset.task_names or dataset.exclude_task_names or dataset.n_tasks: - raise RuntimeError("The template must sample the entire dataset.") - metadata = await PackageDatasetClient().get_dataset_metadata(f"{dataset.name}@{dataset.ref}") - if metadata.version != dataset.ref or len(metadata.task_ids) != 113: - raise RuntimeError("Expected the pinned 113-task dataset.") - selected = random.Random(seed).sample(sorted(metadata.task_ids, key=lambda task: task.get_name()), 5) - dataset.task_names = [task.get_name() for task in selected] - dataset.download_dir = output / "task-cache" - downloads = await TaskClient().download_tasks(selected, output_dir=dataset.download_dir) - records = [] - images = [] - for task_id, path in zip(selected, downloads.paths, strict=True): - task = TaskConfig.model_validate_toml((path / "task.toml").read_text()) - if not task.environment.docker_image: - raise RuntimeError("DeepSWE requires a published task image.") - images.append(task.environment.docker_image) - if task.verifier.environment_mode.value != "separate" or not task.verifier.collect: - raise RuntimeError(f"{task_id.get_name()} requires separate commit-based verification.") - verifier_env = resolve_effective_verifier_env_config(task, None) - plan = resolve_trial_network_plan( - task, agent, config.environment, None, - verifier_mode=task.verifier.environment_mode, env_config=verifier_env, - ) - if plan.agent_phase.network_mode.value != "allowlist" or plan.agent_phase.allowed_hosts != ["api.openai.com"]: - raise RuntimeError("Unexpected agent network policy.") - if plan.verifier_phase.network_mode.value != "no-network": - raise RuntimeError("The verifier must remain offline.") - records.append({ - "name": task_id.get_name(), "ref": task_id.ref, - "agent_timeout_sec": task.agent.timeout_sec, - "verifier_timeout_sec": task.verifier.timeout_sec, - "cpus": task.environment.cpus, "memory_mb": task.environment.memory_mb, - "minimum_storage_mb": task.environment.storage_mb, - "agent_network": plan.agent_phase.model_dump(mode="json"), - "verifier_network": plan.verifier_phase.model_dump(mode="json"), - }) - resolved_tasks = await dataset.get_task_configs() - if {task.name for task in resolved_tasks} != set(dataset.task_names) or len(resolved_tasks) != 5: - raise RuntimeError("Harbor did not resolve exactly the selected five tasks.") - catalog = output / "runtime-images.json" - config.environment.kwargs.update(runtime_catalog=str(catalog), - runtime_catalog_sha256=await prepare_image_runtimes(images, catalog)) - config_path = output / "job.yaml" - # Keep preparation idempotent despite Harbor's timestamp-based job-name default. - config.job_name = "sample-5x2" - serialized = yaml.safe_dump(config.model_dump(mode="json", exclude_none=True), sort_keys=False) - if config_path.exists(): - previous = JobConfig.model_validate(yaml.safe_load(config_path.read_text())) - if previous != config: - raise RuntimeError("Prepared configuration changed; use a fresh preparation directory.") - else: - config_path.write_text(serialized) - manifest = { - "seed": seed, "dataset": metadata.name, "dataset_ref": metadata.version, - "dataset_task_count": 113, "tasks": records, "n_attempts": 2, - "n_trials": 10, "n_concurrent_trials": 5, "diffex_version": PINS["diffex_version"], - "model": agent.model_name, "thinking": policy.thinking, - "subagents": policy.subagents, "evolution_enabled": False, - "task_environment": policy.task_environment, - "live_validation": "not_started", - } - (output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") - print(f"Prepared 5 tasks x 2 attempts = 10 trials; concurrency 5; seed {seed}.") - for record in records: - print(f" {record['name']}: {record['cpus']} CPU, {record['memory_mb']} MiB RAM, {record['minimum_storage_mb']} MiB disk") - print(f"Configuration: {config_path}\nNo sandboxes or model calls have started.") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/evals/deepswe/prepare.py b/evals/deepswe/prepare.py deleted file mode 100644 index b52ac4692..000000000 --- a/evals/deepswe/prepare.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Resolve pinned inputs and validate the smoke configuration without running it.""" - -import argparse -import asyncio -from importlib.metadata import version -import json -from pathlib import Path - -import yaml -from harbor.models.job.config import JobConfig -from harbor.models.task.config import TaskConfig -from harbor.models.task.verifier_mode import resolve_effective_verifier_env_config -from harbor.registry.client.package import PackageDatasetClient -from harbor.tasks.client import TaskClient -from harbor.trial.network_policy import resolve_trial_network_plan - -from diffex_terminal_bench.config import PINS -from diffex_terminal_bench.execution import ExecutionOptions -from deepswe_e2b import prepare_image_runtimes - - -async def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - source = Path(__file__).resolve().parent - config = JobConfig.model_validate(yaml.safe_load((source / "smoke.yaml").read_text())) - if (PINS["diffex_version"], version("harbor"), version("e2b")) != ("0.3.2", "0.24.0", "2.59.0"): - raise RuntimeError("Expected Diffex 0.3.2, Harbor 0.24.0, and E2B 2.59.0.") - if (config.n_attempts, config.n_concurrent_trials, config.retry.max_retries) != (1, 1, 0): - raise RuntimeError("The smoke trial requires one attempt, concurrency one, and zero retries.") - agent = config.agents[0] - policy = ExecutionOptions.model_validate(agent.kwargs) - if policy.task_environment != "preserve": - raise RuntimeError("DeepSWE requires the task image's runtime environment.") - if (agent.model_name, policy.thinking, policy.subagents) != ("openai/gpt-6-luna", "max", "disabled"): - raise RuntimeError("Unexpected model or execution policy.") - if agent.extra_allowed_hosts != ["api.openai.com"] or config.environment.extra_allowed_hosts: - raise RuntimeError("Only the model endpoint may be allowlisted during agent execution.") - if any(value is not None for value in ( - config.environment.override_cpus, config.environment.override_memory_mb, - config.environment.override_storage_mb, agent.override_timeout_sec, - config.verifier.override_timeout_sec, - )): - raise RuntimeError("Do not override task resources or execution deadlines.") - dataset = config.datasets[0] - metadata = await PackageDatasetClient().get_dataset_metadata(f"{dataset.name}@{dataset.ref}") - if metadata.version != dataset.ref or len(metadata.task_ids) != 113: - raise RuntimeError("The dataset must resolve to the pinned 113-task release.") - tasks = await dataset.get_task_configs() - if len(tasks) != 1 or tasks[0].name != "datacurve/abs-stepped-slices": - raise RuntimeError("The smoke selection must contain only abs-stepped-slices.") - selected = [task for task in metadata.task_ids if task.name == "abs-stepped-slices"] - downloads = await TaskClient().download_tasks(selected, output_dir=args.output / "task-cache") - task = TaskConfig.model_validate_toml((downloads.paths[0] / "task.toml").read_text()) - if (task.environment.cpus, task.environment.memory_mb, task.environment.storage_mb) != (2, 8192, 20480): - raise RuntimeError("Unexpected smoke task resource limits.") - if str(task.verifier.environment_mode.value) != "separate" or not task.verifier.collect: - raise RuntimeError("DeepSWE requires separate verification and commit collection.") - verifier_env = resolve_effective_verifier_env_config(task, None) - plan = resolve_trial_network_plan( - task, agent, config.environment, None, - verifier_mode=task.verifier.environment_mode, env_config=verifier_env, - ) - if plan.agent_phase.network_mode.value != "allowlist" or plan.agent_phase.allowed_hosts != ["api.openai.com"]: - raise RuntimeError("Agent execution network isolation was not resolved correctly.") - if plan.verifier_phase.network_mode.value != "no-network": - raise RuntimeError("The verifier must remain offline.") - args.output.mkdir(parents=True, exist_ok=True) - if not task.environment.docker_image: - raise RuntimeError("DeepSWE requires a published task image.") - catalog = args.output.resolve() / "runtime-images.json" - config.environment.kwargs.update(runtime_catalog=str(catalog), runtime_catalog_sha256= - await prepare_image_runtimes([task.environment.docker_image], catalog)) - config.job_name = "smoke" - (args.output / "job.yaml").write_text(yaml.safe_dump(config.model_dump(mode="json", exclude_none=True), sort_keys=False)) - manifest = { - "dataset": metadata.name, "dataset_ref": metadata.version, "dataset_task_count": 113, - "selected_task": tasks[0].model_dump(mode="json", exclude_none=True), - "diffex_version": PINS["diffex_version"], "model": agent.model_name, "thinking": policy.thinking, - "subagents": policy.subagents, "evolution_enabled": False, - "task_environment": policy.task_environment, - "agent_network": plan.agent_phase.model_dump(mode="json"), - "verifier_network": plan.verifier_phase.model_dump(mode="json"), - "agent_timeout_sec": task.agent.timeout_sec, "verifier_timeout_sec": task.verifier.timeout_sec, - "resource_requirements": {"cpus": 2, "memory_mb": 8192, "minimum_storage_mb": 20480}, - "live_validation": "not_started", - } - (args.output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") - print("Prepared abs-stepped-slices: Luna max, no evolution/subagents, separate offline verifier.") - print("Required per sandbox: 2 vCPU, 8 GiB RAM, at least 20 GiB disk; no trial has started.") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/evals/deepswe/run-full.zsh b/evals/deepswe/run-full.zsh deleted file mode 100644 index 2db71495a..000000000 --- a/evals/deepswe/run-full.zsh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/zsh -set -eu -umask 077 -DS_SETUP=${0:A:h} -DS_REPO=${DS_SETUP:h:h} -DS_PREPARED=${DS_PREPARED:-$DS_REPO/.artifacts/deepswe/prepared-113x1-c8-runtime-v3} -cd "$DS_REPO/evals/terminal-bench" -export PYTHONPATH="$DS_SETUP${PYTHONPATH:+:$PYTHONPATH}" -DS_UV=(uv --cache-dir /tmp/diffex-e2b-uv-cache run --locked) -[[ $# -le 1 ]] || { print -u2 'Too many arguments.'; exit 1; } -case "${1:-}" in - --prepare) - "${DS_UV[@]}" python "$DS_SETUP/prepare-full.py" --output "$DS_PREPARED" - exit ;; - --print-config) - "${DS_UV[@]}" harbor run --config "$DS_PREPARED/job.yaml" --print-config - exit ;; - '') ;; - *) print -u2 'Usage: run-full.zsh [--prepare|--print-config]'; exit 1 ;; -esac -"${DS_UV[@]}" python "$DS_SETUP/prepare-full.py" --output "$DS_PREPARED" -if [[ -z "${E2B_API_KEY:-}" ]]; then - read -rs 'E2B_API_KEY?E2B API key: ' - printf '\n' -fi -if [[ -z "${OPENAI_API_KEY:-}" ]]; then - read -rs 'OPENAI_API_KEY?OpenAI API key: ' - printf '\n' -fi -[[ -n "$E2B_API_KEY" && -n "$OPENAI_API_KEY" ]] || { print -u2 'Both API keys are required.'; exit 1; } -export E2B_API_KEY OPENAI_API_KEY -DS_OUTPUT="results/deepswe-1-1-luna-max-113x1-$(date +%Y%m%d-%H%M%S)-$(uuidgen)" -mkdir -p results -mkdir "$DS_OUTPUT" -cp "$DS_PREPARED"/{job.yaml,manifest.json,runtime-images.json} "$DS_OUTPUT/" -cp "$DS_SETUP"/{full.yaml,prepare-full.py,run-full.zsh,deepswe_e2b.py} "$DS_OUTPUT/" -cp -R src/diffex_terminal_bench "$DS_OUTPUT/adapter-source" -git rev-parse HEAD > "$DS_OUTPUT/adapter-commit.txt" -git -C "$DS_REPO" diff --binary -- evals/terminal-bench > "$DS_OUTPUT/adapter-working-tree.diff" -"${DS_UV[@]}" harbor run --config "$DS_OUTPUT/job.yaml" --jobs-dir "$DS_OUTPUT" --job-name full-113x1 --print-config > "$DS_OUTPUT/planned-config.json" -print "Starting 113 DeepSWE trials, concurrency 8. Results: $PWD/$DS_OUTPUT/full-113x1" -"${DS_UV[@]}" harbor run --config "$DS_OUTPUT/job.yaml" --jobs-dir "$DS_OUTPUT" --job-name full-113x1 diff --git a/evals/deepswe/run-sample.zsh b/evals/deepswe/run-sample.zsh deleted file mode 100644 index 4e941ec0a..000000000 --- a/evals/deepswe/run-sample.zsh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/zsh -set -eu -umask 077 -DS_SETUP=${0:A:h} -DS_REPO=${DS_SETUP:h:h} -DS_PREPARED=${DS_PREPARED:-$DS_REPO/.artifacts/deepswe/prepared-5x2-runtime-v3} -cd "$DS_REPO/evals/terminal-bench" -export PYTHONPATH="$DS_SETUP${PYTHONPATH:+:$PYTHONPATH}" -DS_UV=(uv --cache-dir /tmp/diffex-e2b-uv-cache run --locked) -[[ $# -le 1 ]] || { print -u2 'Too many arguments.'; exit 1; } -case "${1:-}" in - --prepare) - "${DS_UV[@]}" python "$DS_SETUP/prepare-sample.py" --output "$DS_PREPARED" - exit ;; - --print-config) - "${DS_UV[@]}" harbor run --config "$DS_PREPARED/job.yaml" --print-config - exit ;; - '') ;; - *) print -u2 'Usage: run-sample.zsh [--prepare|--print-config]'; exit 1 ;; -esac -"${DS_UV[@]}" python "$DS_SETUP/prepare-sample.py" --output "$DS_PREPARED" -if [[ -z "${E2B_API_KEY:-}" ]]; then - read -rs 'E2B_API_KEY?E2B API key: ' - printf '\n' -fi -if [[ -z "${OPENAI_API_KEY:-}" ]]; then - read -rs 'OPENAI_API_KEY?OpenAI API key: ' - printf '\n' -fi -[[ -n "$E2B_API_KEY" && -n "$OPENAI_API_KEY" ]] || { print -u2 'Both API keys are required.'; exit 1; } -export E2B_API_KEY OPENAI_API_KEY -DS_OUTPUT="results/deepswe-1-1-luna-max-5x2-$(date +%Y%m%d-%H%M%S)-$(uuidgen)" -mkdir -p results -mkdir "$DS_OUTPUT" -cp "$DS_PREPARED"/{job.yaml,manifest.json,seed.json,runtime-images.json} "$DS_OUTPUT/" -cp "$DS_SETUP"/{sample.yaml,prepare-sample.py,run-sample.zsh,deepswe_e2b.py} "$DS_OUTPUT/" -cp -R src/diffex_terminal_bench "$DS_OUTPUT/adapter-source" -git rev-parse HEAD > "$DS_OUTPUT/adapter-commit.txt" -git -C "$DS_REPO" diff --binary -- evals/terminal-bench > "$DS_OUTPUT/adapter-working-tree.diff" -"${DS_UV[@]}" harbor run --config "$DS_OUTPUT/job.yaml" --jobs-dir "$DS_OUTPUT" --job-name sample-5x2 --print-config > "$DS_OUTPUT/planned-config.json" -print "Starting 10 DeepSWE trials, concurrency 5. Results: $PWD/$DS_OUTPUT/sample-5x2" -"${DS_UV[@]}" harbor run --config "$DS_OUTPUT/job.yaml" --jobs-dir "$DS_OUTPUT" --job-name sample-5x2 diff --git a/evals/deepswe/run-smoke.zsh b/evals/deepswe/run-smoke.zsh deleted file mode 100644 index 02d644e64..000000000 --- a/evals/deepswe/run-smoke.zsh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/zsh -set -eu -umask 077 -DS_SETUP=${0:A:h} -DS_REPO=${DS_SETUP:h:h} -cd "$DS_REPO/evals/terminal-bench" -export PYTHONPATH="$DS_SETUP${PYTHONPATH:+:$PYTHONPATH}" -DS_UV=(uv --cache-dir /tmp/diffex-e2b-uv-cache run --locked) -[[ $# -le 1 ]] || { print -u2 'Too many arguments.'; exit 1; } -case "${1:-}" in - --print-config) - "${DS_UV[@]}" harbor run --config "$DS_REPO/.artifacts/deepswe/prepared-runtime-v3/job.yaml" --print-config - exit ;; - --prepare) - "${DS_UV[@]}" python "$DS_SETUP/prepare.py" --output "$DS_REPO/.artifacts/deepswe/prepared-runtime-v3" - exit ;; - '') ;; - *) print -u2 'Usage: run-smoke.zsh [--prepare|--print-config]'; exit 1 ;; -esac -[[ -n "${E2B_API_KEY:-}" ]] || { print -u2 'Export E2B_API_KEY in this terminal first.'; exit 1; } -[[ -n "${OPENAI_API_KEY:-}" ]] || { print -u2 'Export OPENAI_API_KEY in this terminal first.'; exit 1; } -DS_OUTPUT="results/deepswe-1-1-luna-max-smoke-$(date +%Y%m%d-%H%M%S)-$(uuidgen)" -mkdir -p results -mkdir "$DS_OUTPUT" -"${DS_UV[@]}" python "$DS_SETUP/prepare.py" --output "$DS_OUTPUT/inputs" -cp "$DS_OUTPUT/inputs/job.yaml" "$DS_OUTPUT/job.yaml" -cp "$DS_SETUP"/{deepswe_e2b.py,prepare.py,verify-smoke.py,run-smoke.zsh} "$DS_OUTPUT/" -cp src/diffex_terminal_bench/{pins.toml,run.cjs} "$DS_OUTPUT/" -cp -R src/diffex_terminal_bench "$DS_OUTPUT/adapter-source" -git rev-parse HEAD > "$DS_OUTPUT/adapter-commit.txt" -git -C "$DS_REPO" diff --binary -- evals/terminal-bench > "$DS_OUTPUT/adapter-working-tree.diff" -"${DS_UV[@]}" harbor run --config "$DS_OUTPUT/job.yaml" --jobs-dir "$DS_OUTPUT" --job-name smoke --print-config > "$DS_OUTPUT/planned-config.json" -print "Starting one DeepSWE smoke trial. Results: $PWD/$DS_OUTPUT/smoke" -print 'Task correctness may fail; smoke qualification checks execution, isolation, and retained evidence.' -"${DS_UV[@]}" harbor run --config "$DS_OUTPUT/job.yaml" --jobs-dir "$DS_OUTPUT" --job-name smoke -"${DS_UV[@]}" python "$DS_SETUP/verify-smoke.py" "$DS_OUTPUT/smoke" diff --git a/evals/deepswe/sample.yaml b/evals/deepswe/sample.yaml deleted file mode 100644 index b1c780777..000000000 --- a/evals/deepswe/sample.yaml +++ /dev/null @@ -1,25 +0,0 @@ -n_attempts: 2 -n_concurrent_trials: 5 -retry: - max_retries: 0 -environment: - import_path: deepswe_e2b:ResourceCheckedE2B - force_build: true - delete: true -agents: - - import_path: diffex_terminal_bench.adapter:DiffexAgent - model_name: openai/gpt-6-luna - extra_allowed_hosts: - - api.openai.com - kwargs: - thinking: max - tools: read,bash,edit,write,update_plan - subagents: disabled - project_resources: context-only - background_process_lifetime: environment - task_environment: preserve - env: - OPENAI_API_KEY: ${OPENAI_API_KEY} -datasets: - - name: datacurve/deep-swe-1-1 - ref: sha256:5affcd534fd90ac85d202d4c63f8b35ddc942140afdd5d60014a21365440a2f5 diff --git a/evals/deepswe/smoke.yaml b/evals/deepswe/smoke.yaml deleted file mode 100644 index 5ad7694bd..000000000 --- a/evals/deepswe/smoke.yaml +++ /dev/null @@ -1,27 +0,0 @@ -n_attempts: 1 -n_concurrent_trials: 1 -retry: - max_retries: 0 -environment: - import_path: deepswe_e2b:ResourceCheckedE2B - force_build: true - delete: true -agents: - - import_path: diffex_terminal_bench.adapter:DiffexAgent - model_name: openai/gpt-6-luna - extra_allowed_hosts: - - api.openai.com - kwargs: - thinking: max - tools: read,bash,edit,write,update_plan - subagents: disabled - project_resources: context-only - background_process_lifetime: environment - task_environment: preserve - env: - OPENAI_API_KEY: ${OPENAI_API_KEY} -datasets: - - name: datacurve/deep-swe-1-1 - ref: sha256:5affcd534fd90ac85d202d4c63f8b35ddc942140afdd5d60014a21365440a2f5 - task_names: - - datacurve/abs-stepped-slices diff --git a/evals/deepswe/steps-itemized.md b/evals/deepswe/steps-itemized.md deleted file mode 100644 index 2fad86012..000000000 --- a/evals/deepswe/steps-itemized.md +++ /dev/null @@ -1,176 +0,0 @@ -# DeepSWE 1.1: smoke trial, inspect evidence, and upload privately - -- Run the commands below in the same zsh terminal. -- Continue only after the preceding command succeeds. -- This workflow runs one fresh attempt at `datacurve/abs-stepped-slices`, with concurrency 1 and zero automatic retries. -- Defaults are Diffex 0.3.2, `openai/gpt-6-luna`, `max` thinking, disabled evolution, disabled subagents, and E2B sandboxes. -- The original task declares 2 vCPU, 8192 MiB RAM, 20480 MiB storage, a 5400-second agent deadline, and an 1800-second verifier deadline. -- No CPU, memory, or task deadline overrides are added. -- A zero verifier reward is allowed for smoke qualification if execution and retained evidence are complete. -- Preparation checks passed locally; live E2B provisioning, network isolation, and separate verification must still be exercised by this smoke trial. - -## Pinned inputs and isolation - -The [Harbor dataset](https://hub.harborframework.com/datasets/datacurve/deep-swe-1-1) contains 113 tasks. -This workflow pins the dataset to `sha256:5affcd534fd90ac85d202d4c63f8b35ddc942140afdd5d60014a21365440a2f5`. -The smoke task pin is `sha256:602b5b3144f5386bc54107a58e9bb9eef276e9cdae98610b482f95eb4e960310`. -Harbor 0.24.0 and E2B 2.59.0 are pinned in the existing Terminal-Bench uv project and lockfile. -All DeepSWE launchers use that shared environment without `--with` dependency overlays. -The existing `diffex_terminal_bench.adapter:DiffexAgent` installs the published Diffex release pinned in `evals/terminal-bench/src/diffex_terminal_bench/pins.toml`. - -[DeepSWE 1.1](https://github.com/datacurve-ai/deep-swe) grades committed changes in a separate pristine verifier sandbox. -The agent's original instruction asks it to create a new branch and commit its completed work. -Do not remove that instruction or replace the commit collector with an uncommitted working-tree diff. -Harbor executes the original `[[verifier.collect]]` command and transfers `model.patch` to the verifier. -The agent does not receive the held-out verifier files or reference solution. - -The original task forbids network access during agent execution and verification. -This configuration adds a disclosed, agent-phase-only exception for `api.openai.com`, because Diffex calls the Responses API from inside the sandbox. -All other destinations remain blocked during agent execution, and the verifier remains offline. -Environment startup and agent installation use the task's default public baseline before the restricted execution phase, allowing Node and npm installation. -Use this as an internal custom-harness evaluation; it does not establish acceptance under DeepSWE's leaderboard policy. - -The adapter uses the checked-in evaluation system prompt and `--no-context-files`, and appends the existing anti-cheating sentence to the user message. -It writes fresh settings with `evolution.enabled=false` and exposes no subagent tools. -The model remains `gpt-6-luna`; `max` is its reasoning-effort setting, not a separate model ID. - -`ResourceCheckedE2B` checks actual capacity after each agent/verifier sandbox starts. -It requires exactly the declared CPU and memory allocation and at least 20480 MiB of reported disk capacity. -It records both checks under each trial's `artifacts/e2b-resources/` directory and rejects an undersized allocation. -Harbor's E2B backend cannot request an exact disk capacity, so this is a minimum-capacity check, not an exact 20 GiB allocation or a free-space check. -The agent and verifier are distinct sandboxes, so allow quota for both within a single trial. - -This E2B workflow does not mount Daytona volumes or activate the Daytona volume-cleanup plugin. -It retains local logs before deleting sandboxes through Harbor's normal lifecycle. -The existing adapter's 30-second bulk log-download limit and connection recovery behavior are unchanged. -Previously observed connection, artifact collection, and shutdown-evidence errors can recur. - -## Run one smoke trial - -1. Open zsh and select the repository: - - ```zsh - cd /Users/jialeyu/Projects/diffex-new - ``` - -2. Load both API keys without echoing them: - - ```zsh - if [[ -z "${E2B_API_KEY:-}" ]]; then - read -rs 'E2B_API_KEY?E2B API key: ' - printf '\n' - fi - export E2B_API_KEY - - if [[ -z "${OPENAI_API_KEY:-}" ]]; then - read -rs 'OPENAI_API_KEY?OpenAI API key: ' - printf '\n' - fi - export OPENAI_API_KEY - - [[ -n "$E2B_API_KEY" && -n "$OPENAI_API_KEY" ]] && printf 'Both keys are set.\n' - ``` - - Key presence is checked locally; successful provisioning and model execution verify actual access. - Never paste keys into YAML or saved commands. - -3. Inspect the configuration without starting a trial: - - ```zsh - zsh evals/deepswe/run-smoke.zsh --print-config - ``` - - Confirm one attempt, concurrency 1, `openai/gpt-6-luna`, `thinking: max`, disabled subagents, and only `api.openai.com` in the agent allowlist. - Confirm the environment import is `deepswe_e2b:ResourceCheckedE2B` and the dataset contains the exact pinned smoke selection. - -4. Optionally repeat preparation without provisioning a sandbox or calling the model: - - ```zsh - zsh evals/deepswe/run-smoke.zsh --prepare - ``` - - Preparation resolves the pinned dataset, downloads the smoke task definition, checks separate verification, and resolves the agent/verifier network policies. - Harbor may warn that the original task's explicit `model.patch` artifact overlaps its standard `/logs/artifacts` collection directory. - The task is preserved unchanged; the post-run checker requires the patch to be retained. - It writes `.artifacts/deepswe/prepared-runtime-v3/manifest.json`. - The launcher repeats these checks automatically before starting the trial. - -5. Start the smoke trial: - - ```zsh - zsh /Users/jialeyu/Projects/diffex-new/evals/deepswe/run-smoke.zsh - ``` - - Keep the terminal running until Harbor and the post-run evidence checker finish. - The launcher creates a unique output directory beneath `evals/terminal-bench/results/deepswe-1-1-luna-max-smoke-*`. - The job is the `smoke/` subdirectory inside that output directory. - It saves the planned configuration, task manifest, setup scripts, adapter commit, adapter working-tree diff, release pins, and native agent launcher. - The maximum agent work period is 90 minutes, with additional provisioning, installation, collection, and verification time. - -## Inspect and qualify the smoke - -1. Use the exact job path printed by the launcher: - - ```zsh - DS_JOB='/Users/jialeyu/Projects/diffex-new/evals/terminal-bench/results/ACTUAL-OUTPUT/smoke' - open "$DS_JOB" - python3 -m json.tool "$DS_JOB/result.json" - ``` - -2. Inspect `smoke-check.json` for `status: passed`. - The launcher checks one completed trial, no execution exception, confirmed shutdown, complete token accounting, a valid ATIF trajectory, model and reasoning settings, disabled evolution/subagents, two distinct resource-checked sandboxes, separate verification, and retained `model.patch`. - A zero reward means the task solution failed verification, even if smoke qualification passed. - The checker does not independently certify firewall behavior or leaderboard compliance. - -3. To repeat the evidence check manually: - - ```zsh - cd /Users/jialeyu/Projects/diffex-new/evals/terminal-bench - uv --cache-dir /tmp/diffex-e2b-uv-cache run --locked \ - python ../deepswe/verify-smoke.py "$DS_JOB" - ``` - -4. Inspect the trial's `result.json`, `trial.log`, `agent/trajectory.json`, native `agent/run-*/sessions/`, `requested.json`, `effective.json`, and `cleanup.json`. - Inspect `verifier/test-stdout.txt`, `verifier/reward.json`, the collected `model.patch`, and both `artifacts/e2b-resources/*.json` records. - Keep the complete local directory, including failures. - -## Upload privately under Diffex - -1. From the existing uv project, log in if needed: - - ```zsh - cd /Users/jialeyu/Projects/diffex-new/evals/terminal-bench - uv --cache-dir /tmp/diffex-e2b-uv-cache run --locked \ - harbor auth login - ``` - -2. Upload the entire native smoke job: - - ```zsh - uv --cache-dir /tmp/diffex-e2b-uv-cache run --locked \ - harbor upload "$DS_JOB" --org diffex --private - ``` - - `--org diffex` assigns ownership to Diffex; `--share-org diffex` alone does not. - Do not add `--share-org diffex` when Diffex already owns the job. - Retry the same upload command and directory after a partial upload. - Save the printed link and confirm the owner, visibility, trial count, and available traces. - -## If the trial fails or is interrupted - -1. Preserve the output and read `job.log`, the trial's exception, and its retained artifacts. -2. If provisioning fails, inspect the E2B build error before changing the original task image. -3. If model calls fail, inspect the restricted `api.openai.com` allowlist and provider errors. -4. If the patch is empty or absent, check whether the agent committed its work as instructed. -5. If shutdown is unverified, keep verification blocked and inspect the E2B dashboard for sandboxes left by this job. -6. If trace collection fails, retain partial usage as incomplete rather than reporting it as a complete total. -7. Upload the original result for diagnosis, then use a fresh output directory for another smoke trial. - -## Before a full evaluation - -Qualify the smoke environment and evidence before scaling. -Prepare a separate full-run configuration that removes `task_names`, preserves the dataset digest and task deadlines, and records the chosen trial count and concurrency. -The pinned dataset has 113 tasks, so five attempts would schedule 565 trials. -Preserve the separate offline verifier and the disclosed agent-only model endpoint exception. -Account for agent and verifier sandbox resources when sizing E2B concurrency. -A full-run launcher is not included here; the final kickoff command above starts only the single smoke trial. diff --git a/evals/deepswe/test_deepswe_e2b.py b/evals/deepswe/test_deepswe_e2b.py deleted file mode 100644 index 38cee26a5..000000000 --- a/evals/deepswe/test_deepswe_e2b.py +++ /dev/null @@ -1,362 +0,0 @@ -"""Offline regressions for image metadata, E2B propagation, and timeout evidence.""" - -import asyncio -import hashlib -import json -import os -from pathlib import Path -import tempfile -from types import SimpleNamespace -import unittest -from unittest.mock import AsyncMock, patch - -import httpx -from e2b.sandbox_async.commands.command import Commands -from harbor.environments.base import ExecResult -from harbor.environments.e2b import E2BEnvironment -from harbor.environments.factory import EnvironmentFactory -from harbor.models.task.config import EnvironmentConfig -from harbor.models.trial.config import EnvironmentConfig as TrialEnvironmentConfig -from harbor.models.trial.paths import TrialPaths -from packaging.version import Version - -from deepswe_e2b import ResourceCheckedE2B, image_runtime, prepare_image_runtimes, registry_get - - -IMAGE = "public.ecr.aws/example/tasks:fixture" - - -class ImageTests(unittest.IsolatedAsyncioTestCase): - def setUp(self): - sleep = patch("deepswe_e2b.registry_sleep", new_callable=AsyncMock) - self.sleep = sleep.start() - self.addCleanup(sleep.stop) - - async def test_throttled_manifest_is_retried_without_restarting_trial(self): - config = json.dumps({"os": "linux", "architecture": "amd64", "config": {"Env": ["PATH=/bin"]}}).encode() - digest = "sha256:" + hashlib.sha256(config).hexdigest() - manifest_requests = 0 - - def respond(request): - nonlocal manifest_requests - if request.url.path == "/token/": - return httpx.Response(200, json={"token": "fixture"}) - if "/manifests/" in request.url.path: - manifest_requests += 1 - if manifest_requests == 1: - return httpx.Response(429, headers={"Retry-After": "2"}) - return httpx.Response(200, json={"config": {"digest": digest}}) - return httpx.Response(200, content=config) - - async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: - runtime = await image_runtime(client, IMAGE) - self.assertEqual(runtime["env"], {"PATH": "/bin"}) - self.assertEqual(manifest_requests, 2) - self.assertIn(2, [call.args[0] for call in self.sleep.await_args_list]) - - async def test_permanent_errors_and_exhausted_throttling_are_bounded(self): - for status, count in ((404, 1), (429, 6), (503, 6)): - calls = [] - - def respond(request): - calls.append(request) - return httpx.Response(status) - - async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: - with self.assertRaises(httpx.HTTPStatusError): - await registry_get(client, "https://public.ecr.aws/fixture") - self.assertEqual(len(calls), count) - - async def test_long_retry_after_fails_preparation_instead_of_retrying_early(self): - calls = [] - - def respond(request): - calls.append(request) - return httpx.Response(429, headers={"Retry-After": "120"}) - - async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: - with self.assertRaises(httpx.HTTPStatusError): - await registry_get(client, "https://public.ecr.aws/fixture") - self.assertEqual(len(calls), 1) - - async def test_transport_failure_retries_safe_read(self): - calls = [] - - def respond(request): - calls.append(request) - if len(calls) == 1: - raise httpx.ReadTimeout("fixture timeout", request=request) - return httpx.Response(200, json={"ok": True}) - - async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: - self.assertEqual((await registry_get(client, "https://public.ecr.aws/fixture")).json(), {"ok": True}) - self.assertEqual(len(calls), 2) - - async def test_registry_config_and_amd64_selection(self): - config = json.dumps({"os": "linux", "architecture": "amd64", "config": { - "Env": ["PYTHONPATH=/app/src", "DENO_DIR=/deno-cache", "VALUE=a=b"], "WorkingDir": "/app", - }}).encode() - config_digest = "sha256:" + hashlib.sha256(config).hexdigest() - manifest = json.dumps({"config": {"digest": config_digest}}).encode() - manifest_digest = "sha256:" + hashlib.sha256(manifest).hexdigest() - - def respond(request): - if request.url.path == "/token/": - return httpx.Response(200, json={"token": "fixture-token"}) - self.assertEqual(request.headers["Authorization"], "Bearer fixture-token") - if request.url.path.endswith("/manifests/fixture"): - return httpx.Response(200, json={"manifests": [ - {"platform": {"os": "linux", "architecture": "arm64"}, "digest": "wrong"}, - {"platform": {"os": "linux", "architecture": "amd64"}, "digest": manifest_digest}, - ]}) - if request.url.path.endswith("/manifests/" + manifest_digest): - return httpx.Response(200, content=manifest) - self.assertTrue(request.url.path.endswith("/blobs/" + config_digest)) - return httpx.Response(200, content=config) - - async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: - runtime = await image_runtime(client, IMAGE) - self.assertEqual(runtime["env"], {"PYTHONPATH": "/app/src", "DENO_DIR": "/deno-cache", "VALUE": "a=b"}) - self.assertEqual(runtime["workdir"], "/app") - self.assertEqual(runtime["manifest_digest"], manifest_digest) - - async def test_corrupt_config_is_rejected(self): - def respond(request): - if request.url.path == "/token/": - return httpx.Response(200, json={"token": "fixture"}) - if "/manifests/" in request.url.path: - return httpx.Response(200, json={"config": {"digest": "sha256:" + "0" * 64}}) - return httpx.Response(200, json={"config": {}}) - - async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: - with self.assertRaisesRegex(ValueError, "configuration digest mismatch"): - await image_runtime(client, IMAGE) - - async def test_unversioned_and_non_ecr_images_are_rejected(self): - async with httpx.AsyncClient() as client: - for image in ("public.ecr.aws/example/task", "registry.example/task:tag"): - with self.assertRaises(ValueError): - await image_runtime(client, image) - - -class EnvironmentTests(unittest.IsolatedAsyncioTestCase): - def setUp(self): - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - self.root = Path(temporary.name) - self.tests = self.root / "tests" - self.tests.mkdir() - (self.tests / "Dockerfile").write_text(f"FROM {IMAGE}\nCOPY test.sh /tests/test.sh\nRUN chmod +x /tests/test.sh\n") - self.paths = TrialPaths(self.root / "trial") - - def environment(self, verifier=False): - return ResourceCheckedE2B( - environment_dir=self.tests, environment_name="fixture", session_id="fixture", - trial_paths=self.paths, task_env_config=EnvironmentConfig( - docker_image=None if verifier else IMAGE, cpus=2, memory_mb=8192, - storage_mb=20480, env={"OVERRIDE": "task"}, - ), - ) - - async def test_agent_and_verifier_restore_runtime_before_start_and_exec(self): - for verifier in (False, True): - environment = self.environment(verifier) - source = self.root / "src" - source.mkdir(exist_ok=True) - (source / "fixture_import.py").write_text("VALUE = 'import-ok'\n") - runtime = {"image": IMAGE, "manifest_digest": "fixture", "config_digest": "fixture", - "workdir": str(self.root), "env": {"PYTHONPATH": str(source), "DENO_DIR": "/deno-cache", "OVERRIDE": "image"}} - - async def start(instance, force_build): - self.assertEqual(instance._startup_env()["PYTHONPATH"], str(source)) - instance._sandbox = SimpleNamespace(sandbox_id="fixture-sandbox") - - async def dispatch(command, *, cwd, env, timeout_sec, user): - process = await asyncio.create_subprocess_exec( - "/bin/sh", "-c", command, cwd=cwd, env=env, - stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, - ) - - async def wait(): - stdout, stderr = await process.communicate() - return SimpleNamespace(stdout=stdout.decode(), stderr=stderr.decode(), exit_code=process.returncode) - - return SimpleNamespace(wait=wait) - - response = httpx.Response(200, request=httpx.Request("GET", "https://api.e2b.dev"), - json={"cpuCount": 2, "memoryMB": 8192, "diskSizeMB": 20480}) - with patch.object(environment, "_load_runtime", return_value=runtime), \ - patch.object(E2BEnvironment, "start", start), \ - patch.object(environment, "_dispatch_command", side_effect=dispatch), \ - patch("httpx.AsyncClient.get", AsyncMock(return_value=response)), \ - patch.dict("os.environ", {"E2B_API_KEY": "fixture"}): - await environment.start(False) - result = await environment.exec("python3 -c 'import fixture_import; print(fixture_import.VALUE)'") - self.assertEqual(result.return_code, 0, result.stderr) - self.assertEqual(result.stdout.strip(), "import-ok") - self.assertEqual(environment._merge_env({"OVERRIDE": "exec"})["OVERRIDE"], "exec") - self.assertEqual(environment._startup_env()["OVERRIDE"], "task") - record = json.loads((self.paths.trial_dir / "artifacts/e2b-runtime/fixture-sandbox.json").read_text()) - self.assertTrue(record["verified"]) - self.assertNotIn("env", record) - - async def test_lost_environment_fails_before_model_execution(self): - environment = self.environment() - runtime = {"image": IMAGE, "workdir": "/app", "env": {"PYTHONPATH": "/app/src"}} - - async def start(instance, force_build): - instance._sandbox = SimpleNamespace(sandbox_id="fixture") - - with patch.object(environment, "_load_runtime", return_value=runtime), \ - patch.object(E2BEnvironment, "start", start), \ - patch.object(environment, "exec", AsyncMock(return_value=ExecResult(stdout="", return_code=0))): - with self.assertRaisesRegex(RuntimeError, "preserve image environment"): - await environment.start(False) - - async def test_sdk_login_shell_preserves_task_path_and_override_precedence(self): - # Exercise the pinned SDK's real run/_start command construction, then - # execute its ProcessConfig locally in place of the remote envd RPC. - environment = self.environment() - profile = self.root / "shell-env" - profile.write_text("export PATH=/usr/bin:/bin\n") - paths = [] - for label in ("image", "exec", "scoped"): - directory = self.root / f"{label} tools with 'quotes' $literal" - directory.mkdir() - executable = directory / "task-only-tool" - executable.write_text(f"#!/bin/sh\nprintf '%s\\n' {label}\n") - executable.chmod(0o755) - paths.append(f"{directory}:/usr/bin:/bin") - environment._persistent_env = {"PATH": paths[0], "BASH_ENV": str(profile)} - commands = object.__new__(Commands) - commands._connection_config = SimpleNamespace(get_request_timeout=lambda value: 30) - commands._envd_version = Version("0.5.0") - commands._rpc = SimpleNamespace(start=lambda request, **kwargs: request.process) - environment._sandbox = SimpleNamespace(commands=commands) - process = None - - async def start(config, timeout): - nonlocal process - self.assertEqual(config.cmd, "/bin/bash") - self.assertEqual(list(config.args[:2]), ["-l", "-c"]) - process = await asyncio.create_subprocess_exec( - config.cmd, *config.args, cwd=config.cwd or None, - env={**os.environ, **config.envs}, - stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, - ) - - async def wait(): - stdout, stderr = await process.communicate() - return SimpleNamespace(stdout=stdout.decode(), stderr=stderr.decode(), exit_code=process.returncode) - - with patch("e2b.sandbox_async.commands.command.as_stream", side_effect=lambda config: config), \ - patch("e2b.sandbox_async.commands.command.first_event", side_effect=start), \ - patch("e2b.sandbox_async.commands.command.extract_start_pid", return_value=1234), \ - patch("e2b.sandbox_async.commands.command.AsyncCommandHandle", return_value=SimpleNamespace(wait=wait)): - command = 'task-only-tool && printf "%s\\n" "$PATH"' - result = await environment.exec(command, cwd=str(self.root)) - self.assertEqual(result.return_code, 0, result.stderr) - self.assertEqual(result.stdout, f"image\n{paths[0]}\n") - result = await environment.exec(command, cwd=str(self.root), env={"PATH": paths[1]}) - self.assertEqual(result.stdout, f"exec\n{paths[1]}\n") - with environment.scoped_exec_env({"PATH": paths[2]}): - result = await environment.exec(command, cwd=str(self.root), env={"PATH": paths[1]}) - self.assertEqual(result.stdout, f"scoped\n{paths[2]}\n") - - async def test_verifier_logs_are_recovered_before_deletion(self): - environment = self.environment(True) - environment._sandbox = SimpleNamespace(sandbox_id="timed-out") - events = [] - - async def download(source, target): - events.append("download") - target.mkdir(parents=True) - (target / "test-stdout.txt").write_text("last test before timeout\n") - - async def stop(instance, delete): - events.append("delete") - - with patch.object(environment, "download_dir", side_effect=download), patch.object(E2BEnvironment, "stop", stop): - await environment.stop(True) - self.assertEqual(events, ["download", "delete"]) - recovery = self.paths.trial_dir / "artifacts/verifier-recovery/timed-out" - self.assertTrue(json.loads((recovery / "recovery.json").read_text())["complete"]) - self.assertIn("last test", (recovery / "logs/test-stdout.txt").read_text()) - self.assertFalse(self.paths.reward_json_path.exists()) - - async def test_failed_recovery_still_deletes_sandbox(self): - environment = self.environment(True) - environment._sandbox = SimpleNamespace(sandbox_id="failed-download") - with patch.object(environment, "download_dir", AsyncMock(side_effect=TimeoutError)), \ - patch.object(E2BEnvironment, "stop", AsyncMock()) as stop, self.assertLogs(level="WARNING"): - await environment.stop(True) - stop.assert_awaited_once_with(True) - record = self.paths.trial_dir / "artifacts/verifier-recovery/failed-download/recovery.json" - self.assertEqual(json.loads(record.read_text()), {"complete": False, "error": "TimeoutError"}) - - def test_verifier_runtime_changes_are_not_silently_ignored(self): - (self.tests / "Dockerfile").write_text(f"FROM {IMAGE}\nENV PYTHONPATH=/other\n") - with self.assertRaisesRegex(ValueError, "Unsupported"): - self.environment(True)._runtime_image() - - async def test_eight_preparers_share_cache_and_agent_verifier_startup_is_offline(self): - catalog = self.root / "runtime-images.json" - runtime = {"image": IMAGE, "env": {"PATH": "/bin"}, "workdir": "/app"} - with patch("deepswe_e2b.image_runtime", AsyncMock(return_value=runtime)) as fetch: - digests = await asyncio.gather(*(prepare_image_runtimes([IMAGE], catalog) for _ in range(8))) - fetch.assert_awaited_once() - self.assertEqual(len(set(digests)), 1) - self.assertEqual(await prepare_image_runtimes([IMAGE], catalog), digests[0]) - fetch.assert_awaited_once() - config = TrialEnvironmentConfig(import_path="deepswe_e2b:ResourceCheckedE2B", kwargs={ - "runtime_catalog": str(catalog), "runtime_catalog_sha256": digests[0], - }) - environments = [EnvironmentFactory.create_environment_from_config( - config=config, environment_dir=self.tests, environment_name="fixture", session_id="fixture", - trial_paths=self.paths, task_env_config=EnvironmentConfig( - docker_image=None if index % 2 == 0 else IMAGE, cpus=2, memory_mb=8192, - storage_mb=20480, env={"OVERRIDE": "task"}, - ), - ) for index in range(8)] - self.assertTrue(all(environment._load_runtime() == runtime for environment in environments)) - - async def start(instance, force_build): - instance._sandbox = SimpleNamespace(sandbox_id=str(id(instance))) - - async def resource_request(url, **kwargs): - self.assertTrue(url.startswith("https://api.e2b.dev/sandboxes/"), url) - return httpx.Response(200, request=httpx.Request("GET", url), - json={"cpuCount": 2, "memoryMB": 8192, "diskSizeMB": 20480}) - - with patch.object(E2BEnvironment, "start", start), \ - patch.object(ResourceCheckedE2B, "exec", AsyncMock(return_value=ExecResult( - stdout="PATH=/bin\0OVERRIDE=task\0", return_code=0))), \ - patch("httpx.AsyncClient.get", side_effect=resource_request) as get, \ - patch.dict("os.environ", {"E2B_API_KEY": "fixture"}): - await asyncio.gather(*(environment.start(False) for environment in environments)) - self.assertEqual(get.await_count, 8) - catalog.write_text(catalog.read_text().replace("/bin", "/changed")) - with self.assertRaisesRegex(ValueError, "catalog changed"): - environments[0]._load_runtime() - with patch("deepswe_e2b.image_runtime", AsyncMock()) as fetch: - with self.assertRaisesRegex(ValueError, "integrity"): - await prepare_image_runtimes([IMAGE], catalog) - fetch.assert_not_awaited() - - async def test_failed_preparation_keeps_successes_and_does_not_cache_failure(self): - catalog = self.root / "runtime-images.json" - images = [IMAGE + "-a", IMAGE + "-b"] - first = {"image": images[0], "env": {}, "workdir": "/app"} - second = {"image": images[1], "env": {}, "workdir": "/app"} - with patch("deepswe_e2b.image_runtime", AsyncMock(side_effect=[first, httpx.ReadTimeout("fixture")])): - with self.assertRaises(httpx.ReadTimeout): - await prepare_image_runtimes(images, catalog) - self.assertEqual(set(json.loads(catalog.read_text())["images"]), {images[0]}) - with patch("deepswe_e2b.image_runtime", AsyncMock(return_value=second)) as fetch: - await prepare_image_runtimes(images, catalog) - self.assertEqual(fetch.await_args.args[1], images[1]) - fetch.assert_awaited_once() - - -if __name__ == "__main__": - unittest.main() diff --git a/evals/deepswe/verify-smoke.py b/evals/deepswe/verify-smoke.py deleted file mode 100644 index c4b03592b..000000000 --- a/evals/deepswe/verify-smoke.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Qualify smoke execution and evidence independently of task correctness.""" - -import argparse -import json -from pathlib import Path -import subprocess -import sys - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("job", type=Path) - args = parser.parse_args() - job = args.job.resolve() - summary = json.loads((job / "result.json").read_text()) - trials = list(job.glob("*/result.json")) - if summary.get("finished_at") is None or summary["n_total_trials"] != 1 or len(trials) != 1: - raise RuntimeError("Expected one finished smoke trial.") - result = json.loads(trials[0].read_text()) - trial = trials[0].parent - if result["task_name"] != "datacurve/abs-stepped-slices": - raise RuntimeError("The job is not the selected DeepSWE smoke task.") - if result.get("exception_info"): - exception = result["exception_info"] - raise RuntimeError(f"Smoke execution failed: {exception['exception_type']}: {exception['exception_message']}") - if result.get("verifier_environment_mode") != "separate" or not result.get("verifier_result"): - raise RuntimeError("Separate verification did not complete.") - metadata = result["agent_result"]["metadata"] - outcome = metadata["diffex_outcome"] - if outcome["status"] != "completed" or outcome.get("agent_execution_stopped") is not True: - raise RuntimeError("Agent completion and shutdown must be confirmed.") - if metadata.get("diffex_accounting", {}).get("complete") is not True: - raise RuntimeError("Token accounting is incomplete.") - run = trial / "agent" / Path(metadata["diffex_run_dir"]).name - requested = json.loads((run / "requested.json").read_text()) - effective = json.loads((run / "effective.json").read_text()) - if (requested["model"], requested["thinking"], requested["subagents"]) != ("gpt-6-luna", "max", "disabled"): - raise RuntimeError("Unexpected recorded execution settings.") - if requested.get("evolution") != {"enabled": False}: - raise RuntimeError("Evolution was not disabled in the requested settings.") - if requested.get("task_environment") != "preserve": - raise RuntimeError("The launcher did not request task environment preservation.") - if effective.get("model") != "gpt-6-luna" or effective.get("thinking") != "max": - raise RuntimeError("Effective model/reasoning differs from the request.") - if effective.get("context_files_disabled") is not True or effective.get("instruction_delivered") is not True: - raise RuntimeError("Prompt isolation evidence is incomplete.") - resources = [json.loads(p.read_text()) for p in (trial / "artifacts/e2b-resources").glob("*.json")] - if len(resources) != 2 or len({r["sandbox_id"] for r in resources}) != 2 or not all(r["verified"] for r in resources): - raise RuntimeError("Expected resource checks for two distinct agent/verifier sandboxes.") - runtime = [json.loads(p.read_text()) for p in (trial / "artifacts/e2b-runtime").glob("*.json")] - if len(runtime) != 2 or not all(r["verified"] for r in runtime): - raise RuntimeError("Expected image environment checks for agent and verifier sandboxes.") - patches = list((trial / "artifacts").rglob("model.patch")) - if not patches: - raise RuntimeError("The commit-derived model.patch artifact is missing.") - trajectory = trial / "agent/trajectory.json" - subprocess.run([sys.executable, "-m", "harbor.utils.trajectory_validator", str(trajectory)], check=True) - report = { - "status": "passed", "trial": result["trial_name"], - "verifier_rewards": result["verifier_result"]["rewards"], - "patch_bytes": [p.stat().st_size for p in patches], - "sandbox_count": len(resources), "accounting_complete": True, - "meaning": "Execution and evidence qualified; verifier rewards separately describe task correctness.", - } - (job / "smoke-check.json").write_text(json.dumps(report, indent=2) + "\n") - print(json.dumps(report, indent=2)) - - -if __name__ == "__main__": - main() diff --git a/evals/swe-atlas/smoke.yaml b/evals/swe-atlas/smoke.yaml deleted file mode 100644 index dc1019de5..000000000 --- a/evals/swe-atlas/smoke.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# One SWE-Atlas Codebase QnA trial with the pinned Diffex 0.3.2 adapter. -n_attempts: 1 -n_concurrent_trials: 1 -retry: - max_retries: 0 -environment: - type: e2b - delete: true - override_cpus: 8 - override_memory_mb: 8192 -verifier: - disable: true -agents: - - name: diffex_terminal_bench.adapter:DiffexAgent - model_name: openai/gpt-6-luna - override_setup_timeout_sec: 630 - kwargs: - thinking: max - tools: read,bash,edit,write,update_plan - subagents: disabled - project_resources: context-only - background_process_lifetime: environment - task_environment: preserve - env: - OPENAI_API_KEY: ${OPENAI_API_KEY} -datasets: - - name: scale-ai/swe-atlas-qna - ref: "1" - task_names: - - scale-ai/task-6905333b74f22949d97ba998 diff --git a/evals/terminal-bench/.gitignore b/evals/terminal-bench/.gitignore deleted file mode 100644 index 5f1cb0a5e..000000000 --- a/evals/terminal-bench/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -.venv/ -__pycache__/ -*.pyc -*.egg-info/ -.pytest_cache/ -.ruff_cache/ -results/ -jobs/ -dist/ diff --git a/evals/terminal-bench/.python-version b/evals/terminal-bench/.python-version deleted file mode 100644 index 24ee5b1be..000000000 --- a/evals/terminal-bench/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.13 diff --git a/evals/terminal-bench/ARTIFACTS.md b/evals/terminal-bench/ARTIFACTS.md deleted file mode 100644 index 6cc922e00..000000000 --- a/evals/terminal-bench/ARTIFACTS.md +++ /dev/null @@ -1,65 +0,0 @@ -# Artifact collection and shutdown verification - -`full.yaml` and the staged benchmark runner use `RecordedDaytonaEnvironment` with the pinned Harbor 0.24.0 and Daytona 0.226.1 APIs. -The backend mounts the shared `diffex-terminal-bench-logs` volume at `/logs`, with a fresh `trials/` subpath for each sandbox creation. -Override `environment.kwargs.artifact_volume` to select a different project volume. -The backend waits up to 120 seconds for volume readiness and records its name, ID, mount path, and subpath in the trial's `artifact-volume.json` before creating the sandbox. -The file retains a `mounts` list with session and environment names, so separate verifiers and restarted environments do not replace the agent's recovery location. -Harbor's Docker-in-Docker backend binds the corresponding `/logs` subdirectories from the sandbox host into the task container. - -Sandbox creation allows an initial attempt plus three startup-timeout retries, with deadlines of 60, 60, 60, and 300 seconds. -There is no retry backoff; each timed-out attempt first receives up to 20 seconds for sandbox lookup and deletion before the next create starts. -The adapter uses one unique sandbox name across these attempts to recover accepted creates after lost responses and prevent duplicate allocation from late requests. -`sandbox-startup.json` records attempt deadlines, outcomes, sandbox names, and cleanup results. -Failed cleanup stops retries and retains any known sandbox reference for Harbor's final cleanup. -Cancellation and non-timeout provider errors do not trigger creation retries. -The fourth creation timeout raises `EnvironmentStartTimeoutError`; Harbor's overall environment deadline still applies, including volume preparation and cleanup overhead. -These retries occur before agent execution and do not rerun evaluation attempts or change `--max-retries 0`. - -Agent home, installed task packages, caches, and mutable harness state live in a private `/tmp/diffex-tb-state-*` directory on sandbox disk. -They remain available to task services through verification and disappear with sandbox deletion. -The exported `state.json` records these paths without linking the state tree into the logs. -Logs, native sessions, transcripts, process identities, and ownership records live under `/logs/agent/run-*` on the volume. - -After requesting cleanup, the adapter retrieves only `cleanup.json`, `stop-requested`, available launcher/CLI identities, `process.json`, and ownership records through a separate small JSON response. -It validates the report against the independent identity and ownership files before starting the bulk download. -Retrieval makes at most three attempts, with a 20-second request limit and a 90-second total budget including exponential backoff, jitter, and any server `Retry-After` delay. -A report that has not yet been published is retried; malformed or negative shutdown evidence is rejected. -Bulk extraction cannot change the independently validated shutdown decision. - -Once shutdown is confirmed, bulk download, trajectory export, or local artifact persistence failures appear in `artifact_errors` and permit Harbor's verifier to run. -Such a trial is still recorded as an artifact collection failure unless another conclusive execution failure or interruption already explains its outcome. -Missing or invalid shutdown evidence raises `DiffexControlStopError` and continues to block verification. -Release qualification still requires valid retained execution evidence; permitting verification does not qualify an incomplete artifact set. - -## Quotas and retention - -Daytona recommends a shared volume with isolated subpaths and allows at most 100 volumes per organization, so a 445-trial run consumes one project volume rather than 445 volumes. -Volume data persists after sandbox deletion and does not count against sandbox disk quota. -Volumes use FUSE and are slower than sandbox disk, which is why package installation and caches remain local. -See the [Daytona volume documentation](https://www.daytona.io/docs/en/volumes/). - -The documented Tier 4 pool is 500 vCPUs, 1,000 GiB RAM, and 5,000 GiB sandbox storage, with 50,000 general API requests and 600 sandbox creations per minute. -Keep job concurrency within the organization's remaining capacity, accounting for other running or stopped sandboxes and each task's resource requirements. -See the [Daytona limits documentation](https://www.daytona.io/docs/en/limits/). -This change does not increase job concurrency or sandbox resources. - -The staged runner automatically prunes a completed Diffex trial's volume prefixes after qualification, before releasing its concurrency slot. -The native Harbor command in the README enables the same behavior with `--plugin diffex_terminal_bench.retention:VolumeCleanupPlugin`. -Pass this CLI option explicitly in other native run commands too; do not rely on YAML to activate the cleanup plugin. -Cleanup requires a completed trial and verifier result, matching local reward and verifier logs, valid native shutdown and execution evidence, a valid trajectory, and no artifact errors or native session gaps. -A zero reward is eligible; failed, interrupted, oracle-only, and incomplete trials retain their remote logs. -All recorded trial sandboxes must be absent before deletion, including separate verifiers and retained task services. -One short-lived Python sandbox mounts only that trial's exact prefixes, compares every remote file's SHA-256 against the local artifact copy, and empties the prefixes only if all files match. -Unknown files, symlinks, or mismatched files preserve remote data for recovery. -Local logs, transcripts, and results remain available. -`volume-cleanup.json` records the decision, prefix locations, deleted file count, and cleanup sandbox deletion status. -Cleanup failures do not change verifier results and retain recovery locations; transport operations use bounded retries. -Remote comparison and pruning allow three attempts within 210 seconds, with a 60-second execution limit per attempt. -The cleanup sandbox is deleted in a shielded finalizer, with a two-minute idle stop, immediate deletion after stopping, and a ten-minute maximum lifetime as fallbacks. -This uses one extra sandbox creation per eligible trial, within the existing worker slot; include that overhead in the Tier 4 creation-rate and compute budget. -The shared volume itself remains available to subsequent trials and concurrent jobs. - -To recover artifacts, select the relevant session in `artifact-volume.json`, mount its volume ID and exact subpath on a temporary sandbox, and download `/logs`. -Delete that recovery sandbox afterward. -Remove only a trial's recorded subpath when its artifacts are no longer needed; do not delete the shared volume while any job uses it. diff --git a/evals/terminal-bench/README.md b/evals/terminal-bench/README.md deleted file mode 100644 index 5aa41d823..000000000 --- a/evals/terminal-bench/README.md +++ /dev/null @@ -1,492 +0,0 @@ -# Diffex Terminal-Bench integration - -This private Python package installs and executes the pinned Diffex release through Harbor. -Execution uses explicit model and harness policies, fresh trial state, and retained native sessions. -Execution outcomes include native stream completion, terminal assistant state, retries, process status, configuration, and cleanup verification. -ATIF export and usage accounting run after evidence recovery, including interrupted trials. -The [staged benchmark release gate](RELEASE_GATE.md) pins dataset inputs and verifies retained evidence before larger Daytona runs. -Use the [offline job exporter](RELEASE_GATE.md#export-retained-trials-for-harbor-hub) to package existing stage results for Harbor Hub without rerunning evaluations. -This integration is not yet ready for leaderboard runs. -The adapter now requires a release with `--background-process-lifetime` and `--background-process-ownership-dir` support. -The checked-in Diffex 0.3.2 pin supports these flags; installation verifies their availability alongside the release integrity. -Source-only lifecycle tests do not qualify the installed release or a live benchmark trial. - -## Host setup - -Use Python 3.13 and uv (validated with uv 0.11.2). -The benchmark has its own Python environment and lockfile, outside the npm workspaces. - -```bash -cd evals/terminal-bench -uv sync --locked -uv run --locked python -c 'from diffex_terminal_bench.adapter import DiffexAgent; print(DiffexAgent.import_path())' -``` - -The import path is `diffex_terminal_bench.adapter:DiffexAgent`. -Harbor `0.24.0`, Daytona `0.226.1`, E2B `2.59.0`, Pydantic `2.14.1`, and PyYAML `6.0.3` are pinned in `pyproject.toml` and `uv.lock`. -DeepSWE uses this same locked environment without runtime dependency overlays. -The adapter follows that release's `BaseInstalledAgent.install(environment)` interface. -See [Harbor's installed-agent documentation](https://www.harborframework.com/docs/agents). - -## Full evaluation with additional sandbox storage - -The native full job uses [full.yaml](full.yaml): 89 tasks, five attempts each, three concurrent trials, and no automatic retries. -It selects `RecordedDaytonaEnvironment` to retain logs and transcripts in one shared Daytona volume with isolated sandbox subpaths. -For an existing command that uses `--env daytona` without this config, replace that option with `--env diffex_terminal_bench.daytona_backend:RecordedDaytonaEnvironment` to enable volume storage. -See [artifact collection and retention](ARTIFACTS.md) for recovery records, retry bounds, and Tier 4 quota considerations. -It requests 20 GiB of disk per sandbox so task files and Harbor's temporary log archive have more room than the failing 10 GiB allocation. -This is an explicit resource override, not an evaluation with the official task resource limits unchanged. -Daytona must allow the requested per-sandbox allocation and enough total storage for concurrent sandboxes. -The extra disk reduces the risk of archive failures; it does not guarantee that an unbounded task cannot fill the disk. - -After exporting `DAYTONA_API_KEY` and `OPENAI_API_KEY`, prepare and inspect a fresh job in zsh: - -```zsh -TB_OUTPUT="results/full-89x5-tb21-$(date +%Y%m%d-%H%M%S)-$(uuidgen)" -TB_STORAGE_MB=20480 -mkdir -p "$TB_OUTPUT" -git rev-parse HEAD > "$TB_OUTPUT/adapter-commit.txt" -TB_RUN=(uv run --locked harbor run --config full.yaml - --plugin diffex_terminal_bench.retention:VolumeCleanupPlugin - --override-storage-mb "$TB_STORAGE_MB" - --jobs-dir "$TB_OUTPUT" --job-name full) -"${TB_RUN[@]}" --print-config > "$TB_OUTPUT/planned-config.json" -cat "$TB_OUTPUT/planned-config.json" -``` - -Start only after reviewing the resolved configuration: - -```zsh -"${TB_RUN[@]}" -``` - -Use a new output directory when changing disk allocation; do not mix results from different resource limits. -The adapter installs the published version in `pins.toml`, so local runtime fixes require a new published release and a reviewed pin update before they are exercised by this job. - -## Installation contract - -The task container must provide Debian/Ubuntu Linux with glibc, root access, Bash, GNU coreutils, and network access to its apt mirrors, nodejs.org, and registry.npmjs.org. -Linux x86_64 and aarch64 have separate pinned Node archive checksums. -The installer fails explicitly on unsupported platforms and refuses to reuse an existing installation. - -[pins.toml](src/diffex_terminal_bench/pins.toml) records Node `22.23.2`, Diffex `0.3.2`, the public npm registry, archive checksums, npm integrity, and a 600-second installation deadline. -The deadline includes apt prerequisites, Node download, npm installation, and all three CLI checks. -GNU `timeout` terminates the installer process group at the deadline and kills it after a ten-second grace period. -Harbor's command transport allows another five seconds to collect the result. - -Setup installs Node under `/opt/diffex-terminal-bench/node` and uses `npm install --ignore-scripts --save-exact --registry=https://registry.npmjs.org/ @diffexai/diffex@0.3.2` with a private prefix. -An isolated home and npm cache prevent host or task npm settings from changing installation. -Node archive SHA-256, registry metadata, installed package version, and npm lockfile integrity must match the checked-in pins. -The `/usr/local/bin/diffex` wrapper always invokes the pinned Node binary. -No monorepo source files or provider credentials are required inside the container. - -Setup then runs `diffex --version`, `diffex --help`, and `diffex --list-models` from `/tmp` with stdin closed and a fresh smoke-state directory. -The setup directory under Harbor's agent logs contains installation stdout/stderr, registry metadata, the resolved npm installation lockfile, and stdout/stderr for each CLI check. -`installation.json` is written only after every check passes and records installed versions, integrity, elapsed time, and the deadline. - -## Run the Linux installation smoke - -Start Docker with Compose support, then run from this directory: - -```bash -uv run --locked python scripts/smoke_install.py -uv run --locked python scripts/smoke_install.py --platform linux/arm64 -``` - -[smoke.toml](smoke.toml) pins Ubuntu 24.04 by platform-specific image digest and defaults to Linux amd64. -An arm64 host needs amd64 emulation for the default check; the second command uses native arm64 where available. -Image pull and container startup have a separate 600-second environment deadline. -The runner invokes the actual Harbor `DiffexAgent.setup()` method, including the 600-second installer deadline. -Only a fresh agent log directory is mounted; the runner inspects Docker's actual mounts to reject access to monorepo source or dependencies. -No API keys or model calls are used. - -Each invocation writes a unique `results/diffex-install-*/` directory. -Inspect `agent/setup/installation.json`, the CLI stdout/stderr files, and `smoke.json` for the image digest, platform, actual mounts, setup wall time, and cleanup result. -The runner removes its container in a `finally` block and fails if Docker still reports a container for that invocation. -Installation failure logs remain available through the mounted agent log directory. -These are installation checks, not Terminal-Bench task scores. - -## Local validation - -```bash -uv run --locked python -m unittest discover -s tests -v -node --test tests/launcher.test.cjs -uv run --locked python scripts/smoke_execution.py --platform linux/arm64 -uv run --locked python scripts/smoke_execution.py --platform linux/amd64 -uv run --locked python scripts/smoke_outcomes.py --platform linux/arm64 -uv run --locked python scripts/smoke_outcomes.py --platform linux/amd64 -uv run --locked python scripts/check_production_contents.py -``` - -The unit checks cover installation pins, configuration validation, explicit credentials, prompt uploads, fresh trial paths, and subagent policy. -They also cover outcome parsing, incomplete streams, transport failures, log-collection failures, and repeated cancellation during recovery. -Converter checks cover deduplication, tool observations, nested child trajectories, auxiliary usage, unknown telemetry, images, and partial messages. -The execution smoke installs the real npm release, then redirects a fixture model to a loopback faux provider with a test-only wrapper. -It exercises shell and file tools, multiline instructions with shell metacharacters and Unicode, repeated trials, and non-root execution without paid API calls. -The fixture checks the actual request credentials without storing their values, verifies empty initial sessions and state, and scans retained artifacts for the fixture secrets. -It checks the supplied eval system prompt and the anti-cheating sentence in the user message against actual provider requests. -Automatic project context, packaged release instructions, project settings, extensions, and `SYSTEM.md` are excluded from the system prompt. -Only the agent log directory is mounted, and each invocation removes its container in a `finally` block. -Generated evidence is retained under `results/diffex-execution-*/`. -Each execution smoke validates ATIF with Harbor's validator and reconciles nonzero input, output, cached-token, and cost totals against every response. -The outcome smoke exercises success, authentication failure, exhausted retry, recovered retry, interrupted JSON output, and a Harbor-style execution timeout as both root and a non-root user. -It uses the pinned npm release with a loopback provider and checks nonzero terminal-error exits and rejection of truncated output even when the process exits zero. -Success, provider-failure, forced-CLI-death, and timeout cases create a TERM-resistant background service and verify that its PID and start time are unchanged across adapter cleanup. -The verifier phase queries that same service and checks that the launcher, CLI, and provider requests have stopped. -The Linux cleanup regression independently creates a detached service with durable handoff evidence, and covers PID reuse, delayed launch, malformed records, and TERM-resistant active work. -Evidence is retained under `results/diffex-outcomes-*/`, including partial stdout, stderr, native sessions, process status, cleanup, and outcomes. -The outcome smoke also validates recoverable ATIF steps and checks that interrupted runs report incomplete accounting. -The packaging check packs every public npm workspace into temporary tarballs with lifecycle scripts disabled, inspects their contents for benchmark paths, and records file counts and SHA-256 hashes in `results/production-contents.json`. -It requires existing production build output, including `packages/coding-agent/dist/cli.js`, and fails if that CLI entrypoint is missing. -It does not build, publish, or regenerate release artifacts. -Run `npm run check` from the repository root after changing code. - -### Source lifecycle validation - -To validate unpublished lifecycle changes, build the development-only image from the repository root: - -```bash -docker build --platform linux/arm64 -f evals/terminal-bench/tests/fixtures/Dockerfile.source -t diffex-adapter-tests:arm64 . -cd evals/terminal-bench -uv run --locked python scripts/smoke_outcomes.py --platform linux/arm64 --source-image diffex-adapter-tests:arm64 -uv run --locked python scripts/smoke_outcomes.py --platform linux/arm64 --source-image diffex-adapter-tests:arm64 --grpc-fixture -docker run --rm --init --user 65534:65534 -v "$PWD:/eval:ro" diffex-adapter-tests:arm64 node --test /eval/tests/cleanup.test.cjs -``` - -Repeat with `linux/amd64` and an `amd64` image tag for emulated coverage on an arm64 host. -Run the cleanup regression without `--user` for root coverage. -The development image installs dependencies without lifecycle scripts and runs the source CLI through `tsx`. -The gRPC fixture uses a known-correct implementation and all seven unmodified verifier tests extracted from the SHA-256-verified dataset archive. -It checks the original process identity before and after verifier RPCs and confirms container deletion afterward. -Every development smoke report is marked `qualification: source-only`. -After changing `pins.toml`, rerun both pinned-release Linux smokes and start a fresh live benchmark trial before declaring integration qualification complete. - -On 2026-09-20, source-only validation passed 16 controlled outcome cases on each architecture, including root and UID 65534 execution. -Both architectures also passed all seven pinned gRPC verifier tests with unchanged service identity and confirmed container deletion. -The Linux cleanup suite passed seven deterministic cases, including forced death before CLI identity publication, and all 142 Python tests and `npm run check` passed. -The retained outcome reports are `results/diffex-outcomes-8dab466d6f5c/smoke.json` (arm64) and `results/diffex-outcomes-284ee6f0c224/smoke.json` (amd64). -The gRPC reports are `results/diffex-outcomes-8d83db9627fb/smoke.json` (arm64) and `results/diffex-outcomes-da1f6f5534a3/smoke.json` (amd64). -These local source checks leave pinned-release smokes and a fresh live benchmark trial pending. - -### Recorded validation - -The following historical smoke records used Diffex 0.2.2. -They do not qualify the updated Diffex 0.3.2 / Terminal-Bench 2.1 configuration. - -On 2026-09-14, both pinned Ubuntu images passed through Harbor 0.23.0 on Docker Desktop with an arm64 host. - -| Container platform | Setup wall time | CLI checks | Cleanup | -| --- | --- | --- | --- | -| Linux amd64, emulated | 38.627 seconds | All three passed | Container removed | -| Linux arm64, native | 14.454 seconds | All three passed | Container removed | - -Both installations reported Diffex 0.2.2, Node 22.23.2, npm 10.9.8, and the pinned npm integrity, with empty stderr for the three CLI checks. -The production content check inspected the current checkout's public `@diffexai/diffex@0.2.4` tarball (1,170 entries) and found no benchmark content. -The five unit tests and repository `npm run check` passed. -These observations validate installation on the specified images; broader task-environment coverage remains a later step. - -On 2026-09-15, the execution smoke passed four controlled trials on each pinned Linux image (amd64 emulated and arm64 native). -Each platform covered two isolated single-agent trials, an enabled-subagent trial, and a non-root trial, with 16 loopback provider requests and successful container cleanup. -All trials preserved the instruction bytes, selected `openai/gpt-5` with `high` thinking, retained the exact release context, executed shell/file tasks in `/task/work`, and retained their native sessions. -The credential-leak scans, ten unit tests, production package-content check, and repository `npm run check` passed. -These controlled provider responses validate execution plumbing; they do not measure model task performance or validate item 3's outcome handling. - -On 2026-09-15, the outcome smoke passed 12 trials on each pinned Linux image, covering all six outcome cases as root and UID 65534. -Authentication failures, exhausted retries, and truncated output from the pinned CLI all exited zero and were correctly rejected by the adapter. -Recovered retries completed successfully, and timeout trials retained their logs and sessions while stopping TERM-resistant detached descendants. -Every successful smoke invocation confirmed container removal. -The prior execution smoke also passed again on both architectures, including enabled subagents. -The 22 adapter unit tests, 11 source print-mode tests, production-content check, and repository `npm run check` passed. -An initial arm64 installation encountered an Ubuntu mirror hash mismatch; the subsequent clean installation passed without changing installation pins or weakening integrity checks. -These results validate controlled execution outcomes and cleanup, not model quality, ATIF export, or the later benchmark release gate. - -On 2026-09-15, ATIF validation passed four execution trials and 12 outcome trials on each pinned Linux image. -The execution trials covered single agents, an enabled child agent, and non-root execution, using 16 loopback model responses per platform. -Each single-agent execution reconciled to 57 input tokens, 21 output tokens, 15 cache hits, and USD 0.0000915 in fixture cost telemetry. -Each enabled-subagent execution reconciled to 133 input tokens, 49 output tokens, 35 cache hits, and USD 0.0002135 across seven root/child responses. -The outcome trials retained valid trajectories for authentication errors, exhausted/recovered retries, truncated output, and timeouts; missing telemetry and interrupted accounting remained explicitly unknown. -Both smoke runners confirmed process and container cleanup on both architectures. -The 37 adapter tests, repository `npm run check`, and production-content inspection passed; the public tarball contained no benchmark files. -These checks use controlled responses without paid model calls and do not establish leaderboard readiness or remote artifact-upload completion. - -### Diffex 0.2.4 pin validation - -On 2026-09-15, Diffex 0.2.4 passed four execution trials and 12 outcome trials on each pinned Linux image (arm64 native and amd64 emulated). -Installation integrity and CLI checks passed, the packaged release context loaded without duplication, and both runners confirmed process and container cleanup. -Execution trajectories reconciled root and child usage; failed and interrupted outcomes retained validated ATIF with incomplete accounting where appropriate. -All 55 adapter tests, `npm run check`, `./test.sh`, and production-content inspection passed. -Dataset preparation verified all 89 Terminal-Bench 2.1 tasks and the ten declared pilot selections from the pinned source archive. -These controlled tests use a loopback provider; a new Daytona oracle and live Diffex trial remain necessary for the updated qualification identity. - -### Diffex 0.2.5 pin validation - -On 2026-09-17, the published Diffex 0.2.5 tarball matched its npm integrity and the checked-in release instructions. -Four execution trials and 12 outcome trials passed on each pinned Linux image (arm64 native and amd64 emulated), with verified installation, validated trajectories, reconciled execution usage, and confirmed container cleanup. -The enabled-subagent allowlist uses the release's renamed `wait_for_agents` tool. -Authentication and exhausted-retry cases returned exit code 1; truncated output remained rejected even with exit code 0. -An initial arm64 setup encountered an Ubuntu mirror hash mismatch; subsequent clean installations passed without weakening checksum checks. -All 120 adapter tests and `npm run check` passed. -These checks used a loopback provider; live Daytona qualification must start a new sequence for the updated pins and adapter identity. - -### Diffex 0.2.6 pin validation - -On 2026-09-18, the published Diffex 0.2.6 tarball matched its npm integrity, and the adapter's release instructions were synchronized with the packaged `dist/AGENTS.md`. -Clean installation checks passed on both pinned Linux images (arm64 native and amd64 emulated), including version, help, model listing, and container cleanup. -The packaged OpenAI catalog supported `gpt-5.6-luna` with `max` thinking, which was selected in `benchmark.toml` at the time. -All 120 adapter tests and `npm run check` passed. -These checks did not call a live model; Daytona qualification requires a fresh sequence for the updated identity. - -### Diffex 0.3.1 pin validation - -On 2026-09-20, the published Diffex 0.3.1 tarball matched its npm integrity and the checked-in release instructions. -Four execution trials and 16 outcome trials passed on each pinned Linux image (arm64 native and amd64 emulated), including root and UID 65534 execution, enabled subagents, validated trajectories, and confirmed container cleanup. -The lifecycle cases verified environment-owned service retention after successful execution, provider failure, and forced agent death. -All 142 adapter tests and `npm run check` passed. -The retained execution reports are `results/diffex-execution-a6196c0988e3/smoke.json` (arm64) and `results/diffex-execution-91c70a4694a1/smoke.json` (amd64). -The retained outcome reports are `results/diffex-outcomes-dea6f3573741/smoke.json` (arm64) and `results/diffex-outcomes-94cec6443269/smoke.json` (amd64). -These checks used a loopback provider; live Daytona qualification requires a fresh sequence for the updated pins and adapter identity. - -## Execute a controlled task - -Start Docker and make the selected provider credential available in the host environment through your normal secret-loading mechanism. -Set the exact `model_name` in `execution.yaml`, then run from this directory with your task path: - -```bash -uv run --locked harbor run --config execution.yaml \ - --path /absolute/path/to/task -``` - -[execution.yaml](execution.yaml) records the default policy and allows 630 seconds for Harbor setup, encompassing the installer's 600-second deadline and transport overhead. -The task's agent timeout remains in effect. -Use `--ak thinking=medium` or `--ak tools=read,bash,edit,write` to change the corresponding policy explicitly. -Subagents must remain disabled; `--ak subagents=enabled` is rejected during preflight. -Inspect accepted options with `uv run --locked harbor agent schema diffex_terminal_bench.adapter:DiffexAgent`. -Unknown options, arbitrary native config, prompt templates, injected skills, and MCP servers are rejected. -Model IDs must include a supported provider prefix; fuzzy patterns and thinking suffixes are rejected. -The adapter also checks the resolved model, thinking level, and active tools against the requested configuration after execution. - -Supported credential environments are: - -| Provider | Credential variables | -| --- | --- | -| `anthropic` | `ANTHROPIC_API_KEY`, `ANTHROPIC_AUTH_TOKEN`, `ANTHROPIC_OAUTH_TOKEN` | -| `openai` | `OPENAI_API_KEY` | -| `google` | `GEMINI_API_KEY` | -| `openrouter` | `OPENROUTER_API_KEY` | -| `deepseek` | `DEEPSEEK_API_KEY` | -| `groq` | `GROQ_API_KEY` | -| `xai` | `XAI_API_KEY` | -| `mistral` | `MISTRAL_API_KEY` | - -Harbor's agent `extra_env` takes precedence over the host environment. -Only the selected provider's credential variables are forwarded, using Harbor's `environment.exec(env=...)` mechanism along with the model and thinking configuration. -Credential values never appear in adapter commands or configuration artifacts. -The launcher starts Diffex with an explicit environment, discarding inherited agent directories, credentials for other providers, and configuration overrides. -Stored OAuth accounts, host credentials files, and custom provider endpoints are not imported. - -### Prompt and harness policy - -The adapter uploads the complete UTF-8 instruction and runs `diffex --mode json --print @instruction.txt` in the task's default working directory and as Harbor's configured agent user. -The CLI receives stdin EOF through `/dev/null` and the launcher's ignored stdin. -Native `@file` transport adds Diffex's standard `` wrapper while retaining all instruction bytes, including leading/trailing whitespace and CRLF line endings. -Instruction contents never become shell command text or command-line flags. -The launcher appends `Do not cheat by using online solutions or hints specific to this task.` after the native file wrapper, separated by a blank line, in the same user message. -The original instruction file and its hash remain unchanged. -The observer extension checks that the first user message contains exactly the wrapper, original instruction, and anti-cheating suffix. -It registers only a `message_start` observer and does not rewrite provider requests. - -Each invocation creates an exclusive `run-/` under Harbor's agent logs. -Its `DIFFEX_CODING_AGENT_DIR` is a fresh private `/tmp/diffex-tb-state-*` directory on sandbox disk, with a fresh home and global `settings.json` containing `{"evolution":{"enabled":false}}` before startup. -Packages and caches remain outside the exported logs and survive until sandbox teardown so retained task services can use them during verification. -No prior settings, credentials, sessions, or evolved harness selections are copied into it. -Startup network updates are disabled with the supported offline setting so the pinned model catalog remains stable. - -The eval adapter passes `--system-prompt` with the checked-in [eval prompt](src/diffex_terminal_bench/eval-system-prompt.txt) and `--no-context-files`. -The entire automatic `` section is omitted, including packaged release instructions and ancestor `AGENTS.md`/`CLAUDE.md` files and their paths. -Task files remain on disk and can be read explicitly by the agent. -The `context-only` project resource setting continues to disable project trust, discovered extensions, skills, prompt templates, and themes; eval launches additionally suppress automatic context files. -The effective system prompt must match the checked-in prompt's SHA-256, and execution evidence must confirm context files are disabled. -This uses supported flags in the pinned Diffex release and requires no product code or release changes. - -All eval configurations require disabled subagents and omit `spawn_agent`, `agent_action`, and `agent_status` from the tool allowlist. -The pinned Diffex 0.3.2 CLI does not propagate `--system-prompt` or `--no-context-files` to child agents, so enabled subagents cannot satisfy this prompt policy. -The adapter rejects attempts to enable them before setup or execution. -The staged runner's `-n/--concurrency` still controls simultaneous independent trials and sandboxes. -Native sessions remain beneath `run-/sessions/`; trajectory conversion continues to support retained child sessions from earlier evaluations. - -### Execution evidence - -Each run directory retains: - -- `requested.json`: pins, model, thinking, tools, subagent policy, project policy, credential variable names, instruction hash, user-message suffix, and system prompt hash. -- `instruction.txt`: the original instruction. -- `eval-system-prompt.txt`: the requested eval system prompt. -- `effective.json`: resolved model, thinking, tools, working directory, session path, prompt hashes, user-message verification, and context-file suppression. -- `system-prompt.txt`: the observed system prompt, without automatic project context. -- `stdout.jsonl`, `stderr.log`, and `process.json`: native JSON events, diagnostics, and process status. -- `launcher.json` and `cli.json`: launcher and root CLI PID/start-time identities. -- `ownership/*.json`: durable shell ownership records published by the CLI. -- `stop-requested` and `cleanup.json`: the launch barrier, verified control shutdown, and explicitly retained environment-owned processes. -- `launcher.stdout.log` and `launcher.stderr.log`: launcher diagnostics, including failures before CLI startup. -- `outcome.json`: execution classification, stream completeness, final assistant stop reason, retry counts, interruption state, shutdown recovery errors, and separate artifact errors. -- `accounting.json`: observed usage, nullable totals, per-model observations, and reasons accounting is incomplete. -- `sessions/`: native root/child history. -- `state.json`: sandbox-local harness state and home paths, without exporting their contents. - -Harbor collects the agent log tree after the run; avoid log filters that exclude these paths. -`trajectory.json` sits directly under Harbor's agent log directory, with embedded child trajectories and optional `trajectory-media/` image files. -Provider usage fields in `AgentContext` contain reconciled sums of recorded usage, with accounting completeness recorded separately. -`AgentContext.metadata.diffex_outcome` contains the execution outcome, and `diffex_run_dir` identifies its artifacts. - -### Execution outcomes and interruption recovery - -A completed execution requires zero CLI and launcher exit codes, no terminating signal or spawn error, matching effective configuration, and verified cessation of agent execution. -Environment-owned services may remain alive for shared-environment verification. -The parser reads JSONL incrementally and requires matching assistant messages, a final `stop` response, closed message/run boundaries, resolved retries, and the native `agent_settled` event. -Malformed or truncated lines, missing completion events, pending tool use, token-limit termination, and unresolved retries cannot count as completed runs. -Earlier provider errors are recoverable when retry events and the subsequent terminal assistant state establish success. -Tool errors alone do not fail execution: the assistant can inspect and recover from them. - -| Evidence | Adapter outcome | Harbor behavior | -| --- | --- | --- | -| Complete successful run, including recovered retry | `completed` | Return normally; verifier determines reward | -| Terminal authentication error | `authentication_failure` | Raise `AgentAuthenticationError` | -| Terminal assistant error or failed retry | `provider_failure` | Raise `DiffexExecutionError` | -| Nonzero process, signal, or spawn error | `process_failure` | Raise `DiffexExecutionError` | -| Missing, truncated, or unfinished native stream | `incomplete` | Raise `DiffexExecutionError` | -| Effective configuration mismatch | `configuration_failure` | Raise `DiffexExecutionError` | -| Unverified agent shutdown or missing shutdown evidence | `cleanup_failure` | Raise `DiffexControlStopError`; block verification | -| Failed artifact collection with verified shutdown | `artifact_collection_failure`, or preserve an existing failure | Record `artifact_errors`; permit verification | -| Harbor task deadline cancels `run()` | `cancelled` | Preserve cancellation; Harbor raises `AgentTimeoutError` | -| Environment transport timeout or failure | `timeout` / `transport_failure` | Preserve the original exception | - -`DiffexExecutionError` subclasses Harbor's `NonZeroAgentExitCodeError`, so infrastructure failures retain exception information rather than becoming ordinary zero-reward results. -The adapter never sets or infers verifier reward. -A completed assistant that produces an incorrect solution returns normally and can earn zero from the verifier. -The pinned `0.3.2` release returns nonzero for terminal assistant errors and aborts in JSON mode as well as text mode. -The adapter also parses native events to classify provider failures and reject incomplete output independently of process exit status. - -The launcher writes stdout and stderr directly to files and retains native sessions during execution. -On success, failure, or cancellation, an independent Harbor exec writes the stop barrier and signals identified launcher and CLI processes individually. -A private pipe prevents CLI startup until the launcher has recorded its PID and start time; launcher death before publication closes the pipe without starting agent execution. -No launcher process-group signal can include a retained service. -After control processes exit, cleanup reads durable ownership records and scans Linux `/proc` for trial-marked processes and identified shell groups. -Linux boot ID, PID, start time, process group, and shell execution tokens distinguish transferred groups from reused process IDs. -Environment-owned groups survive; active or otherwise untransferred work receives TERM, escalating to KILL within a seven-second recovery budget. -Control escalation begins after three seconds, reserving time for active descendants; active-work escalation begins after five seconds. -Already exited zombies cannot execute and are left for sandbox init to reap. -The report records `agent_execution_stopped`, `ownership_validated`, `remaining_agent_pids`, control identities, retained service identities, and the consumed ownership records. -Outcome parsing and gate validation reconcile these records with retained files. -`cleanup_confirmed` in the outcome means pre-verifier agent shutdown, not an empty sandbox. -Missing, malformed, contradictory, or unverified shutdown evidence raises `DiffexControlStopError`, which is outside Harbor's verifier-eligible timeout and nonzero-exit exceptions. -This exception overrides a pending cancellation while retaining its interrupted outcome in the evidence. -Verified shutdown preserves Harbor's existing verification behavior for deadlines and nonzero agent exits without changing their failed classifications. -Task commands must preserve the inherited trial marker or remain in an identified shell group for unfinished descendants to remain discoverable. -Harbor stops the shared environment after verification, including verifier failure or timeout recovery; separate-environment verification keeps Harbor's artifact-based lifecycle. -The benchmark runner independently confirms sandbox deletion after every trial, including abandonment and transport failures. - -Recovery is shielded against repeated cancellation and runs before `run()` returns or re-raises the interruption. -It allows up to 15 seconds for cleanup transport, then retrieves and validates only shutdown evidence with three bounded attempts within 90 seconds. -After shutdown validation, it allows 15 seconds for mounted-log preparation or 30 seconds for bulk remote log download. -Bulk download and local artifact persistence failures cannot invalidate independently confirmed shutdown. -The final summary is written directly to host agent logs after recovery, so later remote log collection cannot overwrite a cancellation classification. -These bounded recovery operations can extend the task deadline; they do not grant the agent additional task execution time. -If the sandbox transport becomes unavailable, available host logs and recovery diagnostics remain, cleanup is not reported as successful, and Harbor's environment teardown remains necessary. -Avoid log filters that discard the retained evidence. - -### ATIF export and accounting - -The converter emits `ATIF-v1.8` using the models and validator shipped with pinned Harbor 0.24.0. -See [Harbor's ATIF documentation](https://www.harborframework.com/docs/agents/trajectory-format) for the schema and validation command. -It writes the final trajectory atomically after cleanup and evidence collection, before returning to Harbor. -Export failure changes an otherwise completed execution to `artifact_collection_failure`; it preserves an existing provider failure or cancellation. -An empty run with no recoverable steps retains accounting diagnostics without fabricating a trajectory. - -Native session message entries are authoritative, including history retained before compaction. -Unmatched root `message_end` events supplement responses absent from persistence, such as discarded retries. -Matching uses provider/model/response ID within a session, falling back to the complete message including its native timestamp when no response ID exists. -Repeated native entry IDs and assistant response identities count once. -`message_start`, `message_update`, `turn_end`, and `agent_end` copies never add another charge for a completed response. -The last unfinished streamed assistant message is retained as partial, with unknown usage. -Malformed JSONL records do not discard intact earlier or later records, but they make accounting incomplete. - -Assistant tool calls and their native `toolResult` observations share an ATIF step and match by tool-call ID. -Completed `tool_execution_end` events recover observations missing from message persistence. -Unresolved observations remain system observations with their original call ID in metadata. -User messages, custom context messages, visible thinking, tool errors, and summary text remain in the trajectory. -Images are decoded into content-addressed files under `trajectory-media/`, with paths relative to `trajectory.json`. - -Child sessions are collected recursively, including hidden `.subagents` directories. -The parent roster's `childSessionId` identifies each child, and its agent name links the child trajectory to the matching `spawn_agent` observation. -Repeated roster snapshots do not duplicate children or their usage. -Nested child trajectories are embedded with resolvable `trajectory_id` references so uploading the root JSON also uploads their steps. -Missing child sessions, missing observations, and incomplete child execution are explicit accounting gaps. -Recoverable orphan child histories are embedded with an unresolved-link diagnostic. -A child marked `interrupted` in the parent roster is a valid terminal state when a successful parent `agent_action` interrupt call establishes intent. -An intentional stop can end during a model response or tool execution; it does not require an additional final assistant reply. -A child marked `failed` in the final parent roster with a nonempty error is also a recorded terminal outcome. -Its trajectory retains `execution_status: failed`, the scheduler's `execution_error`, and native response errors; it does not require a successful final assistant reply. -The parent can finish normally despite that child failure, and the verifier independently determines the task reward. -Missing child sessions, malformed records, unresolved tool calls, unproven interruptions, root execution failures, and cleanup failures still fail qualification. -Missing usage telemetry remains explicitly incomplete and does not erase known usage. -This policy runs in the local Python adapter and needs no new Diffex npm release. -The staged runner requires a clean committed checkout and fresh output for the changed adapter identity; existing runs and retained reports are not rewritten. - -| Native Diffex usage | ATIF step metric | Harbor `AgentContext` | -| --- | --- | --- | -| `input + cacheRead + cacheWrite` | `prompt_tokens` | `n_input_tokens` | -| `output` | `completion_tokens` | `n_output_tokens` | -| `cacheRead` | `cached_tokens` | `n_cache_tokens` | -| `cost.total` | `cost_usd` | `cost_usd` | - -Cache writes contribute to prompt tokens, but are not cache hits. -`metrics.extra.cache_write_tokens`, uncached input, and the native usage object preserve the additional detail. -`totalTokens` is not added separately because it already aggregates token categories. -Native cost is reported as recorded; the converter does not reprice responses or claim it is an independently verified provider invoice. -Every trajectory's final token and cost totals include its descendants; `total_steps` counts its own steps. -The root context totals visit each native session exactly once, without summing child-inclusive parent totals again. - -Compaction and branch-summary entries contribute their `usage` when available, without dropping the earlier responses they summarize. -Their model attribution is `unattributed` because those entries do not identify the model that produced the summary. -Missing summary usage marks accounting incomplete; a compaction start without a persisted summary also leaves an evidence gap. -Work with no native telemetry cannot be reconstructed; an aborted request without telemetry prevents a complete total. -An all-zero native usage object is treated as unverified because Diffex initializes those fields before receiving provider usage. -Missing individual usage fields remain unknown; invalid values are evidence errors. - -`AgentContext.metadata.diffex_accounting` and `accounting.json` retain observed lower bounds, missing fields, and structural gaps. -When execution evidence is valid, totals retain the known usage across the root and all children even if individual requests lack telemetry. -Such totals are partial lower bounds, with `complete: false` in accounting and `accounting_complete: false` in ATIF final-metric metadata. -Fields with no known observations remain `null` in context totals and are omitted from ATIF totals. -Structural evidence gaps still invalidate totals and fail qualification. -`model_usage` contains per-model totals when accounting is complete; otherwise it is empty to prevent Harbor from backfilling incomplete token data as zero. -Per-model observations remain available in `observed_model_usage` even when accounting is partial. -`diffex_trajectory.validated` records successful export, and `accounting_complete` separately records accounting completeness. -Verifier reward is independent of accounting completeness. -The internal qualification gate accepts valid executions with partial accounting, records `usage_complete` and `usage_missing_fields`, and continues the stage. - -For leaderboard validation, require a validated trajectory and complete accounting for every rewarded trial, then upload the entire agent log tree including `trajectory.json` and `trajectory-media/`. -Validate the downloaded/uploaded artifact copy using the command below; a local export alone is not proof of remote upload. -The [release gate](RELEASE_GATE.md) documents staged execution, CI artifact verification, and the separate official submission requirements. - -```bash -uv run --locked python -m harbor.utils.trajectory_validator /path/to/trial/agent/trajectory.json -``` - -## Production exclusion - -This directory is outside every npm workspace and every production package's file allowlist. -The repository root is npm-private, and this Python package carries the `Private :: Do Not Upload` classifier. -Do not publish it to npm or PyPI. -Generated environments, Python caches, `results/`, and Harbor `jobs/` are ignored locally. - -## Updating pins - -Change the Harbor requirement and matching runtime pin together, and keep the Daytona preflight guard and DeepSWE Harbor/E2B guards aligned with the direct requirements. -Regenerate `uv.lock` with `uv lock --upgrade --prerelease if-necessary` and review the dependency changes. -Direct requirements must remain exact stable versions; beta-only OpenTelemetry transitives are required by Daytona. -`[tool.uv].prerelease = "if-necessary"` keeps normal locked syncs consistent with that resolution policy. -Hatchling is separately pinned to `1.32.4` in `[build-system]` and is not included in the runtime lock. -Do not add `uv run --with` overlays: Terminal-Bench and DeepSWE share the checked-in lock. -Update Node checksums from the selected release's [official SHASUMS256.txt](https://nodejs.org/dist/v22.23.2/SHASUMS256.txt). -For Diffex, query the exact release with `npm view @diffexai/diffex@VERSION version dist engines --json --registry=https://registry.npmjs.org/` and review the new version, engine requirement, and integrity. -Rerun the clean Linux installation checks after any pin change. diff --git a/evals/terminal-bench/RELEASE_GATE.md b/evals/terminal-bench/RELEASE_GATE.md deleted file mode 100644 index 2a50406d5..000000000 --- a/evals/terminal-bench/RELEASE_GATE.md +++ /dev/null @@ -1,400 +0,0 @@ -# Terminal-Bench release gate - -This gate qualifies a specific Diffex adapter, dataset, model, and harness configuration for a larger benchmark run. -It does not publish a product release or submit a leaderboard entry. -The implementation supports all qualification stages; live Diffex qualification requires provider credentials and a published release with environment-owned process support. -The checked-in Diffex 0.3.2 pin includes that support; installation validates the required capabilities and release integrity. - -## Configuration and prerequisites - -Use a clean committed checkout, Python 3.13, uv 0.11.2, Git, and a Daytona Cloud account. -Run all commands below from `evals/terminal-bench/`. -No Docker installation is needed for the Daytona runner. -Docker is required for the separate controlled Linux smoke tests. - -```bash -uv sync --locked -uv run --locked python -m unittest discover -s tests -v -uv run --locked python -m diffex_terminal_bench.benchmark --prepare -``` - -[benchmark.toml](benchmark.toml) pins Terminal-Bench 2.1 at Git commit `7131e4375048a0e408a8fb404b5f499d726b695b` from the [upstream repository](https://github.com/harbor-framework/terminal-bench-2-1/tree/7131e4375048a0e408a8fb404b5f499d726b695b). -The archive SHA-256 is `aa992a8848a1f3ed5191a34fa72ed5700d67dd9f968413a0e0a567b6188cb527`. -The runner verifies the archive before extraction, checks all 89 tasks under `tasks/`, and calculates Harbor content digests for every task. -Repository templates outside `tasks/` are excluded. -This source archive pin identifies internal qualification inputs; it does not claim equivalence to an independently versioned Harbor Hub dataset digest. -Each invocation extracts a fresh copy of the verified archive; it does not reuse editable extracted task directories or resolve a mutable dataset alias. -The preparation command writes `results/release-gate/dataset.json` without creating sandboxes or calling a model. - -Harbor 0.24.0 and Daytona SDK 0.226.1 are locked in the benchmark's private Python environment. -DeepSWE shares this lock with E2B 2.59.0; its launchers do not resolve dependency overlays. -Node 22.23.2, Diffex 0.3.2, npm integrity, registry, and Node archive checksums come from [pins.toml](src/diffex_terminal_bench/pins.toml). -The default is `openai/gpt-6-luna`, `xhigh` thinking, tools `read,bash,edit,write,update_plan`, disabled subagents, disabled evolution, and `context-only` project resources. -Enabled subagents are rejected because the pinned CLI does not propagate eval prompt and context-file flags to children. -Every new eval uses the checked-in system prompt without automatic project context and appends the anti-cheating instruction to the user message. -Change and commit the configuration before starting a new qualification sequence. -Every subsequent stage must match the exact recorded identity, including the adapter commit and dependency lock. -The earlier TB 2.0 / Diffex 0.2.2 oracle evidence cannot qualify this configuration. -Start a fresh sequence with `--output results/qualification-tb21` and use that output argument for every stage, recovery, and evidence verification command. - -Load `DAYTONA_API_KEY` through your normal secret-loading mechanism. -The key needs permission to create, inspect, list, and delete sandboxes. -The oracle uses only this key. -Diffex stages additionally require the selected model's credential, such as `OPENAI_API_KEY` for the default model. -A Daytona key does not authenticate OpenAI model requests. -Never put credential values in benchmark configuration or command-line arguments. -The runner records provider environment-variable references and lets Harbor resolve and scrub their values. - -Optional `DAYTONA_TARGET` and `DAYTONA_API_URL` settings are recorded in the run identity. -If omitted, Daytona selects the organization's default region. -The account needs sufficient sandbox resources and access to task dependencies, npm, Node downloads, apt mirrors, and the chosen model endpoint. -Task resource requests vary; the runner uses the dataset's resource limits without overrides. -Provision quota and network access in the account instead of changing benchmark task limits. -Record the created sandbox IDs, snapshots, and actual CPU, memory, and disk from `sandbox-created.json` when diagnosing backend differences. -The pinned dataset includes upstream image tags and network-installed dependencies; archive verification does not make those external services immutable. - -## Run five random tasks - -Use `--sample 5` for a standalone run of five distinct tasks selected uniformly from all 89 pinned tasks, with one trial per task and no prerequisite stages. -Use a clean committed checkout and load the Daytona and model credentials described above. -The command verifies the full dataset automatically, preserves task limits and the configured agent policy, and uses the same bounded concurrency, evidence collection, cleanup, and continue-on-trial-failure behavior as the staged runner. - -```zsh -TB_SAMPLE_OUTPUT="results/sample-tb21-$(date +%Y%m%d-%H%M%S)" -TB_SAMPLE_CONCURRENCY=1 - -uv run --locked python -m diffex_terminal_bench.benchmark \ - --sample 5 -n "$TB_SAMPLE_CONCURRENCY" --output "$TB_SAMPLE_OUTPUT" -``` - -The runner prints the seed and selected task names, then saves them in `$TB_SAMPLE_OUTPUT/sample/gate.json` under `identity.sample` before creating sandboxes. -Omitting `--seed` generates a new seed; add `--seed 42` to repeat a selection with the same dataset and pinned Python runtime. -A repeated selection still requires a fresh output directory and produces new model executions, which can have different outcomes. -The count can be any integer from 1 to 89; `-n` controls simultaneous trials, not the sample size. -Sample runs appear in the local results index and never qualify prerequisite stages. - -After `sample: passed`, verify using the same checkout, configuration, and concurrency: - -```zsh -uv run --locked python -m diffex_terminal_bench.benchmark \ - --verify-sample -n "$TB_SAMPLE_CONCURRENCY" --output "$TB_SAMPLE_OUTPUT" - -open "$TB_SAMPLE_OUTPUT/sample" -``` - -Verification reads the retained selection, reproduces it from the saved seed, and rechecks all selected trial evidence without creating sandboxes or calling a model. -It uses the shared verified dataset cache and may download the pinned archive if that cache is missing. -Export a passing sample as a Harbor job using the same exporter as qualification stages: - -```zsh -TB_SAMPLE_JOB="results/harbor-exports/$(basename "$TB_SAMPLE_OUTPUT")-sample" -uv run --locked python -m diffex_terminal_bench.export_job \ - "$TB_SAMPLE_OUTPUT/sample" --output "$TB_SAMPLE_JOB" -``` - -If trials fail, the remaining selected tasks still run before the command exits nonzero. -Skip `--verify-sample` for a failed run; it requires a passing sample and reports the retained failure without rerunning any tasks. -Use `--allow-partial` on the export command to retain failed terminal results for diagnosis; every started trial still needs its terminal result and lock. -Use `python -m diffex_terminal_bench.recover "$TB_SAMPLE_OUTPUT"` for sandbox recovery after an interruption. -The Harbor authentication and upload commands below also apply to the exported sample job. - -## Ordered stages - -### Refresh and restart with parallel trials - -The adapter runs from the local uv project, and a newly started command reads the current Python source and `benchmark.toml`. -An already running command keeps its loaded code and policy; let it finish or interrupt it once with Ctrl-C and wait for cleanup before restarting. -The environment-owned process policy uses the published Diffex release and integrity recorded in `pins.toml`. -Development source images exercise the lifecycle without qualifying the installed release. - -After updating the checkout, run the following in your Mac's zsh terminal with `DAYTONA_API_KEY` and `OPENAI_API_KEY` exported. -`git status --short` must produce no output. -Use a fresh output directory because the adapter commit, concurrency, and subagent policy must match across the entire sequence; old prerequisites cannot qualify the new configuration. - -```zsh -cd /Users/jialeyu/Projects/diffex-new/evals/terminal-bench -git status --short -uv sync --locked - -TB_OUTPUT="results/qualification-tb21-$(date +%Y%m%d-%H%M%S)" - -# Step 3: verify all dataset inputs. -uv run --locked python -m diffex_terminal_bench.benchmark \ - --prepare -n 3 --output "$TB_OUTPUT" - -# Step 4: oracle, single trial, then three concurrent integration trials. -uv run --locked python -m diffex_terminal_bench.benchmark \ - --through integration -n 3 --output "$TB_OUTPUT" - -# Step 5: after step 4 passes, ten total trials with at most three active. -uv run --locked python -m diffex_terminal_bench.benchmark \ - --stage pilot -n 3 --output "$TB_OUTPUT" - -# Step 6: verify using the same checkout, output, and concurrency. -uv run --locked python -m diffex_terminal_bench.benchmark \ - --verify-through pilot -n 3 --output "$TB_OUTPUT" -open "$TB_OUTPUT/pilot" -``` - -Keep the same terminal or restore the exact `TB_OUTPUT` value when continuing later. -Do not change the checkout, configuration, or concurrency between steps 4-6. -The recorded `gate.json` identity shows `concurrency: 3` and `policy.subagents: disabled`. -Each Diffex run's `requested.json` and `effective.json` show only the configured base tools; native sessions and `trajectory.json` show actual calls. - -### Shared dataset cache and run index - -All runs share content-addressed archives in `results/dataset-cache/` by default, including runs with a custom `--output` directory. -Use `--dataset-cache /path/to/cache` to choose another shared cache. -Each use verifies the pinned SHA-256 and extracts independent temporary task files. -Concurrent preparation uses a per-digest file lock and atomic publication so another run cannot read a partial download. -Locks cover preparation only; benchmark execution proceeds independently. -Existing per-run caches remain untouched and can be deleted when no older benchmark process is using them. - -The runner updates `index.md` in the output directory's parent after preparation, stage start, each completed trial, and stage completion or failure. -It links each qualification run and its stage reports, showing the model, thinking level, dataset, recorded gate status, completed-trial counts, checked-trial counts, failure counts, and full-reward counts. -Smoke-test directories are excluded because they use separate report formats. -The index summarizes recorded reports and does not replace `--verify-through` evidence verification. -An unfinished stage may be running or may have stopped without writing a final report. -Neither caching nor index generation adds model calls. -Rebuild the index from existing local reports without credentials, dataset downloads, or a clean checkout: - -```bash -uv run --locked python -m diffex_terminal_bench.benchmark --refresh-index --results-dir results -open results/index.md -``` - -### Stage sequence - -| Stage | Trials | Required evidence | -| --- | ---: | --- | -| `oracle` | 1 | Official solution earns full reward; logs and sandbox deletion retained | -| `trial` | 1 | Diffex installs and completes the same task with validated ATIF and reconciled usage | -| `integration` | 3 | Git repair, certificate/file work, and log processing with the same configuration | -| `pilot` | 10 | All ten declared tasks complete the integration checks | -| `reliability` | 30 | Three additional passes over the ten tasks with fresh state and no unexplained infrastructure failures | -| `full` | 445 | Five attempts per task across all 89 tasks, after all prerequisites pass | - -The pilot task order is declared before observing model results. -The first trial uses TB 2.1's updated `fix-git` task, followed by `openssl-selfsigned-cert` and `log-summary-date-ranges` for integration. -The remaining pilot tasks are `nginx-request-logging`, `sanitize-git-repo`, `db-wal-recovery`, `build-cython-ext`, `multi-source-data-merger`, `polyglot-c-py`, and `sqlite-db-truncate`, all from the same verified 2.1 archive. -It covers certificates, text processing, Git, service configuration, database recovery, compilation, and data manipulation. -The runner defaults to one concurrent trial and accepts `-n 3` (or `--concurrency 3`) for up to three simultaneous trials. -The limit includes sandbox creation, installation, execution, verification, and the bounded cleanup attempt; a replacement starts only after that slot finishes. -Set `concurrency` in `benchmark.toml` to change the checked-in default. -Use the same concurrency for preparation, every stage, and `--verify-through`, because it is part of the qualification identity. -Stages remain sequential, so the one-task oracle and trial stages each create only one sandbox even with `-n 3`. -There are no Harbor retries. -Task execution and verifier timeouts remain unchanged. -Only Diffex installation receives the existing 630-second setup allowance. - -Run the stages separately to inspect evidence between them: - -```bash -uv run --locked python -m diffex_terminal_bench.benchmark --stage oracle -uv run --locked python -m diffex_terminal_bench.benchmark --stage trial -uv run --locked python -m diffex_terminal_bench.benchmark --stage integration -uv run --locked python -m diffex_terminal_bench.benchmark --stage pilot -uv run --locked python -m diffex_terminal_bench.benchmark --stage reliability -``` - -Alternatively, `--through reliability` runs the first five stages in order and stops on the first failed gate. -Every selected task in that stage is attempted before reporting the failed gate. -Each stage requires a new output directory and refuses to overwrite an earlier attempt. -Use `--output results/qualification-NAME` to start a new sequence after investigating a failure or changing configuration. -Keep failed sequences for diagnosis; do not delete failed trials or edit a report to qualify a stage. -The full run is explicit and is never started by `--through`: - -```bash -uv run --locked python -m diffex_terminal_bench.benchmark --stage full -``` - -Before full execution, run the repository's `npm run check`, controlled Linux smokes, and production-content inspection documented in [README.md](README.md#local-validation), or require the corresponding benchmark CI jobs. -Every previous stage must still have its original artifacts and matching identity. -The gate hashes and revalidates their evidence before allowing progression. - -## Evidence and failure policy - -### Diagnose a stopped trial - -Inspect a retained trial without credentials, downloads, a clean checkout, or another evaluation: - -```zsh -uv run --locked python -m diffex_terminal_bench.benchmark \ - --diagnose-trial "$TB_OUTPUT/pilot/007-build-cython-ext" -``` - -The command reports the Harbor exception, a recorded agent deadline, scheduled retry counts, and observed rate-limit retries without printing raw provider errors. -New live-stage failures also record these diagnostics in `gate.json` and print the evidence directory instead of a generic traceback. -Scheduled backoff is the sum of requested retry delays, not a measurement of all time spent waiting on the provider. - -The retained `qualification-tb21-gpt55-xhigh-20260915-132556` pilot stopped on `build-cython-ext` at its 900-second agent deadline after 68 rate-limit retry events and 210 seconds of scheduled backoff. -Its sandbox cleanup succeeded and the verifier awarded full reward, but timed-out execution still fails qualification. -That run already used concurrency 1; lowering concurrency alone cannot resolve a single trial's provider limit. -Ensure enough available provider token capacity, including other traffic using the same limits, before starting a fresh qualification sequence. -Use `-n 1` throughout that sequence while capacity remains constrained; task deadlines and resource limits stay unchanged. - -### Retained evidence - -Each stage writes `gate.json` before starting remote work and updates it after every completed trial, including failures. -The report separates verified `trials` from `failures`, records `completed_trials` and `expected_trials`, and keeps `failure` as the first observed failure. -Failure entries identify the task, trial index, evidence directory, error type, and phase; a cleanup error following an execution error is retained in `additional_errors`. -It records dataset/archive and per-task digests, adapter commit, Harbor/Daytona/Python versions, dependency lock hash, installation pins, model, thinking, tools, subagent policy, and hashes of adapter/harness files and the eval system prompt. -Each Diffex trial additionally retains the effective system prompt and its SHA-256. -The effective system prompt must exactly match the checked-in eval prompt across isolated trials. -The user-message suffix, disabled automatic context files, and harness source identity are also verified. - -Trial directories contain Harbor `config.json`, `lock.json`, `result.json`, verifier reward and combined stdout/stderr, agent logs, native sessions, and sandbox lifecycle evidence. -Diffex installation evidence includes the three CLI checks and verified npm/Node pins. -The oracle does not install Diffex or call a model, so its Diffex installation, model usage, and ATIF checks are not applicable. - -Evidence inventories preserve relative symlinks that point directly to regular files within the same trial, including Hugging Face cache snapshot links. -Link targets have a separate `symlink:` fingerprint; regular-file content hashes still detect changes to the referenced files. -Absolute, escaping, dangling, chained, and directory symlinks are rejected, including paths that leave the trial and then reenter it. -Exports preserve accepted links and validate containment even for failed trials that have no checked inventory. -This evidence policy runs in the local Python eval harness and requires no new Diffex npm release. -Start subsequent runs from the committed checkout with a fresh output directory; previously recorded failures remain unchanged. - -The gate re-parses native process/stream evidence instead of trusting the outcome summary alone. -The execution policy includes `background_process_lifetime: environment` in the requested/effective configuration and benchmark identity. -Pre-verifier evidence must establish that all agent execution stopped, with every retained task service backed by a durable environment ownership record. -The gate reconciles control identities, ownership files, cleanup reports, outcome summaries, and trajectory metadata. -Retained services do not count as cleanup failures, and verifier reward cannot convert a deadline or provider failure into completed execution. -An unverified control stop raises `DiffexControlStopError` and prevents Harbor from entering verification. -Final cleanup remains Harbor's environment stop followed by independent sandbox-deletion confirmation. -It regenerates ATIF into a temporary directory, validates it with Harbor, and compares it with retained trajectories and image attachments. -Native totals, per-model totals, `accounting.json`, and `AgentContext` must reconcile. -Unknown telemetry remains unknown and cannot satisfy complete accounting. -Valid executions with missing usage telemetry retain their known usage sums as partial totals and continue qualification. -Each checked trial records `usage_complete` and `usage_missing_fields` alongside its usage and verifier reward. -An intentionally interrupted child is valid when its parent roster records `interrupted` and a successful parent interrupt call establishes intent. -An aborted request without usage telemetry makes accounting partial; it does not invalidate the completed parent trial. -A child whose final parent roster records `failed` with a nonempty error is a terminal failure that the parent can recover from. -The exported child trajectory preserves its failed status and error, while missing telemetry remains partial and missing or corrupt evidence still fails validation. -This local adapter policy requires no new Diffex release; commit the change and use a fresh output directory for subsequent staged runs. -Verifier reward must agree with the retained reward file. -A completed Diffex trial with valid evidence and reward zero passes integration checks and remains a zero-reward benchmark result. -An oracle with reward zero fails the smoke. - -Timeouts, unrecovered root provider or process errors, unexplained child interruptions, invalid usage values, missing logs, invalid ATIF, altered configuration, and cleanup failures retain their original evidence and mark the trial as failed. -The runner continues active peers and queued tasks after these failures, in both serial and concurrent stages. -The runner does not convert them to successful executions or silently rerun them. -After every selected task finishes, a stage with any failed trials reports failed qualification and exits nonzero with an evidence summary instead of a trial-error traceback. -Later prerequisite-dependent stages remain blocked by that failed gate. -A legitimate task timeout remains a recorded task result and must not be described as an installation failure. -No failed or interrupted run establishes readiness merely because its process exited zero. - -The backend records sandbox IDs before Harbor tears them down. -Cleanup independently checks known IDs and lists all sandboxes with the trial's unique label, allowing bounded time for asynchronous deletion. -Only a Daytona not-found response establishes absence; authentication or transport errors do not. -An orphan is deleted when possible, but needing recovery still fails qualification. -SIGTERM and explicit stage cancellation stop queued work, cancel active peers, and preserve cleanup and reports. -Trial failures do not cancel peers; every started trial finishes its bounded cleanup attempt before releasing its slot or closing the clients. -Stage-wide failures such as an inability to persist `gate.json` can still stop execution. -Verified trial entries and failure entries stay in declared task order even when they finish out of order. -After a host crash, forced kill, or interrupted CI job, recheck only the sandboxes belonging to the retained sequence: - -```bash -uv run --locked python -m diffex_terminal_bench.recover results/release-gate -``` - -Recovery records `sandbox-recovery.json` and preserves the original cleanup report. -It does not turn a failed sequence into a passing one. -It requires the original output directory and access to the same Daytona account. - -## Export retained trials for Harbor Hub - -The staged runner writes individual Harbor trials and `gate.json`, so its stage directories cannot be passed directly to `harbor upload`. -Export a stage into a new job directory first: - -```zsh -cd /Users/jialeyu/Projects/diffex-new/evals/terminal-bench -uv sync --locked - -TB_JOB="results/harbor-exports/pilot-$(date +%Y%m%d-%H%M%S)" -uv run --locked python -m diffex_terminal_bench.export_job \ - "$TB_OUTPUT/pilot" --output "$TB_JOB" -``` - -The default requires a stopped stage with recorded passing status and complete trial coverage. -To share a failed or incomplete stopped stage for diagnosis, explicitly include every retained terminal result: - -```zsh -uv run --locked python -m diffex_terminal_bench.export_job \ - "$TB_OUTPUT/pilot" --output "$TB_JOB" --allow-partial -``` - -Use one of those export commands, not both with the same destination. -The exporter refuses to overwrite an existing job, copy a running stage, omit a started trial without a terminal result/lock, or accept changed evidence for a checked trial. -It works offline with old recorded configurations and does not require the original clean checkout, provider credentials, or temporary dataset extraction. -It neither reruns trials nor changes source results. - -The job contains Harbor `config.json`, `result.json`, and `lock.json`, plus independent copies of all trial files. -Rewards, exception records, trial IDs, native sessions, ATIF trajectories, and media remain unchanged. -Job totals retain the planned trial count and separately count terminal results, errors, and unrun tasks. -Unknown usage in failed trials remains unknown in those trials; job usage is an observed sum, not a claim of complete accounting. -`analysis.md` explains whether the source gate passed, and `export.json` records source hashes and missing trials. -This packaging check does not requalify the source stage or establish leaderboard eligibility. -The exported configuration retains original task paths for provenance; use the custom staged runner for future evaluations, not `harbor job resume` on an export. - -Harbor's upload archive filters omit custom files at the trial root. -The exporter also places sandbox lifecycle evidence under each trial's `artifacts/diffex-qualification/`, with the source gate and export manifest under the first trial's artifact directory, so those files survive Hub downloads. -Hidden child-session directories remain under `agent/` and are included in Harbor's archives. -Repeated exports of the same source snapshot have a stable job ID; retry an upload from the same exported directory. - -Authenticate and upload the exported job privately: - -```zsh -uv run --locked harbor auth login -uv run --locked harbor upload "$TB_JOB" --private -``` - -To publish it publicly instead, use: - -```zsh -uv run --locked harbor upload "$TB_JOB" --public -``` - -Harbor prints the Hub viewing link after upload. -The export command does not upload anything or change visibility. -See the [Harbor job sharing documentation](https://www.harborframework.com/docs/sharing/jobs) for visibility and download options. - -## CI and remote evidence - -[terminal-bench.yml](../../.github/workflows/terminal-bench.yml) is separate from product CI. -Pull requests and benchmark branch pushes run deterministic adapter/converter/gate tests, dataset verification, controlled Linux execution/outcome smokes, and production-tarball exclusion checks. -These jobs require no Daytona or model credentials. - -For live runs, configure the `terminal-bench` GitHub environment with `DAYTONA_API_KEY` and `OPENAI_API_KEY` secrets for the checked-in default model. -Dispatch the workflow on the committed benchmark branch and select the last qualification stage to run. -If selecting another provider, update the checked-in configuration and workflow's credential environment together. -Live stages never run automatically on pull requests. -Full benchmark execution and official submission are not workflow-dispatch options. -The live job leaves time for cleanup and artifact upload before its six-hour GitHub job limit; use the local stage commands if a sequence needs longer. - -CI retains complete evidence, including hidden child-session directories, even when a stage fails. -Successful uploads record the artifact URL and archive SHA-256 in the job summary. -Successful live runs download the uploaded archive and revalidate all stage evidence, including trajectories and usage. -To check a downloaded artifact yourself, use the same clean adapter commit and configuration: - -```bash -uv run --locked python -m diffex_terminal_bench.benchmark \ - --verify-through reliability --output /path/to/downloaded-evidence -``` - -This command verifies dataset inputs and artifacts without Daytona or model calls. -Local validation is not an upload receipt. -Retain the CI artifact URL/digest or equivalent durable storage receipt with any future submission. - -## Official submission gate - -Rules and availability were checked on 2026-09-15. -The [official Terminal-Bench 2.1 repository](https://github.com/harbor-framework/terminal-bench-2-1#submitting-to-the-leaderboard) currently accepts only maintainer-run submissions; community submissions are closed. -Its published submission instructions require at least five trials per task and public Harbor Hub uploads using `--upload --public`. -Official runs must use the required Hub dataset revision and preserve task timeouts and resources. -The [leaderboard integrity policy](https://www.tbench.ai/news/leaderboard-integrity-update) requires ATIF for passing trials and prohibits cheating and reward hacking. - -Treat this integration's outputs as internal qualification evidence until the applicable submission process is open and its current requirements have been reviewed again. -Before planning an official submission, record the date, source links, availability, required dataset version, repeat count, limits, and artifact-upload requirements in a new review record. -Inspect passing trajectories for prohibited task-specific assistance, solution access, and verifier leakage. -Require remote evidence receipts and validation of every rewarded trial's uploaded ATIF, including child trajectories and media. -Do not infer official eligibility from a passing internal gate or an npm publication. diff --git a/evals/terminal-bench/benchmark.toml b/evals/terminal-bench/benchmark.toml deleted file mode 100644 index c4919e68b..000000000 --- a/evals/terminal-bench/benchmark.toml +++ /dev/null @@ -1,33 +0,0 @@ -# Integration qualification uses the pinned Terminal-Bench 2.1 tasks/ tree. -# Review this file before spending provider or Daytona credits. -model = "openai/gpt-6-luna" -# Maximum simultaneous trials, including cleanup; override with -n/--concurrency. -concurrency = 1 -pilot_tasks = [ - "fix-git", - "openssl-selfsigned-cert", - "log-summary-date-ranges", - "nginx-request-logging", - "sanitize-git-repo", - "db-wal-recovery", - "build-cython-ext", - "multi-source-data-merger", - "polyglot-c-py", - "sqlite-db-truncate", -] - -[dataset] -name = "terminal-bench" -version = "2.1" -git_url = "https://github.com/harbor-framework/terminal-bench-2-1.git" -git_commit = "7131e4375048a0e408a8fb404b5f499d726b695b" -archive_url = "https://codeload.github.com/harbor-framework/terminal-bench-2-1/tar.gz/7131e4375048a0e408a8fb404b5f499d726b695b" -archive_sha256 = "aa992a8848a1f3ed5191a34fa72ed5700d67dd9f968413a0e0a567b6188cb527" -task_count = 89 - -[policy] -thinking = "xhigh" -tools = "read,bash,edit,write,update_plan" -subagents = "disabled" -project_resources = "context-only" -background_process_lifetime = "environment" diff --git a/evals/terminal-bench/execution.yaml b/evals/terminal-bench/execution.yaml deleted file mode 100644 index b8bce5479..000000000 --- a/evals/terminal-bench/execution.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Select an exact provider/model below and supply --path when invoking Harbor. -n_concurrent_trials: 1 -n_attempts: 1 -environment: - type: docker - delete: true -agents: - - import_path: diffex_terminal_bench.adapter:DiffexAgent - model_name: openai/gpt-5 - override_setup_timeout_sec: 630 - kwargs: - thinking: high - tools: read,bash,edit,write,update_plan - subagents: disabled - project_resources: context-only - background_process_lifetime: environment diff --git a/evals/terminal-bench/full.yaml b/evals/terminal-bench/full.yaml deleted file mode 100644 index 4eba260e2..000000000 --- a/evals/terminal-bench/full.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Native full evaluation. This run intentionally overrides task disk limits. -n_attempts: 5 -n_concurrent_trials: 3 -retry: - max_retries: 0 -environment: - import_path: diffex_terminal_bench.daytona_backend:RecordedDaytonaEnvironment - kwargs: - artifact_volume: diffex-terminal-bench-logs - override_storage_mb: 10240 -agents: - - name: diffex_terminal_bench.adapter:DiffexAgent - model_name: openai/gpt-6-luna - kwargs: - thinking: xhigh - tools: read,bash,edit,write,update_plan - # Child agents in the pinned CLI do not inherit eval prompt flags. - subagents: disabled - project_resources: context-only - background_process_lifetime: environment - env: - OPENAI_API_KEY: ${OPENAI_API_KEY} -datasets: - - name: terminal-bench/terminal-bench-2-1 - ref: sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a diff --git a/evals/terminal-bench/pyproject.toml b/evals/terminal-bench/pyproject.toml deleted file mode 100644 index ab21028a7..000000000 --- a/evals/terminal-bench/pyproject.toml +++ /dev/null @@ -1,26 +0,0 @@ -[project] -name = "diffex-terminal-bench" -version = "0.1.0" -description = "Private Harbor integration for Diffex Terminal-Bench evaluations" -requires-python = ">=3.13,<3.14" -dependencies = ["harbor[daytona,e2b]==0.24.0", "daytona==0.226.1", "e2b==2.59.0", "pydantic==2.14.1", "pyyaml==6.0.3"] -classifiers = ["Private :: Do Not Upload"] - -[build-system] -requires = ["hatchling==1.32.4"] -build-backend = "hatchling.build" - -[tool.hatch.build.targets.wheel] -packages = ["src/diffex_terminal_bench"] - -[tool.uv] -default-groups = [] -prerelease = "if-necessary" -# LiteLLM 1.106.4 caps filelock below 4, but its FileLock calls remain compatible. -# 4.2.1 fixes GHSA-v4x8-3wr5-3wq8, GHSA-9vhv-jxww-xfm5, and GHSA-567q-5qvw-pf92. -override-dependencies = ["filelock==4.2.1"] - -[[tool.uv.index]] -name = "pypi" -url = "https://pypi.org/simple" -default = true diff --git a/evals/terminal-bench/scripts/check_production_contents.py b/evals/terminal-bench/scripts/check_production_contents.py deleted file mode 100644 index 6b61b51ff..000000000 --- a/evals/terminal-bench/scripts/check_production_contents.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Pack public npm workspaces without lifecycle scripts and inspect their bytes.""" - -import argparse -import hashlib -import json -from pathlib import Path, PurePosixPath -import subprocess -import tarfile -import tempfile - - -def inspect_packages(root: Path) -> list[dict[str, object]]: - root_manifest = json.loads((root / "package.json").read_text()) - if root_manifest.get("private") is not True: - raise RuntimeError("The repository root must remain npm-private") - manifests = sorted({ - manifest - for pattern in root_manifest["workspaces"] - for directory in root.glob(pattern) - if (manifest := directory / "package.json").is_file() - }) - forbidden = { - "evals", "terminal-bench", "diffex_terminal_bench", ".venv", - "__pycache__", "uv.lock", "jobs", "results", "harbor", - } - reports = [] - with tempfile.TemporaryDirectory(prefix="diffex-production-packs-") as temp: - for manifest_path in manifests: - manifest = json.loads(manifest_path.read_text()) - if manifest.get("private") is True: - continue - if "evals" in manifest_path.relative_to(root).parts: - raise RuntimeError(f"Benchmark became a public workspace: {manifest_path}") - packed = json.loads(subprocess.check_output( - ["npm", "pack", "--ignore-scripts", "--json", "--pack-destination", temp, - "--cache", str(Path(temp) / "npm-cache")], - cwd=manifest_path.parent, text=True, - ))[0] - archive = Path(temp) / packed["filename"] - with tarfile.open(archive) as tar: - members = tar.getmembers() - for member in members: - parts = PurePosixPath(member.name).parts - if forbidden.intersection(parts): - raise RuntimeError(f"Benchmark content in {manifest['name']}: {member.name}") - if manifest["name"] == "@diffexai/diffex": - if "package/dist/cli.js" not in {member.name for member in members}: - raise RuntimeError("CLI dist/cli.js is absent; inspect a prepared checkout") - reports.append({ - "name": manifest["name"], - "version": manifest["version"], - "files": len(members), - "sha256": hashlib.sha256(archive.read_bytes()).hexdigest(), - "benchmark_content_found": False, - }) - if not reports: - raise RuntimeError("No public production packages were inspected") - return reports - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--output", type=Path, default=Path("results/production-contents.json")) - args = parser.parse_args() - repo_root = Path(__file__).resolve().parents[3] - result = inspect_packages(repo_root) - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(json.dumps(result, indent=2) + "\n") - print(f"Inspected {len(result)} production tarballs; no benchmark content: {args.output}") diff --git a/evals/terminal-bench/scripts/smoke_execution.py b/evals/terminal-bench/scripts/smoke_execution.py deleted file mode 100644 index 6f1032662..000000000 --- a/evals/terminal-bench/scripts/smoke_execution.py +++ /dev/null @@ -1,172 +0,0 @@ -"""Run the pinned CLI through Harbor against a loopback faux provider.""" - -import argparse -import asyncio -import hashlib -import json -from pathlib import Path -import shlex -import subprocess -import tempfile -import tomllib -import uuid - -from harbor.environments.docker.docker import DockerEnvironment -from harbor.models.agent.context import AgentContext -from harbor.models.task.config import EnvironmentConfig -from harbor.models.trial.paths import TrialPaths -from harbor.utils.trajectory_validator import TrajectoryValidator - -from diffex_terminal_bench.adapter import DiffexAgent -from diffex_terminal_bench.config import PINS -from diffex_terminal_bench.execution import SYSTEM_PROMPT, SYSTEM_PROMPT_SHA256, USER_MESSAGE_SUFFIX - - -async def smoke(platform: str) -> None: - config = tomllib.loads(Path("smoke.toml").read_text()) - session_id = f"diffex-execution-{uuid.uuid4().hex[:12]}" - trial = TrialPaths(Path("results", session_id).resolve()) - trial.mkdir() - report = {"platform": platform, "image": config["images"][platform], "pins": PINS, - "passed": False, "cleanup_passed": False, "trajectories": []} - print(f"Execution evidence: {trial.trial_dir}", flush=True) - with tempfile.TemporaryDirectory(prefix="diffex-execution-") as temp: - directory = Path(temp) - overlay = directory / "platform.yaml" - overlay.write_text(f"services:\n main:\n platform: {platform}\n") - environment = DockerEnvironment( - environment_dir=directory, environment_name="diffex-execution-smoke", session_id=session_id, - trial_paths=trial, task_env_config=EnvironmentConfig(docker_image=config["images"][platform]), - extra_docker_compose=[overlay], - mounts=[{"type": "bind", "source": str(trial.agent_dir), "target": "/logs/agent"}], - ) - try: - async with asyncio.timeout(config["environment_timeout_sec"]): - await environment.start(force_build=False) - agent = DiffexAgent(logs_dir=trial.agent_dir, model_name="openai/gpt-5", thinking="high", - extra_env={"OPENAI_API_KEY": "fixture-secret-never-log"}) - async with asyncio.timeout(PINS["setup_timeout_sec"] + 15): - await agent.setup(environment) - await environment.upload_file(Path("tests/faux-provider.cjs"), "/tmp/faux-provider.cjs") - wrapper = directory / "diffex" - wrapper.write_text( - "#!/bin/sh\n" - "/opt/diffex-terminal-bench/node/bin/node /tmp/faux-provider.cjs prepare\n" - "exec /usr/local/bin/diffex-release \"$@\"\n" - ) - await agent.exec_as_root(environment, command="mv /usr/local/bin/diffex /usr/local/bin/diffex-release") - await environment.upload_file(wrapper, "/usr/local/bin/diffex") - await agent.exec_as_root(environment, command="chmod 755 /usr/local/bin/diffex && mkdir -p /task/work/.diffex/extensions") - # Automatic context and executable project resources must not - # alter the eval prompt, even when the task supplies those files. - for name, content in { - "AGENTS.md": "Task context marker: DO_NOT_LOAD_TASK_CONTEXT\n", - ".diffex/settings.json": '{"evolution":{"enabled":true},"defaultThinkingLevel":"off"}', - ".diffex/SYSTEM.md": "DO_NOT_LOAD_PROJECT_SYSTEM", - ".diffex/extensions/trap.mjs": 'throw new Error("DO_NOT_LOAD_PROJECT_EXTENSION");', - }.items(): - source = directory / "resource" - source.write_text(content) - await environment.upload_file(source, f"/task/work/{name}") - environment.task_env_config.workdir = "/task/work" - await agent.exec_as_root(environment, command="nohup /opt/diffex-terminal-bench/node/bin/node /tmp/faux-provider.cjs >/tmp/faux-provider.log 2>&1 0 - state = json.loads((run_dir / "state.json").read_text()) - assert state["state_dir"].startswith("/tmp/diffex-tb-state-") - assert not (run_dir / "state").exists() - events = [json.loads(line) for line in (run_dir / "stdout.jsonl").read_text().splitlines()] - assert not any(event.get("isError") for event in events) - assert len(list((run_dir / "sessions").rglob("*.jsonl"))) == 1 - validator = TrajectoryValidator() - assert validator.validate(trial.agent_dir / "trajectory.json"), validator.get_errors() - trajectory = json.loads((trial.agent_dir / "trajectory.json").read_text()) - assert not trajectory.get("subagent_trajectories") - messages = [record["message"] for path in (run_dir / "sessions").rglob("*.jsonl") - for line in path.read_text().splitlines() if (record := json.loads(line)).get("type") == "message" - and record["message"].get("role") == "assistant"] - assert context.metadata["diffex_accounting"]["complete"], context.metadata["diffex_accounting"] - assert context.n_input_tokens == len(messages) * 19 - assert context.n_output_tokens == len(messages) * 7 - assert context.n_cache_tokens == len(messages) * 5 - native_cost = sum(message["usage"]["cost"]["total"] for message in messages) - assert abs(context.cost_usd - native_cost) < 1e-12 and native_cost > 0 - report["trajectories"].append({"run": run_dir.name, "validated": True, "context": context.model_dump()}) - # Contaminate the previous trial after it completes, proving - # the next run never inherits auth, sessions, or harness state. - await current.exec_as_agent(environment, command=shlex.join([ - "/opt/diffex-terminal-bench/node/bin/node", "/tmp/faux-provider.cjs", "contaminate", state["state_dir"], - ])) - await agent.exec_as_root(environment, command="test ! -e /task/work/INJECTED && test \"$(cat cwd.txt)\" = /task/work && test \"$(cat shell.txt)\" = shell-ok && test \"$(cat result.txt)\" = file-ok") - requests = [json.loads(line) for line in (trial.agent_dir / "fixture-requests.jsonl").read_text().splitlines()] - assert requests and all(entry["credential_received"] for entry in requests) - assert all(entry["body"]["reasoning_effort"] == "high" for entry in requests) - for entry in requests: - system_messages = [message["content"] for message in entry["body"]["messages"] - if message.get("role") in ("system", "developer")] - assert system_messages - for content in system_messages: - text = content if isinstance(content, str) else "".join(block["text"] for block in content) - assert text == SYSTEM_PROMPT - for excluded in ("", "\n{instruction}\n\n{USER_MESSAGE_SUFFIX}' - assert any(message.get("role") == "user" and message.get("content") in (expected_prompt, [{"type": "text", "text": expected_prompt}]) - for entry in requests for message in entry["body"]["messages"]) - assert not any(USER_MESSAGE_SUFFIX.strip() in str(message.get("content", "")) - for entry in requests for message in entry["body"]["messages"] - if message.get("role") in ("system", "developer")) - for run_dir in run_dirs: - assert "STALE_HARNESS_SELECTION" not in (run_dir / "system-prompt.txt").read_text() - for path in trial.agent_dir.rglob("*"): - if path.is_file(): - contents = path.read_bytes() - assert b"fixture-secret-never-log" not in contents, path - assert b"unrelated-fixture-secret" not in contents, path - report.update(passed=True, trials=len(run_dirs), provider_requests=len(requests)) - finally: - await environment.stop(delete=True) - remaining = subprocess.check_output(["docker", "ps", "-aq", "--filter", f"label=com.docker.compose.project={session_id}"], text=True).strip() - report["cleanup_passed"] = not remaining - (trial.trial_dir / "smoke.json").write_text(json.dumps(report, indent=2) + "\n") - if remaining: - raise RuntimeError("Execution smoke container was not removed") - print(json.dumps(report, indent=2)) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--platform", choices=("linux/amd64", "linux/arm64"), default="linux/amd64") - args = parser.parse_args() - asyncio.run(smoke(args.platform)) diff --git a/evals/terminal-bench/scripts/smoke_install.py b/evals/terminal-bench/scripts/smoke_install.py deleted file mode 100644 index 09515fcd0..000000000 --- a/evals/terminal-bench/scripts/smoke_install.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Exercise the real Harbor setup contract in a clean Linux container.""" - -import argparse -import asyncio -import json -from pathlib import Path -import subprocess -import tempfile -import time -import tomllib -import uuid - -from harbor.environments.docker.docker import DockerEnvironment -from harbor.models.task.config import EnvironmentConfig -from harbor.models.trial.paths import TrialPaths - -from diffex_terminal_bench.adapter import DiffexAgent -from diffex_terminal_bench.config import PINS - - -async def smoke(config_path: Path, output: Path, platform: str | None) -> None: - config = tomllib.loads(config_path.read_text()) - selected_platform = platform or config["platform"] - if selected_platform not in ("linux/amd64", "linux/arm64"): - raise ValueError("Smoke platform must be linux/amd64 or linux/arm64") - session_id = f"diffex-install-{uuid.uuid4().hex[:12]}" - trial = TrialPaths(output.resolve() / session_id) - trial.mkdir() - report = { - "session_id": session_id, - "image": config["images"][selected_platform], - "platform": selected_platform, - "pins": PINS, - "setup_passed": False, - "cleanup_passed": False, - } - print(f"Installation evidence: {trial.trial_dir}", flush=True) - with tempfile.TemporaryDirectory(prefix="diffex-smoke-environment-") as temp: - environment_dir = Path(temp) - overlay = environment_dir / "platform.yaml" - overlay.write_text(f"services:\n main:\n platform: {selected_platform}\n") - environment = DockerEnvironment( - environment_dir=environment_dir, - environment_name="diffex-install-smoke", - session_id=session_id, - trial_paths=trial, - task_env_config=EnvironmentConfig(docker_image=config["images"][selected_platform]), - extra_docker_compose=[overlay], - mounts=[{ - "type": "bind", "source": str(trial.agent_dir), "target": "/logs/agent", - }], - ) - agent = DiffexAgent(logs_dir=trial.agent_dir) - try: - async with asyncio.timeout(config["environment_timeout_sec"]): - await environment.start(force_build=False) - # Inspect actual mounts, not just the requested Harbor config. - containers = subprocess.check_output( - ["docker", "ps", "-aq", "--filter", f"label=com.docker.compose.project={session_id}"], - text=True, - ).split() - if len(containers) != 1: - raise RuntimeError(f"Expected one smoke container, found {containers}") - inspection = json.loads(subprocess.check_output( - ["docker", "inspect", containers[0]], text=True - ))[0] - report["container_image_id"] = inspection["Image"] - report["mounts"] = inspection["Mounts"] - if len(inspection["Mounts"]) != 1: - raise RuntimeError("Expected only the agent log directory to be mounted") - for mount in inspection["Mounts"]: - if mount["Type"] != "bind" or not Path(mount["Source"]).is_relative_to(trial.trial_dir): - raise RuntimeError(f"Unexpected smoke mount: {mount}") - started = time.monotonic() - async with asyncio.timeout(PINS["setup_timeout_sec"] + 15): - await agent.setup(environment) - report["setup_wall_time_sec"] = round(time.monotonic() - started, 3) - evidence = json.loads((trial.agent_dir / "setup/installation.json").read_text()) - if not evidence["verified"]: - raise RuntimeError("Installer did not produce verified evidence") - expected_arch = "x64" if selected_platform == "linux/amd64" else "arm64" - if evidence["node_arch"] != expected_arch: - raise RuntimeError("Installed Node architecture differs from the smoke platform") - if report["setup_wall_time_sec"] >= PINS["setup_timeout_sec"]: - raise RuntimeError("Harbor setup exceeded the configured deadline") - report["setup_passed"] = True - except Exception as error: - report["error_type"] = type(error).__name__ - raise - finally: - await environment.stop(delete=True) - remaining = subprocess.check_output( - ["docker", "ps", "-aq", "--filter", f"label=com.docker.compose.project={session_id}"], - text=True, - ).strip() - report["cleanup_passed"] = not remaining - (trial.trial_dir / "smoke.json").write_text(json.dumps(report, indent=2) + "\n") - if remaining: - raise RuntimeError(f"Smoke containers were not removed: {remaining}") - print(json.dumps(report, indent=2)) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--config", type=Path, default=Path("smoke.toml")) - parser.add_argument("--output", type=Path, default=Path("results")) - parser.add_argument("--platform", choices=("linux/amd64", "linux/arm64")) - args = parser.parse_args() - asyncio.run(smoke(args.config, args.output, args.platform)) diff --git a/evals/terminal-bench/scripts/smoke_outcomes.py b/evals/terminal-bench/scripts/smoke_outcomes.py deleted file mode 100644 index fd40a5fde..000000000 --- a/evals/terminal-bench/scripts/smoke_outcomes.py +++ /dev/null @@ -1,178 +0,0 @@ -"""Exercise real pinned CLI failures, retry recovery, and timeout cleanup in Harbor.""" - -import argparse -import asyncio -import json -from pathlib import Path -import subprocess -import tempfile -import tomllib -import uuid - -from harbor.agents.installed.base import AgentAuthenticationError -from harbor.environments.docker.docker import DockerEnvironment -from harbor.models.agent.context import AgentContext -from harbor.models.task.config import EnvironmentConfig -from harbor.models.trial.paths import TrialPaths -from harbor.utils.trajectory_validator import TrajectoryValidator - -from diffex_terminal_bench.adapter import DiffexAgent -from diffex_terminal_bench.config import PINS -from diffex_terminal_bench.dataset import load_config, prepare_dataset -from diffex_terminal_bench.gate import sha256 -from diffex_terminal_bench.outcome import DiffexExecutionError - - -async def smoke(platform, source_image=None, grpc_fixture=False): - config = tomllib.loads(Path("smoke.toml").read_text()) - session_id = f"diffex-outcomes-{uuid.uuid4().hex[:12]}" - trial = TrialPaths(Path("results", session_id).resolve()) - trial.mkdir() - report = {"platform": platform, "image": (source_image or config["images"][platform]), "pins": PINS, - "qualification": "source-only" if source_image else "pinned-release", - "passed": False, "cleanup_passed": False, "outcomes": []} - print(f"Outcome evidence: {trial.trial_dir}", flush=True) - with tempfile.TemporaryDirectory(prefix="diffex-outcomes-") as temp: - directory = Path(temp) - overlay = directory / "platform.yaml" - overlay.write_text(f"services:\n main:\n platform: {platform}\n") - environment = DockerEnvironment( - environment_dir=directory, environment_name="diffex-outcomes-smoke", session_id=session_id, - trial_paths=trial, task_env_config=EnvironmentConfig(docker_image=(source_image or config["images"][platform])), - extra_docker_compose=[overlay], - mounts=[{"type": "bind", "source": str(trial.agent_dir), "target": "/logs/agent"}], - ) - try: - await asyncio.wait_for(environment.start(force_build=False), config["environment_timeout_sec"]) - agent = DiffexAgent(logs_dir=trial.agent_dir, model_name="openai/gpt-5", thinking="high", - extra_env={"OPENAI_API_KEY": "fixture-secret-never-log"}) - if source_image: - source_wrapper = directory / "source-cli" - source_wrapper.write_text('#!/bin/sh\nexport TSX_TSCONFIG_PATH=/repo/tsconfig.json\nexec node --import /repo/node_modules/tsx/dist/loader.mjs /repo/packages/coding-agent/src/cli.ts "$@"\n') - await environment.upload_file(source_wrapper, "/usr/local/bin/diffex") - await agent.exec_as_root(environment, command="chmod 755 /usr/local/bin/diffex") - else: - await asyncio.wait_for(agent.setup(environment), PINS["setup_timeout_sec"] + 15) - await environment.upload_file(Path("tests/faux-outcomes.cjs"), "/tmp/faux-outcomes.cjs") - await environment.upload_file(Path("src/diffex_terminal_bench/cleanup.cjs"), "/tmp/faux-cleanup.cjs") - wrapper = directory / "diffex" - wrapper.write_text( - '#!/bin/sh\n' - '/opt/diffex-terminal-bench/node/bin/node /tmp/faux-outcomes.cjs prepare\n' - 'if [ "$(cat /tmp/outcome-scenario)" != interrupted-output ]; then\n' - ' exec /usr/local/bin/diffex-release "$@"\n' - 'fi\n' - '/usr/local/bin/diffex-release "$@"\n' - 'status=$?\n' - '/opt/diffex-terminal-bench/node/bin/node /tmp/faux-outcomes.cjs interrupt-output\n' - 'exit "$status"\n' - ) - await agent.exec_as_root(environment, command="mv /usr/local/bin/diffex /usr/local/bin/diffex-release") - await environment.upload_file(wrapper, "/usr/local/bin/diffex") - await agent.exec_as_root(environment, command="chmod 755 /usr/local/bin/diffex && mkdir -p /task/work && chmod 777 /task/work") - environment.task_env_config.workdir = "/task/work" - await agent.exec_as_root(environment, command="nohup /opt/diffex-terminal-bench/node/bin/node /tmp/faux-outcomes.cjs >/tmp/faux-outcomes.log 2>&1 0 - assert (run_dir / "stderr.log").exists() - assert (run_dir / "sessions/root.jsonl").stat().st_size > 0 - assert (run_dir / "process.json").exists() - validator = TrajectoryValidator() - assert validator.validate(trial.agent_dir / "trajectory.json"), validator.get_errors() - assert context.metadata["diffex_trajectory"]["validated"] - if expected != "completed": - assert not context.metadata["diffex_accounting"]["complete"] - assert context.n_input_tokens is None and context.cost_usd is None - if scenario in ("authentication", "exhausted"): - assert outcome["process"]["exit_code"] == 1, outcome - if scenario == "interrupted-output": - assert outcome["process"]["exit_code"] == 0, outcome - if scenario == "recovered": - assert outcome["recovered_retries"] == 1, outcome - if scenario in ("success", "timeout", "service-failure", "forced-death", "grpc"): - # Harbor transfers mounted log ownership to the host on - # Linux. Verify as the task user using collected evidence. - verification_env = {"DIFFEX_TB_SERVICE_EVIDENCE": json.dumps({ - name: json.loads((run_dir / f"{name}.json").read_text()) - for name in ("service", "launcher", "cli") - })} - verification = await agent.exec_as_agent(environment, command= - f"/opt/diffex-terminal-bench/node/bin/node /tmp/faux-outcomes.cjs verify-service {context.metadata['diffex_run_dir']}", - env=verification_env) - (run_dir / "service-verification.json").write_text(verification.stdout) - assert json.loads(verification.stdout)["verified"] is True - before = json.loads((run_dir / "service-before-exit.json").read_text()) - after = json.loads(verification.stdout) - assert (before["pid"], before["start"]) == (after["pid"], after["start"]) - if grpc_fixture: - verifier = await agent.exec_as_agent(environment, command= - "/opt/verifier/bin/python -m pytest /tests/test_outputs.py -v -p no:cacheprovider", timeout_sec=30) - (run_dir / "verifier.stdout.txt").write_text(verifier.stdout) - assert "7 passed" in verifier.stdout, verifier.stdout - # Check the same process again after all verifier RPCs. - await agent.exec_as_agent(environment, command= - f"/opt/diffex-terminal-bench/node/bin/node /tmp/faux-outcomes.cjs verify-service {context.metadata['diffex_run_dir']}", - env=verification_env) - report["outcomes"].append({"scenario": scenario, "user": user, "outcome": outcome, - "trajectory_validated": True, "accounting": context.metadata["diffex_accounting"]}) - print(f"{scenario} user={user}: {outcome['status']}, cleanup confirmed", flush=True) - for path in trial.agent_dir.rglob("*"): - if path.is_file(): - assert b"fixture-secret-never-log" not in path.read_bytes(), path - report["passed"] = True - finally: - await environment.stop(delete=True) - remaining = subprocess.check_output(["docker", "ps", "-aq", "--filter", f"label=com.docker.compose.project={session_id}"], text=True).strip() - report["cleanup_passed"] = not remaining - (trial.trial_dir / "smoke.json").write_text(json.dumps(report, indent=2) + "\n") - assert not remaining, "Outcome smoke container was not removed" - print(f"Passed {len(report['outcomes'])} controlled outcome trials; container removed") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--platform", choices=("linux/amd64", "linux/arm64"), default="linux/amd64") - parser.add_argument("--source-image", help="Development-only image with /repo source and Linux dependencies; never qualifies release pins") - parser.add_argument("--grpc-fixture", action="store_true", help="Run the known-correct service against all seven pinned verifier tests (requires source image)") - args = parser.parse_args() - if args.grpc_fixture and not args.source_image: - parser.error("--grpc-fixture requires --source-image with the isolated verifier environment") - asyncio.run(smoke(args.platform, args.source_image, args.grpc_fixture)) diff --git a/evals/terminal-bench/smoke.toml b/evals/terminal-bench/smoke.toml deleted file mode 100644 index ad327398c..000000000 --- a/evals/terminal-bench/smoke.toml +++ /dev/null @@ -1,8 +0,0 @@ -# Ubuntu 24.04 platform manifests from index -# sha256:224a1869083a311ef3f13648a154ba79832fbef6364d31493642ca03082da254. -platform = "linux/amd64" -environment_timeout_sec = 600 - -[images] -"linux/amd64" = "ubuntu@sha256:a61567bd31828687156d735ea8eb01ba4e37636e225dd6a48ba94136a70d9d61" -"linux/arm64" = "ubuntu@sha256:ec0b1c9058e44c837a21c3f9d8a3d5e9aaa94ed28edceb18e154af5efecf0950" diff --git a/evals/terminal-bench/src/diffex_terminal_bench/__init__.py b/evals/terminal-bench/src/diffex_terminal_bench/__init__.py deleted file mode 100644 index b3d6a0db9..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Private, installation-only Terminal-Bench integration.""" diff --git a/evals/terminal-bench/src/diffex_terminal_bench/adapter.py b/evals/terminal-bench/src/diffex_terminal_bench/adapter.py deleted file mode 100644 index 6aecb03fa..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/adapter.py +++ /dev/null @@ -1,287 +0,0 @@ -"""Pinned Harbor installation and isolated headless execution.""" - -from importlib.metadata import version -from importlib.resources import as_file, files -import asyncio -import shlex -import hashlib -import json -from pathlib import Path, PurePosixPath -import shutil -import tempfile -import uuid - -from harbor.agents.installed.base import BaseInstalledAgent, NonZeroAgentExitCodeError -from harbor.environments.base import BaseEnvironment -from harbor.models.agent.context import AgentContext - -from diffex_terminal_bench.config import PINS -from diffex_terminal_bench.execution import ExecutionOptions, PROVIDER_ENV, SYSTEM_PROMPT_SHA256, USER_MESSAGE_SUFFIX -from diffex_terminal_bench.lifecycle import read_cleanup -from diffex_terminal_bench.outcome import DiffexControlStopError, parse_outcome, require_completed -from diffex_terminal_bench.transport import retry_transport -from diffex_terminal_bench.trajectory import export_trajectory - - -class DiffexAgent(BaseInstalledAgent): - options_model = ExecutionOptions - - @staticmethod - def name() -> str: - return "diffex" - - def version(self) -> str: - return PINS["diffex_version"] - - async def install(self, environment: BaseEnvironment) -> None: - if version("harbor") != PINS["harbor_version"]: - raise RuntimeError("Harbor version differs from pins.toml; run uv sync --locked") - if self._version not in (None, PINS["diffex_version"]): - raise ValueError("Change pins.toml to select a different Diffex release") - - remote_dir = "/installed-agent/diffex-terminal-bench" - logs_dir = str(self.environment_logs_dir / "setup") - await self.exec_as_root( - environment, - command=f"mkdir -p {remote_dir} {shlex.quote(logs_dir)}", - timeout_sec=10, - ) - for filename in ("install.sh", "verify-install.cjs"): - with as_file(files(__package__).joinpath(filename)) as source: - await environment.upload_file(source, f"{remote_dir}/{filename}") - - # An empty environment keeps task npm configuration and credentials out - # of setup. The wrapper installed below binds Diffex to the pinned Node. - env = { - "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", - "HOME": "/opt/diffex-terminal-bench/home", - "DEBIAN_FRONTEND": "noninteractive", - "NODE_VERSION": PINS["node_version"], - "NODE_SHA256_X64": PINS["node_sha256"]["x64"], - "NODE_SHA256_ARM64": PINS["node_sha256"]["arm64"], - "DIFFEX_VERSION": PINS["diffex_version"], - "DIFFEX_INTEGRITY": PINS["diffex_integrity"], - "NPM_REGISTRY": PINS["npm_registry"], - "HARBOR_VERSION": PINS["harbor_version"], - "SETUP_TIMEOUT_SEC": str(PINS["setup_timeout_sec"]), - "SETUP_LOGS": logs_dir, - } - command = shlex.join([ - "env", "-i", *(f"{key}={value}" for key, value in env.items()), - "timeout", "--signal=TERM", "--kill-after=10", - str(PINS["setup_timeout_sec"]), "bash", f"{remote_dir}/install.sh", - ]) - await self.exec_as_root( - environment, - command=command, - timeout_sec=PINS["setup_timeout_sec"] + 15, - ) - - async def run( - self, instruction: str, environment: BaseEnvironment, context: AgentContext - ) -> None: - if version("harbor") != PINS["harbor_version"] or self._version not in (None, PINS["diffex_version"]): - raise ValueError("Execution requires the checked-in Harbor and Diffex pins") - if not self.model_name or "/" not in self.model_name: - raise ValueError("model_name must be an exact provider/model ID") - provider, model = self.model_name.split("/", 1) - if provider not in PROVIDER_ENV or not model or any(char in model for char in "*?:\n\r"): - raise ValueError("Select a supported provider and exact model ID without a thinking suffix") - if not instruction.strip(): - raise ValueError("The trial instruction must not be empty") - if self.mcp_servers or self.skills_dir: - raise ValueError("Task MCP servers and injected skills are outside the context-only policy") - credentials = { - name: value for name in PROVIDER_ENV[provider] - if (value := self._get_env(name)) - } - if not credentials: - raise ValueError(f"Missing provider credentials: set one of {', '.join(PROVIDER_ENV[provider])}") - options = self.options - if not isinstance(options, ExecutionOptions): - raise TypeError("Expected validated execution options") - tools = options.tools.split(",") - run_dir = self.environment_logs_dir / f"run-{uuid.uuid4().hex}" - requested = { - "pins": PINS, "provider": provider, "model": model, - "thinking": options.thinking, "tools": tools, - "subagents": options.subagents, "project_resources": options.project_resources, - "background_process_lifetime": options.background_process_lifetime, - "evolution": {"enabled": False}, "state_policy": "fresh-per-run", - "credential_env_names": sorted(credentials), - "provider_env_names": sorted({name for names in PROVIDER_ENV.values() for name in names}), - "task_environment": options.task_environment, - "instruction_sha256": hashlib.sha256(instruction.encode("utf-8")).hexdigest(), - "user_message_suffix": USER_MESSAGE_SUFFIX, - "system_prompt_sha256": SYSTEM_PROMPT_SHA256, - } - await self.exec_as_root(environment, command=f"umask 077 && mkdir -p {shlex.quote(str(self.environment_logs_dir))} && mkdir {shlex.quote(str(run_dir))}") - with tempfile.TemporaryDirectory(prefix="diffex-trial-") as temp: - for name, content in (("instruction.txt", instruction), ("requested.json", json.dumps(requested, indent=2))): - source = Path(temp, name) - source.write_text(content, encoding="utf-8") - await environment.upload_file(source, str(run_dir / name)) - for name in ("run.cjs", "cleanup.cjs", "evidence.cjs", "observe.mjs", "eval-system-prompt.txt"): - with as_file(files(__package__).joinpath(name)) as source: - await environment.upload_file(source, str(run_dir / name)) - # Uploads may be root-owned even when the task selects a non-root user. - if environment.default_user is not None: - await self.exec_as_root(environment, command=f"chown -R {shlex.quote(str(environment.default_user))} {shlex.quote(str(run_dir))}") - # Credentials travel only through Harbor's env parameter. Task runtime - # variables can be preserved independently of Diffex state and credentials. - execution = {"interruption": None, "launcher_exit_code": None} - try: - result = await self.exec_as_agent( - environment, - command="umask 077; " + shlex.join(["/opt/diffex-terminal-bench/node/bin/node", str(run_dir / "run.cjs"), str(run_dir)]) - + f" {shlex.quote(str(run_dir / 'launcher.stdout.log'))} 2>{shlex.quote(str(run_dir / 'launcher.stderr.log'))}", - env={**credentials, "DIFFEX_TB_PROVIDER": provider, - "DIFFEX_TB_MODEL": model, "DIFFEX_TB_THINKING": options.thinking}, - ) - execution["launcher_exit_code"] = result.return_code - except NonZeroAgentExitCodeError: - # The persisted process status and stream provide the actual cause. - execution["launcher_exit_code"] = 1 - except asyncio.CancelledError: - execution["interruption"] = "cancelled" - raise - except TimeoutError: - execution["interruption"] = "timeout" - raise - except Exception: - execution["interruption"] = "transport_failure" - raise - finally: - # Harbor cancels run() at the task deadline. An independent exec - # must stop the remote process before Harbor downloads logs/verifies. - finish = asyncio.create_task(self._finish_run( - environment, run_dir, context, execution, - )) - while not finish.done(): - try: - await asyncio.shield(finish) - except asyncio.CancelledError: - # A second cancellation must not abandon remote cleanup. - execution["interruption"] = execution["interruption"] or "cancelled" - outcome = finish.result() - # A deadline can arrive just as recovery completes. - if execution["interruption"]: - outcome.update(status=execution["interruption"], interruption=execution["interruption"], - reason="execution_interrupted") - directory = self.logs_dir / run_dir.name - try: - accounting = export_trajectory(directory, self.logs_dir / "trajectory.json", context, outcome) - context.metadata = {**(context.metadata or {}), "diffex_trajectory": { - "path": "trajectory.json", "validated": True, - "accounting_complete": accounting["complete"], - }} - except Exception as exc: - # Export failure must not replace the original cancellation or - # provider exception, or let a completed run omit its ATIF file. - outcome.setdefault("artifact_errors", []).append({"phase": "trajectory_export", "error": type(exc).__name__}) - context.n_input_tokens = context.n_output_tokens = context.n_cache_tokens = context.cost_usd = None - context.model_usage = {} - context.metadata = {**(context.metadata or {}), "diffex_trajectory": {"validated": False}} - if outcome["status"] == "completed": - outcome.update(status="artifact_collection_failure", reason="trajectory_export_failed") - try: - (directory / "outcome.json").write_text(json.dumps(outcome, indent=2) + "\n") - except OSError as exc: - outcome.setdefault("artifact_errors", []).append({"phase": "outcome_persistence", "error": type(exc).__name__}) - if outcome["status"] == "completed": - outcome.update(status="artifact_collection_failure", reason="outcome_persistence_failed") - # Raise inside finally so even a pending Harbor deadline cannot - # convert unsafe cleanup into a verifier-eligible agent timeout. - if outcome.get("agent_execution_stopped") is not True: - raise DiffexControlStopError("Agent execution stop is unverified; verification is blocked") - if execution["interruption"] == "cancelled": - raise asyncio.CancelledError - require_completed(outcome) - - async def _finish_run(self, environment: BaseEnvironment, run_dir: PurePosixPath, - context: AgentContext, execution: dict) -> dict: - directory = self.logs_dir / run_dir.name - directory.mkdir(parents=True, exist_ok=True) - failures = [] - try: - await asyncio.wait_for(self.exec_as_agent( - environment, - command=shlex.join(["/opt/diffex-terminal-bench/node/bin/node", str(run_dir / "cleanup.cjs"), str(run_dir)]), - timeout_sec=10, - ), timeout=15) - except Exception as exc: - failures.append({"phase": "cleanup", "error": type(exc).__name__}) - # Keep the independently validated evidence outside the bulk download's - # destination so partial extraction cannot overwrite its stop decision. - with tempfile.TemporaryDirectory(prefix="diffex-shutdown-") as temp: - evidence_dir = Path(temp) - cleanup = None - - async def fetch_evidence(): - response = await self.exec_as_agent( - environment, - command=shlex.join(["/opt/diffex-terminal-bench/node/bin/node", str(run_dir / "evidence.cjs"), str(run_dir)]), - timeout_sec=15, - ) - evidence = json.loads(response.stdout) - if evidence is None: - raise TimeoutError("Shutdown evidence is not yet published") - return evidence - - try: - evidence = await retry_transport(fetch_evidence) - if not isinstance(evidence, dict): - raise ValueError("Invalid shutdown evidence bundle") - for name, content in evidence.items(): - path = PurePosixPath(name) - if (name not in ("cleanup.json", "stop-requested", "launcher.json", "cli.json", "process.json") - and not (len(path.parts) == 2 and path.parts[0] == "ownership" - and path.name.endswith(".json") and name == path.as_posix())): - raise ValueError("Unexpected shutdown evidence path") - if not isinstance(content, str): - raise ValueError("Invalid shutdown evidence content") - target = evidence_dir / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(content) - cleanup = read_cleanup(evidence_dir) - except Exception as exc: - failures.append({"phase": "shutdown_evidence", "error": type(exc).__name__}) - - artifact_errors = [] - try: - if environment.capabilities.mounted: - await asyncio.wait_for(environment.prepare_logs_for_host(), timeout=15) - else: - await asyncio.wait_for(environment.download_dir(str(run_dir), directory), timeout=30) - except Exception as exc: - artifact_errors.append({"phase": "log_collection", "error": type(exc).__name__}) - # Retain this bundle even when invalid, for diagnosis. Never infer - # shutdown from files that arrive later with the bulk artifacts. - try: - shutil.copytree(evidence_dir, directory, dirs_exist_ok=True) - except OSError as exc: - artifact_errors.append({"phase": "evidence_persistence", "error": type(exc).__name__}) - outcome = parse_outcome(directory, execution["interruption"], execution["launcher_exit_code"], - shutdown_directory=evidence_dir) - outcome.update(agent_execution_stopped=cleanup is not None, cleanup_confirmed=cleanup is not None) - if cleanup is not None: - outcome["retained_processes"] = cleanup["retained_processes"] - elif not execution["interruption"]: - outcome.update(status="cleanup_failure", reason="process_cleanup_unverified") - if artifact_errors: - outcome["artifact_errors"] = artifact_errors - # Partial downloads may still conclusively establish an agent - # failure. Keep that primary cause alongside artifact errors. - unknown = (outcome["status"] in ("completed", "incomplete") - or (outcome["status"] == "process_failure" and not outcome.get("process")) - or (outcome["status"] == "configuration_failure" - and not all((directory / name).is_file() for name in ("requested.json", "effective.json")))) - if cleanup is not None and not execution["interruption"] and unknown: - outcome.update(status="artifact_collection_failure", reason=f"{artifact_errors[0]['phase']}_failed") - if failures: - outcome["recovery_errors"] = failures - context.metadata = {**(context.metadata or {}), "diffex_outcome": outcome, "diffex_run_dir": str(run_dir)} - # The caller writes the summary to host logs after shielded recovery. - # Do not upload a second copy that could overwrite a later cancellation - # classification when Harbor collects the remote tree again. - return outcome diff --git a/evals/terminal-bench/src/diffex_terminal_bench/benchmark.py b/evals/terminal-bench/src/diffex_terminal_bench/benchmark.py deleted file mode 100644 index a0483b4a5..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/benchmark.py +++ /dev/null @@ -1,368 +0,0 @@ -"""Repeatable Daytona qualification stages with retained evidence.""" - -import argparse -import asyncio -from datetime import datetime, timezone -from importlib.metadata import version -import os -from pathlib import Path -import platform -import secrets -import signal -import subprocess -import sys -import uuid - -from daytona import AsyncDaytona -from harbor.environments.daytona.environment import DaytonaClientManager -from harbor.models.trial.config import AgentConfig, EnvironmentConfig, TaskConfig, TrialConfig -from harbor.publisher.packager import Packager -from harbor.trial.trial import Trial - -from diffex_terminal_bench.adapter import DiffexAgent -from diffex_terminal_bench.config import PINS -from diffex_terminal_bench.dataset import BenchmarkConfig, load_config, prepare_dataset -from diffex_terminal_bench.daytona_backend import confirm_cleanup -from diffex_terminal_bench.diagnostics import TrialExecutionFailure, describe_failure, diagnose_trial -from diffex_terminal_bench.execution import PROVIDER_ENV, SYSTEM_PROMPT_SHA256 -from diffex_terminal_bench.retention import cleanup_trial_volume -from diffex_terminal_bench.gate import STAGES, fingerprint, inventory, read_json, require_previous, sample_tasks, sha256, stage_tasks, validate_trial, verify_stage, write_json -from diffex_terminal_bench.summary import refresh_index, write_index - - -PACKAGE = Path(__file__).resolve().parents[2] -REPOSITORY = PACKAGE.parents[1] - - -class StageFailedError(RuntimeError): - """Expected qualification failure after all started trials are cleaned up.""" - - -def failure_reason(failure: dict) -> str: - try: - return describe_failure(failure["diagnostics"]) if "diagnostics" in failure else failure["error"] - except Exception as exc: - # Diagnostics are supplementary; the recorded trial failure is authoritative. - return f"{failure.get('error', 'UnknownError')} (diagnostic formatting failed: {type(exc).__name__})" - - -def progress(message: str) -> None: - # A detached terminal or broken logging sink must not affect trial execution. - for stream in (sys.stdout, sys.stderr): - try: - print(message, file=stream, flush=True) - return - except Exception: - continue - - -def preflight(config: BenchmarkConfig, *, oracle: bool) -> None: - if version("harbor") != PINS["harbor_version"] or version("daytona") != "0.226.1": - raise ValueError("Run uv sync --locked to install the pinned Harbor/Daytona versions") - if not os.environ.get("DAYTONA_API_KEY"): - raise ValueError("Set DAYTONA_API_KEY through your secret-loading mechanism") - if not oracle and not any(os.environ.get(name) for name in PROVIDER_ENV[config.model.split("/", 1)[0]]): - raise ValueError("Set the selected provider credential through your secret-loading mechanism") - - -def run_identity(config: BenchmarkConfig, digests: dict) -> dict: - status = subprocess.check_output(["git", "status", "--porcelain"], cwd=REPOSITORY, text=True) - if status.strip(): - raise ValueError("Qualification requires a clean committed checkout") - commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=REPOSITORY, text=True).strip() - return {"adapter_commit": commit, "dataset": config.dataset.model_dump(), "task_digests": digests, - "pins": PINS, "harbor_version": version("harbor"), "daytona_version": version("daytona"), - "python_version": platform.python_version(), "uv_lock_sha256": sha256(PACKAGE / "uv.lock"), - "model": config.model, "policy": config.policy.model_dump(include={"thinking", "tools", "subagents", "project_resources", "background_process_lifetime"}), - "pilot_tasks": config.pilot_tasks, "backend": "daytona", - "daytona_target": os.environ.get("DAYTONA_TARGET", "organization-default"), - "daytona_api_url": os.environ.get("DAYTONA_API_URL", "https://app.daytona.io/api"), - "harness_files": {path.name: sha256(path) for path in sorted(Path(__file__).parent.iterdir()) if path.is_file()}, - "system_prompt_sha256": SYSTEM_PROMPT_SHA256, - "timeout_multiplier": 1.0, "resource_overrides": None, "concurrency": config.concurrency, "harbor_retries": 0} - - -def trial_config(config: BenchmarkConfig, task_path: Path, directory: Path, label: str, *, oracle: bool) -> TrialConfig: - agent = AgentConfig(name="oracle") if oracle else AgentConfig( - import_path=DiffexAgent.import_path(), model_name=config.model, - override_setup_timeout_sec=630, - kwargs=config.policy.model_dump(include={"thinking", "tools", "subagents", "project_resources", "background_process_lifetime"}), - # Variable references are serialized, never credential values. Harbor - # also uses these names to scrub accidental credential output from logs. - env={name: f"${{{name}}}" for name in PROVIDER_ENV[config.model.split("/", 1)[0]] if os.environ.get(name)}, - ) - return TrialConfig(task=TaskConfig(path=task_path, source=f"{config.dataset.name}@{config.dataset.version}"), - trial_name=directory.name, trials_dir=directory.parent, agent=agent, - environment=EnvironmentConfig(import_path="diffex_terminal_bench.daytona_backend:RecordedDaytonaEnvironment", - delete=True, kwargs={"labels": {"diffex.gate_trial": label}})) - - -async def run_stage(config: BenchmarkConfig, tasks: dict[str, Path], identity: dict, root: Path, stage: str) -> None: - if stage != "sample": - require_previous(root, stage, identity, config.pilot_tasks, list(tasks)) - selected = stage_tasks(stage, config.pilot_tasks, list(tasks), sample=identity.get("sample")) - directory = root / stage - directory.mkdir(parents=True) # Never overwrite a previous attempt. - report = {"schema_version": 1, "stage": stage, "identity": identity, "identity_sha256": fingerprint(identity), - "started_at": datetime.now(timezone.utc).isoformat(), "passed": False, "trials": [], - "failures": [], "completed_trials": 0, "expected_trials": len(selected)} - write_json(directory / "gate.json", report) - refresh_index(root.parent) - - def checkpoint(*, final: bool = False) -> bool: - try: - write_json(directory / "gate.json", report) - except Exception as exc: - report.setdefault("report_write_errors", []).append({ - "error": type(exc).__name__, "completed_trials": report["completed_trials"], "final": final, - }) - progress(f"{stage}: could not save gate.json ({type(exc).__name__}); retaining results in memory for retry.") - return False - refresh_index(root.parent) - return True - - try: - async with AsyncDaytona() as client: - pending = iter(enumerate(selected, 1)) - - def record_failure(index: int, task: str, exc: BaseException, phase: str) -> None: - failure = {"error": type(exc).__name__, "trial_index": index, - "task": task, "directory": f"{index:03d}-{task}", "phase": phase} - if isinstance(exc, TrialExecutionFailure): - failure["diagnostics"] = exc.details - previous = next((row for row in report["failures"] if row["trial_index"] == index), None) - if previous is not None: - previous.setdefault("additional_errors", []).append(failure) - return - report["failures"].append(failure) - report["failures"].sort(key=lambda row: row["trial_index"]) - report.setdefault("failure", failure) - - async def run_trial(index: int, task: str) -> None: - trial_dir = directory / f"{index:03d}-{task}" - label = uuid.uuid4().hex - trial_dir.mkdir() - write_json(trial_dir / "cleanup-request.json", {"label": label}) - progress(f"{stage} {index}/{len(selected)}: {task}") - try: - if "sha256:" + Packager.compute_content_hash(tasks[task])[0] != identity["task_digests"][task]: - raise ValueError("Task content changed after dataset verification") - trial = await Trial.create(trial_config(config, tasks[task], trial_dir, label, oracle=stage == "oracle")) - await trial.run() - except BaseException as exc: - record_failure(index, task, exc, "execution") - raise - finally: - cleanup = asyncio.create_task(confirm_cleanup(client, trial_dir, label)) - cancelled = False - while not cleanup.done(): - try: - await asyncio.shield(cleanup) - except asyncio.CancelledError: - # Preserve evidence and finish remote deletion even - # when SIGTERM arrives during a cancelled trial. - cancelled = True - except Exception: - break - try: - cleanup_report = cleanup.result() - if cleanup_report.get("confirmed") is not True or cleanup_report.get("unexpected_sandboxes", []): - raise ValueError("Sandbox cleanup failed or required recovery") - except BaseException as exc: - record_failure(index, task, exc, "cleanup") - raise - finally: - if cancelled: - raise asyncio.CancelledError - evidence = validate_trial(trial_dir, task, identity, oracle=stage == "oracle") - await cleanup_trial_volume(client, trial_dir) - report["trials"].append({"task": task, "directory": trial_dir.name, - "evidence": evidence, "files": inventory(trial_dir)}) - report["trials"].sort(key=lambda row: row["directory"]) - - async def worker() -> None: - # A slot includes installation, execution, verification, and - # the cleanup attempt, even when this trial fails. - while True: - item = next(pending, None) - if item is None: - return - index, task = item - try: - await run_trial(index, task) - except (Exception, asyncio.CancelledError) as exc: - if not any(row["trial_index"] == index for row in report["failures"]): - record_failure(index, task, exc, "trial") - # A cancelled stage still stops scheduling. A trial - # raising CancelledError on its own must not lose a slot. - if asyncio.current_task().cancelling(): - raise asyncio.CancelledError from exc - failure = next(row for row in report["failures"] if row["trial_index"] == index) - reason = failure_reason(failure) - progress(f"{stage} {index}/{len(selected)}: failed: {reason}. " - f"Evidence: {directory / failure['directory']}. Continuing remaining trials.") - report["completed_trials"] += 1 - checkpoint() - - # Only stage-level failures or explicit cancellation cancel peers. - # Wait for every started trial's cleanup before closing clients. - async with asyncio.TaskGroup() as group: - for _ in range(min(config.concurrency, len(selected))): - group.create_task(worker()) - if report["completed_trials"] != len(selected): - raise RuntimeError("Stage did not complete every selected trial") - report["passed"] = not report["failures"] - except BaseException as exc: - report.setdefault("failure", {"error": type(exc).__name__}) - raise - finally: - report["finished_at"] = datetime.now(timezone.utc).isoformat() - try: - for attempt in range(3): - if attempt: - await asyncio.sleep(1) - persisted = checkpoint(final=True) - if persisted: - break - if not persisted: - report["passed"] = False - report["incomplete"] = True - progress(f"{stage}: qualification incomplete; final report could not be saved to {directory / 'gate.json'}.") - finally: - # Harbor owns a separate singleton HTTP client; close it even if - # writing the final report fails. - manager = await DaytonaClientManager.get_instance() - await manager._cleanup() - if report["failures"] or not persisted: - reason = "" - if report["failures"]: - failure = report["failure"] - reason = f" First failure: {failure_reason(failure)}. Evidence: {directory / failure['directory']}." - status = "failed" if persisted else "incomplete: final report could not be saved" - raise StageFailedError(f"{stage}: completed {len(selected)} trials with {len(report['failures'])} failed; " - f"qualification {status}.{reason} Report: {directory / 'gate.json'}.") - progress(f"{stage}: passed; evidence: {directory}") - - -async def execute(config: BenchmarkConfig, tasks: dict, identity: dict, root: Path, stage: str, through: bool) -> None: - current = asyncio.current_task() - loop = asyncio.get_running_loop() - loop.add_signal_handler(signal.SIGTERM, current.cancel) - try: - for name in STAGES[:STAGES.index(stage) + 1] if through else (stage,): - await run_stage(config, tasks, identity, root, name) - finally: - loop.remove_signal_handler(signal.SIGTERM) - - -def verify_sample(config: BenchmarkConfig, root: Path, cache: Path) -> None: - directory = root / "sample" - report = read_json(directory / "gate.json") - if not report.get("finished_at"): - raise ValueError(f"sample: no final report; the run is still running or was interrupted. " - f"Wait for completion before verification. Evidence: {directory / 'gate.json'}") - if report.get("passed") is not True: - failure = report.get("failure") or {} - reason = describe_failure(failure["diagnostics"]) if "diagnostics" in failure else failure.get("error", "unknown error") - raise ValueError(f"sample: recorded run did not pass ({report.get('completed_trials', 'unknown')}/" - f"{report.get('expected_trials', 'unknown')} trials completed). " - f"First failure: {failure.get('directory', 'see gate.json')}: {reason}. " - f"Evidence: {directory / 'gate.json'}. --verify-sample requires a passing run. " - "To export retained terminal results, use diffex_terminal_bench.export_job with --allow-partial.") - with prepare_dataset(config, cache) as (tasks, digests): - identity = run_identity(config, digests) - identity["sample"] = report["identity"].get("sample") - verify_stage(root, "sample", identity, config.pilot_tasks, list(tasks)) - refresh_index(root.parent) - print(f"Verified retained sample evidence: {directory}") - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--config", type=Path, default=PACKAGE / "benchmark.toml") - parser.add_argument("--output", type=Path, - help="Output directory (default: a fresh results/sample-tb21-* directory for --random; results/release-gate otherwise)") - parser.add_argument("-n", "--concurrency", type=int, - help="Maximum simultaneous trials, including cleanup (default: config concurrency or 1); use the same value for every stage and verification") - parser.add_argument("--dataset-cache", type=Path, default=PACKAGE / "results/dataset-cache", - help="Shared archive cache; every use verifies the pinned SHA-256") - parser.add_argument("--results-dir", type=Path, help="Directory to scan with --refresh-index (defaults to the output parent)") - parser.add_argument("--seed", type=int, help="Reproducible task selection for --sample or --random (default: generate and record a seed)") - mode = parser.add_mutually_exclusive_group(required=True) - mode.add_argument("--prepare", action="store_true", help="Verify dataset and report stage sizes without creating sandboxes") - mode.add_argument("--refresh-index", action="store_true", help="Rebuild index.md from local reports without downloads or API calls") - mode.add_argument("--diagnose-trial", type=Path, help="Explain a retained trial failure without downloads or API calls") - mode.add_argument("--stage", choices=STAGES) - mode.add_argument("--verify-through", choices=STAGES, help="Recheck retained or downloaded stage evidence without API calls") - mode.add_argument("--through", choices=STAGES[:-1], help="Run prerequisite stages in order; stops on first failure") - mode.add_argument("--sample", type=int, metavar="COUNT", help="Run COUNT distinct random tasks once, without prerequisite stages") - mode.add_argument("--random", action="store_true", help="Run exactly one random task without prerequisites, using a fresh output directory by default") - mode.add_argument("--verify-sample", action="store_true", help="Recheck the saved sample selection and evidence without sandbox or model calls") - args = parser.parse_args() - if args.random: - args.sample = 1 - if args.seed is not None and args.sample is None: - parser.error("--seed requires --sample or --random") - if args.sample is not None and args.sample < 1: - parser.error("--sample must be a positive integer") - if args.results_dir is not None and not args.refresh_index: - parser.error("--results-dir requires --refresh-index") - if args.concurrency is not None and args.concurrency < 1: - parser.error("--concurrency must be a positive integer") - os.umask(0o077) - default_output = PACKAGE / "results/release-gate" - if args.random: - timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") - default_output = PACKAGE / "results" / f"sample-tb21-{timestamp}-{uuid.uuid4().hex}" - root = (args.output or default_output).resolve() - if args.diagnose_trial: - directory = args.diagnose_trial.resolve() - print(f"{directory.name}: {describe_failure(diagnose_trial(directory))}") - print(f"Retained evidence: {directory}") - return - if args.refresh_index: - directory = args.results_dir.resolve() if args.results_dir is not None else root.parent - print(f"Updated run index: {write_index(directory)}") - return - config = load_config(args.config) - if args.sample is not None and args.sample > config.dataset.task_count: - parser.error(f"--sample cannot exceed the dataset's {config.dataset.task_count} tasks") - if args.concurrency is not None: - config = BenchmarkConfig.model_validate({**config.model_dump(), "concurrency": args.concurrency}) - if args.verify_sample: - try: - verify_sample(config, root, args.dataset_cache.resolve()) - except (OSError, ValueError, KeyError, TypeError, AttributeError) as exc: - raise SystemExit(f"Sample verification failed: {exc}") from None - return - stage = "sample" if args.sample is not None else args.stage or args.through - if not args.prepare and not args.verify_through: - preflight(config, oracle=stage == "oracle") - with prepare_dataset(config, args.dataset_cache.resolve()) as (tasks, digests): - if args.prepare: - write_json(root / "dataset.json", {"dataset": config.dataset.model_dump(), "task_digests": digests, - "model": config.model, "policy": config.policy.model_dump(), "concurrency": config.concurrency, - "stages": {name: len(stage_tasks(name, config.pilot_tasks, list(tasks))) for name in STAGES}}) - refresh_index(root.parent) - print(f"Verified {len(tasks)} pinned tasks: {root / 'dataset.json'}") - return - identity = run_identity(config, digests) - if args.sample is not None: - seed = args.seed if args.seed is not None else secrets.randbits(64) - selected = sample_tasks(list(tasks), args.sample, seed) - identity["sample"] = {"count": args.sample, "seed": seed, "tasks": selected} - print(f"Sample seed: {seed}; selected tasks ({len(selected)}): {', '.join(selected)}", flush=True) - print(f"Sample output: {root}", flush=True) - if args.verify_through: - require_previous(root, args.verify_through, identity, config.pilot_tasks, list(tasks), include_current=True) - refresh_index(root.parent) - print(f"Verified retained evidence through {args.verify_through}: {root}") - return - try: - asyncio.run(execute(config, tasks, identity, root, stage, bool(args.through))) - except StageFailedError as exc: - raise SystemExit(str(exc)) from None - - -if __name__ == "__main__": - main() diff --git a/evals/terminal-bench/src/diffex_terminal_bench/cleanup.cjs b/evals/terminal-bench/src/diffex_terminal_bench/cleanup.cjs deleted file mode 100644 index d58070256..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/cleanup.cjs +++ /dev/null @@ -1,149 +0,0 @@ -// Linux-only recovery, run as the agent user through an independent Harbor exec. -const { readFileSync, writeFileSync, readdirSync, existsSync } = require("node:fs"); -const { join } = require("node:path"); -const { setTimeout: delay } = require("node:timers/promises"); - -function identity(pid) { - try { - const stat = readFileSync(`/proc/${pid}/stat`, "utf8"); - const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" "); - return fields[0] === "Z" ? null : fields[19]; - } catch (error) { - if (error.code === "ENOENT" || error.code === "ESRCH") return null; - throw error; - } -} - -function signal(pid, start, name) { - if (!start || identity(pid) !== start) return; - try { process.kill(pid, name); } catch (error) { if (error.code !== "ESRCH") throw error; } -} - -function snapshot() { - const processes = new Map(); - for (const entry of readdirSync("/proc")) { - if (!/^\d+$/.test(entry) || Number(entry) === process.pid) continue; - try { - const stat = readFileSync(`/proc/${entry}/stat`, "utf8"); - const fields = stat.slice(stat.lastIndexOf(")") + 2).trim().split(/\s+/); - if (fields[0] === "Z") continue; - const uid = Number(readFileSync(`/proc/${entry}/status`, "utf8").match(/^Uid:\s+(\d+)/m)?.[1]); - let env; - try { env = readFileSync(`/proc/${entry}/environ`, "utf8").split("\0"); } - catch (error) { - if (error.code !== "EACCES") throw error; - if (uid !== process.getuid()) continue; - } - processes.set(Number(entry), { pid: Number(entry), start: fields[19], group: Number(fields[2]), env }); - } catch (error) { - if (!["ENOENT", "ESRCH"].includes(error.code)) throw error; - } - } - return processes; -} - -function ownership(runDir) { - const directory = join(runDir, "ownership"); - if (!existsSync(directory)) return []; - return readdirSync(directory).filter((name) => name.endsWith(".json")).map((name) => { - const record = JSON.parse(readFileSync(join(directory, name), "utf8")); - if (record.version !== 1 || typeof record.executionId !== "string" || !record.executionId || - name !== `${record.executionId}.json` || - !Number.isSafeInteger(record.processGroupId) || record.processGroupId <= 1 || - !Number.isSafeInteger(record.supervisorPid) || record.supervisorPid <= 1 || - !((record.owner === "environment" && record.state === "background") || - (record.owner === "session" && ["active", "terminated"].includes(record.state))) || - !Array.isArray(record.members) || (record.owner === "environment" && !record.members.length) || - record.members.some((member) => !Number.isSafeInteger(member.pid) || member.pid <= 1 || - member.processGroupId !== record.processGroupId || typeof member.startTime !== "string" || !member.startTime)) { - throw new Error("Invalid shell ownership record"); - } - return record; - }); -} - -async function cleanup(runDir) { - // This barrier must precede discovery, including when the launcher has not started. - writeFileSync(join(runDir, "stop-requested"), "", { mode: 0o600 }); - const controls = new Map(); - const deadline = Date.now() + 7000; - const boot = readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim(); - const seen = new Map(); - let remaining = []; - let retained = []; - let records = []; - let stopped = false; - let quietScans = 0; - do { - // Re-read identities to close delayed spawn/publication races. Do not - // inspect handoffs until their writers have stopped publishing them. - for (const name of ["launcher", "cli"]) { - const path = join(runDir, `${name}.json`); - if (!existsSync(path)) continue; - const control = JSON.parse(readFileSync(path, "utf8")); - if (Object.keys(control).sort().join(",") !== "pid,start" || !Number.isSafeInteger(control.pid) || control.pid <= 1 || - typeof control.start !== "string" || !control.start) throw new Error("Invalid control identity"); - controls.set(name, control); - } - // Leave time for TERM-resistant active descendants after control shutdown. - const force = Date.now() >= deadline - 4000; - for (const control of controls.values()) signal(control.pid, control.start, force ? "SIGKILL" : "SIGTERM"); - remaining = [...controls.values()].filter((control) => identity(control.pid) === control.start).map((control) => control.pid); - if (!remaining.length) { - stopped = true; - records = ownership(runDir); - const processes = snapshot(); - const owned = new Map(); - for (const record of records) { - const members = [...processes.values()].filter((member) => member.group === record.processGroupId); - const known = members.some((member) => record.members.some((previous) => - previous.pid === member.pid && previous.startTime === `${boot}:${member.start}`) || - member.env?.includes(`DIFFEX_SHELL_EXECUTION_ID=${record.executionId}`)); - if (!known) continue; // A recycled group ID is not ownership evidence. - for (const member of members) { - if (owned.has(member.pid) && owned.get(member.pid).executionId !== record.executionId) { - throw new Error("Conflicting shell ownership records"); - } - if ([...controls.values()].some((control) => control.pid === member.pid && control.start === member.start)) { - throw new Error("Control process claimed by shell ownership"); - } - owned.set(member.pid, record); - } - } - retained = []; - for (const member of processes.values()) { - const record = owned.get(member.pid); - if (record?.owner === "environment") { - retained.push({ pid: member.pid, startTime: `${boot}:${member.start}`, - processGroupId: member.group, executionId: record.executionId, owner: "environment" }); - } else if (record || member.env?.includes(`DIFFEX_TB_RUN_DIR=${runDir}`)) { - seen.set(member.pid, member.start); - } else if (!member.env) { - remaining.push(member.pid); // Never claim success with uninspectable same-user work. - } - } - for (const [pid, start] of seen) signal(pid, start, Date.now() >= deadline - 2000 ? "SIGKILL" : "SIGTERM"); - remaining.push(...[...seen].filter(([pid, start]) => identity(pid) === start).map(([pid]) => pid)); - // A parent can fork after /proc enumeration and before its signal. - // Scan again after all discovered active work has exited. - quietScans = remaining.length ? 0 : quietScans + 1; - if (quietScans >= 2) break; - } - await delay(100); - } while (Date.now() < deadline); - const report = { - schema_version: 1, phase: "pre-verifier", stop_requested: true, - agent_execution_stopped: stopped && remaining.length === 0 && quietScans >= 2, - ownership_validated: stopped, remaining_agent_pids: [...new Set(remaining)], - control_processes: [...controls].map(([role, control]) => ({ role, ...control })), - retained_processes: retained, ownership_records: records, - }; - writeFileSync(join(runDir, "cleanup.json"), JSON.stringify(report), { mode: 0o600 }); - if (!report.agent_execution_stopped) process.exitCode = 1; -} - -module.exports = { identity }; -if (require.main === module) cleanup(process.argv[2]).catch((error) => { - writeFileSync(join(process.argv[2], "cleanup.json"), JSON.stringify({ schema_version: 1, phase: "pre-verifier", agent_execution_stopped: false, ownership_validated: false, error: error.code ?? error.name, path: error.path }), { mode: 0o600 }); - process.exitCode = 1; -}); diff --git a/evals/terminal-bench/src/diffex_terminal_bench/config.py b/evals/terminal-bench/src/diffex_terminal_bench/config.py deleted file mode 100644 index 60e20b5f5..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/config.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Checked-in installation pins shared by the adapter and smoke runner.""" - -from importlib.resources import files -import tomllib - - -PINS = tomllib.loads(files(__package__).joinpath("pins.toml").read_text()) diff --git a/evals/terminal-bench/src/diffex_terminal_bench/dataset.py b/evals/terminal-bench/src/diffex_terminal_bench/dataset.py deleted file mode 100644 index 9eac6c0dc..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/dataset.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Download verified benchmark inputs independently of mutable registries.""" - -from contextlib import contextmanager -import fcntl -import os -from pathlib import Path -import re -import shutil -import tarfile -import tempfile -import tomllib -from urllib.request import urlopen - -from harbor.models.task.task import Task -from harbor.publisher.packager import Packager -from pydantic import BaseModel, ConfigDict, Field, field_validator - -from diffex_terminal_bench.execution import ExecutionOptions, PROVIDER_ENV -from diffex_terminal_bench.gate import sha256, stage_tasks - - -class DatasetPin(BaseModel): - model_config = ConfigDict(extra="forbid") - name: str - version: str - git_url: str - git_commit: str = Field(pattern=r"^[0-9a-f]{40}$") - archive_url: str = Field(pattern=r"^https://") - archive_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") - task_count: int = Field(ge=10) - - -class BenchmarkConfig(BaseModel): - model_config = ConfigDict(extra="forbid") - model: str - concurrency: int = Field(default=1, ge=1, strict=True) - pilot_tasks: list[str] - dataset: DatasetPin - policy: ExecutionOptions - - @field_validator("model") - @classmethod - def exact_model(cls, value: str) -> str: - provider, separator, model = value.partition("/") - if not separator or provider not in PROVIDER_ENV or not model or re.search(r"[\s*?:]", model): - raise ValueError("Select an exact supported provider/model") - return value - - -def load_config(path: Path) -> BenchmarkConfig: - return BenchmarkConfig.model_validate(tomllib.loads(path.read_text())) - - -@contextmanager -def prepare_dataset(config: BenchmarkConfig, cache: Path): - """Re-extract the verified archive each time; never trust an edited cache.""" - cache.mkdir(parents=True, exist_ok=True) - archive = cache / f"{config.dataset.archive_sha256}.tar.gz" - with tempfile.TemporaryDirectory(prefix="diffex-tb-inputs-") as temp: - # Serialize publication and extraction for this digest, not execution. - # Separate runs can share the archive but never their extracted tasks. - with (cache / f"{config.dataset.archive_sha256}.lock").open("a") as lock: - fcntl.flock(lock, fcntl.LOCK_EX) - if not archive.exists(): - with tempfile.TemporaryDirectory(dir=cache, prefix=".download-") as staging: - downloaded = Path(staging) / archive.name - with downloaded.open("wb") as download, urlopen(config.dataset.archive_url, timeout=60) as response: - shutil.copyfileobj(response, download) - download.flush() - os.fsync(download.fileno()) - if sha256(downloaded) != config.dataset.archive_sha256: - raise ValueError("Dataset download SHA-256 differs from benchmark.toml") - # Readers can never observe an incomplete published archive. - os.replace(downloaded, archive) - if sha256(archive) != config.dataset.archive_sha256: - raise ValueError("Cached dataset archive SHA-256 differs from benchmark.toml") - with tarfile.open(archive) as tar: - tar.extractall(temp, filter="data") - roots = list(Path(temp).iterdir()) - if len(roots) != 1 or not roots[0].is_dir(): - raise ValueError("Expected a single dataset archive root") - # TB 2.1 keeps benchmark tasks under tasks/. Repository templates and - # leaderboard tooling outside this directory are not benchmark inputs. - tasks = {path.name: path for path in sorted((roots[0] / "tasks").iterdir()) if Task.is_valid_dir(path)} - if len(tasks) != config.dataset.task_count: - raise ValueError("Dataset task count differs from the pin") - stage_tasks("pilot", config.pilot_tasks, list(tasks)) - digests = {name: "sha256:" + Packager.compute_content_hash(path)[0] for name, path in tasks.items()} - yield tasks, digests diff --git a/evals/terminal-bench/src/diffex_terminal_bench/daytona_backend.py b/evals/terminal-bench/src/diffex_terminal_bench/daytona_backend.py deleted file mode 100644 index 18bf86a33..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/daytona_backend.py +++ /dev/null @@ -1,215 +0,0 @@ -"""Retain sandbox identities and independently verify Harbor's cleanup.""" - -import asyncio -from datetime import datetime, timezone -from pathlib import Path -from uuid import uuid4 - -from daytona import AsyncDaytona, CreateSandboxFromImageParams, CreateSandboxFromSnapshotParams, ListSandboxesQuery, VolumeMount -from daytona.common.errors import DaytonaConflictError, DaytonaNotFoundError, DaytonaTimeoutError -from harbor.environments.daytona import DaytonaEnvironment -from harbor.trial.errors import EnvironmentStartTimeoutError - -from diffex_terminal_bench.gate import read_json, write_json -from diffex_terminal_bench.transport import retry_transport - - -class RecordedDaytonaEnvironment(DaytonaEnvironment): - async def _create_sandbox( - self, params: CreateSandboxFromImageParams | CreateSandboxFromSnapshotParams, - daytona: AsyncDaytona | None = None, - ) -> None: - # Harbor has no public hook for SDK volume mounts. This creation - # boundary covers both direct sandboxes and the DinD host (whose /logs - # paths Harbor bind-mounts into the task container). - if daytona is None: - if self._client_manager is None: - raise RuntimeError("Daytona client manager is not initialized") - daytona = await self._client_manager.get_client() - name = self._kwargs.get("artifact_volume", "diffex-terminal-bench-logs") - if not isinstance(name, str) or not name.strip(): - raise ValueError("artifact_volume must be a nonempty volume name") - - async def get_volume(): - try: - return await daytona.volume.get(name, create=True) - except DaytonaConflictError: - # Another concurrent trial or runner created the shared volume. - return await daytona.volume.get(name) - - async with asyncio.timeout(120): - volume = await retry_transport(get_volume) - while volume.state != "ready": - if volume.state not in ("pending_create", "creating"): - raise RuntimeError(f"Artifact volume is not usable: {volume.state}") - await asyncio.sleep(1) - volume = await retry_transport(lambda: daytona.volume.get(name)) - - # One project volume, not one per trial: Daytona permits 100 volumes - # per organization. A random subpath isolates concurrent jobs/retries. - mount = VolumeMount(volume_id=volume.id, mount_path="/logs", subpath=f"trials/{uuid4().hex}") - params.volumes = [*(params.volumes or []), mount] - path = self.trial_paths.trial_dir / "artifact-volume.json" - record = read_json(path) if path.exists() else {"mounts": []} - record["mounts"].append({ - "session_id": self.session_id, "environment_name": self.environment_name, - "volume_id": volume.id, "volume_name": name, - "mount_path": mount.mount_path, "subpath": mount.subpath, - }) - write_json(path, record) - await self._create_with_retries(params, daytona) - - async def _create_with_retries( - self, params: CreateSandboxFromImageParams | CreateSandboxFromSnapshotParams, - daytona: AsyncDaytona, - ) -> None: - # Use a stable, unique name to recover an accepted create whose SDK - # response timed out. Reusing it also prevents late requests from - # allocating multiple sandboxes for this creation sequence. - params = params.model_copy(deep=True) - params.name = params.name or f"diffex-start-{uuid4().hex}" - params.labels = {**(params.labels or {}), **self._sandbox_labels()} - path = self.trial_paths.trial_dir / "sandbox-startup.json" - record = read_json(path) if path.exists() else {"attempts": []} - for attempt, timeout in enumerate((60, 60, 60, 300), start=1): - entry = {"session_id": self.session_id, "name": params.name, - "attempt": attempt, "timeout_sec": timeout, "status": "starting"} - record["attempts"].append(entry) - write_json(path, record) - self.logger.info("Creating sandbox: attempt %s/4, timeout %ss", attempt, timeout) - try: - async with asyncio.timeout(timeout): - self._sandbox = await daytona.create(params=params, timeout=timeout) - except (TimeoutError, DaytonaTimeoutError) as exc: - entry["status"] = "timed_out" - await self._cleanup_start_attempt(daytona, params.name, entry, path, record) - if attempt == 4: - raise EnvironmentStartTimeoutError( - "Sandbox creation timed out after four attempts (60, 60, 60, 300 seconds)" - ) from exc - except BaseException: - # Cancellation and permanent provider errors must not start a - # replacement sandbox. Recover any accepted create first. - entry["status"] = "failed" - try: - await self._cleanup_start_attempt(daytona, params.name, entry, path, record) - except Exception: - self.logger.exception("Could not clean up failed sandbox creation %s", params.name) - raise - else: - entry.update(status="started", sandbox_id=self._sandbox.id) - write_json(path, record) - return - - async def _cleanup_start_attempt(self, daytona: AsyncDaytona, name: str, - entry: dict, path: Path, record: dict) -> None: - async def cleanup(): - async with asyncio.timeout(20): - try: - sandbox = await daytona.get(name, request_timeout=10) - except DaytonaNotFoundError: - entry["cleanup"] = "absent" - return - # Retain the reference if deletion fails so Harbor stop() can - # retry it. Never create a replacement after failed cleanup. - self._sandbox = sandbox - entry["sandbox_id"] = sandbox.id - try: - await daytona.delete(sandbox, wait=True, timeout=10) - except DaytonaNotFoundError: - pass - self._sandbox = None - entry["cleanup"] = "deleted" - - task = asyncio.create_task(cleanup()) - cancelled = False - try: - while not task.done(): - try: - await asyncio.shield(task) - except asyncio.CancelledError: - cancelled = True - task.result() - except Exception as exc: - entry.update(cleanup="failed", cleanup_error=type(exc).__name__) - raise - finally: - write_json(path, record) - if cancelled: - raise asyncio.CancelledError - - async def start(self, force_build: bool) -> None: - try: - await super().start(force_build) - finally: - # Harbor exposes no public creation-result API. Read its - # pinned sandbox reference before stop() deliberately clears it. - if self._sandbox is not None: - path = self.trial_paths.trial_dir / "sandbox-created.json" - record = read_json(path) if path.exists() else {"sandboxes": []} - sandbox = self._sandbox - record["sandboxes"].append({"id": sandbox.id, "snapshot": sandbox.snapshot, - "cpu": sandbox.cpu, "memory_gib": sandbox.memory, - "disk_gib": sandbox.disk, "labels": sandbox.labels}) - write_json(path, record) - - -async def confirm_cleanup(client: AsyncDaytona, directory: Path, label: str, *, report_path: Path | None = None) -> dict: - """Delete only this trial's orphans; recovered leaks still fail the gate.""" - report = {"confirmed": False, "unexpected_sandboxes": [], "absent_ids": [], "label": label} - - async def is_absent(identifier: str) -> bool: - try: - await client.get(identifier, request_timeout=10) - except DaytonaNotFoundError: - if identifier not in report["absent_ids"]: - report["absent_ids"].append(identifier) - return True - if identifier in report["absent_ids"]: - report["absent_ids"].remove(identifier) - return False - - try: - # List is paginated by the SDK. The unique trial label also catches a - # sandbox created remotely during a timed-out create request. - async with asyncio.timeout(120): - created_path = directory / "sandbox-created.json" - created = read_json(created_path)["sandboxes"] if created_path.exists() else [] - for identifier in sorted({row["id"] for row in created}): - # Deletion may be asynchronous even when Harbor stop returns. - for attempt in range(10): - if await is_absent(identifier): - break - if attempt == 9: - break - await asyncio.sleep(2) - remaining = [sandbox async for sandbox in client.list( - ListSandboxesQuery(labels={"diffex.gate_trial": label}), request_timeout=20)] - for sandbox in remaining: - # Listings can retain deleted sandboxes after get() returns - # not-found. Confirm existence before classifying a leak. - if await is_absent(sandbox.id): - continue - try: - await client.delete(sandbox, wait=True, timeout=60) - except DaytonaNotFoundError: - # Harbor's deletion may finish between lookup and delete. - # A fresh lookup must independently establish absence. - pass - else: - report["unexpected_sandboxes"].append(sandbox.id) - if not await is_absent(sandbox.id): - raise RuntimeError("Sandbox remains after recovery deletion") - async for sandbox in client.list( - ListSandboxesQuery(labels={"diffex.gate_trial": label}), request_timeout=20): - if not await is_absent(sandbox.id): - raise RuntimeError("Sandbox appeared after cleanup") - if not {row["id"] for row in created}.issubset(report["absent_ids"]): - raise RuntimeError("Known sandbox deletion could not be confirmed") - report["confirmed"] = True - except Exception as exc: - report["error"] = type(exc).__name__ - finally: - report["checked_at"] = datetime.now(timezone.utc).isoformat() - write_json(report_path or directory / "sandbox-cleanup.json", report) - return report diff --git a/evals/terminal-bench/src/diffex_terminal_bench/diagnostics.py b/evals/terminal-bench/src/diffex_terminal_bench/diagnostics.py deleted file mode 100644 index 293799ee3..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/diagnostics.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Describe retained trial failures without exposing provider error text.""" - -import json -from pathlib import Path -import re - - -class TrialExecutionFailure(ValueError): - """A rejected Harbor execution with safe, actionable retained diagnostics.""" - - def __init__(self, directory: Path, result: dict): - self.details = diagnose_trial(directory, result) - super().__init__(describe_failure(self.details)) - - -def diagnose_trial(directory: Path, result: dict | None = None) -> dict: - if result is None: - result = json.loads((directory / "result.json").read_text()) - if not isinstance(result, dict): - raise ValueError("Expected a Harbor trial result object") - exception = result.get("exception_info") or {} - name = exception.get("exception_type") if isinstance(exception, dict) else None - details = {"harbor_exception": name if isinstance(name, str) and re.fullmatch(r"[A-Za-z][A-Za-z0-9_]{0,99}", name) else None, - "harbor_failed": bool(exception), "finished": bool(result.get("finished_at")), "retry_attempts": 0, - "rate_limit_retries": 0, "scheduled_retry_delay_sec": 0.0, - "retry_evidence_complete": True} - if details["harbor_exception"] == "AgentTimeoutError": - timeout = re.fullmatch(r"Agent execution timed out after ([0-9]+(?:\.[0-9]+)?) seconds", - str(exception.get("exception_message", ""))) - if timeout and len(timeout[1]) < 16: - details["agent_timeout_sec"] = float(timeout[1]) - streams = sorted((directory / "agent").glob("run-*/stdout.jsonl")) - if not streams: - details["retry_evidence_complete"] = False - for stream in streams: - try: - with stream.open() as source: - for line in source: - try: - event = json.loads(line) - except ValueError: - details["retry_evidence_complete"] = False - continue - if not line.endswith("\n") or not isinstance(event, dict): - details["retry_evidence_complete"] = False - continue - if event.get("type") != "auto_retry_start": - continue - details["retry_attempts"] += 1 - if re.search(r"rate.?limit|too many requests|\b429\b", str(event.get("errorMessage", "")), re.IGNORECASE): - details["rate_limit_retries"] += 1 - delay = event.get("delayMs") - if type(delay) in (int, float) and 0 <= delay <= 86_400_000: - details["scheduled_retry_delay_sec"] += delay / 1000 - except (OSError, UnicodeError): - details["retry_evidence_complete"] = False - return details - - -def describe_failure(details: dict) -> str: - reason = details.get("harbor_exception") - if not reason: - reason = "Harbor trial failed" if details["harbor_failed"] else ( - "Harbor trial has no finish timestamp" if not details["finished"] else "Harbor trial finished without a recorded exception") - if "agent_timeout_sec" in details: - reason += f" at the {details['agent_timeout_sec']:g}s task deadline" - if details["retry_attempts"]: - reason += (f"; observed {details['retry_attempts']} scheduled retries" - f" ({details['rate_limit_retries']} rate-limit retries," - f" {details['scheduled_retry_delay_sec']:g}s scheduled backoff)") - if details["rate_limit_retries"]: - reason += "; provider throttling was observed; ensure available provider capacity before retrying" - if not details["retry_evidence_complete"]: - reason += "; retry evidence is incomplete" - return reason diff --git a/evals/terminal-bench/src/diffex_terminal_bench/eval-system-prompt.txt b/evals/terminal-bench/src/diffex_terminal_bench/eval-system-prompt.txt deleted file mode 100644 index a024a16ac..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/eval-system-prompt.txt +++ /dev/null @@ -1,39 +0,0 @@ -Available tools: -- read: Read file contents -- bash: Execute bash commands (ls, grep, find, etc.) -- edit: Make precise file edits with exact text replacement, including multiple disjoint edits in one call -- write: Create or overwrite files -- update_plan: track progress with an ordered task checklist - -Tool usage guidelines: -- read: - - Use read to examine files instead of cat or sed. -- bash: - - Use bash for file operations like ls, rg, find - - You can inspect DIFFEX_* environment variables for current model and session details. -- edit: - - Use edit for precise changes (edits[].oldText must match exactly) - - When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls - - Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit. - - Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions. -- write: - - Use write only for new files or complete rewrites. -- update_plan: - - Use update_plan for complex, ambiguous, dependent, long-running, or multi-part work, when the user requests a plan, or when checkpoints would improve collaboration; skip it for simple or immediately executable work. - - Write short, ordered, outcome-focused steps with observable evidence where useful; do not narrate commands or pad the checklist with obvious actions. - - Include final verification when correctness requires it. - - Update the checklist at meaningful milestones and mark completed work before moving on. - - Explain material replanning, omit explanations for routine progress, and use clear: true only when abandoning the checklist. - - Publishing the checklist communicates progress but does not execute, schedule, or prove completion of its steps. - -Conversational Style: -- Keep responses concise and direct, adapting detail and terminology to the user. -- No emojis anywhere. -- No fluff or cheerful filler text (e.g., "Thanks @user" not "Thanks so much @user!"). -- Show file paths clearly when working with files - -Quality: -- Follow repository instructions, conventions, and existing tooling. - -Diffex documentation (read the documentation website only when the user asks about Diffex itself, its SDK, extensions, or skills): -- Documentation website: https://diffex.ai/docs/overview/ diff --git a/evals/terminal-bench/src/diffex_terminal_bench/evidence.cjs b/evals/terminal-bench/src/diffex_terminal_bench/evidence.cjs deleted file mode 100644 index 61857ee6d..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/evidence.cjs +++ /dev/null @@ -1,26 +0,0 @@ -// Transfer shutdown evidence without traversing logs, sessions, or agent state. -const { readFileSync, readdirSync } = require("node:fs"); -const { join } = require("node:path"); - -const directory = process.argv[2]; -const evidence = {}; -for (const name of ["cleanup.json", "stop-requested", "launcher.json", "cli.json", "process.json"]) { - try { - evidence[name] = readFileSync(join(directory, name), "utf8"); - } catch (error) { - if (error.code !== "ENOENT") throw error; - } -} -// A lost cleanup RPC response can arrive before the remote process has -// published its report. Absence is retryable; a negative report is not. -if (!("cleanup.json" in evidence) || !("stop-requested" in evidence)) { - process.stdout.write("null"); -} else { - let ownership = []; - try { ownership = readdirSync(join(directory, "ownership")); } - catch (error) { if (error.code !== "ENOENT") throw error; } - for (const name of ownership.filter((name) => name.endsWith(".json")).sort()) { - evidence[`ownership/${name}`] = readFileSync(join(directory, "ownership", name), "utf8"); - } - process.stdout.write(JSON.stringify(evidence)); -} diff --git a/evals/terminal-bench/src/diffex_terminal_bench/execution.py b/evals/terminal-bench/src/diffex_terminal_bench/execution.py deleted file mode 100644 index 3f168e74c..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/execution.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Explicit execution policy for the pinned Diffex release.""" - -import hashlib -from importlib.resources import files -from typing import Literal - -from harbor.agents.options import InstalledAgentOptions -from pydantic import field_validator - - -PROVIDER_ENV = { - "anthropic": ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_OAUTH_TOKEN"), - "openai": ("OPENAI_API_KEY",), - "google": ("GEMINI_API_KEY",), - "openrouter": ("OPENROUTER_API_KEY",), - "deepseek": ("DEEPSEEK_API_KEY",), - "groq": ("GROQ_API_KEY",), - "xai": ("XAI_API_KEY",), - "mistral": ("MISTRAL_API_KEY",), -} -BUILTIN_TOOLS = ("read", "bash", "edit", "write", "update_plan", "grep", "find", "ls") -SUBAGENT_TOOLS = ("spawn_agent", "agent_action", "agent_status") -USER_MESSAGE_SUFFIX = "\nDo not cheat by using online solutions or hints specific to this task." -SYSTEM_PROMPT = files(__package__).joinpath("eval-system-prompt.txt").read_text(encoding="utf-8") -SYSTEM_PROMPT_SHA256 = hashlib.sha256(SYSTEM_PROMPT.encode("utf-8")).hexdigest() - - -class ExecutionOptions(InstalledAgentOptions): - thinking: Literal["off", "minimal", "low", "medium", "high", "xhigh", "max"] = "high" - tools: str = "read,bash,edit,write,update_plan" - # The pinned CLI does not propagate prompt/context flags to child agents. - subagents: Literal["disabled"] = "disabled" - project_resources: Literal["context-only"] = "context-only" - background_process_lifetime: Literal["environment"] = "environment" - task_environment: Literal["isolated", "preserve"] = "isolated" - # Arbitrary native config and prompt templates could defeat trial isolation - # or alter the benchmark instruction. Reject them in Harbor preflight too. - config: None = None - prompt_template_path: None = None - - @field_validator("tools") - @classmethod - def validate_tools(cls, value: str) -> str: - names = value.split(",") - if not names or len(set(names)) != len(names) or any(name not in BUILTIN_TOOLS for name in names): - raise ValueError("tools must be unique comma-separated built-in tool names") - return value diff --git a/evals/terminal-bench/src/diffex_terminal_bench/export_job.py b/evals/terminal-bench/src/diffex_terminal_bench/export_job.py deleted file mode 100644 index 553b1e3c8..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/export_job.py +++ /dev/null @@ -1,180 +0,0 @@ -"""Export retained qualification trials as a Harbor job without executing them.""" - -import argparse -from collections import Counter -from datetime import datetime -from importlib.metadata import version -import os -from pathlib import Path -import shutil -import tempfile -from uuid import NAMESPACE_URL, uuid5 - -from harbor.models.job.config import JobConfig, RetryConfig -from harbor.models.job.lock import HarborLockInfo, JobLock, TrialLock -from harbor.models.job.result import JobResult, JobStats -from harbor.models.trial.config import TrialConfig -from harbor.models.trial.result import TrialResult - -from diffex_terminal_bench.config import PINS -from diffex_terminal_bench.gate import fingerprint, inventory, read_json, stage_tasks, write_json - - -def export_job(source: Path, output: Path, *, allow_partial: bool = False) -> Path: - """Copy immutable trial evidence and synthesize only job-level metadata.""" - if version("harbor") != PINS["harbor_version"]: - raise ValueError("Run uv sync --locked before exporting") - if source.is_symlink() or output.is_symlink(): - raise ValueError("Export paths must not be symlinks") - source, output = source.resolve(), output.resolve() - if output == source or output.is_relative_to(source) or source.is_relative_to(output): - raise ValueError("Export destination must be separate from the source stage") - if output.exists(): - raise FileExistsError(f"Export destination already exists: {output}") - before = inventory(source) - gate = read_json(source / "gate.json") - identity = gate["identity"] - if gate["identity_sha256"] != fingerprint(identity): - raise ValueError("Gate identity fingerprint does not match") - if identity["harbor_version"] != version("harbor"): - raise ValueError("Export requires the Harbor version recorded by the stage") - if not gate.get("finished_at"): - raise ValueError("Stage has not stopped; wait for its final gate.json before exporting") - selected = stage_tasks(gate["stage"], identity["pilot_tasks"], list(identity["task_digests"]), sample=identity.get("sample")) - planned = {f"{index:03d}-{task}": task for index, task in enumerate(selected, 1)} - checked = {row["directory"]: row for row in gate["trials"]} - if len(checked) != len(gate["trials"]) or not checked.keys() <= planned.keys(): - raise ValueError("Gate contains duplicate or unexpected trials") - directories = sorted(path for path in source.iterdir() if path.is_dir()) - if not directories or any(path.name not in planned for path in directories): - raise ValueError("Stage has no trials or contains unexpected trial directories") - results = [] - locks = [] - common = None - for directory in directories: - # Even failed/unchecked trials must have links confined to this trial, - # not merely to the encompassing stage scanned above. - trial_files = inventory(directory) - if directory.name in checked: - row = checked[directory.name] - if row["task"] != planned[directory.name] or trial_files != row["files"]: - raise ValueError(f"{directory.name}: retained checked evidence changed") - # Never silently omit a started trial, including failed or cancelled work. - result = TrialResult.model_validate_json((directory / "result.json").read_text()) - config = TrialConfig.model_validate_json((directory / "config.json").read_text()) - lock = TrialLock.model_validate_json((directory / "lock.json").read_text()) - if not result.finished_at: - raise ValueError(f"{directory.name}: trial has no terminal result") - if (result.trial_name != directory.name or config.trial_name != directory.name - or result.task_name.rsplit("/", 1)[-1] != planned[directory.name] - or result.config.model_dump() != config.model_dump() - or lock.task.digest != identity["task_digests"][planned[directory.name]]): - raise ValueError(f"{directory.name}: trial identity or configuration differs from retained evidence") - values = config.model_dump(exclude={"task", "trial_name", "trials_dir", "job_id", "source_trial"}) - # This unique sandbox ownership label belongs to the trial, not the job. - labels = values["environment"]["kwargs"].get("labels", {}) - labels.pop("diffex.gate_trial", None) - if not labels: - values["environment"]["kwargs"].pop("labels", None) - if common is not None and common != values: - raise ValueError("Stage mixes execution configurations") - common = values - results.append(result) - locks.append(lock) - if len({result.id for result in results}) != len(results): - raise ValueError("Stage contains duplicate trial IDs") - if not checked.keys() <= {path.name for path in directories}: - raise ValueError("A checked trial directory is missing") - complete = len(results) == len(selected) - if gate.get("passed") is True and (not complete or len(checked) != len(selected) - or any(result.exception_info for result in results)): - raise ValueError("Passing gate has incomplete coverage or errored trials") - if (not complete or gate.get("passed") is not True) and not allow_partial: - raise ValueError("Stage failed or is incomplete; use --allow-partial to export all retained terminal results") - - job_id = uuid5(NAMESPACE_URL, "diffex-terminal-bench-export:" + fingerprint(before)) - counts = Counter(selected) - # Original task paths can be expired temporary extractions. Preserve them - # as provenance; this export is for viewing/upload, not Harbor job resume. - sample = results[0].config.task - tasks = {planned[result.trial_name]: result.config.task for result in results} - job_values = {key: value for key, value in common.items() if key in JobConfig.model_fields} - job_config = JobConfig( - **job_values, agents=[results[0].config.agent], job_name=output.name, jobs_dir=output.parent, - n_concurrent_trials=identity["concurrency"], retry=RetryConfig(max_retries=identity["harbor_retries"]), - n_attempts=next(iter(counts.values())), - tasks=[tasks.get(task) or sample.model_copy(update={"path": sample.path.parent / task}) for task in counts], - ) - stats = JobStats.from_trial_results(results, n_total_trials=len(selected)) - job_result = JobResult(id=job_id, started_at=datetime.fromisoformat(gate["started_at"]), - finished_at=datetime.fromisoformat(gate["finished_at"]), - n_total_trials=len(selected), stats=stats) - job_lock = JobLock(created_at=job_result.started_at, harbor=HarborLockInfo(version=identity["harbor_version"]), - n_concurrent_trials=job_config.n_concurrent_trials, retry=job_config.retry, trials=locks) - manifest = {"schema_version": 1, "job_id": str(job_id), "source_stage": str(source), - "stage": gate["stage"], "recorded_gate_passed": gate.get("passed") is True, - "qualification_revalidated": False, "planned_trials": len(selected), "exported_trials": len(results), - "missing_trials": sorted(planned.keys() - {path.name for path in directories}), - "source_files": before} - output.parent.mkdir(parents=True, exist_ok=True) - with tempfile.TemporaryDirectory(prefix=".diffex-export-", dir=output.parent) as temporary: - staging = Path(temporary) / output.name - staging.mkdir() - for directory in directories: - destination = staging / directory.name - shutil.copytree(directory, destination, symlinks=True) - if inventory(destination) != inventory(directory): - raise ValueError("Trial evidence changed while copying") - # Harbor's archive allowlist omits custom trial-root files. Retain - # them under artifacts so cleanup evidence survives Hub downloads. - provenance = destination / "artifacts/diffex-qualification" - provenance.mkdir(parents=True, exist_ok=False) - for name in ("cleanup-request.json", "sandbox-created.json", "sandbox-cleanup.json", "sandbox-recovery.json"): - path = directory / name - if path.is_file(): - shutil.copy2(path, provenance / path.name) - provenance = staging / directories[0].name / "artifacts/diffex-qualification" - shutil.copy2(source / "gate.json", provenance / "gate.json") - write_json(provenance / "export.json", manifest) - for name, model in (("config.json", job_config), ("result.json", job_result), ("lock.json", job_lock)): - write_json(staging / name, model.model_dump(mode="json")) - write_json(staging / "export.json", manifest) - (staging / "analysis.md").write_text( - "# Diffex qualification export\n\n" - f"Stage: {gate['stage']}.\n" - f"Recorded gate passed: {manifest['recorded_gate_passed']}.\n" - f"Retained terminal trials: {len(results)} of {len(selected)} planned.\n" - f"Errored trials: {stats.n_errored_trials}.\n\n" - "This is a snapshot of retained evidence, not a new qualification or evaluation.\n" - "Failures, rewards, trial IDs, sessions, and trajectories are preserved.\n" - "Missing trials were not run and must not be interpreted as successful.\n" - "Job usage totals are observed sums; failed trials can have unknown usage.\n" - "Use this directory for upload and viewing, not `harbor job resume`.\n" - f"The source gate and file hashes are in `{directories[0].name}/artifacts/diffex-qualification/`.\n" - ) - if inventory(source) != before: - raise ValueError("Source stage changed during export; wait for all writers to stop") - if output.exists(): - raise FileExistsError(f"Export destination already exists: {output}") - staging.rename(output) - return output - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("stage", type=Path, help="Retained stage directory containing gate.json") - parser.add_argument("--output", type=Path, required=True, help="New Harbor job directory; never overwrites") - parser.add_argument("--allow-partial", action="store_true", help="Include every terminal trial from a stopped failed/incomplete stage") - args = parser.parse_args() - os.umask(0o077) - try: - destination = export_job(args.stage, args.output, allow_partial=args.allow_partial) - except (OSError, ValueError, KeyError) as exc: - parser.exit(1, f"Export failed: {exc}\n") - report = read_json(destination / "export.json") - print(f"Exported {report['exported_trials']}/{report['planned_trials']} trials: {destination}") - print("No evaluations or uploads were performed. Use harbor upload on the exported directory.") - - -if __name__ == "__main__": - main() diff --git a/evals/terminal-bench/src/diffex_terminal_bench/gate.py b/evals/terminal-bench/src/diffex_terminal_bench/gate.py deleted file mode 100644 index 90b72f93d..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/gate.py +++ /dev/null @@ -1,287 +0,0 @@ -"""Fail-closed evidence checks for staged benchmark qualification.""" - -from collections import Counter -import hashlib -import json -import math -import os -from pathlib import Path -import random -import stat -import tempfile - -from harbor.models.agent.context import AgentContext -from harbor.models.trial.config import AgentConfig, EnvironmentConfig, TrialConfig -from harbor.utils.trajectory_validator import TrajectoryValidator - -from diffex_terminal_bench.config import PINS -from diffex_terminal_bench.diagnostics import TrialExecutionFailure -from diffex_terminal_bench.execution import PROVIDER_ENV, SUBAGENT_TOOLS -from diffex_terminal_bench.outcome import parse_outcome -from diffex_terminal_bench.trajectory import CONTEXT_FIELDS, export_trajectory - - -STAGES = ("oracle", "trial", "integration", "pilot", "reliability", "full") - - -def read_json(path: Path) -> dict: - value = json.loads(path.read_text()) - if not isinstance(value, dict): - raise ValueError(f"Expected JSON object: {path.name}") - return value - - -def write_json(path: Path, value: dict) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - temporary = path.with_suffix(".tmp") - temporary.write_text(json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n") - temporary.replace(path) - - -def sha256(path: Path) -> str: - with path.open("rb") as stream: - return hashlib.file_digest(stream, "sha256").hexdigest() - - -def fingerprint(value: dict) -> str: - return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() - - -def sample_tasks(all_tasks: list[str], count: int, seed: int) -> list[str]: - if type(count) is not int or not 1 <= count <= len(all_tasks): - raise ValueError(f"Sample count must be between 1 and {len(all_tasks)}") - if type(seed) is not int: - raise ValueError("Sample seed must be an integer") - if len(set(all_tasks)) != len(all_tasks): - raise ValueError("Sample population must contain distinct tasks") - return random.Random(seed).sample(sorted(all_tasks), count) - - -def stage_tasks(stage: str, pilot: list[str], all_tasks: list[str], *, sample: dict | None = None) -> list[str]: - if stage == "sample": - if not isinstance(sample, dict) or set(sample) != {"count", "seed", "tasks"}: - raise ValueError("Missing or invalid sample selection") - selected = sample_tasks(all_tasks, sample["count"], sample["seed"]) - if sample["tasks"] != selected: - raise ValueError("Sample tasks do not match the recorded seed and dataset") - return selected - if len(pilot) != 10 or len(set(pilot)) != 10 or not set(pilot).issubset(all_tasks): - raise ValueError("The pilot must select ten distinct pinned tasks") - if stage in ("oracle", "trial"): - return pilot[:1] - if stage == "integration": - return pilot[:3] - if stage == "pilot": - return pilot - if stage == "reliability": - return pilot * 3 - if stage == "full": - return all_tasks * 5 - raise ValueError(f"Unknown stage: {stage}") - - -def inventory(directory: Path) -> dict[str, str]: - """Hash retained files and portable links without following link targets. - - Cache snapshots may link directly to regular files inside the evidence root. - Bind the link text separately from file contents so retargeting, replacing a - link with a file, and editing its target all change the inventory. - """ - if directory.is_symlink(): - raise ValueError("Evidence root must not be a symlink") - directory = directory.resolve(strict=True) - result = {} - for path in sorted(directory.rglob("*")): - relative = path.relative_to(directory).as_posix() - if path.is_symlink(): - link = os.readlink(path) - target = path.parent - if Path(link).is_absolute(): - raise ValueError(f"Evidence symlinks must be relative: {relative}") - # Walk the original components, including '..', before resolving. - # Normalizing first could hide traversal through an external path - # or a directory symlink. Chained links are deliberately rejected. - try: - for part in Path(link).parts: - if not stat.S_ISDIR(target.lstat().st_mode): - raise ValueError(f"Evidence symlinks must not traverse other links or files: {relative}") - target = target.parent if part == ".." else target / part - if not target.is_relative_to(directory): - raise ValueError(f"Evidence symlinks must stay inside their root: {relative}") - if not stat.S_ISREG(target.lstat().st_mode) or not path.is_file(): - raise ValueError(f"Evidence symlinks must point directly to regular files: {relative}") - except OSError as exc: - raise ValueError(f"Evidence symlinks must have existing regular-file targets: {relative}") from exc - result[relative] = "symlink:" + fingerprint({"target": link}) - elif path.is_file(): - result[relative] = sha256(path) - return result - - -def require_previous(root: Path, stage: str, identity: dict, pilot: list[str], tasks: list[str], *, include_current: bool = False) -> None: - """Recheck every prior artifact, not just a cached 'passed' flag.""" - for previous in STAGES[:STAGES.index(stage) + int(include_current)]: - verify_stage(root, previous, identity, pilot, tasks) - - -def verify_stage(root: Path, stage: str, identity: dict, pilot: list[str], tasks: list[str]) -> None: - directory = root / stage - report = read_json(directory / "gate.json") - if report.get("passed") is not True or report.get("identity") != identity: - raise ValueError(f"{stage}: missing qualification for this exact configuration") - selected = stage_tasks(stage, pilot, tasks, sample=identity.get("sample")) - trials = report.get("trials", []) - if Counter(row["task"] for row in trials) != Counter(selected): - raise ValueError(f"{stage}: trial coverage does not match the stage") - if len({row["directory"] for row in trials}) != len(trials): - raise ValueError(f"{stage}: duplicate trial evidence") - if stage == "sample": - if (report.get("stage") != stage or report.get("identity_sha256") != fingerprint(identity) - or not report.get("finished_at") or report.get("failure") or report.get("failures")): - raise ValueError("Sample report is unfinished, failed, or inconsistent") - if [(row["task"], row["directory"]) for row in trials] != [ - (task, f"{index:03d}-{task}") for index, task in enumerate(selected, 1) - ]: - raise ValueError("Sample trial order differs from the recorded selection") - for row in trials: - trial_dir = directory / row["directory"] - if trial_dir.resolve().parent != directory.resolve(): - raise ValueError("Trial evidence must be a direct child of its stage") - if inventory(trial_dir) != row["files"]: - raise ValueError(f"{stage}: retained evidence changed or is missing") - validate_trial(trial_dir, row["task"], identity, oracle=stage == "oracle") - - -def validate_trial(directory: Path, task: str, identity: dict, *, oracle: bool = False) -> dict: - """A wrong solution is a result; broken execution or missing evidence is not.""" - result = read_json(directory / "result.json") - if result.get("exception_info") or not result.get("finished_at"): - raise TrialExecutionFailure(directory, result) - lock = read_json(directory / "lock.json") - if lock["task"]["digest"] != identity["task_digests"][task]: - raise ValueError("Task digest differs from the pinned dataset") - config = TrialConfig.model_validate_json((directory / "config.json").read_text()) - credentials = config.agent.env - allowed = PROVIDER_ENV[identity["model"].split("/", 1)[0]] - if (oracle and credentials) or (not oracle and not credentials) or any( - key not in allowed or value != f"${{{key}}}" for key, value in credentials.items() - ): - raise ValueError("Credentials must use selected-provider environment references") - agent_config = AgentConfig(name="oracle") if oracle else AgentConfig( - import_path="diffex_terminal_bench.adapter:DiffexAgent", model_name=identity["model"], - override_setup_timeout_sec=630, kwargs=identity["policy"], env=credentials) - label = read_json(directory / "cleanup-request.json")["label"] - expected_config = TrialConfig(task=config.task, trial_name=config.trial_name, trials_dir=config.trials_dir, - agent=agent_config, environment=EnvironmentConfig( - import_path="diffex_terminal_bench.daytona_backend:RecordedDaytonaEnvironment", delete=True, - kwargs={"labels": {"diffex.gate_trial": label}})) - if config.model_dump() != expected_config.model_dump(): - raise ValueError("Harbor configuration changed task limits, policy, or isolation") - for key in ("agent", "environment", "timeout_multiplier", "agent_timeout_multiplier", "verifier_timeout_multiplier", - "agent_setup_timeout_multiplier", "environment_build_timeout_multiplier", "install_only"): - if lock.get(key) != config.model_dump(mode="json", exclude_none=True).get(key): - raise ValueError(f"Harbor lock does not match configuration: {key}") - cleanup = read_json(directory / "sandbox-cleanup.json") - if cleanup.get("confirmed") is not True or cleanup.get("unexpected_sandboxes") != []: - raise ValueError("Sandbox cleanup failed or required recovery") - if not read_json(directory / "sandbox-created.json").get("sandboxes"): - raise ValueError("Missing sandbox creation evidence") - rewards = (result.get("verifier_result") or {}).get("rewards") - if not isinstance(rewards, dict) or not rewards or any( - type(value) not in (int, float) or not math.isfinite(value) for value in rewards.values() - ): - raise ValueError("Missing or invalid verifier reward") - verifier = directory / "verifier" - reward_path = verifier / "reward.json" - saved_rewards = read_json(reward_path) if reward_path.exists() else {"reward": float((verifier / "reward.txt").read_text())} - if saved_rewards != rewards: - raise ValueError("Verifier reward does not reconcile with Harbor") - # Harbor redirects both verifier streams to test-stdout.txt. - if not (verifier / "test-stdout.txt").is_file(): - raise ValueError("Missing combined verifier log: test-stdout.txt") - if oracle: - if any(value != 1 for value in rewards.values()): - raise ValueError("Oracle smoke must earn full reward") - (directory / "agent/oracle.txt").read_bytes() - return {"rewards": rewards, "usage": "not_applicable", "trajectory": "not_applicable"} - - agent = directory / "agent" - installation = read_json(agent / "setup/installation.json") - expected = {"verified": True, "harbor_version": PINS["harbor_version"], - "background_process_lifetime": "environment", - "node_version": f"v{PINS['node_version']}", "diffex_version": PINS["diffex_version"], - "npm_integrity": PINS["diffex_integrity"], "npm_registry": PINS["npm_registry"]} - if any(installation.get(key) != value for key, value in expected.items()): - raise ValueError("Installation evidence differs from pins") - if not 0 <= installation["elapsed_sec"] < PINS["setup_timeout_sec"]: - raise ValueError("Installation missed its deadline") - for command in ("version", "help", "list-models"): - if not (agent / f"setup/{command}.stdout.txt").read_text().strip(): - raise ValueError(f"Missing install check: {command}") - (agent / f"setup/{command}.stderr.txt").read_bytes() - runs = sorted(agent.glob("run-*")) - if len(runs) != 1: - raise ValueError("Expected one isolated Diffex run") - run = runs[0] - outcome = read_json(run / "outcome.json") - if outcome.get("status") != "completed" or outcome.get("recovery_errors"): - raise ValueError("Diffex did not complete cleanly") - native_outcome = parse_outcome(run, launcher_exit_code=outcome.get("launcher_exit_code")) - if native_outcome["status"] != "completed": - raise ValueError("Native execution evidence does not establish completion") - if any(outcome.get(key) != native_outcome.get(key) for key in ( - "agent_execution_stopped", "cleanup_confirmed", "retained_processes")): - raise ValueError("Outcome differs from process ownership evidence") - requested, effective = read_json(run / "requested.json"), read_json(run / "effective.json") - provider, model = identity["model"].split("/", 1) - policy = identity["policy"] - tools = policy["tools"].split(",") + (list(SUBAGENT_TOOLS) if policy["subagents"] == "enabled" else []) - for key, value in {"provider": provider, "model": model, "thinking": policy["thinking"], - "subagents": policy["subagents"], "project_resources": policy["project_resources"], - "background_process_lifetime": policy["background_process_lifetime"], - "pins": PINS, "tools": tools, "evolution": {"enabled": False}, "state_policy": "fresh-per-run"}.items(): - if requested.get(key) != value: - raise ValueError(f"Requested configuration differs from gate identity: {key}") - if sha256(run / "instruction.txt") != requested["instruction_sha256"]: - raise ValueError("Retained instruction changed") - if sha256(run / "system-prompt.txt") != effective["system_prompt_sha256"]: - raise ValueError("System prompt hash mismatch") - if effective["system_prompt_sha256"] != identity["system_prompt_sha256"]: - raise ValueError("Eval system prompt identity mismatch") - for name in ("stderr.log", "launcher.stdout.log", "launcher.stderr.log", "sessions/root.jsonl"): - (run / name).read_bytes() - validator = TrajectoryValidator() - if not validator.validate(agent / "trajectory.json"): - raise ValueError("Invalid ATIF trajectory") - with tempfile.TemporaryDirectory(prefix="diffex-accounting-") as temp: - # Export uses a temporary copy because it also writes accounting.json. - # Retained evidence must never be repaired implicitly by the gate. - source = Path(temp) / "run" - source.mkdir() - for name in ("requested.json", "stdout.jsonl", "system-prompt.txt"): - (source / name).write_bytes((run / name).read_bytes()) - for path in (run / "sessions").rglob("*.jsonl"): - target = source / path.relative_to(run) - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(path.read_bytes()) - context = AgentContext() - accounting = export_trajectory(source, Path(temp) / "trajectory.json", context, outcome) - if accounting["gaps"]: - raise ValueError(f"Native evidence is incomplete: {', '.join(accounting['gaps'])}") - if accounting != read_json(run / "accounting.json"): - raise ValueError("Usage does not reconcile with native sessions") - if read_json(Path(temp) / "trajectory.json") != read_json(agent / "trajectory.json"): - raise ValueError("ATIF differs from authoritative native evidence") - for path in (Path(temp) / "trajectory-media").glob("*"): - if sha256(path) != sha256(agent / "trajectory-media" / path.name): - raise ValueError("Trajectory media changed") - saved_context = result.get("agent_result") or {} - if (saved_context.get("metadata") or {}).get("diffex_accounting") != accounting: - raise ValueError("Harbor accounting metadata differs from native evidence") - if any(saved_context.get(field) != getattr(context, field) for field in CONTEXT_FIELDS): - raise ValueError("Harbor usage differs from native usage") - if saved_context.get("model_usage") != context.model_dump(mode="json")["model_usage"]: - raise ValueError("Per-model usage differs from native usage") - return {"rewards": rewards, "usage": accounting["totals"], "usage_complete": accounting["complete"], - "usage_missing_fields": accounting["missing_fields"], "trajectory": "validated", - "system_prompt_sha256": effective["system_prompt_sha256"], "installation": installation} diff --git a/evals/terminal-bench/src/diffex_terminal_bench/install.sh b/evals/terminal-bench/src/diffex_terminal_bench/install.sh deleted file mode 100644 index b428648a1..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/install.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -umask 022 - -exec > >(tee "$SETUP_LOGS/install.stdout.log") -exec 2> >(tee "$SETUP_LOGS/install.stderr.log" >&2) -export SETUP_STARTED_AT -SETUP_STARTED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ) -SECONDS=0 -prefix=/opt/diffex-terminal-bench -script_dir=$(cd -- "$(dirname -- "$0")" && pwd) - -if [ "$(uname -s)" != Linux ] || ! command -v apt-get >/dev/null; then - echo "Setup requires Debian/Ubuntu Linux with glibc and root access" >&2 - exit 1 -fi -case "$(uname -m)" in - x86_64) arch=x64; checksum=$NODE_SHA256_X64 ;; - aarch64) arch=arm64; checksum=$NODE_SHA256_ARM64 ;; - *) echo "Unsupported Linux architecture: $(uname -m)" >&2; exit 1 ;; -esac -export NODE_ARCH="$arch" NODE_SHA256="$checksum" - -# Harbor's sandbox log transfers also require these base-image utilities. -for utility in tar gzip; do - if ! command -v "$utility" >/dev/null; then - echo "Setup requires $utility in the task image" >&2 - exit 1 - fi -done - -# Refuse to reuse an earlier install or overwrite a task-provided command. -mkdir "$prefix" -if [ -e /usr/local/bin/diffex ] || [ -L /usr/local/bin/diffex ]; then - echo "A task-provided /usr/local/bin/diffex already exists" >&2 - exit 1 -fi -mkdir -p "$HOME" "$prefix/node" "$prefix/cli" "$prefix/work" -cd "$prefix/work" -packages=() -for package in ca-certificates wget; do - if [ "$(dpkg-query -W -f='${Status}' "$package" 2>/dev/null || true)" != "install ok installed" ]; then - packages+=("$package") - fi -done -if [ "${#packages[@]}" -gt 0 ]; then - apt_options=(-o Acquire::Retries=3 -o Acquire::http::No-Cache=true -o Acquire::https::No-Cache=true - -o Acquire::http::Timeout=20 -o Acquire::https::Timeout=20 -o "Dir::State::lists=$prefix/apt-lists") - for attempt in 1 2 3; do - # A retry must refresh metadata after a mirror returns a stale archive URL. - # Keep task-owned indexes untouched and never install from a partial update. - rm -rf "$prefix/apt-lists" - mkdir -p "$prefix/apt-lists/partial" - if apt-get "${apt_options[@]}" -o APT::Update::Error-Mode=any update && - apt-get "${apt_options[@]}" install -y --no-install-recommends "${packages[@]}"; then - break - fi - if [ "$attempt" -eq 3 ]; then - echo "Failed to install prerequisites after $attempt attempts" >&2 - exit 100 - fi - echo "Retrying prerequisite installation with fresh package indexes" >&2 - sleep "$((attempt * 2))" - done -fi -archive="node-v$NODE_VERSION-linux-$arch.tar.gz" -wget --https-only --tries=3 --timeout=20 --output-document="$archive" \ - "https://nodejs.org/dist/v$NODE_VERSION/$archive" -printf '%s %s\n' "$checksum" "$archive" | sha256sum --check --strict -tar -xzf "$archive" -C "$prefix/node" --strip-components=1 -rm "$archive" -export PATH="$prefix/node/bin:$PATH" -test "$(node --version)" = "v$NODE_VERSION" - -touch "$prefix/npm-user.npmrc" "$prefix/npm-global.npmrc" -export NPM_CONFIG_USERCONFIG="$prefix/npm-user.npmrc" -export NPM_CONFIG_GLOBALCONFIG="$prefix/npm-global.npmrc" -export NPM_CONFIG_CACHE="$prefix/npm-cache" -export NPM_CONFIG_REGISTRY="$NPM_REGISTRY" NPM_CONFIG_IGNORE_SCRIPTS=true -npm view "@diffexai/diffex@$DIFFEX_VERSION" version dist engines \ - --registry="$NPM_REGISTRY" --json > "$SETUP_LOGS/npm-metadata.json" -node "$script_dir/verify-install.cjs" metadata -npm install --prefix "$prefix/cli" --ignore-scripts --save-exact \ - --no-audit --no-fund --registry="$NPM_REGISTRY" "@diffexai/diffex@$DIFFEX_VERSION" -node "$script_dir/verify-install.cjs" installed -cp "$prefix/cli/package-lock.json" "$SETUP_LOGS/npm-install-lock.json" - -cat > /usr/local/bin/diffex <<'WRAPPER' -#!/bin/sh -exec /opt/diffex-terminal-bench/node/bin/node /opt/diffex-terminal-bench/cli/node_modules/@diffexai/diffex/dist/cli.js "$@" -WRAPPER -chmod 755 /usr/local/bin/diffex -export DIFFEX_CODING_AGENT_DIR="$prefix/smoke-state" -cd /tmp -for flag in version help list-models; do - diffex "--$flag" "$SETUP_LOGS/$flag.stdout.txt" 2> "$SETUP_LOGS/$flag.stderr.txt" - test -s "$SETUP_LOGS/$flag.stdout.txt" -done -export SETUP_ELAPSED_SEC="$SECONDS" -node "$script_dir/verify-install.cjs" complete diff --git a/evals/terminal-bench/src/diffex_terminal_bench/lifecycle.py b/evals/terminal-bench/src/diffex_terminal_bench/lifecycle.py deleted file mode 100644 index 6441f1236..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/lifecycle.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Validate pre-verifier control shutdown separately from environment teardown.""" - -import json -from pathlib import Path - - -def read_cleanup(directory: Path) -> dict: - report = json.loads((directory / "cleanup.json").read_text()) - if (not isinstance(report, dict) or type(report.get("schema_version")) is not int or report["schema_version"] != 1 - or report.get("phase") != "pre-verifier" or report.get("stop_requested") is not True - or report.get("agent_execution_stopped") is not True - or report.get("ownership_validated") is not True - or report.get("remaining_agent_pids") != [] or not (directory / "stop-requested").is_file()): - raise ValueError("Agent execution stop is unverified") - controls = report.get("control_processes") - records = report.get("ownership_records") - retained = report.get("retained_processes") - if not all(isinstance(value, list) for value in (controls, records, retained)): - raise ValueError("Missing ownership or control evidence") - expected = [] - for role in ("launcher", "cli"): - path = directory / f"{role}.json" - if path.exists(): - control = json.loads(path.read_text()) - if (not isinstance(control, dict) or set(control) != {"pid", "start"} - or type(control.get("pid")) is not int or control["pid"] <= 1 - or not isinstance(control.get("start"), str) or not control["start"]): - raise ValueError("Invalid control identity") - expected.append({"role": role, **control}) - if controls != expected: - raise ValueError("Control identities differ from shutdown evidence") - process_path = directory / "process.json" - if process_path.exists(): - process = json.loads(process_path.read_text()) - roles = {control["role"] for control in controls} - if "launcher" not in roles or (type(process.get("exit_code")) is int - and not process.get("spawn_error") and "cli" not in roles): - raise ValueError("Missing launched control identity") - saved = [] - for path in sorted((directory / "ownership").glob("*.json")): - record = json.loads(path.read_text()) - if path.name != f"{record['executionId']}.json": - raise ValueError("Ownership filename differs from execution identity") - saved.append(record) - if sorted(records, key=lambda record: record["executionId"]) != sorted(saved, key=lambda record: record["executionId"]): - raise ValueError("Ownership records differ from shutdown evidence") - by_id = {} - for record in records: - if (type(record.get("version")) is not int or record["version"] != 1 or not isinstance(record.get("executionId"), str) - or not record["executionId"] or record["executionId"] in by_id - or type(record.get("processGroupId")) is not int or record["processGroupId"] <= 1 - or type(record.get("supervisorPid")) is not int or record["supervisorPid"] <= 1 - or (record.get("owner"), record.get("state")) not in ( - ("environment", "background"), ("session", "active"), ("session", "terminated")) - or not isinstance(record.get("members"), list) - or (record["owner"] == "environment" and not record["members"])): - raise ValueError("Invalid ownership record") - for member in record["members"]: - if (type(member.get("pid")) is not int or member["pid"] <= 1 - or member.get("processGroupId") != record["processGroupId"] - or not isinstance(member.get("startTime"), str) or not member["startTime"]): - raise ValueError("Invalid owned process identity") - by_id[record["executionId"]] = record - seen = set() - for member in retained: - record = by_id.get(member.get("executionId"), {}) - if (member.get("owner") != "environment" or record.get("owner") != "environment" - or member.get("processGroupId") != record.get("processGroupId") - or type(member.get("pid")) is not int or member["pid"] <= 1 or member["pid"] in seen - or not isinstance(member.get("startTime"), str) or not member["startTime"] - or any(control["pid"] == member["pid"] for control in controls)): - raise ValueError("Retained process lacks unambiguous environment ownership") - seen.add(member["pid"]) - return report diff --git a/evals/terminal-bench/src/diffex_terminal_bench/observe.mjs b/evals/terminal-bench/src/diffex_terminal_bench/observe.mjs deleted file mode 100644 index e2ccb486e..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/observe.mjs +++ /dev/null @@ -1,36 +0,0 @@ -import { createHash } from "node:crypto"; -import { readFileSync, writeFileSync } from "node:fs"; -import { join } from "node:path"; - -// Observe the first user message without registering hooks that alter input, -// provider requests, or tools. Such hooks would disable subagents in 0.2.2. -export default function observe(api) { - const runDir = process.env.DIFFEX_TB_RUN_DIR; - const rootSession = join(runDir, "sessions", "root.jsonl"); - let recorded = false; - api.on("message_start", (event, ctx) => { - if (ctx.sessionManager.getSessionFile() !== rootSession || recorded || event.message.role !== "user") return; - recorded = true; - const instructionPath = join(runDir, "instruction.txt"); - const instruction = readFileSync(instructionPath, "utf8"); - const requested = JSON.parse(readFileSync(join(runDir, "requested.json"), "utf8")); - const prompt = typeof event.message.content === "string" ? event.message.content : - event.message.content.filter((block) => block.type === "text").map((block) => block.text).join(""); - const systemPrompt = ctx.getSystemPrompt(); - writeFileSync(join(runDir, "system-prompt.txt"), systemPrompt, { mode: 0o600 }); - writeFileSync(join(runDir, "effective.json"), JSON.stringify({ - provider: ctx.model?.provider, model: ctx.model?.id, - thinking: api.getThinkingLevel(), tools: api.getActiveTools(), cwd: ctx.cwd, - root_session: rootSession, - background_process_lifetime: process.argv.includes("--background-process-lifetime") - ? process.argv[process.argv.indexOf("--background-process-lifetime") + 1] : null, - instruction_transport: "native-file", - instruction_delivered: prompt === `\n${instruction}\n\n${requested.user_message_suffix}`, - user_message_suffix: requested.user_message_suffix, - instruction_sha256: createHash("sha256").update(instruction).digest("hex"), - prompt_sha256: createHash("sha256").update(prompt).digest("hex"), - system_prompt_sha256: createHash("sha256").update(systemPrompt).digest("hex"), - context_files_disabled: process.argv.includes("--no-context-files"), - }, null, 2), { mode: 0o600 }); - }); -} diff --git a/evals/terminal-bench/src/diffex_terminal_bench/outcome.py b/evals/terminal-bench/src/diffex_terminal_bench/outcome.py deleted file mode 100644 index fbf7a905a..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/outcome.py +++ /dev/null @@ -1,165 +0,0 @@ -"""Classify execution evidence independently of verifier rewards and CLI versions.""" - -import json -from pathlib import Path -import re - -from harbor.agents.installed.base import AgentAuthenticationError, NonZeroAgentExitCodeError - -from diffex_terminal_bench.lifecycle import read_cleanup -from diffex_terminal_bench.execution import SYSTEM_PROMPT_SHA256, USER_MESSAGE_SUFFIX - - -class DiffexExecutionError(NonZeroAgentExitCodeError): - """An agent infrastructure failure, not a completed task with reward zero.""" - - -class DiffexControlStopError(RuntimeError): - """Block Harbor verification when agent execution may still be running.""" - - -def parse_outcome(directory: Path, interruption: str | None = None, - launcher_exit_code: int | None = None, *, shutdown_directory: Path | None = None) -> dict: - result = {"status": "incomplete", "reason": "missing_evidence", "stream_complete": False, - "assistant_stop_reason": None, "retry_attempts": 0, "recovered_retries": 0, - "launcher_exit_code": launcher_exit_code, "interruption": interruption} - try: - process = json.loads((directory / "process.json").read_text()) - if not isinstance(process, dict): - raise ValueError("invalid_process") - result["process"] = process - except (OSError, ValueError): - process = {} - try: - cleanup = read_cleanup(shutdown_directory if shutdown_directory is not None else directory) - result.update(cleanup_confirmed=True, agent_execution_stopped=True, - retained_processes=cleanup["retained_processes"]) - except (OSError, ValueError, KeyError, TypeError, AttributeError): - result.update(cleanup_confirmed=False, agent_execution_stopped=False) - # Parse incrementally so long runs do not require the whole stream in memory. - active = False - ended = False - settled = False - pending_retry = False - failed_retry = False - open_message = False - last_assistant = None - terminal = None - stream_error = None - saw_session = False - try: - with (directory / "stdout.jsonl").open(encoding="utf-8") as stream: - for line in stream: - if not line.endswith("\n"): - raise ValueError("truncated_line") - event = json.loads(line) - if not isinstance(event, dict) or not isinstance(event.get("type"), str): - raise ValueError("invalid_event") - kind = event["type"] - if settled: - raise ValueError("event_after_settled") - if kind == "session": - if saw_session: - raise ValueError("duplicate_session") - saw_session = True - elif kind == "agent_start": - if active or open_message: - raise ValueError("overlapping_run") - active, ended, terminal = True, False, None - last_assistant = None - elif kind == "message_start": - if not active or open_message: - raise ValueError("unexpected_message") - open_message = True - elif kind == "message_end": - if not active or not open_message: - raise ValueError("unexpected_message_end") - open_message = False - message = event.get("message") - if not isinstance(message, dict): - raise ValueError("invalid_message") - if message.get("role") == "assistant": - last_assistant = message - elif kind == "message_update": - if not active or not open_message: - raise ValueError("unexpected_update") - elif kind == "agent_end": - if not active or open_message: - raise ValueError("unexpected_agent_end") - messages = event.get("messages") - if not isinstance(messages, list) or not messages or not isinstance(messages[-1], dict): - raise ValueError("missing_terminal_message") - terminal = messages[-1] - if terminal.get("role") != "assistant" or terminal != last_assistant: - raise ValueError("terminal_message_mismatch") - active, ended = False, True - pending_retry = pending_retry or event.get("willRetry") is True - elif kind == "auto_retry_start": - pending_retry, ended = True, False - result["retry_attempts"] += 1 - elif kind == "auto_retry_end": - if not isinstance(event.get("success"), bool): - raise ValueError("invalid_retry_outcome") - pending_retry = False - failed_retry = not event["success"] - if event["success"]: - result["recovered_retries"] += 1 - elif kind == "agent_settled": - if active or not ended or pending_retry or open_message: - raise ValueError("premature_settled") - settled = True - elif kind == "error": - raise ValueError("unhandled_error_event") - elif kind in ("turn_start", "turn_end", "tool_execution_start", "tool_execution_update", "tool_execution_end"): - if not active: - raise ValueError("event_outside_run") - except (OSError, ValueError, UnicodeError) as exc: - # Never copy provider text or malformed raw output into summary/errors. - stream_error = type(exc).__name__ - complete = saw_session and ended and settled and not active and not open_message and not pending_retry and stream_error is None - result["stream_complete"] = complete - if terminal: - result["assistant_stop_reason"] = terminal.get("stopReason") - if interruption: - result.update(status=interruption, reason="execution_interrupted") - elif not result.get("cleanup_confirmed"): - result.update(status="cleanup_failure", reason="process_cleanup_unverified") - elif terminal and terminal.get("stopReason") in ("error", "aborted"): - authentication = re.search(r"\b401\b|authentication_error|invalid[ _-]api[ _-]key|unauthorized", - str(terminal.get("errorMessage", "")), re.IGNORECASE) - result.update(status="authentication_failure" if authentication else "provider_failure", - reason="terminal_assistant_error") - elif failed_retry: - result.update(status="provider_failure", reason="retry_failed") - elif process.get("exit_code") != 0 or process.get("signal") or process.get("spawn_error") or launcher_exit_code != 0: - result.update(status="process_failure", reason="unsuccessful_process") - elif not complete or not terminal or terminal.get("stopReason") != "stop": - result.update(status="incomplete", reason="unfinished_json_stream") - else: - try: - requested = json.loads((directory / "requested.json").read_text()) - effective = json.loads((directory / "effective.json").read_text()) - if not isinstance(requested, dict) or not isinstance(effective, dict): - raise ValueError("invalid_configuration") - matches = all(effective.get(key) == requested[key] for key in ("provider", "model", "thinking", "instruction_sha256", "background_process_lifetime", "user_message_suffix", "system_prompt_sha256")) - matches = matches and requested["user_message_suffix"] == USER_MESSAGE_SUFFIX - matches = matches and requested["system_prompt_sha256"] == SYSTEM_PROMPT_SHA256 - matches = matches and requested.get("background_process_lifetime") == "environment" - matches = matches and effective.get("context_files_disabled") is True and effective.get("instruction_delivered") is True - matches = matches and sorted(effective["tools"]) == sorted(requested["tools"]) - except (OSError, ValueError, KeyError, TypeError): - matches = False - result.update(status="completed" if matches else "configuration_failure", - reason="completed" if matches else "effective_configuration_mismatch") - return result - - -def require_completed(outcome: dict) -> None: - if outcome.get("agent_execution_stopped") is not True: - raise DiffexControlStopError("Agent execution stop is unverified; verification is blocked") - if outcome["status"] == "completed": - return - message = f"Diffex execution {outcome['status']}: {outcome['reason']}; see retained run artifacts" - if outcome["status"] == "authentication_failure": - raise AgentAuthenticationError(message) - raise DiffexExecutionError(message) diff --git a/evals/terminal-bench/src/diffex_terminal_bench/pins.toml b/evals/terminal-bench/src/diffex_terminal_bench/pins.toml deleted file mode 100644 index 788788490..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/pins.toml +++ /dev/null @@ -1,13 +0,0 @@ -harbor_version = "0.24.0" -node_version = "22.23.2" -diffex_version = "0.3.2" -npm_registry = "https://registry.npmjs.org/" -diffex_integrity = "sha512-iULuVLn4H6Xbeo7GI3sg+4JslJjwgVp4Jv6bbWLHvYN9ZnIdRZTUu+xq4HKs4b7zn84oLjj248oIWSE2VVGLpg==" -# Must match the instructions shipped by diffex_version, not the current checkout. -release_agents_sha256 = "6f194cd7f04d80e08cd3c8b22096de33ba5dbb2ba42176567c466579f8ec26db" -setup_timeout_sec = 600 - -[node_sha256] -# Official node-v22.23.2-linux-.tar.gz archives. -x64 = "b294a556e639d64338823920e5866c21c02741742d2e1529ee1a225c1ec9252a" -arm64 = "013b59cfd2819703a6f4a14ab891fc46fc2a4e3f5bcd92de3fb4929b43e35b30" diff --git a/evals/terminal-bench/src/diffex_terminal_bench/prune_volume.py b/evals/terminal-bench/src/diffex_terminal_bench/prune_volume.py deleted file mode 100644 index 0fa029d3a..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/prune_volume.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Run inside a disposable sandbox with only the completed trial's prefixes.""" - -import hashlib -import json -from pathlib import Path -import stat -import sys - - -def prune(roots: list[Path], expected: dict[str, str]) -> int: - # Check every prefix before deleting anything. Never follow remote links. - files = [] - directories = [] - for root in roots: - if root.is_symlink() or not root.is_dir(): - raise ValueError("Missing volume mount") - for path in root.rglob("*"): - mode = path.lstat().st_mode - if stat.S_ISDIR(mode): - directories.append(path) - continue - if not stat.S_ISREG(mode): - raise ValueError("Volume contains a link or special file") - relative = path.relative_to(root).as_posix() - with path.open("rb") as stream: - digest = hashlib.file_digest(stream, "sha256").hexdigest() - if expected.get(relative) != digest: - raise ValueError("Volume file has no matching local copy") - files.append(path) - for path in files: - path.unlink() - for path in sorted(directories, key=lambda item: len(item.parts), reverse=True): - path.rmdir() - if any(any(root.iterdir()) for root in roots): - raise ValueError("Volume prefix is not empty after cleanup") - return len(files) - - -if __name__ == "__main__": - request = json.loads(Path(sys.argv[1]).read_text()) - count = prune([Path(root) for root in request["roots"]], request["expected"]) - print(json.dumps({"deleted_files": count, "confirmed": True})) diff --git a/evals/terminal-bench/src/diffex_terminal_bench/recover.py b/evals/terminal-bench/src/diffex_terminal_bench/recover.py deleted file mode 100644 index fa12caf84..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/recover.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Recover only sandboxes named by retained benchmark trial labels.""" - -import argparse -import asyncio -from pathlib import Path - -from daytona import AsyncDaytona - -from diffex_terminal_bench.daytona_backend import confirm_cleanup -from diffex_terminal_bench.gate import read_json - - -async def recover(root: Path) -> None: - failed = [] - async with AsyncDaytona() as client: - for request in sorted(root.glob("*/*/cleanup-request.json")): - directory = request.parent - report = await confirm_cleanup(client, directory, read_json(request)["label"], - report_path=directory / "sandbox-recovery.json") - if not report["confirmed"]: - failed.append(directory.name) - if failed: - raise RuntimeError(f"Unconfirmed sandbox cleanup: {', '.join(failed)}") - print("Confirmed no remaining sandboxes for retained trial labels") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("output", type=Path) - args = parser.parse_args() - if not args.output.is_dir(): - parser.error("The output directory does not exist") - asyncio.run(recover(args.output)) diff --git a/evals/terminal-bench/src/diffex_terminal_bench/release-agents.md b/evals/terminal-bench/src/diffex_terminal_bench/release-agents.md deleted file mode 100644 index ae39c99d1..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/release-agents.md +++ /dev/null @@ -1,17 +0,0 @@ -# Development Rules - -## Code Quality - -- Before editing or drawing conclusions, read enough surrounding code to understand the relevant behavior and dependencies. -- Read affected files in full for broad changes; do not rely on search snippets alone for investigations or audits. - -## Commands - -- After changes, run relevant tests and checks, including any tests you create or modify. -- Resolve failures caused by your changes and report any remaining failures or checks you could not run. -- Write substantial or multiline ad-hoc scripts to a temporary file, run and revise them as needed, and remove your temporary files when done. -- Simple one-line scripts may run directly in the shell. - -## User Override - -- Explicit user instructions take precedence over these defaults. diff --git a/evals/terminal-bench/src/diffex_terminal_bench/retention.py b/evals/terminal-bench/src/diffex_terminal_bench/retention.py deleted file mode 100644 index d4fe8be5d..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/retention.py +++ /dev/null @@ -1,172 +0,0 @@ -"""Reclaim completed trial prefixes after verification and local validation.""" - -import asyncio -import json -import logging -import math -from pathlib import Path -import re -from uuid import uuid4 - -from daytona import AsyncDaytona, CreateSandboxFromSnapshotParams, VolumeMount -from daytona.common.errors import DaytonaNotFoundError -from harbor.job import Job -from harbor.models.job.result import JobResult -from harbor.trial.hooks import TrialEvent, TrialHookEvent -from harbor.utils.trajectory_validator import TrajectoryValidator - -from diffex_terminal_bench.gate import inventory, read_json, write_json -from diffex_terminal_bench.outcome import parse_outcome -from diffex_terminal_bench.transport import retry_transport - - -LOGGER = logging.getLogger(__name__) - - -def validate_retention(directory: Path) -> dict[str, str]: - result = read_json(directory / "result.json") - if result.get("exception_info") or not result.get("finished_at"): - raise ValueError("Trial did not complete") - rewards = (result.get("verifier_result") or {}).get("rewards") - if not isinstance(rewards, dict) or not rewards or any( - type(value) not in (int, float) or not math.isfinite(value) for value in rewards.values() - ): - raise ValueError("Verification did not complete") - verifier = directory / "verifier" - saved = read_json(verifier / "reward.json") if (verifier / "reward.json").exists() else { - "reward": float((verifier / "reward.txt").read_text())} - if saved != rewards or not (verifier / "test-stdout.txt").is_file(): - raise ValueError("Verifier evidence is incomplete") - runs = list((directory / "agent").glob("run-*")) - if len(runs) != 1: - raise ValueError("Expected one completed Diffex run") - outcome = read_json(runs[0] / "outcome.json") - if outcome.get("status") != "completed" or outcome.get("artifact_errors") or outcome.get("recovery_errors"): - raise ValueError("Execution or artifact collection failed") - native = parse_outcome(runs[0], launcher_exit_code=outcome.get("launcher_exit_code")) - if native["status"] != "completed" or not native["cleanup_confirmed"]: - raise ValueError("Native completion and shutdown are not established") - if not TrajectoryValidator().validate(directory / "agent/trajectory.json"): - raise ValueError("Invalid local trajectory") - if read_json(runs[0] / "accounting.json").get("gaps"): - raise ValueError("Incomplete native sessions") - files = inventory(directory) - # Harbor stores the convention /logs/artifacts directory by source path. - expected = {} - for name, digest in files.items(): - if name.startswith(("agent/", "verifier/", "user-agent/")): - expected[name] = digest - elif name.startswith("artifacts/logs/artifacts/"): - expected[name.removeprefix("artifacts/logs/")] = digest - return expected - - -async def cleanup_trial_volume(client: AsyncDaytona, directory: Path) -> dict: - report = {"status": "retained", "confirmed": False} - sandbox = None - try: - expected = validate_retention(directory) - mounts = read_json(directory / "artifact-volume.json")["mounts"] - created = read_json(directory / "sandbox-created.json")["sandboxes"] - if not mounts or len(mounts) != len(created): - raise ValueError("Incomplete sandbox creation records") - for mount in mounts: - if not re.fullmatch(r"trials/[0-9a-f]{32}", mount["subpath"]) or mount["mount_path"] != "/logs": - raise ValueError("Invalid trial volume prefix") - # Retained services and separate verifiers must all be gone first. - for row in created: - async def confirm_absent(): - try: - await client.get(row["id"], request_timeout=10) - except DaytonaNotFoundError: - return - # Harbor's delete request returns before deletion settles. - raise TimeoutError("Trial sandbox deletion is still pending") - - await retry_transport(confirm_absent) - roots = [f"/mnt/trial-{index}" for index in range(len(mounts))] - report["mounts"] = mounts - report["cleanup_label"] = uuid4().hex - report["status"] = "cleaning" - write_json(directory / "volume-cleanup.json", report) - # Never retry a create: a lost response could allocate another sandbox. - # Automatic stop/delete also bounds leaks if the runner loses this RPC. - sandbox = await client.create(CreateSandboxFromSnapshotParams( - language="python", os_user="root", auto_stop_interval=2, auto_delete_interval=0, ttl_minutes=10, - labels={"diffex.volume_cleanup": report["cleanup_label"]}, - volumes=[VolumeMount(volume_id=mount["volume_id"], subpath=mount["subpath"], mount_path=root) - for mount, root in zip(mounts, roots, strict=True)], - ), timeout=60) - report["sandbox_id"] = sandbox.id - write_json(directory / "volume-cleanup.json", report) - await retry_transport(lambda: sandbox.fs.upload_file( - Path(__file__).with_name("prune_volume.py").read_bytes(), "/tmp/prune-volume.py")) - payload = json.dumps({"roots": roots, "expected": expected}).encode() - await retry_transport(lambda: sandbox.fs.upload_file(payload, "/tmp/prune-request.json")) - - async def prune(): - response = await sandbox.process.exec("python3 /tmp/prune-volume.py /tmp/prune-request.json", timeout=60) - if response.exit_code != 0: - raise ValueError("Remote artifact comparison or cleanup failed") - value = json.loads(response.result) - if value.get("confirmed") is not True: - raise ValueError("Remote cleanup was not confirmed") - return value - - report.update(await retry_transport(prune, request_timeout=65, total_timeout=210)) - report["status"] = "cleaned" - except Exception as exc: - report["status"] = "retained" if sandbox is None else "cleanup_failed" - report["error"] = type(exc).__name__ - LOGGER.info("Retaining volume recovery records for %s: %s", directory.name, exc) - finally: - if sandbox is not None: - async def delete(): - try: - await client.delete(sandbox, wait=True, timeout=15) - except DaytonaNotFoundError: - pass - - deletion = asyncio.create_task(retry_transport(delete)) - cancelled = False - while not deletion.done(): - try: - await asyncio.shield(deletion) - except asyncio.CancelledError: - cancelled = True - except Exception: - break - try: - deletion.result() - report["cleanup_sandbox_deleted"] = True - except Exception as exc: - report["cleanup_sandbox_deleted"] = False - report["sandbox_error"] = type(exc).__name__ - if cancelled: - report["interrupted"] = True - try: - write_json(directory / "volume-cleanup.json", report) - except OSError: - LOGGER.warning("Could not save volume cleanup report for %s", directory, exc_info=True) - if sandbox is not None and cancelled: - raise asyncio.CancelledError - return report - - -class VolumeCleanupPlugin: - async def on_job_start(self, job: Job) -> None: - job.add_hook(TrialEvent.END, self.on_trial_end) - - async def on_trial_end(self, event: TrialHookEvent) -> None: - if (event.config.environment.import_path != "diffex_terminal_bench.daytona_backend:RecordedDaytonaEnvironment" - or not event.config.environment.delete): - return - directory = event.config.trials_dir / event.config.trial_name - try: - async with AsyncDaytona() as client: - await cleanup_trial_volume(client, directory) - except Exception: - LOGGER.warning("Volume cleanup unavailable for %s", directory, exc_info=True) - - async def on_job_end(self, job_result: JobResult) -> None: - pass diff --git a/evals/terminal-bench/src/diffex_terminal_bench/run.cjs b/evals/terminal-bench/src/diffex_terminal_bench/run.cjs deleted file mode 100644 index 7b1232872..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/run.cjs +++ /dev/null @@ -1,98 +0,0 @@ -// Launch the installed CLI with the explicitly selected environment policy. -const { spawn } = require("node:child_process"); -const { mkdirSync, mkdtempSync, readFileSync, writeFileSync, openSync, closeSync, existsSync } = require("node:fs"); -const { join } = require("node:path"); -const { identity } = require("./cleanup.cjs"); - -const runDir = process.argv[2]; -writeFileSync(join(runDir, "launcher.json"), JSON.stringify({ pid: process.pid, start: identity(process.pid) }), { mode: 0o600 }); -if (existsSync(join(runDir, "stop-requested"))) process.exit(1); -const config = JSON.parse(readFileSync(join(runDir, "requested.json"), "utf8")); -if (process.env.DIFFEX_TB_PROVIDER !== config.provider || process.env.DIFFEX_TB_MODEL !== config.model || - process.env.DIFFEX_TB_THINKING !== config.thinking) throw new Error("Harbor environment differs from requested.json"); -// Keep packages, caches, credentials, and mutable harness state on sandbox -// disk. Preserve them through verification for retained task services. -const state = mkdtempSync("/tmp/diffex-tb-state-"); -writeFileSync(join(runDir, "state.json"), JSON.stringify({ state_dir: state, home: join(state, "home") }), { mode: 0o600 }); -mkdirSync(join(state, "home"), { mode: 0o700 }); -mkdirSync(join(runDir, "sessions"), { mode: 0o700 }); -writeFileSync(join(state, "settings.json"), JSON.stringify({ evolution: { enabled: false } }), { mode: 0o600 }); -const env = { - ...(config.task_environment === "preserve" ? Object.fromEntries( - Object.entries(process.env).filter(([name]) => !name.startsWith("DIFFEX_") && - !config.provider_env_names.includes(name)), - ) : {}), - PATH: process.env.PATH, - HOME: config.task_environment === "preserve" ? process.env.HOME : join(state, "home"), - LANG: "C.UTF-8", - DIFFEX_CODING_AGENT_DIR: state, - DIFFEX_SESSION_DIR: join(runDir, "sessions"), - DIFFEX_OFFLINE: "1", - DIFFEX_TB_RUN_DIR: runDir, -}; -for (const name of config.credential_env_names) env[name] = process.env[name]; -const args = [ - "--mode", "json", "--provider", process.env.DIFFEX_TB_PROVIDER, "--model", process.env.DIFFEX_TB_MODEL, - "--thinking", process.env.DIFFEX_TB_THINKING, "--tools", config.tools.join(","), - "--session", join(runDir, "sessions", "root.jsonl"), - "--session-dir", join(runDir, "sessions"), - "--background-process-lifetime", config.background_process_lifetime, - "--background-process-ownership-dir", join(runDir, "ownership"), - "--no-approve", "--no-extensions", "--no-skills", "--no-prompt-templates", "--no-themes", - "--no-context-files", "--system-prompt", join(runDir, "eval-system-prompt.txt"), - "--extension", join(runDir, "observe.mjs"), "--print", `@${join(runDir, "instruction.txt")}`, - config.user_message_suffix, -]; -const stdout = openSync(join(runDir, "stdout.jsonl"), "wx", 0o600); -const stderr = openSync(join(runDir, "stderr.log"), "wx", 0o600); -// Even an uncatchable launcher kill leaves explicit unknown process status. -writeFileSync(join(runDir, "process.json"), JSON.stringify({ - exit_code: null, signal: null, spawn_error: null, interrupted: false, -}), { mode: 0o600 }); -// The child cannot execute the CLI before its identity is durable. If the -// launcher dies during publication, EOF on the private pipe closes the gate. -// exec preserves this PID and start time through the installed CLI wrapper. -const child = spawn("/bin/sh", ["-c", - 'read -r ready <&3 && [ "$ready" = start ] || exit 1; exec 3<&-; exec /usr/local/bin/diffex "$@"', - "diffex", ...args], { - cwd: process.cwd(), env, stdio: ["ignore", stdout, stderr, "pipe"], detached: true, -}); -const childStart = child.pid ? identity(child.pid) : null; -if (child.pid) writeFileSync(join(runDir, "cli.json"), JSON.stringify({ pid: child.pid, start: childStart }), { mode: 0o600 }); -closeSync(stdout); -closeSync(stderr); -let spawnError = null; -let interrupted = false; -let killTimer; -function stop() { - if (interrupted) return; - interrupted = true; - if (!child.pid) return; - // Signal only the identified CLI. Group signals could reach retained task - // services; independent recovery handles active descendants after CLI exit. - if (identity(child.pid) === childStart) { - try { process.kill(child.pid, "SIGTERM"); } catch (error) { if (error.code !== "ESRCH") throw error; } - } - killTimer = setTimeout(() => { - if (identity(child.pid) === childStart) { - try { process.kill(child.pid, "SIGKILL"); } catch (error) { if (error.code !== "ESRCH") throw error; } - } - }, 5000); -} -for (const signal of ["SIGTERM", "SIGINT", "SIGHUP"]) process.on(signal, stop); -child.on("error", (error) => { spawnError = error.code ?? error.name; }); -child.stdio[3].on("error", (error) => { spawnError = error.code ?? error.name; }); -child.on("close", (code, signal) => { - clearTimeout(killTimer); - writeFileSync(join(runDir, "process.json"), JSON.stringify({ - exit_code: code, signal, spawn_error: spawnError, interrupted, - }), { mode: 0o600 }); - process.exitCode = interrupted ? 1 : (code ?? 1); -}); -// Close the race with a stop request written during synchronous initialization. -if (existsSync(join(runDir, "stop-requested"))) { - child.stdio[3].end(); - stop(); -} else { - child.stdio[3].end("start\n"); -} diff --git a/evals/terminal-bench/src/diffex_terminal_bench/summary.py b/evals/terminal-bench/src/diffex_terminal_bench/summary.py deleted file mode 100644 index ff6229c3d..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/summary.py +++ /dev/null @@ -1,128 +0,0 @@ -"""Local, rebuildable navigation for retained qualification evidence.""" - -from datetime import datetime, timezone -import fcntl -from html import escape -import json -import os -from pathlib import Path -import sys -import tempfile -from urllib.parse import quote - -from diffex_terminal_bench.gate import STAGES - - -REPORT_STAGES = (*STAGES, "sample") - - -def read_report(path: Path) -> dict: - if path.is_symlink(): - raise ValueError("Summary reports must not be symlinks") - report = json.loads(path.read_text()) - if not isinstance(report, dict): - raise ValueError("Expected an object") - return report - - -def cell(value: object) -> str: - text = escape(str(value)).replace("\n", " ").replace("\r", " ") - for character in ("\\", "|", "[", "]", "*", "_", "`"): - text = text.replace(character, "\\" + character) - return text - - -def run_row(root: Path) -> str | None: - dataset_path = root / "dataset.json" - gates = [root / stage / "gate.json" for stage in REPORT_STAGES] - if not dataset_path.exists() and not any(path.exists() for path in gates): - return None - model, thinking, dataset = "unknown", "unknown", "unknown" - sizes = {} - if dataset_path.exists(): - try: - prepared = read_report(dataset_path) - dataset = f"{prepared['dataset']['name']}@{prepared['dataset']['version']}" - model = prepared.get("model", model) - thinking = prepared.get("policy", {}).get("thinking", thinking) - sizes = prepared.get("stages", {}) - except (OSError, ValueError, KeyError, TypeError, AttributeError): - dataset = "unreadable dataset report" - stages = [] - for stage, path in zip(REPORT_STAGES, gates): - if not path.exists(): - stages.append("not started") - continue - try: - if path.parent.is_symlink(): - raise ValueError("Stage must not be a symlink") - gate = read_report(path) - identity = gate["identity"] - model = identity["model"] - thinking = identity["policy"]["thinking"] - dataset = f"{identity['dataset']['name']}@{identity['dataset']['version']}" - trials = gate["trials"] - full_reward = sum(bool(rewards) and all(value == 1 for value in rewards.values()) - for trial in trials for rewards in [trial["evidence"]["rewards"]]) - if "completed_trials" in gate and not gate.get("finished_at"): - status = "unfinished" - elif gate.get("failure") or (gate.get("finished_at") and gate.get("passed") is not True): - status = "failed" - elif gate.get("passed") is True: - status = "passed" - else: - status = "unfinished" - total = sizes.get(stage) - progress = f"{len(trials)}/{total}" if total is not None else str(len(trials)) - label = f"{status}; {progress} checked; {full_reward} full reward" - if "completed_trials" in gate: - label = (f"{status}; {gate['completed_trials']}/{gate['expected_trials']} completed; " - f"{len(trials)} checked; {len(gate['failures'])} failed; {full_reward} full reward") - except (OSError, ValueError, KeyError, TypeError, AttributeError): - label = "unreadable report" - stages.append(f"[{cell(label)}]({quote(f'{root.name}/{stage}/gate.json', safe='/')})") - return "| " + " | ".join([ - f"[{cell(root.name)}]({quote(root.name, safe='')}/)", - cell(model), cell(thinking), cell(dataset), *stages, - ]) + " |" - - -def write_index(results: Path) -> Path: - """Read only small reports; never load sessions or contact a provider.""" - results.mkdir(parents=True, exist_ok=True) - destination = results / "index.md" - with (results / ".index.lock").open("a") as lock: - fcntl.flock(lock, fcntl.LOCK_EX) - rows = [] - directories = [root for root in results.iterdir() if root.is_dir() and not root.is_symlink()] - for root in sorted(directories, key=lambda path: (path.stat().st_mtime, path.name), reverse=True): - row = run_row(root) - if row is not None: - rows.append(row) - lines = [ - "# Benchmark runs", "", - f"Updated: {datetime.now(timezone.utc).isoformat()}", "", - "These are recorded summaries, not a new verification of the retained evidence.", - "A passed gate checks execution and evidence; full reward counts task solutions among checked trials.", - "Unfinished means running or interrupted before a final report was written.", - "Refresh with `python -m diffex_terminal_bench.benchmark --refresh-index --results-dir PATH`.", "", - "| Run | Model | Thinking | Dataset | " + " | ".join(REPORT_STAGES) + " |", - "| " + " | ".join(["---"] * (4 + len(REPORT_STAGES))) + " |", - *rows, "", - ] - with tempfile.TemporaryDirectory(dir=results, prefix=".index-") as staging: - temporary = Path(staging) / "index.md" - temporary.write_text("\n".join(lines)) - os.replace(temporary, destination) - return destination - - -def refresh_index(results: Path) -> None: - # A navigation failure must never hide a trial exception or skip cleanup. - try: - write_index(results) - except Exception as exc: - try: - print(f"Could not refresh run index ({type(exc).__name__}); use --refresh-index to retry.", file=sys.stderr) - except Exception: - pass # The warning stream is also optional navigation output. diff --git a/evals/terminal-bench/src/diffex_terminal_bench/trajectory.py b/evals/terminal-bench/src/diffex_terminal_bench/trajectory.py deleted file mode 100644 index 9774d1bcd..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/trajectory.py +++ /dev/null @@ -1,438 +0,0 @@ -"""Convert retained Diffex evidence to validated, self-contained Harbor ATIF.""" - -from collections import Counter, defaultdict -import base64 -from datetime import datetime, timezone -import hashlib -import json -import math -from pathlib import Path - -from harbor.models.agent.context import AgentContext, ModelUsage -from harbor.models.trajectories.trajectory import Trajectory -from harbor.utils.trajectory_validator import TrajectoryValidator - -from diffex_terminal_bench.config import PINS - - -FIELDS = ("prompt_tokens", "completion_tokens", "cached_tokens", "cost_usd") -CONTEXT_FIELDS = ("n_input_tokens", "n_output_tokens", "n_cache_tokens", "cost_usd") - - -def read_records(path: Path, gaps: list[str]) -> list[dict]: - """Keep every intact record, even when a later record is torn or corrupt.""" - records = [] - try: - with path.open("rb") as stream: - for line in stream: - try: - record = json.loads(line) - if not isinstance(record, dict): - raise ValueError("invalid record") - records.append(record) - if not line.endswith(b"\n"): - gaps.append(f"unterminated_record:{path.name}") - except (ValueError, UnicodeError): - gaps.append(f"invalid_record:{path.name}") - except OSError: - gaps.append(f"missing_file:{path.name}") - return records - - -def message_key(message: dict) -> str: - # Response IDs are scoped to the native session. For providers without an - # ID, the exact final message (including timestamp) identifies a response. - if message.get("role") == "assistant" and message.get("responseId"): - return json.dumps([message.get("provider"), message.get("model"), message["responseId"]]) - return json.dumps(message, sort_keys=True, ensure_ascii=False) - - -def metrics_for(usage: object) -> dict: - invalid = usage is not None and not isinstance(usage, dict) - usage = usage if isinstance(usage, dict) else {} - values = {} - for key in ("input", "output", "cacheRead", "cacheWrite"): - value = usage.get(key) - values[key] = value if type(value) is int and value >= 0 else None - invalid |= value is not None and values[key] is None - cost = usage.get("cost", {}) - invalid |= cost is not None and not isinstance(cost, dict) - cost = cost.get("total") if isinstance(cost, dict) else None - invalid |= cost is not None and not (type(cost) in (int, float) and math.isfinite(cost) and cost >= 0) - cost = cost if type(cost) in (int, float) and math.isfinite(cost) and cost >= 0 else None - inputs = [values[key] for key in ("input", "cacheRead", "cacheWrite")] - # Native providers initialize usage to zero before receiving telemetry. - # An all-zero record cannot establish that an inference was free/empty. - if all(value == 0 for value in values.values()): - values = dict.fromkeys(values) - inputs, cost = [None], None - return { - "prompt_tokens": sum(inputs) if None not in inputs else None, - "completion_tokens": values["output"], "cached_tokens": values["cacheRead"], - "cost_usd": cost, - "extra": {"input_tokens_uncached": values["input"], "cache_write_tokens": values["cacheWrite"], - "native_usage": usage, **({"invalid_usage": True} if invalid else {})}, - } - - -class Session: - def __init__(self, path: Path, records: list[dict], gaps: list[str], export_dir: Path): - self.path = path - self.records = records - self.gaps = gaps - self.header = next((record for record in records if record.get("type") == "session"), {}) - self.id = self.header.get("id") or hashlib.sha256(str(path).encode()).hexdigest() - self.children: dict[str, dict] = {} - self.steps: list[dict] = [] - self.calls: dict[str, dict] = {} - self.export_dir = export_dir - - def supplement(self, events: list[dict]) -> None: - # Match occurrences, not just a set: identical legitimate messages can - # occur twice. Never sum message_start/update/turn_end/agent_end copies. - available = Counter(message_key(record["message"]) for record in self.records - if record.get("type") == "message") - pending = None - tool_results = {} - compactions = sum(record.get("type") == "compaction" for record in self.records) - for index, event in enumerate(events): - kind = event.get("type") - if kind in ("message_start", "message_update"): - pending = event.get("message") - elif kind == "message_end" and isinstance(event.get("message"), dict): - pending = None - message = event["message"] - key = message_key(message) - if available[key]: - available[key] -= 1 - else: - self.records.append({"type": "message", "id": f"event-{index}", "message": message}) - elif kind == "auto_compaction_start": - if compactions: - compactions -= 1 - else: - self.gaps.append("compaction_telemetry_missing") - elif kind == "tool_execution_end" and isinstance(event.get("result"), dict): - tool_results[event.get("toolCallId")] = event - known_results = {record["message"].get("toolCallId") for record in self.records - if record.get("type") == "message" and record["message"].get("role") == "toolResult"} - for call_id, event in tool_results.items(): - if call_id not in known_results: - self.records.append({"type": "message", "id": f"tool-event-{call_id}", "message": { - "role": "toolResult", "toolCallId": call_id, "toolName": event.get("toolName"), - "content": event["result"].get("content", []), "details": event["result"].get("details"), - "isError": event.get("isError"), - }}) - if isinstance(pending, dict) and pending.get("role") == "assistant": - # A persisted final message can outlive a torn stdout message_end. - if not any(record.get("message", {}).get("timestamp") == pending.get("timestamp") - and record.get("message", {}).get("role") == "assistant" for record in self.records): - self.records.append({"type": "message", "id": "partial-stream-message", - "message": pending, "partial": True}) - self.gaps.append("unfinished_model_response") - # Supplemental retry messages must appear before their successful - # successors, even if persistence discarded the failed response. - def timestamp(record): - value = record.get("timestamp") - if value: - return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() - return record.get("message", {}).get("timestamp", float("inf")) / 1000 - self.records.sort(key=timestamp) - - def convert(self, requested: dict, system_prompt: str | None) -> dict: - model = "/".join([requested.get("provider", "unknown"), requested.get("model", "unknown")]) - thinking = requested.get("thinking") - if system_prompt is not None: - self.steps.append({"source": "system", "message": system_prompt}) - seen = set() - responses = set() - for record in self.records: - kind = record.get("type") - entry_id = record.get("id") - if entry_id and entry_id in seen: - continue - seen.add(entry_id) - if kind == "model_change": - model = f"{record['provider']}/{record['modelId']}" - elif kind == "thinking_level_change": - thinking = record.get("thinkingLevel") - elif kind == "custom" and record.get("customType") == "diffex.subagents": - for child in record.get("data", {}).get("agents", []): - if child.get("childSessionId"): - self.children[child["childSessionId"]] = child - elif kind in ("compaction", "branch_summary"): - metrics = metrics_for(record.get("usage")) - self.steps.append({"source": "agent", "message": record.get("summary", ""), - "model_name": "unattributed", "metrics": metrics, "extra": {"native_type": kind, "native_entry_id": entry_id, - "model_attribution": "unavailable"}}) - elif kind == "custom_message": - self.steps.append({"source": "user", "message": self.content(record.get("content", "")), - "extra": {"native_type": kind, "custom_type": record.get("customType")}}) - elif kind == "message": - message = record["message"] - role = message.get("role") - extra = {"native_entry_id": entry_id, "native_role": role} - if role == "toolResult": - call_id = message.get("toolCallId") - observation = {"content": self.content(message.get("content", "")), - "extra": {**extra, "is_error": message.get("isError"), - "details": message.get("details")}} - if call_id in self.calls: - observation["source_call_id"] = call_id - self.calls[call_id].setdefault("observation", {"results": []})["results"].append(observation) - else: - self.gaps.append("orphan_tool_result") - observation["extra"]["unresolved_call_id"] = call_id - self.steps.append({"source": "system", "message": "Recovered tool result", - "observation": {"results": [observation]}}) - continue - step = {"source": "agent" if role == "assistant" else "user", - "message": self.content(message.get("content", "")), "extra": extra} - if message.get("timestamp") is not None: - step["timestamp"] = datetime.fromtimestamp(message["timestamp"] / 1000, timezone.utc).isoformat() - if role == "assistant": - key = message_key(message) - if key in responses: - continue - responses.add(key) - step.update(model_name=f"{message.get('provider', model.split('/')[0])}/{message.get('model', model.split('/', 1)[1])}", - reasoning_effort=thinking, llm_call_count=1, - metrics=metrics_for(None if record.get("partial") else message.get("usage"))) - extra.update(stop_reason=message.get("stopReason"), response_id=message.get("responseId"), - partial=record.get("partial", False), error_message=message.get("errorMessage")) - blocks = message.get("content", []) - if isinstance(blocks, list): - reasoning = [block["thinking"] for block in blocks if block.get("type") == "thinking" and "thinking" in block] - if reasoning: - step["reasoning_content"] = "\n".join(reasoning) - calls = [] - for block in blocks: - if block.get("type") != "toolCall": - continue - if not block.get("id") or not block.get("name") or not isinstance(block.get("arguments"), dict): - extra.setdefault("incomplete_tool_calls", []).append(block) - continue - calls.append({"tool_call_id": block["id"], "function_name": block["name"], - "arguments": block["arguments"]}) - if calls: - step["tool_calls"] = calls - for call in calls: - self.calls[call["tool_call_id"]] = step - self.steps.append(step) - for step in self.steps: - if step.get("metrics", {}).get("extra", {}).get("invalid_usage"): - self.gaps.append("invalid_usage") - for index, step in enumerate(self.steps, 1): - step["step_id"] = index - return {"schema_version": "ATIF-v1.8", "session_id": self.id, "trajectory_id": self.id, - "agent": {"name": "diffex", "version": PINS["diffex_version"], "model_name": model}, - "steps": self.steps, "extra": {"native_session": self.path.name}} - - def content(self, content: object) -> str | list[dict]: - if isinstance(content, str): - return content - parts = [] - for block in content if isinstance(content, list) else []: - if block.get("type") == "text": - parts.append({"type": "text", "text": block.get("text", "")}) - elif block.get("type") == "image": - data = base64.b64decode(block["data"], validate=True) - name = f"trajectory-media/{hashlib.sha256(data).hexdigest()}" - path = self.export_dir / name - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(data) - parts.append({"type": "image", "source": {"media_type": block["mimeType"], - "path": name}}) - elif block.get("type") not in ("thinking", "toolCall"): - self.gaps.append("unsupported_content") - if all(part["type"] == "text" for part in parts): - return "\n".join(part["text"] for part in parts) - return parts - - -def export_trajectory(directory: Path, destination: Path, context: AgentContext, outcome: dict) -> dict: - """Export before returning to Harbor, including failure/cancellation paths.""" - gaps: list[str] = [] - requested = {} - try: - requested = json.loads((directory / "requested.json").read_text()) - except (OSError, ValueError): - gaps.append("missing_requested_configuration") - sessions = [] - root_path = directory / "sessions/root.jsonl" - for path in [root_path, *sorted(path for path in (directory / "sessions").rglob("*.jsonl") if path != root_path)]: - local_gaps: list[str] = [] - sessions.append(Session(path, read_records(path, local_gaps), local_gaps, destination.parent)) - root = sessions[0] - root.supplement(read_records(directory / "stdout.jsonl", gaps)) - prompt_path = directory / "system-prompt.txt" - documents = {session.id: session.convert(requested, prompt_path.read_text() if session is root and prompt_path.exists() else None) - for session in sessions} - if len(documents) != len(sessions): - raise ValueError("duplicate native session IDs") - root_document = documents[root.id] - linked = set() - interrupted = set() - failed = set() - for session in sessions: - document = documents[session.id] - children = [] - for child_id, child in session.children.items(): - child_document = documents.get(child_id) - if child_document is None or not child_document["steps"]: - session.gaps.append("missing_child_trajectory") - continue - if child_id == root.id or child_id in linked: - raise ValueError("cyclic or multiply-owned child session") - linked.add(child_id) - children.append(child_document) - # A recorded scheduler failure is terminal even when the child - # never produced a final answer. Keep the error in its trajectory; - # session integrity and tool-result checks still apply below. - error = child.get("error") - if child.get("status") == "failed" and isinstance(error, str) and error.strip(): - failed.add(child_id) - child_document["extra"].update(execution_status="failed", execution_error=error) - # Shutdown can also mark children interrupted. Accept a deliberate - # stop only when the parent retained a successful interrupt call. - if child.get("status") == "interrupted" and any( - call["function_name"] == "agent_action" - and call["arguments"].get("action") == "interrupt" - and call["arguments"].get("target") == child.get("name") - and any(result.get("source_call_id") == call["tool_call_id"] - and result.get("extra", {}).get("is_error") is False - for result in step.get("observation", {}).get("results", [])) - for step in document["steps"] for call in step.get("tool_calls", []) - ): - interrupted.add(child_id) - child_document["extra"]["execution_status"] = "interrupted" - attached = False - for step in document["steps"]: - for call in step.get("tool_calls", []): - if call["function_name"] != "spawn_agent" or call["arguments"].get("name") != child.get("name"): - continue - results = step.setdefault("observation", {"results": []})["results"] - result = next((result for result in results if result.get("source_call_id") == call["tool_call_id"]), None) - if result is None: - result = {"source_call_id": call["tool_call_id"], "extra": {"tool_result_missing": True}} - results.append(result) - result.setdefault("subagent_trajectory_ref", []).append({"trajectory_id": child_id}) - attached = True - break - if attached: - break - if not attached: - session.gaps.append("unlinked_child_trajectory") - if children: - document["subagent_trajectories"] = children - # Retain children whose parent metadata was lost on interruption. - for session in sessions[1:]: - if session.id not in linked and documents[session.id]["steps"]: - root_document.setdefault("subagent_trajectories", []).append(documents[session.id]) - gaps.append("unlinked_child_trajectory") - assistants = [step for step in session.steps if step.get("extra", {}).get("native_role") == "assistant"] - stop_reason = assistants[-1]["extra"].get("stop_reason") if assistants else None - # Auth/setup cancellation can arrive as an error before streaming starts. - # The parent must still prove that it deliberately interrupted this child. - if (session.id in interrupted and stop_reason == "error" - and assistants[-1]["extra"].get("error_message") == "This operation was aborted"): - stop_reason = "aborted" - if (stop_reason != "stop" and session.id not in failed - and not (session.id in interrupted and stop_reason in (None, "aborted", "toolUse"))): - session.gaps.append("child_execution_incomplete") - for session in sessions: - last_assistant = next((step for step in reversed(session.steps) - if step.get("extra", {}).get("native_role") == "assistant"), None) - for step in session.steps: - # A deliberately aborted response may contain unfinished calls - # that were never dispatched. Earlier calls still need results. - aborted = (session.id in interrupted and step is last_assistant - and step.get("extra", {}).get("stop_reason") == "aborted") - if step.get("extra", {}).get("incomplete_tool_calls") and not aborted: - session.gaps.append("incomplete_tool_call") - for call in step.get("tool_calls", []): - results = [result for result in step.get("observation", {}).get("results", []) - if result.get("source_call_id") == call["tool_call_id"]] - if not results and not aborted: - session.gaps.append("missing_tool_result") - if call["function_name"] == "spawn_agent": - if any(not result.get("extra", {}).get("is_error") and not result.get("subagent_trajectory_ref") for result in results): - session.gaps.append("missing_child_link") - if outcome.get("status") != "completed" or not outcome.get("stream_complete"): - gaps.append("execution_incomplete") - for session in sessions: - gaps.extend(f"{session.id}:{gap}" for gap in session.gaps) - observed = dict.fromkeys(FIELDS, 0) - known = set() - missing = set() - by_model = defaultdict(lambda: dict.fromkeys(FIELDS, 0)) - for session in sessions: - own = dict.fromkeys(FIELDS, 0) - own_known = set() - own_missing = set() - for step in session.steps: - if "metrics" not in step: - continue - for field in FIELDS: - value = step["metrics"].get(field) - if value is None: - missing.add(field) - own_missing.add(field) - else: - known.add(field) - own_known.add(field) - own[field] += value - observed[field] += value - by_model[step.get("model_name", "unattributed")][field] += value - documents[session.id]["final_metrics"] = { - **{f"total_{field}": None if gaps or (field in own_missing and field not in own_known) else value - for field, value in own.items()}, - "total_steps": len(session.steps), - "extra": {"scope": "this_session", "observed_usage": own, "accounting_complete": not gaps and not own_missing}, - } - # Each nested trajectory includes its descendants in its totals. Global - # usage above still visits each native session exactly once. - def aggregate(document): - metrics = document["final_metrics"] - for child in document.get("subagent_trajectories", []): - aggregate(child) - for field in FIELDS: - key = f"total_{field}" - values = [value for value in (metrics[key], child["final_metrics"][key]) if value is not None] - metrics[key] = sum(values) if values else None - metrics["extra"]["accounting_complete"] &= child["final_metrics"]["extra"]["accounting_complete"] - metrics["extra"]["scope"] = "session_and_descendants" - aggregate(root_document) - # Missing telemetry affects completeness, not the usage already recorded. - # Keep unknown-only fields null and retain known sums as partial totals. - totals = {field: None if gaps or (field in missing and field not in known) else value for field, value in observed.items()} - accounting = {"complete": not gaps and not missing, "gaps": sorted(set(gaps)), - "missing_fields": sorted(missing), "observed_usage": observed, - "observed_model_usage": dict(by_model), "totals": totals} - root_document["extra"].update(accounting=accounting, execution_status=outcome.get("status"), - background_process_lifetime=requested.get("background_process_lifetime"), - agent_execution_stopped=outcome.get("agent_execution_stopped"), - retained_processes=outcome.get("retained_processes", [])) - root_document["final_metrics"] = { - **{f"total_{field}": value for field, value in totals.items()}, "total_steps": len(root.steps), - "extra": {"scope": "root_and_all_descendants", "observed_usage": observed, "accounting_complete": accounting["complete"]}, - } - for field, target in zip(FIELDS, CONTEXT_FIELDS): - setattr(context, target, totals[field]) - # Harbor's ModelUsage defaults missing token fields to zero. Suppress its - # automatic ATIF backfill when accounting cannot establish complete totals. - context.model_usage = {model: ModelUsage(**dict(zip(CONTEXT_FIELDS, (values[field] for field in FIELDS)))) - for model, values in by_model.items()} if accounting["complete"] else {} - context.metadata = {**(context.metadata or {}), "diffex_accounting": accounting} - (directory / "accounting.json").write_text(json.dumps(accounting, indent=2, allow_nan=False) + "\n") - if not root.steps: - raise ValueError("no recoverable ATIF steps") - trajectory = Trajectory.model_validate(root_document) - destination.parent.mkdir(parents=True, exist_ok=True) - temporary = destination.with_suffix(".tmp") - temporary.write_text(trajectory.model_dump_json(indent=2, exclude_none=True) + "\n") - validator = TrajectoryValidator() - if not validator.validate(temporary): - raise ValueError("ATIF validation failed") - temporary.replace(destination) - return accounting diff --git a/evals/terminal-bench/src/diffex_terminal_bench/transport.py b/evals/terminal-bench/src/diffex_terminal_bench/transport.py deleted file mode 100644 index 358f1da19..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/transport.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Bounded retries for idempotent Daytona evidence and volume requests.""" - -import asyncio -from collections.abc import Awaitable, Callable -import random - -from daytona.common.errors import DaytonaConnectionError, DaytonaError, DaytonaTimeoutError - - -async def retry_transport[T](operation: Callable[[], Awaitable[T]], *, request_timeout: float = 20, total_timeout: float = 90) -> T: - # The outer bound includes backoff and all requests; each request is also - # bounded so a lost connection cannot consume the entire retry budget. - async with asyncio.timeout(total_timeout): - for attempt in range(3): - try: - async with asyncio.timeout(request_timeout): - return await operation() - except Exception as exc: - transient = isinstance(exc, (TimeoutError, ConnectionError, DaytonaConnectionError, DaytonaTimeoutError)) - if isinstance(exc, DaytonaError): - transient |= exc.status_code in (408, 429, 500, 502, 503, 504) - if not transient or attempt == 2: - raise - # Cleanup may still be finishing its seven-second TERM/KILL - # cycle after a lost RPC response. Cover that publication lag. - delay = 4 * 2 ** attempt + random.uniform(0, 0.25) - if isinstance(exc, DaytonaError): - for key, value in exc.headers.items(): - if key.lower().startswith("retry-after"): - try: - delay = max(delay, float(value)) - except (ValueError, TypeError): - pass - await asyncio.sleep(delay) - raise TimeoutError("Transport retry budget exhausted") diff --git a/evals/terminal-bench/src/diffex_terminal_bench/verify-install.cjs b/evals/terminal-bench/src/diffex_terminal_bench/verify-install.cjs deleted file mode 100644 index 5fa6e9537..000000000 --- a/evals/terminal-bench/src/diffex_terminal_bench/verify-install.cjs +++ /dev/null @@ -1,48 +0,0 @@ -const assert = require("node:assert/strict"); -const { execFileSync } = require("node:child_process"); -const { readFileSync, writeFileSync } = require("node:fs"); -const { join } = require("node:path"); - -const env = process.env; -const logs = env.SETUP_LOGS; -const prefix = "/opt/diffex-terminal-bench/cli"; -const metadata = JSON.parse(readFileSync(join(logs, "npm-metadata.json"), "utf8")); -const tarball = `${env.NPM_REGISTRY}@diffexai/diffex/-/diffex-${env.DIFFEX_VERSION}.tgz`; -assert.equal(metadata.version, env.DIFFEX_VERSION, "Registry version mismatch"); -assert.equal(metadata.dist.integrity, env.DIFFEX_INTEGRITY, "Registry integrity mismatch"); -assert.equal(metadata.dist.tarball, tarball, "Registry tarball mismatch"); -if (process.argv[2] === "metadata") process.exit(0); - -const manifest = JSON.parse(readFileSync(join(prefix, "node_modules/@diffexai/diffex/package.json"), "utf8")); -const lock = JSON.parse(readFileSync(join(prefix, "package-lock.json"), "utf8")); -const installed = lock.packages["node_modules/@diffexai/diffex"]; -assert.equal(manifest.version, env.DIFFEX_VERSION, "Installed version mismatch"); -assert.equal(installed.version, env.DIFFEX_VERSION, "Lock version mismatch"); -assert.equal(installed.integrity, env.DIFFEX_INTEGRITY, "Installed integrity mismatch"); -assert.equal(installed.resolved, tarball, "Installed registry mismatch"); -if (process.argv[2] === "installed") process.exit(0); -assert.equal(process.argv[2], "complete"); -assert.equal(readFileSync(join(logs, "version.stdout.txt"), "utf8").trim(), env.DIFFEX_VERSION); -const help = readFileSync(join(logs, "help.stdout.txt"), "utf8"); -for (const flag of ["--background-process-lifetime", "--background-process-ownership-dir"]) { - assert.ok(help.includes(flag), `Pinned Diffex release lacks ${flag}; publish and pin the corrected release`); -} -assert.ok(Number(env.SETUP_ELAPSED_SEC) < Number(env.SETUP_TIMEOUT_SEC), "Setup exceeded deadline"); -writeFileSync(join(logs, "installation.json"), `${JSON.stringify({ - schema_version: 1, - verified: true, - background_process_lifetime: "environment", - started_at: env.SETUP_STARTED_AT, - elapsed_sec: Number(env.SETUP_ELAPSED_SEC), - setup_timeout_sec: Number(env.SETUP_TIMEOUT_SEC), - harbor_version: env.HARBOR_VERSION, - node_version: process.version, - node_arch: env.NODE_ARCH, - node_sha256: env.NODE_SHA256, - npm_version: execFileSync("npm", ["--version"], { encoding: "utf8" }).trim(), - diffex_version: manifest.version, - npm_registry: env.NPM_REGISTRY, - npm_integrity: installed.integrity, - npm_tarball: installed.resolved, - smoke_commands: ["diffex --version", "diffex --help", "diffex --list-models"], -}, null, 2)}\n`); diff --git a/evals/terminal-bench/tests/cleanup.test.cjs b/evals/terminal-bench/tests/cleanup.test.cjs deleted file mode 100644 index 955002f5c..000000000 --- a/evals/terminal-bench/tests/cleanup.test.cjs +++ /dev/null @@ -1,171 +0,0 @@ -// Run in a Linux container: node --test /eval/tests/cleanup.test.cjs. -const assert = require("node:assert/strict"); -const { spawn } = require("node:child_process"); -const { once } = require("node:events"); -const { mkdtempSync, mkdirSync, readFileSync, writeFileSync, rmSync } = require("node:fs"); -const { createServer, connect } = require("node:net"); -const { tmpdir } = require("node:os"); -const { join } = require("node:path"); -const { setTimeout: delay } = require("node:timers/promises"); -const { test } = require("node:test"); -const { identity } = require("../src/diffex_terminal_bench/cleanup.cjs"); - -if (process.argv[2] === "service") { - process.on("SIGTERM", () => {}); - const server = createServer((socket) => socket.end(String(process.pid))); - server.listen(0, "127.0.0.1", () => process.send(server.address().port)); -} else { - async function fixture(t) { - const runDir = mkdtempSync(join(tmpdir(), "diffex-cleanup-")); - mkdirSync(join(runDir, "ownership")); - const children = []; - t.after(async () => { - for (const child of children) { - if (child.exitCode === null && child.signalCode === null) { - child.kill("SIGKILL"); - await once(child, "exit"); - } - } - rmSync(runDir, { recursive: true, force: true }); - }); - return { - runDir, - async service(executionId, marked = true) { - const child = spawn(process.execPath, [__filename, "service"], { - detached: true, stdio: ["ignore", "ignore", "inherit", "ipc"], - env: { ...process.env, ...(marked ? { DIFFEX_TB_RUN_DIR: runDir } : {}), - DIFFEX_SHELL_EXECUTION_ID: executionId }, - }); - children.push(child); - const [port] = await once(child, "message", { signal: t.signal }); - const start = identity(child.pid); - const member = { pid: child.pid, processGroupId: child.pid, - startTime: `${readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim()}:${start}` }; - return { child, port, start, member, executionId }; - }, - publish(service, owner = "environment", overrides = {}) { - const record = { version: 1, executionId: service.executionId, owner, - state: owner === "environment" ? "background" : "active", processGroupId: service.child.pid, - supervisorPid: process.pid, members: [service.member], ...overrides }; - writeFileSync(join(runDir, `ownership/${service.executionId}.json`), JSON.stringify(record)); - return record; - }, - async cleanup(expectedCode = 0) { - const child = spawn(process.execPath, [join(__dirname, "../src/diffex_terminal_bench/cleanup.cjs"), runDir], { stdio: "inherit" }); - children.push(child); - assert.equal((await once(child, "exit", { signal: t.signal }))[0], expectedCode); - return JSON.parse(readFileSync(join(runDir, "cleanup.json"), "utf8")); - }, - }; - } - - async function verify(service) { - assert.equal(identity(service.child.pid), service.start, "the original service must survive cleanup"); - const socket = connect(service.port, "127.0.0.1"); - try { - assert.equal(String((await once(socket, "data"))[0]), String(service.child.pid)); - } finally { - socket.destroy(); - } - } - - const options = { skip: process.platform !== "linux", timeout: 15000 }; - test("pre-verifier cleanup preserves a transferred marked service", options, async (t) => { - const f = await fixture(t); - const service = await f.service("service"); - f.publish(service); - await verify(service); - const report = await f.cleanup(); - assert.equal(report.agent_execution_stopped, true); - assert.deepEqual(report.remaining_agent_pids, []); - assert.equal(report.retained_processes[0].pid, service.child.pid); - await verify(service); - assert.deepEqual(await f.cleanup(), report, "cleanup is idempotent"); - }); - - test("forced CLI death preserves handoffs and kills TERM-resistant unfinished work", options, async (t) => { - const f = await fixture(t); - const service = await f.service("service"); - f.publish(service); - const active = await f.service("active"); - f.publish(active, "session"); - const cli = await f.service("control"); - writeFileSync(join(f.runDir, "cli.json"), JSON.stringify({ pid: cli.child.pid, start: cli.start })); - cli.child.kill("SIGKILL"); - await once(cli.child, "exit"); - const report = await f.cleanup(); - assert.equal(identity(active.child.pid), null); - assert.equal(report.control_processes[0].pid, cli.child.pid); - await verify(service); - }); - - test("cleanup stops TERM-resistant control activity before allowing verification", options, async (t) => { - const f = await fixture(t); - const service = await f.service("service"); - f.publish(service); - const cli = await f.service("control"); - writeFileSync(join(f.runDir, "cli.json"), JSON.stringify({ pid: cli.child.pid, start: cli.start })); - const report = await f.cleanup(); - assert.equal(report.agent_execution_stopped, true); - assert.equal(identity(cli.child.pid), null); - await verify(service); - }); - - test("PID reuse cannot signal an unrelated process or authorize a stale handoff", options, async (t) => { - const f = await fixture(t); - const unrelated = await f.service("unrelated", false); - writeFileSync(join(f.runDir, "launcher.json"), JSON.stringify({ pid: unrelated.child.pid, start: "recycled" })); - const active = await f.service("active"); - f.publish(active, "environment", { executionId: "stale", members: [{ ...active.member, startTime: "old-boot:0" }] }); - // The filename and token must refer to the stale execution, not the new one. - const stale = join(f.runDir, "ownership/active.json"); - writeFileSync(join(f.runDir, "ownership/stale.json"), readFileSync(stale)); - rmSync(stale); - const report = await f.cleanup(); - assert.deepEqual(report.retained_processes, []); - assert.equal(identity(active.child.pid), null); - await verify(unrelated); - }); - - test("invalid or contradictory handoff evidence fails closed without killing the service", options, async (t) => { - const f = await fixture(t); - const service = await f.service("service"); - f.publish(service, "environment", { state: "active" }); - assert.equal((await f.cleanup(1)).agent_execution_stopped, false); - await verify(service); - f.publish(service); - writeFileSync(join(f.runDir, "ownership/conflict.json"), JSON.stringify({ - ...f.publish(service), executionId: "conflict", owner: "session", state: "active", - })); - assert.equal((await f.cleanup(1)).agent_execution_stopped, false); - await verify(service); - }); - - test("stop barrier prevents a delayed real launcher from spawning the CLI", options, async (t) => { - const f = await fixture(t); - await f.cleanup(); - const launcher = spawn(process.execPath, [join(__dirname, "../src/diffex_terminal_bench/run.cjs"), f.runDir]); - assert.equal((await once(launcher, "exit", { signal: t.signal }))[0], 1); - const report = await f.cleanup(); - assert.equal(report.control_processes[0].role, "launcher"); - assert.equal(report.agent_execution_stopped, true); - }); - - test("launcher death before PID publication cannot release the CLI gate", options, async (t) => { - const f = await fixture(t); - writeFileSync(join(f.runDir, "requested.json"), JSON.stringify({ provider: "openai", model: "fixture", - thinking: "off", tools: ["bash"], credential_env_names: [], background_process_lifetime: "environment", - user_message_suffix: "\nDo not cheat by using online solutions or hints specific to this task." })); - const launcher = spawn(process.execPath, ["--require", join(__dirname, "fixtures/kill-before-cli-publication.cjs"), - join(__dirname, "../src/diffex_terminal_bench/run.cjs"), f.runDir], { - env: { ...process.env, DIFFEX_TB_PROVIDER: "openai", DIFFEX_TB_MODEL: "fixture", DIFFEX_TB_THINKING: "off" }, - }); - assert.equal((await once(launcher, "exit", { signal: t.signal }))[1], "SIGKILL"); - const child = JSON.parse(readFileSync(join(f.runDir, "gated-child.json"), "utf8")); - // EOF must stop the waiting child even before independent cleanup runs. - const deadline = Date.now() + 2000; - while (identity(child.pid) === child.start && Date.now() < deadline) await delay(10); - assert.notEqual(identity(child.pid), child.start); - assert.equal((await f.cleanup()).agent_execution_stopped, true); - }); -} diff --git a/evals/terminal-bench/tests/faux-outcomes.cjs b/evals/terminal-bench/tests/faux-outcomes.cjs deleted file mode 100644 index 22541a9a7..000000000 --- a/evals/terminal-bench/tests/faux-outcomes.cjs +++ /dev/null @@ -1,133 +0,0 @@ -// Outcome smoke fixture. All model responses stay on container loopback. -const { createServer } = require("node:http"); -const { createServer: tcpServer, connect } = require("node:net"); -const assert = require("node:assert/strict"); -const { once } = require("node:events"); -const { setTimeout: delay } = require("node:timers/promises"); -const { readFileSync, writeFileSync } = require("node:fs"); -const { join, basename } = require("node:path"); - -const { identity } = require("./faux-cleanup.cjs"); - -const action = process.argv[2]; -if (action === "health") { - (async () => { - for (let attempt = 0; attempt < 20; attempt++) { - try { - if ((await fetch("http://127.0.0.1:8766/health", { signal: AbortSignal.timeout(1000) })).ok) return; - } catch {} - await delay(100); - } - process.exitCode = 1; - })(); -} else if (action === "verify-service") { - (async () => { - const runDir = process.argv[3]; - const evidence = JSON.parse(process.env.DIFFEX_TB_SERVICE_EVIDENCE); - const service = evidence.service; - assert.equal(identity(service.pid), service.start); - for (const name of ["launcher", "cli"]) { - const control = evidence[name]; - assert.notEqual(identity(control.pid), control.start); - } - const socket = connect(service.port, "127.0.0.1"); - if (service.protocol === "grpc") await once(socket, "connect"); - else assert.equal(String((await once(socket, "data"))[0]), String(service.pid)); - socket.destroy(); - const provider = await (await fetch(`http://127.0.0.1:8766/state/${basename(runDir)}`)).json(); - assert.equal(provider.active, 0, "provider activity must stop before verification"); - console.log(JSON.stringify({ ...service, provider, verified: true })); - })().catch((error) => { console.error(error); process.exitCode = 1; }); -} else if (action === "prepare") { - const state = process.env.DIFFEX_CODING_AGENT_DIR; - const runDir = process.env.DIFFEX_TB_RUN_DIR; - const scenario = readFileSync("/tmp/outcome-scenario", "utf8").trim(); - writeFileSync(join(state, "settings.json"), JSON.stringify({ evolution: { enabled: false }, retry: { enabled: true, maxRetries: 1, baseDelayMs: 1 } })); - writeFileSync(join(state, "models.json"), JSON.stringify({ providers: { openai: { - baseUrl: `http://127.0.0.1:8766/${scenario}/${basename(runDir)}/v1`, api: "openai-completions", models: [{ - id: "gpt-5", name: "Controlled outcome model", reasoning: true, input: ["text"], - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: 128000, maxTokens: 4096, - }], - } } })); - process.exit(0); -} else if (action === "interrupt-output") { - const runDir = process.env.DIFFEX_TB_RUN_DIR; - if (readFileSync("/tmp/outcome-scenario", "utf8").trim() === "interrupted-output") { - const path = join(runDir, "stdout.jsonl"); - // Model a dropped final frame while preserving process success. - const lines = readFileSync(path, "utf8").trimEnd().split("\n"); - writeFileSync(path, lines.filter((line) => JSON.parse(line).type !== "agent_end").join("\n") + '\n{"type":'); - } - process.exit(0); -} else if (action === "linger") { - // Successful shell completion transfers this TERM-resistant service. - process.on("SIGTERM", () => {}); - const runDir = process.env.DIFFEX_TB_RUN_DIR; - const server = tcpServer((socket) => socket.end(String(process.pid))); - server.listen(0, "127.0.0.1", () => writeFileSync(join(runDir, "service.json"), JSON.stringify({ - pid: process.pid, start: identity(process.pid), port: server.address().port, - }))); -} else { - const counts = new Map(); - const states = new Map(); - createServer(async (request, response) => { - if (request.url === "/health") { response.end("ok"); return; } - if (request.url.startsWith("/state/")) { - response.end(JSON.stringify(states.get(request.url.split("/")[2]))); - return; - } - let text = ""; - for await (const chunk of request) text += chunk; - const body = JSON.parse(text); - const scenario = request.url.split("/")[1]; - const count = (counts.get(request.url) ?? 0) + 1; - counts.set(request.url, count); - const run = request.url.split("/")[2]; - const state = states.get(run) ?? { requests: 0, active: 0 }; - states.set(run, state); - state.requests++; - state.active++; - response.on("close", () => state.active--); - if (["success", "service-failure", "timeout", "forced-death", "grpc"].includes(scenario) && count === 2) { - const runDir = join("/logs/agent", run); - const service = JSON.parse(readFileSync(join(runDir, "service.json"), "utf8")); - const socket = connect(service.port, "127.0.0.1"); - await once(socket, "connect"); - socket.destroy(); - writeFileSync(join(runDir, "service-before-exit.json"), JSON.stringify(service)); - if (scenario === "forced-death") { - const cli = JSON.parse(readFileSync(join(runDir, "cli.json"), "utf8")); - assert.equal(identity(cli.pid), cli.start); - process.kill(cli.pid, "SIGKILL"); - response.destroy(); - return; - } - } - if (scenario === "service-failure" && count > 1) { - response.writeHead(400, { "Content-Type": "application/json" }); - response.end(JSON.stringify({ error: { message: "Controlled failure after handoff" } })); - return; - } - if (scenario === "authentication" || scenario === "exhausted" || (scenario === "recovered" && count === 1)) { - response.writeHead(scenario === "authentication" ? 401 : 503, { "Content-Type": "application/json" }); - response.end(JSON.stringify({ error: { message: scenario === "authentication" ? "Controlled invalid API key" : "Controlled service overloaded" } })); - return; - } - if (scenario === "timeout" && count > 1) return; - let tool = count === 1 && ["timeout", "success", "service-failure", "forced-death"].includes(scenario) - ? { name: "bash", arguments: JSON.stringify({ command: 'nohup /opt/diffex-terminal-bench/node/bin/node /tmp/faux-outcomes.cjs linger >/dev/null 2>&1 & while [ ! -f "$DIFFEX_TB_RUN_DIR/service.json" ]; do sleep 0.01; done; printf tool-ok' }) } - : null; - if (scenario === "grpc" && count === 1) tool = { name: "bash", arguments: JSON.stringify({ command: - 'cd /app && /opt/verifier/bin/python -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. kv-store.proto && nohup /opt/verifier/bin/python /app/server.py >/app/server.log 2>&1 & while [ ! -f "$DIFFEX_TB_RUN_DIR/service.json" ]; do sleep 0.01; done' }) }; - response.writeHead(200, { "Content-Type": "text/event-stream" }); - const send = (delta, finish_reason = null) => response.write(`data: ${JSON.stringify({ - id: `fixture-${count}`, object: "chat.completion.chunk", model: body.model, - choices: [{ index: 0, delta, finish_reason }], - })}\n\n`); - send({ role: "assistant" }); - if (tool) send({ tool_calls: [{ index: 0, id: `call-${count}`, type: "function", function: tool }] }); - else send({ content: "controlled-ok" }); - send({}, tool ? "tool_calls" : "stop"); - response.end("data: [DONE]\n\n"); - }).listen(8766, "127.0.0.1"); -} diff --git a/evals/terminal-bench/tests/faux-provider.cjs b/evals/terminal-bench/tests/faux-provider.cjs deleted file mode 100644 index fd6bf679f..000000000 --- a/evals/terminal-bench/tests/faux-provider.cjs +++ /dev/null @@ -1,76 +0,0 @@ -// Controlled Linux smoke fixture. Never used by the benchmark adapter. -const { createServer } = require("node:http"); -const { appendFileSync, existsSync, readFileSync, readdirSync, writeFileSync } = require("node:fs"); -const { join } = require("node:path"); - -if (process.argv[2] === "contaminate") { - const state = process.argv[3]; - writeFileSync(join(state, "auth.json"), '{"stale":true}'); - writeFileSync(join(state, "SYSTEM.md"), "STALE_HARNESS_SELECTION"); - const selections = join(state, "harness-state/selections"); - if (existsSync(selections)) { - for (const name of readdirSync(selections).filter((name) => name.endsWith(".json"))) { - writeFileSync(join(selections, name), '{"stale_harness_selection":true}'); - } - } - process.exit(0); -} - -if (process.argv[2] === "prepare") { - const state = process.env.DIFFEX_CODING_AGENT_DIR; - const runDir = process.env.DIFFEX_TB_RUN_DIR; - writeFileSync(join(runDir, "fixture-start.json"), JSON.stringify({ - state_entries: readdirSync(state).sort(), - session_entries: readdirSync(process.env.DIFFEX_SESSION_DIR), - settings: JSON.parse(readFileSync(join(state, "settings.json"), "utf8")), - unrelated_credential_present: Boolean(process.env.ANTHROPIC_API_KEY), - })); - // Only redirect the fixture model to loopback. The release's CLI, tools, - // prompt construction, persistence, and adapter extension run unchanged. - writeFileSync(join(state, "models.json"), JSON.stringify({ providers: { openai: { - baseUrl: "http://127.0.0.1:8765/v1", api: "openai-completions", models: [{ - id: "gpt-5", name: "Controlled smoke model", reasoning: true, input: ["text"], - cost: { input: 1, output: 2, cacheRead: 0.5, cacheWrite: 1 }, - contextWindow: 128000, maxTokens: 4096, - }], - } } })); - process.exit(0); -} - -let requestNumber = 0; -createServer(async (request, response) => { - if (request.url === "/health") { response.end("ok"); return; } - let text = ""; - for await (const chunk of request) text += chunk; - const body = JSON.parse(text); - appendFileSync("/logs/agent/fixture-requests.jsonl", `${JSON.stringify({ - body, credential_received: request.headers.authorization === "Bearer fixture-secret-never-log", - })}\n`); - const completed = body.messages.filter((message) => message.role === "tool").length; - const isChild = JSON.stringify(body.messages.find((message) => message.role === "user")?.content).includes("CONTROLLED_CHILD_TASK"); - const enabled = body.tools.some((tool) => tool.function.name === "spawn_agent"); - let tool; - if (completed === 0 && isChild) { - tool = { name: "write", arguments: JSON.stringify({ path: "child.txt", content: "child-ok\n" }) }; - } else if (completed === 0) { - tool = { name: "bash", arguments: JSON.stringify({ command: "pwd > cwd.txt; printf shell-ok > shell.txt" }) }; - } else if (completed === 1 && !isChild) { - tool = { name: "write", arguments: JSON.stringify({ path: "result.txt", content: "file-ok\n" }) }; - } else if (completed === 2 && enabled && !isChild) { - tool = { name: "spawn_agent", arguments: JSON.stringify({ name: "helper", task: "CONTROLLED_CHILD_TASK" }) }; - } - response.writeHead(200, { "Content-Type": "text/event-stream" }); - const id = `completion-${++requestNumber}`; - const send = (delta, finish_reason = null) => response.write(`data: ${JSON.stringify({ - id, object: "chat.completion.chunk", created: 1, model: body.model, - choices: [{ index: 0, delta, finish_reason }], - })}\n\n`); - send({ role: "assistant" }); - if (tool) send({ tool_calls: [{ index: 0, id: `call-${requestNumber}`, type: "function", function: tool }] }); - else send({ content: "controlled-ok" }); - send({}, tool ? "tool_calls" : "stop"); - response.write(`data: ${JSON.stringify({ id, object: "chat.completion.chunk", choices: [], - usage: { prompt_tokens: 19, completion_tokens: 7, total_tokens: 26, prompt_tokens_details: { cached_tokens: 5 } }, - })}\n\n`); - response.end("data: [DONE]\n\n"); -}).listen(8765, "127.0.0.1"); diff --git a/evals/terminal-bench/tests/fixtures/Dockerfile.source b/evals/terminal-bench/tests/fixtures/Dockerfile.source deleted file mode 100644 index 1aff9db77..000000000 --- a/evals/terminal-bench/tests/fixtures/Dockerfile.source +++ /dev/null @@ -1,11 +0,0 @@ -FROM node:22.23.2-bookworm-slim@sha256:48e4b67d85f87bd551df43704e24d252f56cc5f8e9718841aace50f19948f0f9 -WORKDIR /repo -COPY package.json package-lock.json tsconfig.json tsconfig.base.json ./ -COPY packages ./packages -RUN npm ci --ignore-scripts --no-audit --no-fund -RUN npm run hydrate:model-data -RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && sed -i "s|http://deb.debian.org|https://deb.debian.org|g" /etc/apt/sources.list.d/debian.sources && apt-get update && apt-get install -y --no-install-recommends python3 python3-pip python3-venv git procps && rm -rf /var/lib/apt/lists/* -RUN python3 -m venv /opt/verifier && /opt/verifier/bin/pip install pytest==8.4.2 grpcio==1.75.1 grpcio-tools==1.75.1 -RUN mkdir -p /opt/diffex-terminal-bench/node/bin /opt/diffex-terminal-bench/cli/node_modules/@diffexai/diffex/dist && ln -s /usr/local/bin/node /opt/diffex-terminal-bench/node/bin/node -ENV TSX_TSCONFIG_PATH=/repo/tsconfig.json -WORKDIR /task diff --git a/evals/terminal-bench/tests/fixtures/grpc_server.py b/evals/terminal-bench/tests/fixtures/grpc_server.py deleted file mode 100644 index cc8d422a9..000000000 --- a/evals/terminal-bench/tests/fixtures/grpc_server.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Known-correct service for the pinned verifier, with process identity evidence.""" - -from concurrent import futures -import json -import os -from pathlib import Path -import threading - -import grpc -import kv_store_pb2 -import kv_store_pb2_grpc - - -class Server(kv_store_pb2_grpc.KVStoreServicer): - def __init__(self): - self.values = {} - self.lock = threading.Lock() - - def GetVal(self, request, context): - with self.lock: - return kv_store_pb2.GetValResponse(val=self.values.get(request.key, 0)) - - def SetVal(self, request, context): - with self.lock: - self.values[request.key] = request.value - return kv_store_pb2.SetValResponse(val=request.value) - - -server = grpc.server(futures.ThreadPoolExecutor(max_workers=4)) -kv_store_pb2_grpc.add_KVStoreServicer_to_server(Server(), server) -assert server.add_insecure_port("127.0.0.1:5328") == 5328 -server.start() -stat = Path(f"/proc/{os.getpid()}/stat").read_text().rsplit(")", 1)[1].split() -Path(os.environ["DIFFEX_TB_RUN_DIR"], "service.json").write_text(json.dumps({ - "pid": os.getpid(), "start": stat[19], "port": 5328, "protocol": "grpc", -})) -server.wait_for_termination() diff --git a/evals/terminal-bench/tests/fixtures/kill-before-cli-publication.cjs b/evals/terminal-bench/tests/fixtures/kill-before-cli-publication.cjs deleted file mode 100644 index b7228e47d..000000000 --- a/evals/terminal-bench/tests/fixtures/kill-before-cli-publication.cjs +++ /dev/null @@ -1,12 +0,0 @@ -// Terminate the real launcher at its spawn-to-publication boundary. -const fs = require("node:fs"); -const { basename, dirname, join } = require("node:path"); - -const write = fs.writeFileSync; -fs.writeFileSync = (path, ...args) => { - if (basename(path) === "cli.json") { - write(join(dirname(path), "gated-child.json"), ...args); - process.kill(process.pid, "SIGKILL"); - } - return write(path, ...args); -}; diff --git a/evals/terminal-bench/tests/fixtures/kv-store.proto b/evals/terminal-bench/tests/fixtures/kv-store.proto deleted file mode 100644 index 6e401bc3f..000000000 --- a/evals/terminal-bench/tests/fixtures/kv-store.proto +++ /dev/null @@ -1,10 +0,0 @@ -syntax = "proto3"; - -service KVStore { - rpc GetVal (GetValRequest) returns (GetValResponse) {} - rpc SetVal (SetValRequest) returns (SetValResponse) {} -} -message GetValRequest { string key = 1; } -message GetValResponse { int32 val = 1; } -message SetValRequest { string key = 1; int32 value = 2; } -message SetValResponse { int32 val = 1; } diff --git a/evals/terminal-bench/tests/launcher-environment.test.cjs b/evals/terminal-bench/tests/launcher-environment.test.cjs deleted file mode 100644 index 26569243f..000000000 --- a/evals/terminal-bench/tests/launcher-environment.test.cjs +++ /dev/null @@ -1,54 +0,0 @@ -// Run in a disposable Linux container: this fixture owns /usr/local/bin/diffex. -const assert = require("node:assert/strict"); -const { spawnSync } = require("node:child_process"); -const { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } = require("node:fs"); -const { join, resolve } = require("node:path"); -const { test } = require("node:test"); - -test("real launcher preserves task imports and caches while isolating Diffex state", { - skip: process.env.DIFFEX_LAUNCHER_CONTAINER_TEST !== "1", -}, (t) => { - assert.equal(existsSync("/usr/local/bin/diffex"), false, "Use a disposable container without Diffex installed"); - const root = mkdtempSync("/tmp/launcher-env-"); - t.after(() => { - rmSync("/usr/local/bin/diffex"); - rmSync(root, { recursive: true, force: true }); - }); - const home = join(root, "task-home"); - const source = join(root, "src"); - const cache = join(root, "deno-cache"); - for (const directory of [home, source, cache]) mkdirSync(directory); - writeFileSync(join(source, "task_module.py"), 'VALUE = "import-ok"\n'); - writeFileSync(join(cache, "metadata"), "cache-ok"); - writeFileSync(join(home, "home-cache"), "home-ok"); - writeFileSync("/usr/local/bin/diffex", `#!/bin/sh -set -eu -python3 -c 'import task_module; print(task_module.VALUE)' -cat "$DENO_DIR/metadata" "$HOME/home-cache" -test -z "\${ANTHROPIC_API_KEY:-}" -test -z "\${DIFFEX_MODEL:-}" -test "$OPENAI_API_KEY" = selected-fixture -test "$DIFFEX_OFFLINE" = 1 -test "$DIFFEX_CODING_AGENT_DIR" != /stale -`, { mode: 0o755 }); - const run = join(root, "run"); - mkdirSync(run); - writeFileSync(join(run, "requested.json"), JSON.stringify({ - provider: "openai", model: "fixture", thinking: "high", tools: ["bash"], - credential_env_names: ["OPENAI_API_KEY"], provider_env_names: ["OPENAI_API_KEY", "ANTHROPIC_API_KEY"], - task_environment: "preserve", background_process_lifetime: "environment", user_message_suffix: "fixture", - })); - const result = spawnSync(process.execPath, [resolve(__dirname, "../src/diffex_terminal_bench/run.cjs"), run], { - encoding: "utf8", env: { - PATH: process.env.PATH, HOME: home, PYTHONPATH: source, DENO_DIR: cache, - DIFFEX_TB_PROVIDER: "openai", DIFFEX_TB_MODEL: "fixture", DIFFEX_TB_THINKING: "high", - DIFFEX_MODEL: "stale", DIFFEX_CODING_AGENT_DIR: "/stale", DIFFEX_OFFLINE: "0", - OPENAI_API_KEY: "selected-fixture", ANTHROPIC_API_KEY: "unselected-fixture", - }, - }); - const state = JSON.parse(readFileSync(join(run, "state.json"), "utf8")); - t.after(() => rmSync(state.state_dir, { recursive: true, force: true })); - assert.equal(result.status, 0, result.stderr + readFileSync(join(run, "stderr.log"), "utf8")); - assert.equal(readFileSync(join(run, "stdout.jsonl"), "utf8"), "import-ok\ncache-okhome-ok"); - assert.deepEqual(JSON.parse(readFileSync(join(state.state_dir, "settings.json"), "utf8")), { evolution: { enabled: false } }); -}); diff --git a/evals/terminal-bench/tests/launcher.test.cjs b/evals/terminal-bench/tests/launcher.test.cjs deleted file mode 100644 index 90cb679c8..000000000 --- a/evals/terminal-bench/tests/launcher.test.cjs +++ /dev/null @@ -1,71 +0,0 @@ -const assert = require("node:assert/strict"); -const { spawnSync } = require("node:child_process"); -const { EventEmitter } = require("node:events"); -const { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } = require("node:fs"); -const { tmpdir } = require("node:os"); -const { join, resolve } = require("node:path"); -const { test } = require("node:test"); -const { runInNewContext } = require("node:vm"); - -test("launcher preserves task packages outside exported logs and isolates each run", (t) => { - const root = mkdtempSync(join(tmpdir(), "diffex-launcher-test-")); - t.after(() => rmSync(root, { recursive: true, force: true })); - const states = []; - for (let index = 0; index < 2; index++) { - const logs = join(root, `run-${index}`); - mkdirSync(logs); - writeFileSync(join(logs, "requested.json"), JSON.stringify({ - provider: "openai", model: "fixture", thinking: "high", tools: ["bash"], - credential_env_names: [], background_process_lifetime: "environment", - user_message_suffix: "\nDo not cheat by using online solutions or hints specific to this task.", - })); - const child = new EventEmitter(); - child.pid = 12345; - const gate = new EventEmitter(); - gate.end = (message) => assert.equal(message, "start\n"); - child.stdio = [null, null, null, gate]; - let childEnv; - runInNewContext(readFileSync(resolve(__dirname, "../src/diffex_terminal_bench/run.cjs"), "utf8"), { - require: (name) => { - if (name === "./cleanup.cjs") return { identity: () => "100" }; - if (name === "node:child_process") return { spawn: (command, args, options) => { - childEnv = options.env; - assert.equal(command, "/bin/sh"); - assert.ok(args.includes(join(logs, "ownership"))); - assert.ok(args.includes("--no-context-files")); - assert.equal(args[args.indexOf("--system-prompt") + 1], join(logs, "eval-system-prompt.txt")); - assert.equal(args.at(-2), `@${join(logs, "instruction.txt")}`); - assert.equal(args.at(-1), "\nDo not cheat by using online solutions or hints specific to this task."); - return child; - } }; - return require(name); - }, - process: { argv: ["node", "run.cjs", logs], pid: 12344, cwd: () => root, on: () => {}, - env: { PATH: process.env.PATH, DIFFEX_TB_PROVIDER: "openai", DIFFEX_TB_MODEL: "fixture", DIFFEX_TB_THINKING: "high" } }, - setTimeout, clearTimeout, - }); - const state = JSON.parse(readFileSync(join(logs, "state.json"), "utf8")); - t.after(() => rmSync(state.state_dir, { recursive: true, force: true })); - states.push(state.state_dir); - assert.equal(childEnv.HOME, state.home); - assert.equal(childEnv.DIFFEX_CODING_AGENT_DIR, state.state_dir); - assert.equal(childEnv.DIFFEX_SESSION_DIR, join(logs, "sessions")); - assert.ok(state.state_dir.startsWith("/tmp/diffex-tb-state-")); - assert.equal(existsSync(join(logs, "state")), false); - const cache = join(state.home, ".cache"); - assert.equal(existsSync(cache), false); - mkdirSync(cache); - writeFileSync(join(cache, "package"), "task dependency"); - const fixture = join(__dirname, "faux-provider.cjs"); - const prepare = spawnSync(process.execPath, [fixture, "prepare"], { env: childEnv, encoding: "utf8" }); - assert.equal(prepare.status, 0, prepare.stderr); - assert.equal(JSON.parse(readFileSync(join(logs, "fixture-start.json"), "utf8")).settings.evolution.enabled, false); - child.emit("close", 0, null); - assert.equal(readFileSync(join(cache, "package"), "utf8"), "task dependency"); - assert.equal(JSON.parse(readFileSync(join(logs, "process.json"), "utf8")).exit_code, 0); - const contaminate = spawnSync(process.execPath, [fixture, "contaminate", state.state_dir], { encoding: "utf8" }); - assert.equal(contaminate.status, 0, contaminate.stderr); - assert.deepEqual(JSON.parse(readFileSync(join(state.state_dir, "auth.json"), "utf8")), { stale: true }); - } - assert.notEqual(states[0], states[1]); -}); diff --git a/evals/terminal-bench/tests/test_benchmark.py b/evals/terminal-bench/tests/test_benchmark.py deleted file mode 100644 index 8a35616a8..000000000 --- a/evals/terminal-bench/tests/test_benchmark.py +++ /dev/null @@ -1,269 +0,0 @@ -"""Deterministic stage planning, dataset integrity, and cleanup checks.""" - -import asyncio -from pathlib import Path -import os -import shutil -import tarfile -import tempfile -import unittest -from unittest.mock import AsyncMock, Mock, patch - -from daytona import AsyncDaytona -from daytona.common.errors import DaytonaNotFoundError -from harbor.models.task.task import Task -from harbor.models.trial.config import TrialConfig - -from diffex_terminal_bench.benchmark import PACKAGE, preflight, run_stage, trial_config -from diffex_terminal_bench.dataset import load_config, prepare_dataset -from diffex_terminal_bench.daytona_backend import confirm_cleanup -from diffex_terminal_bench.gate import read_json, sha256, write_json -from diffex_terminal_bench.recover import recover - - -class BenchmarkTests(unittest.TestCase): - def setUp(self): - self.config = load_config(PACKAGE / "benchmark.toml") - - def test_configuration_preserves_task_resources_timeouts_and_isolation(self): - with patch.dict(os.environ, {"OPENAI_API_KEY": "secret"}, clear=True): - config = trial_config(self.config, Path("/task"), Path("/runs/one"), "unique", oracle=False) - restored = TrialConfig.model_validate_json(config.model_dump_json()) - self.assertEqual(restored.timeout_multiplier, 1) - self.assertIsNone(restored.agent.override_timeout_sec) - self.assertIsNone(restored.agent.max_timeout_sec) - self.assertIsNone(restored.environment.override_cpus) - self.assertIsNone(restored.environment.override_memory_mb) - self.assertIsNone(restored.environment.override_storage_mb) - self.assertTrue(restored.environment.delete) - self.assertFalse(restored.verifier.disable) - self.assertEqual(restored.agent.env, {"OPENAI_API_KEY": "${OPENAI_API_KEY}"}) - self.assertNotIn("secret", config.model_dump_json()) - self.assertEqual(config.environment.kwargs["labels"], {"diffex.gate_trial": "unique"}) - oracle = trial_config(self.config, Path("/task"), Path("/runs/one"), "unique", oracle=True) - self.assertEqual(oracle.agent.name, "oracle") - self.assertEqual(oracle.agent.env, {}) - - def test_missing_credentials_fail_before_remote_work(self): - with patch.dict(os.environ, {}, clear=True), self.assertRaisesRegex(ValueError, "DAYTONA_API_KEY"): - preflight(self.config, oracle=True) - with patch.dict(os.environ, {"DAYTONA_API_KEY": "fixture"}, clear=True): - preflight(self.config, oracle=True) - with self.assertRaisesRegex(ValueError, "provider credential"): - preflight(self.config, oracle=False) - - def test_dataset_corruption_and_changed_extraction_are_rejected(self): - with tempfile.TemporaryDirectory() as temp: - root = Path(temp) - tree = root / "tree" - for name in self.config.pilot_tasks: - path = tree / "tasks" / name - (path / "environment").mkdir(parents=True) - (path / "environment/Dockerfile").write_text("FROM ubuntu:24.04\n") - (path / "task.toml").write_text('version = "1.0"\n') - (path / "instruction.md").write_text("fixture") - (path / "tests").mkdir() - (path / "tests/test.sh").write_text("exit 0\n") - self.assertTrue(Task.is_valid_dir(path)) - # The upstream repository also contains a valid analysis task - # template outside tasks/. It must never enter the benchmark. - shutil.copytree(tree / "tasks" / self.config.pilot_tasks[0], tree / "analysis-task-template") - archive = root / "fixture.tar.gz" - with tarfile.open(archive, "w:gz") as tar: - tar.add(tree, arcname="dataset") - config = self.config.model_copy(deep=True) - config.dataset.archive_sha256 = sha256(archive) - config.dataset.task_count = 10 - cache = root / "cache" - cache.mkdir() - saved = cache / f"{config.dataset.archive_sha256}.tar.gz" - shutil.copyfile(archive, saved) - with prepare_dataset(config, cache) as (tasks, digests): - self.assertEqual(len(tasks), 10) - self.assertEqual(set(tasks), set(self.config.pilot_tasks)) - self.assertNotIn("analysis-task-template", tasks) - (tasks[config.pilot_tasks[0]] / "instruction.md").write_text("changed") - with prepare_dataset(config, cache) as (tasks, repeated): - self.assertEqual(digests, repeated) - self.assertEqual((tasks[config.pilot_tasks[0]] / "instruction.md").read_text(), "fixture") - saved.write_bytes(b"corrupt") - with self.assertRaisesRegex(ValueError, "SHA-256"), prepare_dataset(config, cache): - self.fail("corrupt archive accepted") - - -class CleanupTests(unittest.IsolatedAsyncioTestCase): - def setUp(self): - temp = tempfile.TemporaryDirectory() - self.addCleanup(temp.cleanup) - self.directory = Path(temp.name) - write_json(self.directory / "sandbox-created.json", {"sandboxes": [{"id": "known"}]}) - self.client = Mock(spec=AsyncDaytona) - self.client.get = AsyncMock(side_effect=DaytonaNotFoundError("gone")) - self.client.delete = AsyncMock() - self.queries = [] - - async def listing(query, request_timeout): - self.queries.append(query) - for value in []: - yield value - - self.client.list = listing - - async def test_confirmed_absence_requires_api_evidence(self): - report = await confirm_cleanup(self.client, self.directory, "trial-only") - self.assertTrue(report["confirmed"]) - self.assertEqual(report["absent_ids"], ["known"]) - self.assertEqual(report["unexpected_sandboxes"], []) - self.assertTrue(all(query.labels == {"diffex.gate_trial": "trial-only"} for query in self.queries)) - self.client.delete.assert_not_called() - - async def test_api_errors_never_count_as_absence(self): - self.client.get.side_effect = RuntimeError("transport unavailable") - report = await confirm_cleanup(self.client, self.directory, "trial-only") - self.assertFalse(report["confirmed"]) - self.assertEqual(report["error"], "RuntimeError") - self.assertEqual(read_json(self.directory / "sandbox-cleanup.json"), report) - - async def test_recovery_preserves_original_evidence(self): - trial_dir = self.directory / "oracle/one" - write_json(trial_dir / "cleanup-request.json", {"label": "unique"}) - write_json(trial_dir / "sandbox-cleanup.json", {"confirmed": False}) - original = (trial_dir / "sandbox-cleanup.json").read_bytes() - with patch("diffex_terminal_bench.recover.AsyncDaytona") as client: - client.return_value.__aenter__ = AsyncMock(return_value=self.client) - client.return_value.__aexit__ = AsyncMock(return_value=False) - await recover(self.directory) - self.assertEqual((trial_dir / "sandbox-cleanup.json").read_bytes(), original) - self.assertTrue(read_json(trial_dir / "sandbox-recovery.json")["confirmed"]) - - async def test_orphan_recovery_is_retained_as_gate_failure(self): - orphan = Mock(id="orphan") - self.client.get.side_effect = [DaytonaNotFoundError("gone"), orphan, DaytonaNotFoundError("gone")] - calls = 0 - - async def listing(query, request_timeout): - nonlocal calls - calls += 1 - if calls == 1: - yield orphan - - self.client.list = listing - report = await confirm_cleanup(self.client, self.directory, "trial-only") - self.assertTrue(report["confirmed"]) - self.assertEqual(report["unexpected_sandboxes"], ["orphan"]) - self.client.delete.assert_awaited_once_with(orphan, wait=True, timeout=60) - - async def test_stale_list_entries_are_checked_by_id_before_recovery(self): - for identifier in ("known", "unrecorded"): - with self.subTest(identifier=identifier): - sandbox = Mock(id=identifier) - - async def listing(query, request_timeout): - self.queries.append(query) - yield sandbox - - self.client.list = listing - self.client.delete.side_effect = DaytonaNotFoundError("already deleted") - report = await confirm_cleanup(self.client, self.directory, "trial-only") - self.assertTrue(report["confirmed"]) - self.assertEqual(report["unexpected_sandboxes"], []) - self.assertEqual(set(report["absent_ids"]), {"known", identifier}) - self.assertEqual(len(report["absent_ids"]), len(set(report["absent_ids"]))) - self.client.delete.assert_not_awaited() - self.assertEqual(len(self.queries), 2) - self.assertTrue(all(query.labels == {"diffex.gate_trial": "trial-only"} for query in self.queries)) - self.queries.clear() - - async def test_deletion_race_requires_independent_absence_confirmation(self): - sandbox = Mock(id="unrecorded") - - async def listing(query, request_timeout): - yield sandbox - - self.client.list = listing - self.client.delete.side_effect = DaytonaNotFoundError("already deleted") - for observation, confirmed in ((DaytonaNotFoundError("gone"), True), - (sandbox, False), (RuntimeError("transport unavailable"), False)): - with self.subTest(observation=type(observation).__name__, confirmed=confirmed): - self.client.get.side_effect = [DaytonaNotFoundError("gone"), sandbox, observation, - DaytonaNotFoundError("gone")] - report = await confirm_cleanup(self.client, self.directory, "trial-only") - self.assertEqual(report["confirmed"], confirmed) - self.assertEqual(report["unexpected_sandboxes"], []) - - async def test_list_lookup_errors_do_not_establish_absence(self): - async def listing(query, request_timeout): - yield Mock(id="unrecorded") - - self.client.list = listing - self.client.get.side_effect = [DaytonaNotFoundError("gone"), RuntimeError("transport unavailable")] - report = await confirm_cleanup(self.client, self.directory, "trial-only") - self.assertFalse(report["confirmed"]) - self.assertEqual(report["error"], "RuntimeError") - self.client.delete.assert_not_awaited() - - async def test_final_listing_rechecks_previously_absent_ids(self): - calls = 0 - - async def listing(query, request_timeout): - nonlocal calls - calls += 1 - if calls == 2: - yield Mock(id="known") - - self.client.list = listing - self.client.get.side_effect = [DaytonaNotFoundError("gone"), Mock(id="known")] - report = await confirm_cleanup(self.client, self.directory, "trial-only") - self.assertFalse(report["confirmed"]) - self.assertEqual(report["error"], "RuntimeError") - self.assertNotIn("known", report["absent_ids"]) - - async def test_asynchronous_deletion_gets_a_bounded_grace_period(self): - self.client.get.side_effect = [Mock(), DaytonaNotFoundError("gone")] - with patch("diffex_terminal_bench.daytona_backend.asyncio.sleep", new_callable=AsyncMock) as sleep: - report = await confirm_cleanup(self.client, self.directory, "trial-only") - self.assertTrue(report["confirmed"]) - self.assertEqual(report["unexpected_sandboxes"], []) - sleep.assert_awaited_once_with(2) - - async def test_no_remote_trial_starts_without_prerequisites(self): - config = load_config(PACKAGE / "benchmark.toml") - with patch("diffex_terminal_bench.benchmark.Trial.create", new_callable=AsyncMock) as create: - with self.assertRaises(OSError): - await run_stage(config, {name: Path(name) for name in config.pilot_tasks}, {}, self.directory, "full") - create.assert_not_called() - - async def test_cancellation_during_cleanup_cannot_pass_a_stage(self): - config = load_config(PACKAGE / "benchmark.toml") - started, finish = asyncio.Event(), asyncio.Event() - - async def cleanup(*args): - started.set() - await finish.wait() - return {"confirmed": True} - - manager = Mock(_cleanup=AsyncMock()) - trial = Mock(run=AsyncMock()) - with patch("diffex_terminal_bench.benchmark.AsyncDaytona") as client, \ - patch("diffex_terminal_bench.benchmark.Trial.create", new_callable=AsyncMock, return_value=trial), \ - patch("diffex_terminal_bench.benchmark.Packager.compute_content_hash", return_value=("fixture", [])), \ - patch("diffex_terminal_bench.benchmark.confirm_cleanup", side_effect=cleanup), \ - patch("diffex_terminal_bench.benchmark.DaytonaClientManager.get_instance", new_callable=AsyncMock, return_value=manager): - client.return_value.__aenter__ = AsyncMock(return_value=self.client) - client.return_value.__aexit__ = AsyncMock(return_value=False) - identity = {"task_digests": {name: "sha256:fixture" for name in config.pilot_tasks}} - task = asyncio.create_task(run_stage(config, {name: Path(name) for name in config.pilot_tasks}, - identity, self.directory, "oracle")) - await started.wait() - task.cancel() - await asyncio.sleep(0) - finish.set() - with self.assertRaises(asyncio.CancelledError): - await task - report = read_json(self.directory / "oracle/gate.json") - self.assertFalse(report["passed"]) - self.assertEqual(report["failure"]["error"], "CancelledError") - - -if __name__ == "__main__": - unittest.main() diff --git a/evals/terminal-bench/tests/test_concurrency.py b/evals/terminal-bench/tests/test_concurrency.py deleted file mode 100644 index d5f2bad87..000000000 --- a/evals/terminal-bench/tests/test_concurrency.py +++ /dev/null @@ -1,405 +0,0 @@ -"""Bounded parallel stages through the runner's Harbor/Daytona boundary.""" - -import asyncio -from contextlib import redirect_stderr -import io -from pathlib import Path -import tempfile -import unittest -from unittest.mock import AsyncMock, Mock, patch - -from diffex_terminal_bench.benchmark import PACKAGE, StageFailedError, main, run_identity, run_stage -from diffex_terminal_bench.dataset import BenchmarkConfig, load_config -from diffex_terminal_bench.diagnostics import TrialExecutionFailure -from diffex_terminal_bench.gate import read_json, write_json - - -class ConcurrencyConfigTests(unittest.TestCase): - def test_config_requires_positive_integer(self): - values = load_config(PACKAGE / "benchmark.toml").model_dump() - values.pop("concurrency") - self.assertEqual(BenchmarkConfig.model_validate(values).concurrency, 1) - for value in (0, -1, True, 1.5, "3"): - with self.subTest(value=value), self.assertRaises(ValueError): - BenchmarkConfig.model_validate({**values, "concurrency": value}) - - def test_cli_rejects_invalid_limits_before_remote_work(self): - for value in ("0", "-1", "1.5"): - with self.subTest(value=value), patch("sys.argv", ["benchmark", "--stage", "pilot", "-n", value]), \ - patch("diffex_terminal_bench.benchmark.preflight") as preflight, redirect_stderr(io.StringIO()): - with self.assertRaises(SystemExit) as error: - main() - self.assertEqual(error.exception.code, 2) - preflight.assert_not_called() - - def test_cli_override_is_prepared_and_part_of_identity(self): - for flag in ("-n", "--concurrency"): - with self.subTest(flag=flag), tempfile.TemporaryDirectory() as temp, \ - patch("sys.argv", ["benchmark", "--prepare", flag, "3", "--output", temp]), \ - patch("diffex_terminal_bench.benchmark.refresh_index"), \ - patch("diffex_terminal_bench.benchmark.prepare_dataset") as prepare: - config = load_config(PACKAGE / "benchmark.toml") - tasks = {name: Path(name) for name in config.pilot_tasks} - prepare.return_value.__enter__.return_value = (tasks, {}) - main() - selected = prepare.call_args.args[0] - self.assertEqual(selected.concurrency, 3) - self.assertEqual(read_json(Path(temp) / "dataset.json")["concurrency"], 3) - with patch("diffex_terminal_bench.benchmark.subprocess.check_output", side_effect=["", "commit"]): - self.assertEqual(run_identity(selected, {})["concurrency"], 3) - - -class ParallelStageTests(unittest.IsolatedAsyncioTestCase): - def setUp(self): - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - self.root = Path(temporary.name) - self.config = load_config(PACKAGE / "benchmark.toml") - self.config.concurrency = 3 - self.tasks = {name: Path(name) for name in self.config.pilot_tasks} - self.identity = {"task_digests": {name: "sha256:fixture" for name in self.tasks}, "concurrency": 3} - self.started = {index: asyncio.Event() for index in range(1, 11)} - self.finish_run = {index: asyncio.Event() for index in range(1, 11)} - self.cleaning = {index: asyncio.Event() for index in range(1, 11)} - self.finish_cleanup = {index: asyncio.Event() for index in range(1, 11)} - self.active = set() - self.cleaned = set() - self.configs = [] - self.peak = 0 - self.failure_phase = None - self.cleanup_failure = False - - async def create(config): - index = int(config.trial_name.split("-", 1)[0]) - self.configs.append(config) - self.active.add(index) - self.peak = max(self.peak, len(self.active)) - self.started[index].set() - if index == 2 and self.failure_phase == "create": - await self.finish_run[index].wait() - raise ValueError("creation failed") - - async def run(): - await self.finish_run[index].wait() - if index == 2 and self.failure_phase == "run": - raise ValueError("execution failed") - if index == 2 and self.failure_phase == "cancel": - raise asyncio.CancelledError - - return Mock(run=run) - - async def cleanup(client, directory, label): - index = int(directory.name.split("-", 1)[0]) - self.cleaning[index].set() - await self.finish_cleanup[index].wait() - self.active.remove(index) - self.cleaned.add(index) - if index == 2 and self.failure_phase == "cleanup": - raise ValueError("cleanup failed") - if index == 2 and self.cleanup_failure: - return {"confirmed": False, "error": "RuntimeError"} - return {"confirmed": True} - - def validate(directory, *args, **kwargs): - index = int(directory.name.split("-", 1)[0]) - self.assertIn(index, self.cleaned) - if index in (2, 4) and self.failure_phase == "multiple": - raise ValueError("invalid evidence") - if index == 2 and self.failure_phase == "validate": - raise ValueError("invalid evidence") - if index == 2 and self.failure_phase == "timeout": - result = {"finished_at": "2026-09-15T00:00:00Z", "exception_info": { - "exception_type": "AgentTimeoutError", "exception_message": "Agent execution timed out after 900.0 seconds"}} - write_json(directory / "result.json", result) - raise TrialExecutionFailure(directory, result) - return {"rewards": {"reward": 1}} - - async def close(*args): - self.assertEqual(self.active, set()) - - self.manager = Mock(_cleanup=AsyncMock(side_effect=close)) - patches = { - "require_previous": Mock(), - "refresh_index": Mock(), - "Trial.create": AsyncMock(side_effect=create), - "Packager.compute_content_hash": Mock(return_value=("fixture", [])), - "confirm_cleanup": AsyncMock(side_effect=cleanup), - "validate_trial": Mock(side_effect=validate), - "DaytonaClientManager.get_instance": AsyncMock(return_value=self.manager), - } - for name, replacement in patches.items(): - current = patch(f"diffex_terminal_bench.benchmark.{name}", replacement) - current.start() - self.addCleanup(current.stop) - client = patch("diffex_terminal_bench.benchmark.AsyncDaytona") - mocked = client.start() - self.addCleanup(client.stop) - mocked.return_value.__aexit__ = AsyncMock(side_effect=close) - - async def wait_events(self, events, indices): - async with asyncio.timeout(5): - await asyncio.gather(*(events[index].wait() for index in indices)) - - def start_stage(self): - task = asyncio.create_task(run_stage(self.config, self.tasks, self.identity, self.root, "pilot")) - self.addAsyncCleanup(self.stop_stage, task) - return task - - async def stop_stage(self, task): - for event in self.finish_cleanup.values(): - event.set() - if not task.done(): - task.cancel() - await asyncio.gather(task, return_exceptions=True) - - async def test_three_slots_include_cleanup_and_keep_declared_order(self): - task = self.start_stage() - await self.wait_events(self.started, (1, 2, 3)) - self.finish_run[3].set() - await self.wait_events(self.cleaning, (3,)) - self.assertFalse(self.started[4].is_set()) - self.finish_cleanup[3].set() - await self.wait_events(self.started, (4,)) - report = read_json(self.root / "pilot/gate.json") - self.assertEqual([row["task"] for row in report["trials"]], [self.config.pilot_tasks[2]]) - for event in (*self.finish_run.values(), *self.finish_cleanup.values()): - event.set() - await asyncio.wait_for(task, 5) - report = read_json(self.root / "pilot/gate.json") - self.assertTrue(report["passed"]) - self.assertEqual([row["task"] for row in report["trials"]], self.config.pilot_tasks) - self.assertEqual(self.peak, 3) - self.assertEqual(self.cleaned, set(range(1, 11))) - self.assertEqual(len({config.environment.kwargs["labels"]["diffex.gate_trial"] for config in self.configs}), 10) - self.manager._cleanup.assert_awaited_once() - - async def test_one_slot_retains_serial_execution(self): - self.config.concurrency = 1 - for event in (*self.finish_run.values(), *self.finish_cleanup.values()): - event.set() - await asyncio.wait_for(self.start_stage(), 5) - self.assertEqual(self.peak, 1) - self.assertEqual(self.cleaned, set(range(1, 11))) - - async def test_failure_continues_queue_without_cancelling_peers(self): - self.failure_phase = "validate" - await self.check_failure() - - async def test_creation_failure_cleans_started_trials(self): - self.failure_phase = "create" - await self.check_failure() - - async def test_execution_failure_cleans_started_trials(self): - self.failure_phase = "run" - await self.check_failure() - - async def test_cleanup_exception_continues_queue(self): - self.failure_phase = "cleanup" - await self.check_failure() - - async def test_unconfirmed_cleanup_continues_queue(self): - self.cleanup_failure = True - await self.check_failure() - - async def test_execution_and_cleanup_failures_are_both_retained(self): - self.failure_phase = "run" - self.cleanup_failure = True - await self.check_failure() - failure = read_json(self.root / "pilot/gate.json")["failures"][0] - self.assertEqual(failure["phase"], "execution") - self.assertEqual(failure["additional_errors"][0]["phase"], "cleanup") - - async def test_serial_stage_records_every_failure_and_finishes_queue(self): - self.config.concurrency = 1 - self.failure_phase = "multiple" - for event in (*self.finish_run.values(), *self.finish_cleanup.values()): - event.set() - with self.assertRaisesRegex(StageFailedError, "completed 10 trials with 2 failed"): - await asyncio.wait_for(self.start_stage(), 5) - report = read_json(self.root / "pilot/gate.json") - self.assertEqual([row["trial_index"] for row in report["failures"]], [2, 4]) - self.assertEqual(len(report["trials"]), 8) - self.assertEqual(report["completed_trials"], 10) - self.assertEqual(self.cleaned, set(range(1, 11))) - self.assertEqual(self.peak, 1) - - async def test_harbor_timeout_retains_cause_after_cleanup(self): - self.failure_phase = "timeout" - await self.check_failure() - failure = read_json(self.root / "pilot/gate.json")["failure"] - self.assertEqual(failure["diagnostics"]["harbor_exception"], "AgentTimeoutError") - self.assertEqual(failure["diagnostics"]["agent_timeout_sec"], 900) - - async def test_broken_failure_formatter_does_not_cancel_peers(self): - self.failure_phase = "timeout" - with patch("diffex_terminal_bench.benchmark.describe_failure", side_effect=ValueError("bad diagnostics")): - await self.check_failure(formatted=False) - - async def test_broken_progress_streams_do_not_cancel_peers(self): - self.failure_phase = "run" - with patch("diffex_terminal_bench.benchmark.print", create=True, side_effect=BrokenPipeError): - await self.check_failure() - - async def test_broken_progress_streams_do_not_fail_successful_trials(self): - for event in (*self.finish_run.values(), *self.finish_cleanup.values()): - event.set() - with patch("diffex_terminal_bench.benchmark.print", create=True, side_effect=BrokenPipeError): - await asyncio.wait_for(self.start_stage(), 5) - report = read_json(self.root / "pilot/gate.json") - self.assertTrue(report["passed"]) - self.assertEqual(report["completed_trials"], 10) - self.assertEqual(self.cleaned, set(range(1, 11))) - - async def test_checkpoint_failure_preserves_failed_trial_and_continues_peers(self): - self.failure_phase = "run" - await self.check_report_write_failure() - - async def test_checkpoint_failure_can_recover_without_failing_trials(self): - await self.check_report_write_failure() - - async def test_final_report_write_is_retried(self): - await self.check_report_write_failure(final_only=True) - - async def test_persistent_checkpoint_failure_finishes_every_trial(self): - self.failure_phase = "run" - await self.check_report_write_failure(permanent=True) - - async def test_persistent_final_write_cannot_pass_qualification(self): - await self.check_report_write_failure(final_only=True, permanent=True) - - async def test_report_write_failure_cannot_mask_stage_cancellation(self): - def save(path, value): - if path.name == "gate.json" and value.get("finished_at"): - raise OSError("disk full") - write_json(path, value) - - with patch("diffex_terminal_bench.benchmark.write_json", side_effect=save): - task = self.start_stage() - await self.wait_events(self.started, (1, 2, 3)) - task.cancel() - for event in self.finish_cleanup.values(): - event.set() - with self.assertRaises(asyncio.CancelledError): - await asyncio.wait_for(task, 5) - self.assertEqual(self.cleaned, {1, 2, 3}) - self.assertFalse(self.started[4].is_set()) - self.manager._cleanup.assert_awaited_once() - - async def check_report_write_failure(self, *, final_only=False, permanent=False): - failures = 0 - - def save(path, value): - nonlocal failures - if path.name == "gate.json" and value["completed_trials"] > 0: - if (not final_only or value.get("finished_at")) and (permanent or failures == 0): - failures += 1 - raise OSError("disk full") - write_json(path, value) - - with patch("diffex_terminal_bench.benchmark.write_json", side_effect=save): - task = self.start_stage() - await self.wait_events(self.started, (1, 2, 3)) - self.finish_run[2].set() - self.finish_cleanup[2].set() - await self.wait_events(self.started, (4,)) - self.assertFalse(self.cleaning[1].is_set()) - self.assertFalse(self.cleaning[3].is_set()) - for event in (*self.finish_run.values(), *self.finish_cleanup.values()): - event.set() - if permanent or self.failure_phase: - with self.assertRaises(StageFailedError) as error: - await asyncio.wait_for(task, 5) - self.assertIn("completed 10", str(error.exception)) - if permanent: - self.assertIn("incomplete", str(error.exception)) - if self.failure_phase: - self.assertIn("ValueError", str(error.exception)) - else: - await asyncio.wait_for(task, 5) - self.assertEqual(self.cleaned, set(range(1, 11))) - self.assertEqual(self.peak, 3) - self.manager._cleanup.assert_awaited_once() - report = read_json(self.root / "pilot/gate.json") - if permanent: - self.assertFalse(report["passed"]) - self.assertNotIn("finished_at", report) - self.assertGreaterEqual(failures, 3) - else: - self.assertEqual(failures, 1) - self.assertEqual(report["passed"], self.failure_phase is None) - self.assertEqual(report["completed_trials"], 10) - self.assertEqual(len(report["trials"]), 9 if self.failure_phase else 10) - self.assertEqual(len(report["failures"]), 1 if self.failure_phase else 0) - self.assertEqual(report["report_write_errors"][0]["error"], "OSError") - - async def check_failure(self, *, formatted=True): - task = self.start_stage() - await self.wait_events(self.started, (1, 2, 3)) - self.finish_run[2].set() - await self.wait_events(self.cleaning, (2,)) - self.assertFalse(self.started[4].is_set()) - self.finish_cleanup[2].set() - await self.wait_events(self.started, (4,)) - self.assertFalse(task.done()) - self.assertFalse(self.cleaning[1].is_set()) - self.assertFalse(self.cleaning[3].is_set()) - self.manager._cleanup.assert_not_awaited() - interim = read_json(self.root / "pilot/gate.json") - self.assertEqual(interim["failures"][0]["trial_index"], 2) - for event in (*self.finish_run.values(), *self.finish_cleanup.values()): - event.set() - with self.assertRaises(StageFailedError) as error: - await asyncio.wait_for(task, 5) - report = read_json(self.root / "pilot/gate.json") - self.assertFalse(report["passed"]) - self.assertEqual(report["failure"]["error"], "TrialExecutionFailure" if self.failure_phase == "timeout" else "ValueError") - self.assertEqual(report["failure"]["trial_index"], 2) - self.assertEqual(report["failure"]["task"], self.config.pilot_tasks[1]) - self.assertEqual(report["failure"]["directory"], f"002-{self.config.pilot_tasks[1]}") - if self.failure_phase == "timeout": - self.assertIn(str(self.root / "pilot" / report["failure"]["directory"]), str(error.exception)) - self.assertIn("AgentTimeoutError at the 900s task deadline" if formatted else "TrialExecutionFailure", - str(error.exception)) - self.assertEqual(report["failure"]["diagnostics"]["harbor_exception"], "AgentTimeoutError") - self.assertEqual(self.cleaned, set(range(1, 11))) - self.assertEqual(len(report["trials"]), 9) - self.assertEqual(len(report["failures"]), 1) - self.assertEqual(report["completed_trials"], 10) - self.assertEqual(self.peak, 3) - - async def test_repeated_cancellation_waits_for_all_cleanup(self): - task = self.start_stage() - await self.wait_events(self.started, (1, 2, 3)) - task.cancel() - await self.wait_events(self.cleaning, (1, 2, 3)) - task.cancel() - await asyncio.sleep(0) - self.assertFalse(task.done()) - self.manager._cleanup.assert_not_awaited() - for event in self.finish_cleanup.values(): - event.set() - with self.assertRaises(asyncio.CancelledError): - await asyncio.wait_for(task, 5) - report = read_json(self.root / "pilot/gate.json") - self.assertFalse(report["passed"]) - self.assertEqual(report["failure"]["error"], "CancelledError") - self.assertEqual(self.cleaned, {1, 2, 3}) - self.assertFalse(self.started[4].is_set()) - - async def test_trial_cancellation_does_not_cancel_the_worker(self): - self.failure_phase = "cancel" - task = self.start_stage() - await self.wait_events(self.started, (1, 2, 3)) - self.finish_run[2].set() - await self.wait_events(self.cleaning, (2,)) - for event in (*self.finish_run.values(), *self.finish_cleanup.values()): - event.set() - with self.assertRaisesRegex(StageFailedError, "completed 10 trials with 1 failed"): - await asyncio.wait_for(task, 5) - self.assertFalse(read_json(self.root / "pilot/gate.json")["passed"]) - self.assertEqual(self.cleaned, set(range(1, 11))) - self.assertTrue(self.started[4].is_set()) - - -if __name__ == "__main__": - unittest.main() diff --git a/evals/terminal-bench/tests/test_dataset_cache.py b/evals/terminal-bench/tests/test_dataset_cache.py deleted file mode 100644 index 57ca41426..000000000 --- a/evals/terminal-bench/tests/test_dataset_cache.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Shared downloads remain verified, atomic, and independent across runs.""" - -from concurrent.futures import ThreadPoolExecutor -from io import BytesIO -from pathlib import Path -import subprocess -import sys -import tarfile -import tempfile -import threading -import unittest -from unittest.mock import patch - -from diffex_terminal_bench.benchmark import PACKAGE -from diffex_terminal_bench.dataset import load_config, prepare_dataset -from diffex_terminal_bench.gate import sha256 - - -class DatasetCacheTests(unittest.TestCase): - def setUp(self): - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - self.root = Path(temporary.name) - self.config = load_config(PACKAGE / "benchmark.toml") - tree = self.root / "tree" - for name in self.config.pilot_tasks: - task = tree / "tasks" / name - (task / "environment").mkdir(parents=True) - (task / "environment/Dockerfile").write_text("FROM ubuntu:24.04\n") - (task / "task.toml").write_text('version = "1.0"\n') - (task / "instruction.md").write_text("fixture") - (task / "tests").mkdir() - (task / "tests/test.sh").write_text("exit 0\n") - self.archive = self.root / "fixture.tar.gz" - with tarfile.open(self.archive, "w:gz") as archive: - archive.add(tree, arcname="dataset") - self.config.dataset.archive_sha256 = sha256(self.archive) - self.config.dataset.task_count = 10 - self.cache = self.root / "cache" - - def test_concurrent_runs_download_once_and_extract_separate_tasks(self): - barrier = threading.Barrier(4) - - def prepare(_): - with prepare_dataset(self.config, self.cache) as (tasks, digests): - barrier.wait(timeout=10) - path = tasks[self.config.pilot_tasks[0]] / "instruction.md" - self.assertEqual(path.read_text(), "fixture") - path.write_text("changed in this run") - return path, digests - - with patch("diffex_terminal_bench.dataset.urlopen", side_effect=lambda *a, **kw: BytesIO(self.archive.read_bytes())) as download: - with ThreadPoolExecutor(max_workers=4) as pool: - results = list(pool.map(prepare, range(4))) - download.assert_called_once_with(self.config.dataset.archive_url, timeout=60) - self.assertEqual(len({path for path, _ in results}), 4) - self.assertTrue(all(digests == results[0][1] for _, digests in results)) - self.assertTrue(all(not path.exists() for path, _ in results)) - self.assertEqual(len(list(self.cache.glob("*.tar.gz"))), 1) - self.assertFalse(list(self.cache.glob(".download-*"))) - - def test_invalid_download_is_never_published_and_retry_succeeds(self): - with patch("diffex_terminal_bench.dataset.urlopen", return_value=BytesIO(b"partial archive")): - with self.assertRaisesRegex(ValueError, "download SHA-256"), prepare_dataset(self.config, self.cache): - self.fail("corrupt download accepted") - self.assertFalse(list(self.cache.glob("*.tar.gz"))) - self.assertFalse(list(self.cache.glob(".download-*"))) - with patch("diffex_terminal_bench.dataset.urlopen", return_value=BytesIO(self.archive.read_bytes())): - with prepare_dataset(self.config, self.cache) as (tasks, _): - self.assertEqual(len(tasks), 10) - saved = self.cache / f"{self.config.dataset.archive_sha256}.tar.gz" - saved.write_bytes(b"tampered after publication") - with patch("diffex_terminal_bench.dataset.urlopen") as download: - with self.assertRaisesRegex(ValueError, "Cached dataset archive SHA-256"), prepare_dataset(self.config, self.cache): - self.fail("corrupt shared cache accepted") - download.assert_not_called() - - def test_prepare_cli_reuses_shared_cache_across_output_directories(self): - self.cache.mkdir() - (self.cache / f"{self.config.dataset.archive_sha256}.tar.gz").write_bytes(self.archive.read_bytes()) - original = load_config(PACKAGE / "benchmark.toml") - config = self.root / "benchmark.toml" - config.write_text((PACKAGE / "benchmark.toml").read_text() - .replace(original.dataset.archive_sha256, self.config.dataset.archive_sha256) - .replace(f"task_count = {original.dataset.task_count}", "task_count = 10")) - for name in ("first", "second"): - output = self.root / "results" / name - result = subprocess.run([sys.executable, "-m", "diffex_terminal_bench.benchmark", "--prepare", - "--config", str(config), "--dataset-cache", str(self.cache), "--output", str(output)], - capture_output=True, text=True, timeout=30) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("Verified 10 pinned tasks", result.stdout) - self.assertTrue((output / "dataset.json").is_file()) - self.assertFalse((output / "dataset-cache").exists()) - index = (self.root / "results/index.md").read_text() - self.assertIn("[first](first/)", index) - self.assertIn("[second](second/)", index) - self.assertIn(self.config.model, index) - self.assertIn(self.config.policy.thinking, index) - - -if __name__ == "__main__": - unittest.main() diff --git a/evals/terminal-bench/tests/test_daytona_backend.py b/evals/terminal-bench/tests/test_daytona_backend.py deleted file mode 100644 index 875e8b794..000000000 --- a/evals/terminal-bench/tests/test_daytona_backend.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Volume isolation at the pinned Harbor sandbox creation boundary.""" - -import asyncio -import json -from pathlib import Path -import tempfile -from types import SimpleNamespace -import unittest -from unittest.mock import AsyncMock, patch - -from daytona import CreateSandboxFromImageParams, CreateSandboxFromSnapshotParams, Image -from daytona.common.errors import DaytonaConflictError, DaytonaError -from harbor.models.job.config import JobConfig -from harbor.models.task.config import EnvironmentConfig -from harbor.models.trial.paths import TrialPaths -import yaml - -from diffex_terminal_bench.daytona_backend import RecordedDaytonaEnvironment - - -class VolumeTests(unittest.IsolatedAsyncioTestCase): - def setUp(self): - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - self.root = Path(temporary.name) - (self.root / "Dockerfile").write_text("FROM ubuntu:24.04\n") - self.environment = RecordedDaytonaEnvironment( - environment_dir=self.root, environment_name="fixture", session_id="fixture", - trial_paths=TrialPaths(self.root / "trial"), task_env_config=EnvironmentConfig(), - ) - self.volume = SimpleNamespace(id="volume-id", state="ready") - self.client = SimpleNamespace(volume=SimpleNamespace(get=AsyncMock(return_value=self.volume))) - parent = patch.object(RecordedDaytonaEnvironment, "_create_with_retries", new_callable=AsyncMock) - self.create = parent.start() - self.addCleanup(parent.stop) - - async def test_full_445_trials_share_one_volume_with_isolated_prefixes(self): - params = [CreateSandboxFromSnapshotParams(snapshot="fixture") for _ in range(445)] - # Exercise the same creation hook for every trial, with no live API. - for request in params: - await self.environment._create_sandbox(request, self.client) - await asyncio.sleep(0) - mounts = [request.volumes[0] for request in params] - self.assertEqual({mount.volume_id for mount in mounts}, {"volume-id"}) - self.assertEqual({mount.mount_path for mount in mounts}, {"/logs"}) - self.assertEqual(len({mount.subpath for mount in mounts}), 445) - self.assertTrue(all(mount.subpath.startswith("trials/") and ".." not in mount.subpath for mount in mounts)) - self.assertEqual({call.args[0] for call in self.client.volume.get.call_args_list}, {"diffex-terminal-bench-logs"}) - record = json.loads((self.root / "trial/artifact-volume.json").read_text()) - self.assertEqual([entry["subpath"] for entry in record["mounts"]], [mount.subpath for mount in mounts]) - - async def test_separate_verifier_preserves_agent_recovery_location(self): - verifier = RecordedDaytonaEnvironment( - environment_dir=self.root, environment_name="verifier", session_id="verifier-session", - trial_paths=self.environment.trial_paths, task_env_config=EnvironmentConfig(), - ) - for environment in (self.environment, verifier): - await environment._create_sandbox(CreateSandboxFromSnapshotParams(snapshot="fixture"), self.client) - entries = json.loads((self.root / "trial/artifact-volume.json").read_text())["mounts"] - self.assertEqual([entry["session_id"] for entry in entries], ["fixture", "verifier-session"]) - self.assertNotEqual(entries[0]["subpath"], entries[1]["subpath"]) - - async def test_waits_for_readiness_after_concurrent_volume_creation(self): - self.client.volume.get.side_effect = [DaytonaConflictError("already created"), - SimpleNamespace(id="volume-id", state="pending_create"), - SimpleNamespace(id="volume-id", state="creating"), self.volume] - with patch("diffex_terminal_bench.daytona_backend.asyncio.sleep"): - await self.environment._create_sandbox(CreateSandboxFromImageParams(image=Image.base("ubuntu:24.04")), self.client) - self.create.assert_awaited_once() - self.assertEqual(self.client.volume.get.await_count, 4) - - async def test_unusable_volume_prevents_sandbox_creation(self): - self.volume.state = "error" - with self.assertRaisesRegex(RuntimeError, "not usable"): - await self.environment._create_sandbox(CreateSandboxFromSnapshotParams(snapshot="fixture"), self.client) - self.create.assert_not_awaited() - - async def test_volume_quota_error_is_not_retried_or_bypassed(self): - self.client.volume.get.side_effect = DaytonaError("volume quota", status_code=400) - with self.assertRaises(DaytonaError): - await self.environment._create_sandbox(CreateSandboxFromSnapshotParams(snapshot="fixture"), self.client) - self.assertEqual(self.client.volume.get.await_count, 1) - self.create.assert_not_awaited() - - async def test_provisioning_wait_is_bounded(self): - self.volume.state = "creating" - timeout = asyncio.timeout - with patch("diffex_terminal_bench.daytona_backend.asyncio.timeout", side_effect=lambda seconds: timeout(0.01)): - with self.assertRaises(TimeoutError): - await self.environment._create_sandbox(CreateSandboxFromSnapshotParams(snapshot="fixture"), self.client) - self.create.assert_not_awaited() - - async def test_dind_client_manager_and_custom_volume(self): - self.environment._client_manager = SimpleNamespace(get_client=AsyncMock(return_value=self.client)) - self.environment._kwargs["artifact_volume"] = "dedicated-project-logs" - params = CreateSandboxFromImageParams(image=Image.base("docker:28.3.3-dind")) - await self.environment._create_sandbox(params) - self.client.volume.get.assert_awaited_once_with("dedicated-project-logs", create=True) - self.create.assert_awaited_once_with(params, self.client) - - def test_native_full_job_selects_volume_backend(self): - config = JobConfig.model_validate(yaml.safe_load((Path(__file__).resolve().parents[1] / "full.yaml").read_text())) - self.assertEqual(config.environment.import_path, "diffex_terminal_bench.daytona_backend:RecordedDaytonaEnvironment") - self.assertEqual(config.environment.kwargs["artifact_volume"], "diffex-terminal-bench-logs") - - -if __name__ == "__main__": - unittest.main() diff --git a/evals/terminal-bench/tests/test_diagnostics.py b/evals/terminal-bench/tests/test_diagnostics.py deleted file mode 100644 index 0e0935d55..000000000 --- a/evals/terminal-bench/tests/test_diagnostics.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Replay a timed-out Harbor trial through the offline CLI and live-stage gate.""" - -from contextlib import redirect_stdout -import io -import json -from pathlib import Path -import tempfile -import unittest -from unittest.mock import patch - -from diffex_terminal_bench.benchmark import StageFailedError, main -from diffex_terminal_bench.diagnostics import TrialExecutionFailure, describe_failure, diagnose_trial -from diffex_terminal_bench.gate import validate_trial, write_json - - -class FailureDiagnosticsTests(unittest.TestCase): - def setUp(self): - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - self.directory = Path(temporary.name) / "007-build-cython-ext" - self.result = {"finished_at": "2026-09-15T18:22:36Z", "exception_info": { - "exception_type": "AgentTimeoutError", "exception_message": "Agent execution timed out after 900.0 seconds", - "exception_traceback": "private traceback"}, "verifier_result": {"rewards": {"reward": 1.0}}} - write_json(self.directory / "result.json", self.result) - self.stream = self.directory / "agent/run-fixture/stdout.jsonl" - self.stream.parent.mkdir(parents=True) - events = [{"type": "auto_retry_start", "errorMessage": "Rate limit reached for private-org. secret-token", - "delayMs": 2000}, {"type": "auto_retry_start", "errorMessage": "HTTP 429", "delayMs": 4000}, - {"type": "auto_retry_start", "errorMessage": "Network error", "delayMs": 2000}] - self.stream.write_text("".join(json.dumps(event) + "\n" for event in events)) - - def test_timeout_with_full_reward_is_rejected_with_safe_cause(self): - with self.assertRaises(TrialExecutionFailure) as error: - validate_trial(self.directory, "build-cython-ext", {}) - self.assertIn("AgentTimeoutError at the 900s task deadline", str(error.exception)) - self.assertIn("3 scheduled retries (2 rate-limit retries, 8s scheduled backoff)", str(error.exception)) - self.assertNotIn("private", str(error.exception)) - self.assertNotIn("secret", json.dumps(error.exception.details)) - - def test_offline_cli_explains_old_trials_without_identity_or_provider(self): - output = io.StringIO() - with patch("sys.argv", ["benchmark", "--diagnose-trial", str(self.directory)]), \ - patch("diffex_terminal_bench.benchmark.preflight") as preflight, \ - patch("diffex_terminal_bench.benchmark.prepare_dataset") as prepare, \ - patch("diffex_terminal_bench.benchmark.run_identity") as identity, redirect_stdout(output): - main() - self.assertIn("007-build-cython-ext: AgentTimeoutError", output.getvalue()) - self.assertIn(str(self.directory), output.getvalue()) - preflight.assert_not_called() - prepare.assert_not_called() - identity.assert_not_called() - - def test_truncated_evidence_retains_observed_retry_counts(self): - with self.stream.open("a") as output: - output.write('{"type":') - details = diagnose_trial(self.directory) - self.assertFalse(details["retry_evidence_complete"]) - self.assertEqual(details["retry_attempts"], 3) - - def test_missing_stream_is_unknown_and_raw_exception_text_is_not_reported(self): - self.stream.unlink() - self.result["exception_info"] = {"exception_type": "secret\nprovider text", "exception_message": "secret"} - details = diagnose_trial(self.directory, self.result) - self.assertFalse(details["retry_evidence_complete"]) - self.assertIsNone(details["harbor_exception"]) - self.assertNotIn("secret", describe_failure(details)) - - def test_expected_cli_stage_failure_exits_nonzero_without_traceback(self): - with patch("sys.argv", ["benchmark", "--stage", "pilot", "--output", str(self.directory)]), \ - patch("diffex_terminal_bench.benchmark.preflight"), \ - patch("diffex_terminal_bench.benchmark.prepare_dataset") as prepare, \ - patch("diffex_terminal_bench.benchmark.run_identity", return_value={}), \ - patch("diffex_terminal_bench.benchmark.execute", side_effect=StageFailedError("pilot: AgentTimeoutError")): - prepare.return_value.__enter__.return_value = ({}, {}) - with self.assertRaises(SystemExit) as error: - main() - self.assertEqual(error.exception.code, "pilot: AgentTimeoutError") - self.assertTrue(error.exception.__suppress_context__) - - -if __name__ == "__main__": - unittest.main() diff --git a/evals/terminal-bench/tests/test_evidence.py b/evals/terminal-bench/tests/test_evidence.py deleted file mode 100644 index 80763719d..000000000 --- a/evals/terminal-bench/tests/test_evidence.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Exercise the remote evidence collector without a provider or sandbox API.""" - -from importlib.resources import files -import json -from pathlib import Path -import subprocess -import tempfile -import unittest - -from diffex_terminal_bench.lifecycle import read_cleanup -from test_outcome import retain_service, write_evidence - - -class EvidenceTests(unittest.TestCase): - def test_collector_only_transfers_shutdown_records(self): - with tempfile.TemporaryDirectory() as temp: - remote, local = Path(temp) / "remote", Path(temp) / "local" - write_evidence(remote) - report = retain_service(remote) - cache = remote / "state/home/.cache/packages" - cache.parent.mkdir(parents=True) - with cache.open("wb") as stream: - stream.truncate(3 * 1024 ** 3) - script = files("diffex_terminal_bench").joinpath("evidence.cjs") - result = subprocess.run(["node", str(script), str(remote)], capture_output=True, - text=True, check=True, timeout=5) - evidence = json.loads(result.stdout) - self.assertEqual(set(evidence), {"cleanup.json", "stop-requested", "launcher.json", - "cli.json", "process.json", "ownership/service.json"}) - self.assertLess(len(result.stdout), 4096) - for name, content in evidence.items(): - target = local / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(content) - self.assertEqual(read_cleanup(local), report) - - def test_missing_required_evidence_is_pending(self): - with tempfile.TemporaryDirectory() as temp: - result = subprocess.run(["node", str(files("diffex_terminal_bench").joinpath("evidence.cjs")), temp], - capture_output=True, text=True, timeout=5) - self.assertEqual(result.returncode, 0) - self.assertIsNone(json.loads(result.stdout)) - - -if __name__ == "__main__": - unittest.main() diff --git a/evals/terminal-bench/tests/test_execution.py b/evals/terminal-bench/tests/test_execution.py deleted file mode 100644 index 46039f3fa..000000000 --- a/evals/terminal-bench/tests/test_execution.py +++ /dev/null @@ -1,190 +0,0 @@ -"""Execution contract tests without provider calls.""" - -import hashlib -import json -from importlib.resources import files -import os -from pathlib import Path, PurePosixPath -import tempfile -import unittest -from unittest.mock import create_autospec, patch - -from harbor.environments.base import BaseEnvironment, ExecResult -from harbor.models.agent.context import AgentContext -from harbor.models.job.config import JobConfig -import yaml - -from diffex_terminal_bench.adapter import DiffexAgent -from diffex_terminal_bench.benchmark import PACKAGE, run_identity, trial_config -from diffex_terminal_bench.config import PINS -from diffex_terminal_bench.dataset import load_config -from diffex_terminal_bench.execution import SYSTEM_PROMPT, SYSTEM_PROMPT_SHA256, USER_MESSAGE_SUFFIX - - -class ExecutionTests(unittest.IsolatedAsyncioTestCase): - def setUp(self): - # These tests isolate the launch contract; recovery has separate tests - # and a Linux smoke using the real launcher and provider. - async def finish_run(agent, environment, run_dir, context, execution): - (agent.logs_dir / run_dir.name).mkdir(parents=True, exist_ok=True) - return {"status": "completed", "agent_execution_stopped": True} - - finish = patch.object(DiffexAgent, "_finish_run", autospec=True, side_effect=finish_run) - finish.start() - self.addCleanup(finish.stop) - export = patch("diffex_terminal_bench.adapter.export_trajectory", return_value={"complete": True}) - export.start() - self.addCleanup(export.stop) - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - self.logs_dir = Path(temporary.name) - - def test_pinned_release_context_and_example_config(self): - # This fixture belongs to the pinned npm release. Current source - # instructions may change before the benchmark selects a new release. - release_context = files("diffex_terminal_bench").joinpath("release-agents.md").read_bytes() - self.assertEqual(hashlib.sha256(release_context).hexdigest(), PINS["release_agents_sha256"]) - config = JobConfig.model_validate(yaml.safe_load((Path(__file__).resolve().parents[1] / "execution.yaml").read_text())) - self.assertEqual(config.agents[0].import_path, DiffexAgent.import_path()) - self.assertEqual(config.agents[0].model_name, "openai/gpt-5") - DiffexAgent.preflight(config.agents[0].kwargs) - - def make_environment(self): - environment = create_autospec(BaseEnvironment, instance=True) - environment.default_user = None - environment.exec.return_value = ExecResult(return_code=0, stdout="", stderr="") - return environment - - def test_run_identity_uses_eval_system_prompt(self): - config = load_config(PACKAGE / "benchmark.toml") - with patch("diffex_terminal_bench.benchmark.subprocess.check_output", side_effect=["", "a" * 40]), \ - patch("diffex_terminal_bench.benchmark.sha256", return_value="new-source-digest"): - identity = run_identity(config, {}) - self.assertEqual(identity["system_prompt_sha256"], SYSTEM_PROMPT_SHA256) - self.assertEqual(identity["harness_files"]["eval-system-prompt.txt"], "new-source-digest") - - async def test_prompt_environment_and_fresh_trial_paths(self): - environment = self.make_environment() - uploads = {} - - async def upload(source, target): - uploads[target] = Path(source).read_bytes() - - environment.upload_file.side_effect = upload - instruction = "\n --literal @file 'quote' \"double\" $HOME $(touch /tmp/injected) `pwd`\nUnicode: 雪\n\n" - with patch.dict(os.environ, {"OPENAI_API_KEY": "host-secret", "ANTHROPIC_API_KEY": "unrelated-secret"}, clear=True): - agent = DiffexAgent( - logs_dir=self.logs_dir, model_name="openai/gpt-5", thinking="high", - extra_env={"OPENAI_API_KEY": "selected-secret", "DIFFEX_CODING_AGENT_DIR": "/stale"}, - environment_logs_dir=PurePosixPath("/logs/agent custom"), - ) - for _ in range(2): - await agent.run(instruction, environment, AgentContext()) - prompts = [value.decode() for key, value in uploads.items() if key.endswith("instruction.txt")] - self.assertEqual(prompts, [instruction, instruction]) - system_prompts = [value.decode() for key, value in uploads.items() if key.endswith("eval-system-prompt.txt")] - self.assertEqual(system_prompts, [SYSTEM_PROMPT, SYSTEM_PROMPT]) - self.assertNotIn("", SYSTEM_PROMPT) - self.assertNotIn("