diff --git a/.gitattributes b/.gitattributes index d8a52c2ec..30c9e0222 100644 --- a/.gitattributes +++ b/.gitattributes @@ -9,6 +9,9 @@ # Shell scripts should keep LF *.sh text eol=lf +# Terminal snapshots intentionally retain empty rows to preserve screen height. +scripts/daytona-behaviors/baselines/*.txt whitespace=-blank-at-eof + # Common binary assets *.png binary *.jpg binary diff --git a/.github/workflows/daytona-behaviors.yml b/.github/workflows/daytona-behaviors.yml new file mode 100644 index 000000000..45d196fbd --- /dev/null +++ b/.github/workflows/daytona-behaviors.yml @@ -0,0 +1,346 @@ +name: Daytona Behaviors + +on: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + actions: read + +env: + DIFFEX_SOURCE_COMMIT: ${{ github.sha }} + +jobs: + prepare: + # Dispatch remains available for qualification; pushes spend nothing until approved. + if: ${{ github.event_name == 'workflow_dispatch' || vars.DAYTONA_BEHAVIORS_QUALIFIED == 'true' }} + runs-on: ubuntu-24.04 + timeout-minutes: 35 + steps: + - name: Checkout exact event commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Setup Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22.19.0' + - name: Install pinned npm and locked dependencies + run: | + npm install -g npm@11.16.0 --ignore-scripts + npm ci --ignore-scripts + - name: Freeze, build offline, prepare and verify exact candidate bytes + run: | + npm run hydrate:model-data + node scripts/release-artifacts.mjs freeze .artifacts/release + npm run build:offline + node scripts/release-artifacts.mjs prepare .artifacts/release + node scripts/release-artifacts.mjs verify .artifacts/release + git diff --exit-code HEAD + - name: Test behavior harness and failure-path orchestration without credentials + env: + PYTHONDONTWRITEBYTECODE: '1' + run: | + python3 -m venv "$RUNNER_TEMP/behavior-test-venv" + "$RUNNER_TEMP/behavior-test-venv/bin/python" -m pip install -r scripts/daytona-behaviors/requirements.txt + "$RUNNER_TEMP/behavior-test-venv/bin/python" -m unittest discover -s scripts/daytona-behaviors -p 'test_*.py' + "$RUNNER_TEMP/behavior-test-venv/bin/python" -m unittest discover -s scripts -p 'test_verify_candidate_artifacts.py' + node --test scripts/artifact-installation.test.mjs scripts/artifact-acceptance.test.mjs scripts/daytona-behaviors-workflow.test.mjs scripts/daytona-behaviors/fixture.test.mjs scripts/daytona-behaviors/live-guard.test.mjs + - name: Bundle checked behavior harness without rebuilding the candidate + run: | + mkdir -p .artifacts/daytona-bundle + tar -czf .artifacts/daytona-bundle/bundle.tar.gz \ + scripts/artifact-installation.mjs \ + scripts/verify-candidate-artifacts.py \ + scripts/fixtures/artifact-handoff.mjs \ + scripts/daytona-behaviors/installed.py \ + scripts/daytona-behaviors/install.mjs \ + scripts/daytona-behaviors/fixture.mjs \ + scripts/daytona-behaviors/live-guard.mjs \ + scripts/daytona-behaviors/baselines + sha256sum .artifacts/daytona-bundle/bundle.tar.gz > .artifacts/daytona-bundle/SHA256SUMS + - name: Retain exact candidate bytes (not confidential) + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: daytona-release-${{ github.sha }}-${{ github.run_attempt }} + path: .artifacts/release/ + include-hidden-files: true + if-no-files-found: error + retention-days: 30 + - name: Retain exact behavior bundle (not confidential) + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: daytona-bundle-${{ github.sha }}-${{ github.run_attempt }} + path: .artifacts/daytona-bundle/ + if-no-files-found: error + retention-days: 30 + + deterministic: + needs: prepare + runs-on: ubuntu-24.04 + timeout-minutes: 35 + environment: daytona-behaviors + env: + DAYTONA_BEHAVIOR_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }}-deterministic + DAYTONA_SNAPSHOT: ${{ vars.DAYTONA_BEHAVIORS_SNAPSHOT }} + steps: + - name: Checkout exact event commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Setup Node for artifact verification only + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22.19.0' + - name: Install pinned Daytona SDK in an isolated environment + run: | + python3 -m venv "$RUNNER_TEMP/daytona-venv" + "$RUNNER_TEMP/daytona-venv/bin/python" -m pip install -r scripts/daytona-behaviors/requirements.txt + - name: Download exact candidate bytes + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: daytona-release-${{ github.sha }}-${{ github.run_attempt }} + path: .artifacts/release + - name: Download exact behavior bundle + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: daytona-bundle-${{ github.sha }}-${{ github.run_attempt }} + path: .artifacts/daytona-bundle + - name: Verify retained source, tarballs and harness bytes + run: | + node scripts/release-artifacts.mjs verify .artifacts/release + sha256sum -c .artifacts/daytona-bundle/SHA256SUMS + test -n "$DAYTONA_SNAPSHOT" + - name: Run deterministic installed-artifact behaviors in Daytona + id: run + timeout-minutes: 25 + env: + DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }} + run: | + "$RUNNER_TEMP/daytona-venv/bin/python" scripts/daytona-behaviors/daytona_runner.py run \ + --artifacts .artifacts/release --bundle .artifacts/daytona-bundle/bundle.tar.gz \ + --snapshot "$DAYTONA_SNAPSHOT" --output .artifacts/daytona-deterministic \ + --mode deterministic --commit "$DIFFEX_SOURCE_COMMIT" --run-id "$DAYTONA_BEHAVIOR_RUN_ID" \ + --timeout-seconds 1200 + - name: Recover evidence and confirm sandbox cleanup without relaunching + id: recover + if: ${{ always() && steps.run.outcome != 'skipped' }} + timeout-minutes: 5 + env: + DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }} + run: | + "$RUNNER_TEMP/daytona-venv/bin/python" scripts/daytona-behaviors/daytona_runner.py recover \ + --output .artifacts/daytona-deterministic + - name: Record execution and cleanup outcomes + if: always() + env: + EXECUTION: ${{ steps.run.outcome }} + RECOVERY: ${{ steps.recover.outcome }} + run: | + mkdir -p .artifacts/daytona-deterministic + jq -n --arg commit "$DIFFEX_SOURCE_COMMIT" --arg runId "$DAYTONA_BEHAVIOR_RUN_ID" \ + --arg mode deterministic --arg execution "$EXECUTION" --arg recovery "$RECOVERY" \ + '{commit: $commit, runId: $runId, mode: $mode, execution: $execution, recovery: $recovery}' \ + > .artifacts/daytona-deterministic/workflow-job.json + - name: Guard all deterministic evidence before publication + id: sanitize + if: always() + env: + DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }} + run: | + python3 scripts/daytona-behaviors/sanitize_evidence.py \ + --root .artifacts/daytona-deterministic --output .artifacts/daytona-deterministic-sanitized \ + --mode deterministic + - name: Retain sanitized deterministic evidence even on failure + if: ${{ always() && steps.sanitize.outputs.safe_to_upload == 'true' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: daytona-evidence-deterministic-${{ github.sha }}-${{ github.run_attempt }} + path: .artifacts/daytona-deterministic-sanitized/ + include-hidden-files: true + if-no-files-found: error + retention-days: 30 + + live: + needs: prepare + runs-on: ubuntu-24.04 + timeout-minutes: 35 + environment: daytona-behaviors + env: + DAYTONA_BEHAVIOR_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }}-live + DAYTONA_SNAPSHOT: ${{ vars.DAYTONA_BEHAVIORS_SNAPSHOT }} + steps: + - name: Checkout exact event commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Setup Node for artifact verification only + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22.19.0' + - name: Install pinned Daytona SDK in an isolated environment + run: | + python3 -m venv "$RUNNER_TEMP/daytona-venv" + "$RUNNER_TEMP/daytona-venv/bin/python" -m pip install -r scripts/daytona-behaviors/requirements.txt + - name: Download exact candidate bytes + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: daytona-release-${{ github.sha }}-${{ github.run_attempt }} + path: .artifacts/release + - name: Download exact behavior bundle + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: daytona-bundle-${{ github.sha }}-${{ github.run_attempt }} + path: .artifacts/daytona-bundle + - name: Verify retained source, tarballs and harness bytes + run: | + node scripts/release-artifacts.mjs verify .artifacts/release + sha256sum -c .artifacts/daytona-bundle/SHA256SUMS + test -n "$DAYTONA_SNAPSHOT" + - name: Run bounded live installed-artifact behaviors in Daytona + id: run + timeout-minutes: 25 + env: + DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: | + "$RUNNER_TEMP/daytona-venv/bin/python" scripts/daytona-behaviors/daytona_runner.py run \ + --artifacts .artifacts/release --bundle .artifacts/daytona-bundle/bundle.tar.gz \ + --snapshot "$DAYTONA_SNAPSHOT" --output .artifacts/daytona-live \ + --mode live --commit "$DIFFEX_SOURCE_COMMIT" --run-id "$DAYTONA_BEHAVIOR_RUN_ID" \ + --timeout-seconds 1200 + - name: Recover evidence and confirm sandbox cleanup without relaunching + id: recover + if: ${{ always() && steps.run.outcome != 'skipped' }} + timeout-minutes: 5 + env: + DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }} + run: | + "$RUNNER_TEMP/daytona-venv/bin/python" scripts/daytona-behaviors/daytona_runner.py recover \ + --output .artifacts/daytona-live + - name: Record execution and cleanup outcomes + if: always() + env: + EXECUTION: ${{ steps.run.outcome }} + RECOVERY: ${{ steps.recover.outcome }} + run: | + mkdir -p .artifacts/daytona-live + jq -n --arg commit "$DIFFEX_SOURCE_COMMIT" --arg runId "$DAYTONA_BEHAVIOR_RUN_ID" \ + --arg mode live --arg execution "$EXECUTION" --arg recovery "$RECOVERY" \ + '{commit: $commit, runId: $runId, mode: $mode, execution: $execution, recovery: $recovery}' \ + > .artifacts/daytona-live/workflow-job.json + - name: Guard all live evidence before publication + id: sanitize + if: always() + env: + DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: | + python3 scripts/daytona-behaviors/sanitize_evidence.py \ + --root .artifacts/daytona-live --output .artifacts/daytona-live-sanitized \ + --mode live + - name: Retain sanitized live evidence even on failure + if: ${{ always() && steps.sanitize.outputs.safe_to_upload == 'true' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: daytona-evidence-live-${{ github.sha }}-${{ github.run_attempt }} + path: .artifacts/daytona-live-sanitized/ + include-hidden-files: true + if-no-files-found: error + retention-days: 30 + + daytona-behaviors: + needs: [prepare, deterministic, live] + if: ${{ always() && (github.event_name == 'workflow_dispatch' || vars.DAYTONA_BEHAVIORS_QUALIFIED == 'true') }} + runs-on: ubuntu-24.04 + timeout-minutes: 10 + env: + GH_TOKEN: ${{ github.token }} + PREPARE_RESULT: ${{ needs.prepare.result }} + DETERMINISTIC_RESULT: ${{ needs.deterministic.result }} + LIVE_RESULT: ${{ needs.live.result }} + steps: + - name: Aggregate both modes, missing evidence and cleanup failures + run: | + mkdir -p .artifacts/daytona-summary + failed=0 + for result in "$PREPARE_RESULT" "$DETERMINISTIC_RESULT" "$LIVE_RESULT"; do + if [[ "$result" != success ]]; then failed=1; fi + done + for mode in deterministic live; do + evidence=".artifacts/daytona-summary/$mode" + if ! gh run download "$GITHUB_RUN_ID" --repo "$GITHUB_REPOSITORY" \ + --name "daytona-evidence-$mode-$DIFFEX_SOURCE_COMMIT-$GITHUB_RUN_ATTEMPT" --dir "$evidence"; then + echo "::error::Missing $mode evidence artifact" + failed=1 + continue + fi + if ! jq --exit-status --arg mode "$mode" \ + '.schemaVersion == 1 and .mode == $mode and .safe_to_upload == true and .outcome == "clean" and .credential_matches == 0' \ + "$evidence/publication-safety.json" > /dev/null; then + echo "::error::Missing or failed $mode credential publication guard" + failed=1 + fi + if ! jq --exit-status --arg commit "$DIFFEX_SOURCE_COMMIT" --arg mode "$mode" \ + --arg runId "$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-$mode" \ + '.commit == $commit and .mode == $mode and .runId == $runId and .execution == "success" and .recovery == "success"' \ + "$evidence/workflow-job.json" > /dev/null; then + echo "::error::$mode execution or cleanup did not succeed" + failed=1 + fi + if ! jq --exit-status --arg commit "$DIFFEX_SOURCE_COMMIT" --arg mode "$mode" \ + --arg runId "$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-$mode" \ + '.commit == $commit and .mode == $mode and .run_id == $runId and .outcome == "passed" and .execution == "passed" and .exit_code == 0 and .installed_outcome == "passed" and .reports_complete == true and .cleanup == "confirmed"' \ + "$evidence/orchestration.json" > /dev/null; then + echo "::error::Missing, incomplete or failed $mode orchestration/cleanup result" + failed=1 + fi + if [[ "$mode" == deterministic ]]; then + expected='["artifact-integrity","npm-package-integrity","bun-package-integrity","npm-installation","npm-print","npm-builtin-auth-transport","npm-terminal-tools-cancel-resume","npm-artifact-terminal","bun-installation","bun-print","bun-builtin-auth-transport","bun-terminal-tools-cancel-resume","bun-artifact-terminal"]' + else + expected='["artifact-integrity","npm-package-integrity","npm-live-openai"]' + fi + if ! jq --exit-status --arg commit "$DIFFEX_SOURCE_COMMIT" --arg mode "$mode" --argjson expected "$expected" \ + '.schemaVersion == 1 and .commit == $commit and .mode == $mode and .outcome == "passed" and (.artifactHashes | type == "object" and length > 0) and ((.scenarios | map(.id) | sort) == ($expected | sort)) and all(.scenarios[]; .commit == $commit and .outcome == "passed")' \ + "$evidence/evidence/results.json" > /dev/null; then + echo "::error::Missing, incomplete or failed $mode installed scenarios" + failed=1 + fi + for result in orchestration.xml evidence/results.xml; do + if [[ ! -s "$evidence/$result" ]]; then + echo "::error::Missing $mode $result" + failed=1 + fi + done + done + if ! jq --slurp --exit-status '.[0].artifactHashes == .[1].artifactHashes' \ + .artifacts/daytona-summary/deterministic/evidence/results.json \ + .artifacts/daytona-summary/live/evidence/results.json > /dev/null; then + echo '::error::Both modes must test the same candidate tarball hashes' + failed=1 + fi + jq -n --arg commit "$DIFFEX_SOURCE_COMMIT" --arg prepare "$PREPARE_RESULT" \ + --arg deterministic "$DETERMINISTIC_RESULT" --arg live "$LIVE_RESULT" --argjson failed "$failed" \ + '{commit: $commit, jobs: {prepare: $prepare, deterministic: $deterministic, live: $live}, passed: ($failed == 0)}' \ + > .artifacts/daytona-summary/summary.json + { + echo '## Daytona installed-artifact behaviors' + echo "Commit: \`$DIFFEX_SOURCE_COMMIT\`" + echo "Prepare: $PREPARE_RESULT; deterministic: $DETERMINISTIC_RESULT; live: $LIVE_RESULT" + echo "Evidence and confirmed cleanup gate: $([[ "$failed" == 0 ]] && echo passed || echo failed)" + } >> "$GITHUB_STEP_SUMMARY" + exit "$failed" + - name: Retain sanitized aggregate evidence even on failure + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: daytona-summary-${{ github.sha }}-${{ github.run_attempt }} + path: .artifacts/daytona-summary/ + include-hidden-files: true + if-no-files-found: error + retention-days: 30 diff --git a/docs/daytona-behaviors-qualification.md b/docs/daytona-behaviors-qualification.md new file mode 100644 index 000000000..ca1a84fe7 --- /dev/null +++ b/docs/daytona-behaviors-qualification.md @@ -0,0 +1,96 @@ +# Daytona behavior qualification — 2026-10-27 + +## Status + +**Real-service CLI qualification passed. GitHub Actions qualification remains pending.** + +The snapshot, installed package scenarios, live OpenAI transport, intentional assertion failure, overlapping CLI runs, active cancellation and runner-loss recovery were exercised against Daytona. This is not evidence that the GitHub workflow or its 30-day artifact retention has executed successfully. + +At qualification time, remote `main` was `c79aa4ccf251507c39cbc132353231cc08ca01fe` and did not contain `daytona-behaviors.yml`. The user's no-push restriction was retained. `DAYTONA_BEHAVIORS_QUALIFIED` remains `false`; automatic spending has not been enabled. + +## Inputs + +| Input | Identity | +| --- | --- | +| Frozen candidate source | `d025b29e554c13c4835a2d592adf47c39d49b752` | +| Package | `@diffexai/diffex@0.4.0` | +| Reviewed tarball SHA-256 | `da75c4042adbc0146b213682cb2a2477a5f1c97c6853b7179557648b70b47b92` | +| Corrected scenario harness | `6efd24323c7d14bb85e2dfe62a774f85c7643a4a` | +| Corrected harness bundle SHA-256 | `d5b57d1e15f1b39e1c0734747b7080a4c590a8dea6e34461816f02415a4df29e` | +| Publication guard | `805454eb6` | +| Snapshot | `diffex-behaviors-v1-e94e1af7cbc7` | +| Snapshot ID | `2a851315-6915-40e8-9b38-0446b7d329f5` | +| Dockerfile SHA-256 | `e94e1af7cbc73ce2b1a9b85e79974f73a39d69f9187f58daeb885bed90fa5f3a` | + +The candidate was frozen, built offline, packaged and verified once. Corrected harness bundles reused those retained candidate bytes rather than silently rebuilding or replacing a failed candidate. Candidate and harness commits are recorded separately; the corrected harness is not represented as part of the older frozen source commit. + +The real sandbox reported Node `22.23.2`, npm `11.16.0`, Bun `1.3.11`, Python `3.11.2`, tmux `3.3a`, ripgrep `13.0.0` and fd `8.6.0`. Snapshot and sandbox resources were checked as 2 vCPU, 4 GiB RAM and 8 GiB disk. Creation requests used the 45-minute wall-clock TTL. + +## Successful installed-package runs + +Both modes used the same retained tarball and corrected harness bundle. + +| Run directory | Result | Evidence | +| --- | --- | --- | +| `deterministic-v4/` | All 13 scenarios passed | npm and Bun installation/inventory checks; help/version/models/resolution; print; built-in OpenAI API-key transport fixture; real read/write/edit/bash; cancellation/recovery; clean exit; persisted-session resume; 80×24, 120×40 and resize baselines; existing artifact login/terminal acceptance | +| `live-v3/` | All 3 scenarios passed | Retained artifact/toolchain integrity, npm installation integrity, real OpenAI print and interactive read/write task | + +The live run used `openai/gpt-4.1-mini`, made **4 provider requests**, produced **98 output tokens** total (maximum **38** in one request), and reported approximately **$0.00205** provider usage. The harness capped every request at 512 output tokens and the smoke at three minutes after installation. It observed 41 streamed text events, successful real `read` and `write` tools, exact input/output file bytes and application exit code 0. + +The reported cost is for the final passing live run, not an account billing total. Earlier failed or superseded harness runs remain retained separately. + +## Failure and lifecycle rehearsals + +- **Intentional assertion failure:** `intentional-failure/` used a separately identified bundle with a deliberately wrong 80×24 baseline. Both package-manager terminal scenarios failed with `Layout differs: startup-80x24.txt`; the runner exited nonzero, retained JSON/JUnit/captures, and confirmed deletion. The approved bundle was not modified. +- **Two overlapping runs:** `overlap-a/` and `overlap-b/` had simultaneous sandbox lifetimes, distinct sandbox IDs and ownership labels, successful complete deterministic results, separate evidence and confirmed deletion. These were overlapping CLI invocations, not GitHub workflow runs. +- **Active cancellation:** `cancel-active/` received SIGTERM after remote execution began and before completion. It remained failed, explicitly reported incomplete final results, and confirmed deletion. Missing results did not become a pass. +- **Runner loss and recovery:** the initial `live-v1/` and `deterministic-v2/` supervisors were killed after an evidence-transport incompatibility was diagnosed. `recover` retrieved completed remote reports without relaunching tests, preserved their failed assertions and confirmed deletion. +- **Independent final audit:** fresh lookups by sandbox ID and name, plus ownership-label queries, confirmed that **all 10 sandboxes created during this qualification were deleted**. The reusable toolchain snapshot is intentionally retained. + +Fake-client tests additionally cover ambiguous creation, unavailable lookups, upload/input failures, ownership mismatch, cleanup failures, corrupt or missing evidence, invalid JUnit and interruption. These tests are not represented as real Daytona service-outage rehearsals. + +## Issues discovered and fixed + +Failures were preserved; assertions were not automatically rerun. + +1. The SDK returns a string-backed enum for snapshot state. Stringifying it produced `SnapshotState.ACTIVE` rather than `active`; readiness now compares the actual enum value correctly. +2. SDK `0.226.1` rejects streaming downloads of empty files. The runner now materializes an empty file locally only after verifying its declared zero size and SHA-256. Nonempty evidence still uses streaming downloads. +3. Polling tolerated changed log bytes but final collection did not previously require completeness. Final evidence now requires complete size/hash verification, matching JSON identity/evidence references, and parsed JUnit with matching scenarios and outcomes. +4. The sandbox's tmux could report a dead pane without a usable pane exit status. A PTY-preserving launcher now independently records the actual application child's return code; zero exit and a dead pane are both required. This does not replace the application's terminal, session or UI implementation. +5. The first live model copied the marker but omitted its trailing newline. The fixture now contains an unambiguous marker with no trailing newline; byte-for-byte output verification remains required. +6. Actions artifacts in this public repository are not confidential. A fail-closed publication guard now scans every evidence file, including launcher/install logs, before publishing a separate sanitized tree. Credential findings fail the producing job; unexpected guard failures block upload entirely. Authentication files and symlinks are excluded. Hidden evidence files are retained. + +## Evidence and secret handling + +Local evidence roots, relative to the repository: + +```text +.artifacts/daytona-qualification/ +.artifacts/daytona-qualification-final-sanitized/ +``` + +Useful records include: + +- `qualification-summary.json` +- `preparation-report.json`, `SHA256SUMS` and `logs/` +- `snapshot-v1/snapshot.json` +- `deterministic-v4/{state.json,orchestration.json,orchestration.xml,evidence/}` +- `live-v3/{state.json,orchestration.json,orchestration.xml,evidence/}` +- `rehearsals.json` and `interruption-rehearsal.json` +- `independent-deletion-audit.json` +- The sanitized root's `publication-safety.json` + +The final publication guard scanned **955 files** and found **zero exact matches** for either real credential. Keys were loaded privately from the owner's external credential file, never placed in command arguments, printed or committed. The Daytona key stayed outside sandboxes; only live execution received the OpenAI key. GitHub stores both as encrypted, main-only environment secrets. + +These ignored local directories are not a 30-day GitHub retention guarantee. No qualification evidence was uploaded through Actions because the workflow was not present on remote `main`. Treat any future Actions evidence as non-confidential sanitized data, not a secret store. + +## Remaining GitHub boundary + +After explicit authorization to push the reviewed commits, follow [the operator procedure](daytona-behaviors.md#qualification-and-enabling-automatic-runs): + +1. Push through the repository's normal review/protection process; do not bypass branch protection. +2. Dispatch the workflow on its exact remote `main` commit and require both modes plus aggregate to pass with 30-day artifacts. +3. Rehearse workflow cancellation and two overlapping workflow runs; confirm per-run evidence retention and sandbox deletion. +4. Review the retained results before setting `DAYTONA_BEHAVIORS_QUALIFIED=true`. + +A successful CLI qualification does not authorize skipping those GitHub checks or enabling the qualification variable early. diff --git a/docs/daytona-behaviors.md b/docs/daytona-behaviors.md new file mode 100644 index 000000000..0503e74fd --- /dev/null +++ b/docs/daytona-behaviors.md @@ -0,0 +1,299 @@ +# Daytona installed-artifact behaviors + +## Status and scope + +The workflow and harness are implementation, not remote qualification evidence. +A maintainer must provision Daytona access, prepare a snapshot, supply an approved live-provider credential and run the qualification below before enabling automatic runs. +Local fixture tests, SDK mocks and workflow validation do not establish that Daytona creation, upload, PTY execution, live inference or deletion works against the real service. +Do not mark the integration qualified until those operations and their failure paths have retained evidence. + +The [2026-10-27 qualification record](daytona-behaviors-qualification.md) documents successful real-service CLI runs and lifecycle rehearsals. GitHub Actions execution, retention and workflow-level cancellation/overlap qualification remain pending; automatic execution is still disabled. + +This is a standalone acceptance gate for the packaged Diffex CLI, not the hosted product's runtime image, a release publisher or a replacement for native artifact acceptance. +It installs the exact reviewed candidate tarballs into isolated consumers outside the repository. +The deterministic mode controls inference while exercising installed behavior; the live mode uses a bounded real OpenAI request path. +A passing deterministic run is not live-provider evidence. +The snapshot is toolchain-only; each run uploads its candidate artifacts and checked harness rather than baking repository source or credentials into a snapshot. +Version 1 covers the Linux installed CLI only; hosted-runtime behavior and subscription/OAuth login qualification are out of scope. +The workflow performs no automatic reverts and changes no release or publication gates. + +## Workflow contract + +`.github/workflows/daytona-behaviors.yml` listens to every push to `main` and to manual dispatch. +There are no path filters or concurrency cancellation: overlapping pushes retain independent run IDs and sandboxes. +Every checkout explicitly uses `github.sha`, including a manually dispatched run's selected commit. +A later movement of `main` must not change the source under test. + +Automatic pushes are gated by the **repository variable** `DAYTONA_BEHAVIORS_QUALIFIED` being exactly `true`. +Leave it unset or `false` until qualification passes; unqualified pushes skip the gated jobs rather than reporting remote success. +Manual dispatch bypasses this variable so qualification and repair remain possible. +The workflow never sets its own qualification variable. + +1. **Prepare, once, with a 35-minute timeout.** Install locked dependencies with lifecycle scripts disabled, hydrate model data once, then run the existing artifact pipeline: + + ```sh + npm run hydrate:model-data + node scripts/release-artifacts.mjs freeze .artifacts/release + npm run build:offline + node scripts/release-artifacts.mjs prepare .artifacts/release + node scripts/release-artifacts.mjs verify .artifacts/release + git diff --exit-code HEAD + ``` + + This preserves the frozen source/model inputs, manifests and exact reviewed npm tarballs. + There is no second candidate build or repack in either behavior job. + Preparation also runs the targeted Node fixture/workflow tests and Python fake-client/report tests without credentials. + The separate harness bundle contains only: + + ```text + scripts/artifact-installation.mjs + scripts/verify-candidate-artifacts.py + scripts/fixtures/artifact-handoff.mjs + scripts/daytona-behaviors/installed.py + scripts/daytona-behaviors/install.mjs + scripts/daytona-behaviors/fixture.mjs + scripts/daytona-behaviors/live-guard.mjs + scripts/daytona-behaviors/baselines/ + ``` + + Bundle paths start at `scripts/`; no package source tree is needed. + The workflow retains a `SHA256SUMS` sidecar and verifies the downloaded bundle before execution. + +2. **Deterministic and live, in separate jobs.** Both depend only on prepare and use the GitHub environment `daytona-behaviors`. + Each downloads the same retained release and bundle, re-verifies them and creates its own sandbox. + Each job has a 35-minute timeout, a 25-minute execution step, a 1,200-second runner deadline and a separate bounded recovery step. + A run ID includes the workflow run ID, attempt and mode, so retry evidence cannot be confused with an earlier attempt. + +3. **Recovery and evidence, including failure.** Once execution starts, the always-run recovery step invokes `recover --output DIR`. + Recovery retrieves available evidence and confirms cleanup; it does not launch or retry behavior execution. + The workflow records execution and recovery outcomes separately in `workflow-job.json`, then always runs the final all-files credential publication guard before any per-mode evidence upload. + The guard scans the entire local output root, including `installed.log`, `launcher.log`, nested `install.txt` files, binary captures and hidden files; it does not rely on the installed suite's earlier filtering. + It reads actual credential bytes only from `DAYTONA_API_KEY` (both modes) and `OPENAI_API_KEY` (live only) in its environment, never from credential files or command-line arguments. + Auth files/directories, symlinks and special files are excluded without opening them. + Exact credential bytes in file contents and path names are redacted before a separate sanitized publication copy is written; the original local recovery output is never uploaded. + `publication-safety.json` records only sanitized paths, credential variable names, counts and hashes of sanitized bytes, not credential values or hashes of original secret-bearing bytes. + A credential match fails the guard and producing job even if the sanitized evidence can be retained for diagnosis. + Only a fully completed scan, final byte/path verification and safety report set `safe_to_upload=true`; unexpected failures or missing scan credentials block upload rather than falling back to raw evidence. + These exact-byte checks are defense in depth, not a general detector for arbitrary credentials or encoded secret transformations: the harness must never write secrets to evidence or logs. + A recovered or sanitized transcript does not turn a failed run into a pass. + +4. **Aggregate, including missing results.** The final job runs even when dependencies fail or skip on an enabled run. + It requires successful prepare, deterministic and live jobs; exact commit/mode/run-ID markers; successful execution and recovery; passing orchestration JSON with confirmed deletion; the complete expected installed scenario inventory; matching candidate tarball hashes in both modes; and nonempty orchestration and installed-suite XML reports. + Missing artifacts, incomplete results, execution failure, credential-guard failure or cleanup failure fail the aggregate. + The aggregate requires a clean publication safety report and checks the producing jobs' outcomes, so a failed sanitizer cannot be hidden by otherwise passing scenario reports. + Evidence-upload failure also leaves the producing job failed. + Aggregate validation does not echo evidence JSON or credential values into job logs or the step summary. + Per-mode and aggregate uploads include hidden sanitized files, preserving `.orchestration-exit` and other digest-manifest members; excluded auth files stay excluded. + +GitHub artifacts retain the release, bundle, per-mode evidence and aggregate evidence for **30 days**. +Download evidence before expiry if a longer-lived qualification record is required. +This repository is public: GitHub Actions artifacts can be downloaded by authenticated users with repository read access and are **not confidential storage**. +The release and bundle artifacts contain no credentials; per-mode and aggregate evidence must contain only the guarded, sanitized publication copies. +They are workflow artifacts, not npm or release assets; that distinction does not make them private. +Never rely on GitHub log masking or artifact access controls to protect keys. + +## Configure access and a snapshot + +A repository administrator and a Daytona account owner must perform this setup. +Do not put API keys in the Dockerfile, bundle, checked files, command-line arguments or repository variables. + +Create the GitHub environment `daytona-behaviors` and configure: + +| Setting | Location | Purpose | +| --- | --- | --- | +| `DAYTONA_API_KEY` | Environment secret | Local orchestration calls to Daytona; never passed as the sandbox's model credential. | +| `OPENAI_API_KEY` | Environment secret | Approved, restricted live-provider key; injected only into live execution and the live publication guard. | +| `DAYTONA_BEHAVIORS_SNAPSHOT` | Environment variable | Prepared toolchain snapshot name passed explicitly as `--snapshot`. | +| `DAYTONA_BEHAVIORS_QUALIFIED` | Repository variable | Set to `true` only after reviewing a successful qualification. | + +The deterministic job and recovery steps do not receive `OPENAI_API_KEY`. +Each publication guard receives only its mode's credential whitelist, scoped to that step; credential values never appear in its arguments, report or status output. +The prepare and aggregate jobs receive neither credential. +Provider credentials must be test-only, revocable and covered by a spending limit; do not use personal login state or production agent configuration. +Review the current model, request/token/time limits and allowed provider destination in the checked live harness before approval. +Its application-level guard is not a claim of an account-wide network firewall. + +Install the pinned orchestration SDK in a local virtual environment: + +```sh +python3 -m venv /tmp/diffex-daytona-venv +/tmp/diffex-daytona-venv/bin/python -m pip install -r scripts/daytona-behaviors/requirements.txt +``` + +`requirements.txt` pins `daytona==0.226.1`. +Supply `DAYTONA_API_KEY` through an approved local secret mechanism, then prepare the snapshot from the checked `scripts/daytona-behaviors/Dockerfile`: + +```sh +/tmp/diffex-daytona-venv/bin/python scripts/daytona-behaviors/daytona_runner.py prepare-snapshot \ + --name diffex-behaviors-toolchain-v1 \ + --output .artifacts/daytona-snapshot +``` + +Inspect the preparation evidence and the resulting snapshot's readiness in Daytona before setting `DAYTONA_BEHAVIORS_SNAPSHOT`. +Record the checked Dockerfile commit, snapshot name/identity, architecture, actual Node/npm/Bun and helper versions, preparation outcome and owner. +An existing production snapshot must not be assumed suitable. +If the toolchain changes, prepare and qualify a new snapshot rather than silently changing an approved baseline. +Snapshot preparation is an explicit operator action, not part of each push. +The versioned definition pins Node **22.23.2**, npm **11.16.0** and Bun **1.3.11**, and installs Python, tmux, ripgrep and fd. +Each sandbox uses **2 vCPU, 4 GiB RAM and 8 GiB disk**; the runner checks snapshot and sandbox resources and verifies installed toolchain versions. +A **45-minute wall-clock TTL** (`ttl_minutes=45`) destroys the sandbox independently of activity or state; stopped-time auto-delete is not a substitute. +See [Daytona wall-clock TTL](https://www.daytona.io/docs/en/sandboxes/#wall-clock-ttl). +The live scenario defaults to `openai/gpt-4.1-mini`, with a shared six-request budget across print/TUI processes, 512 output tokens per request and a three-minute smoke deadline after installation. +Assertions are never automatically rerun. + +Restrict the `daytona-behaviors` environment's deployment branch policy to **branch `main` only**; do not allow other branches or tags to access these secrets. +Qualification dispatches must therefore run from `main` after the reviewed workflow lands there through the normal review process. +The implementation does not push or merge itself. +After qualification, protection rules must permit trusted `main` pushes without required manual approval if fully automatic execution is intended. +Do not broaden the deployment branch policy or remove security controls without administrator review; a workflow waiting for environment approval has not executed remotely. + +## Runner interface and local reproduction + +Use retained verified artifacts rather than rebuilding a failing candidate. +Extract the release artifact and behavior-bundle artifact into the same directory layout used by the workflow, then verify: + +```sh +node scripts/release-artifacts.mjs verify .artifacts/release +sha256sum -c .artifacts/daytona-bundle/SHA256SUMS +``` + +Artifact verification requires a checkout of the matching source commit and its unchanged lockfile and distribution policy. +`DIFFEX_SOURCE_COMMIT` may be set to enforce the expected commit explicitly. + +Run one mode with a fresh output directory and unique run ID: + +```sh +/tmp/diffex-daytona-venv/bin/python scripts/daytona-behaviors/daytona_runner.py run \ + --artifacts .artifacts/release \ + --bundle .artifacts/daytona-bundle/bundle.tar.gz \ + --snapshot diffex-behaviors-toolchain-v1 \ + --output .artifacts/daytona-deterministic \ + --mode deterministic \ + --commit "$(git rev-parse HEAD)" \ + --run-id "manual-$(date +%s)-$$-deterministic" +``` + +`--bundle` accepts a bundle directory or a tar.gz archive. +For a live run use `--mode live`, a separate output directory/run ID and the approved `OPENAI_API_KEY` environment secret. +The Daytona API key belongs only on the orchestration machine; it must not be supplied as an in-sandbox environment variable. + +Expected local output: + +- `state.json`: durable identity written before sandbox creation, used for recovery. +- `orchestration.json` and `orchestration.xml`: orchestration outcomes and cleanup evidence. +- `evidence/results.json` and `evidence/results.xml`: incremental installed-suite results. +- Available per-scenario transcripts, events, terminal captures/baseline comparisons and other diagnostic files produced by the harness. +- `workflow-job.json`: CI-only record of exact source, run identity, execution and recovery outcomes. +- `publication-safety.json`: final guard report in the separate sanitized publication directory, never an authorization to upload the original output. + +Do not delete or hand-edit `state.json` while a sandbox may exist. +For interrupted execution, retained output or cleanup failure: + +```sh +/tmp/diffex-daytona-venv/bin/python scripts/daytona-behaviors/daytona_runner.py recover \ + --output .artifacts/daytona-deterministic +``` + +Recovery must not create a replacement sandbox or re-execute tests. +It must preserve failure evidence and positively confirm deletion; an attempted delete is not enough. +If credentials, runner termination or service failure prevent confirmation, treat cleanup as failed and use the saved sandbox/run identity with the Daytona owner to reconcile resources. +CI's always-run step is best effort under cancellation or runner loss, not a guarantee that GitHub will execute cleanup. +Recover manually and confirm no run-owned sandboxes remain before closing the incident. + +Before sharing any local evidence, run the same publication guard after recovery with the approved credentials available only in the environment: + +```sh +python3 scripts/daytona-behaviors/sanitize_evidence.py \ + --root .artifacts/daytona-deterministic \ + --output .artifacts/daytona-deterministic-sanitized \ + --mode deterministic +``` + +Use `--mode live` and a separate fresh publication destination for live evidence. +A nonzero exit is a security failure, even if a completed sanitized copy and its safety report exist. +If the scan did not complete, do not publish any evidence; never substitute the raw directory. +Retain or share only a completed sanitized copy, review it for unrelated sensitive data and treat it as public-readable. + +## Local executable checks and baseline review + +To exercise an already installed executable without creating a Daytona sandbox: + +```sh +python3 scripts/daytona-behaviors/installed.py \ + --executable /path/to/installed/node_modules/.bin/diffex \ + --output /tmp/diffex-local-behaviors \ + --mode deterministic \ + --commit "$(git rev-parse HEAD)" +``` + +Use a fresh output directory and the required local Node, tmux and search-helper toolchain. +This executable-only mode does not establish reviewed-tarball installation integrity, both package-manager paths or remote qualification. +For an intentional local baseline refresh, append `--update-baselines` to that command. +Only a local reviewer may regenerate baselines, inspect the raw and normalized terminal captures, review the baseline diff and submit intentional changes through normal review. +CI never passes `--update-baselines` and must fail rather than automatically accepting a changed layout. + +## Qualification and enabling automatic runs + +Qualification requires a user with Daytona access and approved provider spending authority. +The implementation must remain **unqualified** until that user supplies and reviews real evidence. + +1. Leave `DAYTONA_BEHAVIORS_QUALIFIED` unset or `false`; configure the main-only environment and prepare the toolchain snapshot. + Wait until the reviewed workflow and harness land on `main`; these instructions do not authorize a push or merge. +2. Dispatch a passing qualification run from `main`: + + ```sh + gh workflow run daytona-behaviors.yml --ref main + ``` + + Record the resolved 40-character commit and workflow run URL; the workflow checks out that event's SHA, not a later branch tip. + Require successful deterministic **and** live results against the same retained candidate hashes, the complete expected behavior inventory and reviewed terminal baselines. + Confirm the live path actually reached the approved provider rather than a fixture. + Verify provider request/token/time limits and that neither evidence nor logs contain credentials or unrelated user state. +3. Require an **intentional product-assertion failure**, not just a missing credential or transport error. + An operator can copy the retained harness into a separate local bundle, deliberately corrupt one expected terminal baseline and run the deterministic remote CLI against the unchanged retained candidate: + + ```sh + bad_bundle="$(mktemp -d /tmp/diffex-daytona-bad-baseline.XXXXXX)" + tar -xzf .artifacts/daytona-bundle/bundle.tar.gz -C "$bad_bundle" + printf 'intentional qualification mismatch\n' > "$bad_bundle/scripts/daytona-behaviors/baselines/startup-80x24.txt" + /tmp/diffex-daytona-venv/bin/python scripts/daytona-behaviors/daytona_runner.py run \ + --artifacts .artifacts/release --bundle "$bad_bundle" \ + --snapshot diffex-behaviors-toolchain-v1 \ + --output .artifacts/daytona-intentional-failure --mode deterministic \ + --commit "$(git rev-parse HEAD)" --run-id "failure-$(date +%s)-$$" + ``` + + Expect a nonzero exit and a recorded baseline assertion failure, retained JSON/XML and captures, and confirmed sandbox deletion. + Run `recover --output .artifacts/daytona-intentional-failure` if needed; it must not relaunch or turn the failed execution into a pass. + Record the deliberately altered bundle identity separately; never replace the original retained bundle, regenerate its approved checksum or commit the bad baseline. + Run the final publication guard on the local CLI output and retain only its completed sanitized copy with the qualification record; this rehearsal needs no new workflow inputs. +4. Require a **cancellation rehearsal**: dispatch another run from `main`, wait until a sandbox and behavior execution exist, then cancel that workflow run. + Retain the run URL, sandbox/run identities and whatever incremental artifacts were uploaded. + Recover available saved output without relaunching; if cancellation prevented artifact upload, use the recorded run labels and the Daytona owner to reconcile resources. + Confirm deletion of every sandbox created by that run; missing evidence or an attempted delete alone does not satisfy qualification. +5. Require **two overlapping workflow runs**: dispatch twice from `main` without waiting for the first to finish, and verify that both actually execute concurrently rather than queue behind environment approvals. + For each run separately, verify the resolved source commit, unique run/attempt/mode identities, release and bundle artifacts, deterministic/live results, aggregate evidence and confirmed deletion of every owned sandbox. + Confirm that neither run cancels the other, overwrites the other's artifacts, mixes evidence or deletes the other's active sandbox. +6. Also exercise creation/upload/execution/download failures, missing/incomplete result evidence and cleanup failure through operator-controlled remote rehearsals. + Retain proof that failures stay failed, recovery does not relaunch and deletion is positively confirmed. + Local mock tests are supplementary, not substitutes for these remote rehearsals. +7. Review all passing, intentional-failure, cancellation and overlapping-run evidence, snapshot/toolchain identities, sandbox IDs and cleanup confirmations. + Keep a qualification record with owner, date, commits, workflow run URLs, snapshot, artifact/bundle hashes, provider/model, bounded cost and failure-path evidence. + Ensure no orphaned run-owned sandboxes remain. +8. Only after those checks pass, an administrator enables subsequent pushes: + + ```sh + gh variable set DAYTONA_BEHAVIORS_QUALIFIED --body true + ``` + + Observe the next `main` push and require both modes plus aggregate to execute automatically. + Setting the variable does not retroactively execute skipped pushes. + +To suspend automatic spending without removing dispatch or evidence: + +```sh +gh variable set DAYTONA_BEHAVIORS_QUALIFIED --body false +``` + +Requalify after material snapshot, SDK, credential policy or harness changes. +Neither a static workflow test nor setting the variable proves remote qualification. diff --git a/scripts/artifact-acceptance.mjs b/scripts/artifact-acceptance.mjs index f8f60f200..ac4188808 100644 --- a/scripts/artifact-acceptance.mjs +++ b/scripts/artifact-acceptance.mjs @@ -1,6 +1,7 @@ +import { assertContained, installConsumer, isolatedEnvironment, verifyTarballs } from "./artifact-installation.mjs"; +export { assertContained, installConsumer, isolatedEnvironment, verifyTarballs } from "./artifact-installation.mjs"; import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; -import { createHash } from "node:crypto"; import { cpSync, existsSync, @@ -13,20 +14,14 @@ import { writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; -import { dirname, isAbsolute, join, relative, resolve } from "node:path"; +import { dirname, join, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { cliDistribution } from "./cli-distribution.mjs"; import { getPublicWorkspacePackages } from "./release-packages.mjs"; const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); -const digest = (bytes) => createHash("sha256").update(bytes).digest("hex"); const readJson = (file) => JSON.parse(readFileSync(file, "utf8")); -export function assertContained(file, directory) { - const local = relative(realpathSync(directory), realpathSync(file)); - assert(local === "" || (!local.startsWith("..") && !isAbsolute(local)), `Path escapes isolated consumer: ${file}`); -} - export function packageClosure(name, packages) { const result = []; const visiting = new Set(); @@ -52,51 +47,6 @@ export function packageClosure(name, packages) { return result; } -export function verifyTarballs(directory, records, packages) { - assert.equal(records.length, packages.length, "Incomplete tarball inventory"); - assert.equal(new Set(records.map((pkg) => pkg.name)).size, packages.length, "Duplicate tarball identity"); - for (const pkg of packages) { - const record = records.find((entry) => entry.name === pkg.name); - assert(record && record.version === pkg.version, `Missing tarball: ${pkg.name}`); - assert.equal(record.filename, `${pkg.name.replace("@", "").replace("/", "-")}-${pkg.version}.tgz`); - assert.equal( - digest(readFileSync(join(directory, record.filename))), - record.sha256, - `Tarball checksum mismatch: ${pkg.name}`, - ); - } -} - -export function isolatedEnvironment(directory) { - const env = {}; - for (const key of ["PATH", "SystemRoot", "WINDIR", "COMSPEC", "PATHEXT", "LANG"]) { - if (process.env[key]) env[key] = process.env[key]; - } - for (const dir of ["home", "config", "cache", "prefix", "project", "agent", "temp"]) - mkdirSync(join(directory, dir), { recursive: true }); - return { - ...env, - HOME: join(directory, "home"), - USERPROFILE: join(directory, "home"), - XDG_CONFIG_HOME: join(directory, "config"), - XDG_CACHE_HOME: join(directory, "cache"), - npm_config_cache: join(directory, "cache", "npm"), - npm_config_prefix: join(directory, "prefix"), - npm_config_userconfig: join(directory, "empty.npmrc"), - npm_config_globalconfig: join(directory, "empty-global.npmrc"), - npm_config_registry: "https://registry.npmjs.org", - npm_config_update_notifier: "false", - BUN_INSTALL_CACHE_DIR: join(directory, "cache", "bun"), - TMPDIR: join(directory, "temp"), - TMP: join(directory, "temp"), - TEMP: join(directory, "temp"), - DIFFEX_CODING_AGENT_DIR: join(directory, "agent"), - DIFFEX_NO_LOCAL_LLM: "1", - DIFFEX_OFFLINE: "1", - TERM: "xterm-256color", - }; -} - function walk(directory) { return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { const path = join(directory, entry.name); @@ -154,67 +104,6 @@ export const operations = { "@diffexai/diffex-coding-agent-server": `const reporter = api.createDurableUsageReporter({ endpoint: "http://127.0.0.1:1/usage", sharedSecret: "local-artifact-fixture-only-32-bytes", outboxDirectory: path.join(process.env.DIFFEX_CODING_AGENT_DIR, "outbox"), fetch: () => { throw new Error("Unexpected usage"); } }); await reporter.close();`, }; -export function installConsumer( - directory, - manager, - selected, - records, - tarballDirectory, - cacheDirectory = join(directory, "cache"), -) { - mkdirSync(directory, { recursive: true }); - const env = isolatedEnvironment(directory); - mkdirSync(cacheDirectory, { recursive: true }); - env.npm_config_cache = join(cacheDirectory, "npm"); - env.BUN_INSTALL_CACHE_DIR = join(cacheDirectory, "bun"); - writeFileSync(join(directory, ".npmrc"), "@diffexai:registry=http://127.0.0.1:9/\n"); - const dependencies = Object.fromEntries( - selected.map((pkg) => [ - pkg.name, - `file:${join(tarballDirectory, records.find((record) => record.name === pkg.name).filename)}`, - ]), - ); - writeFileSync( - join(directory, "package.json"), - JSON.stringify({ private: true, type: "module", dependencies, overrides: dependencies }), - ); - execFileSync( - manager, - manager === "npm" - ? ["install", "--omit=dev", "--ignore-scripts", "--no-audit", "--no-fund"] - : ["install", "--production", "--ignore-scripts", "--network-concurrency=8"], - { cwd: directory, env, stdio: "inherit", timeout: 600_000 }, - ); - for (const pkg of selected) { - const installed = join(directory, "node_modules", pkg.name); - assertContained(installed, directory); - const info = readJson(join(installed, "package.json")); - assert.equal(info.name, pkg.name); - assert.equal(info.version, pkg.version); - const record = records.find((entry) => entry.name === pkg.name); - const inventory = JSON.parse( - execFileSync("tar", ["-xOf", join(tarballDirectory, record.filename), "package/distribution-files.json"], { - encoding: "utf8", - maxBuffer: 10 * 1024 * 1024, - }), - ); - assert.deepEqual( - readJson(join(installed, "distribution-files.json")), - inventory, - "Installed inventory differs from the checked tarball", - ); - for (const file of inventory.files) { - assertContained(join(installed, file.path), installed); - assert.equal( - digest(readFileSync(join(installed, file.path))), - file.sha256, - `Installed bytes differ: ${pkg.name}/${file.path}`, - ); - } - } - return env; -} - export function acceptArtifacts(artifactDirectory, outputDirectory, { managers = ["npm", "bun"] } = {}) { assert( managers.length && managers.every((manager) => ["npm", "bun"].includes(manager)), diff --git a/scripts/artifact-installation.mjs b/scripts/artifact-installation.mjs new file mode 100644 index 000000000..ed266bd07 --- /dev/null +++ b/scripts/artifact-installation.mjs @@ -0,0 +1,153 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { lstatSync, mkdirSync, readFileSync, realpathSync, writeFileSync } from "node:fs"; +import { homedir, tmpdir } from "node:os"; +import { basename, isAbsolute, join, relative, resolve, sep } from "node:path"; +const digest = (bytes) => createHash("sha256").update(bytes).digest("hex"); +const readJson = (file) => JSON.parse(readFileSync(file, "utf8")); + +export function resolveArtifactDirectory(value) { + assert(typeof value === "string" && value.length > 0 && !value.includes("\0"), "Invalid artifact directory"); + const directory = resolve(value); + for (const base of [process.cwd(), tmpdir(), homedir(), "/tmp"]) { + const root = resolve(base); + if (!directory.startsWith(root + sep)) continue; + let current = root; + for (const component of relative(root, directory).split(sep)) { + current = join(current, component); + const entry = lstatSync(current, { throwIfNoEntry: false }); + assert(!entry?.isSymbolicLink(), "Artifact directory must not traverse symlinks"); + assert(!entry || entry.isDirectory(), "Artifact directory contains a non-directory component"); + } + return directory; + } + throw new Error("Artifact directory must be inside the workspace, home, or temporary directory"); +} + +export function assertContained(file, directory) { + const local = relative(realpathSync(directory), realpathSync(file)); + assert(local === "" || (!local.startsWith("..") && !isAbsolute(local)), `Path escapes isolated consumer: ${file}`); +} + +export function verifyTarballs(directory, records, packages) { + directory = resolveArtifactDirectory(directory); + assert.equal(records.length, packages.length, "Incomplete tarball inventory"); + assert.equal(new Set(records.map((pkg) => pkg.name)).size, packages.length, "Duplicate tarball identity"); + for (const pkg of packages) { + const record = records.find((entry) => entry.name === pkg.name); + assert(record && record.version === pkg.version, `Missing tarball: ${pkg.name}`); + assert(typeof record.filename === "string" && basename(record.filename) === record.filename, "Invalid tarball filename"); + assert.equal(record.filename, `${pkg.name.replace("@", "").replace("/", "-")}-${pkg.version}.tgz`); + assert.equal( + digest(readFileSync(join(directory, record.filename))), + record.sha256, + `Tarball checksum mismatch: ${pkg.name}`, + ); + } +} + +export function isolatedEnvironment(directory) { + directory = resolveArtifactDirectory(directory); + // Read literal allowlisted names: dynamic environment lookups can accidentally + // admit credentials when this helper is reused by installation/lifecycle code. + const env = Object.fromEntries(Object.entries({ + PATH: process.env.PATH, + SystemRoot: process.env.SystemRoot, + WINDIR: process.env.WINDIR, + COMSPEC: process.env.COMSPEC, + PATHEXT: process.env.PATHEXT, + LANG: process.env.LANG, + }).filter(([, value]) => typeof value === "string")); + for (const dir of ["home", "config", "cache", "prefix", "project", "agent", "temp"]) + mkdirSync(resolveArtifactDirectory(join(directory, dir)), { recursive: true }); + return { + ...env, + HOME: join(directory, "home"), + USERPROFILE: join(directory, "home"), + XDG_CONFIG_HOME: join(directory, "config"), + XDG_CACHE_HOME: join(directory, "cache"), + npm_config_cache: join(directory, "cache", "npm"), + npm_config_prefix: join(directory, "prefix"), + npm_config_userconfig: join(directory, "empty.npmrc"), + npm_config_globalconfig: join(directory, "empty-global.npmrc"), + npm_config_registry: "https://registry.npmjs.org", + npm_config_update_notifier: "false", + BUN_INSTALL_CACHE_DIR: join(directory, "cache", "bun"), + TMPDIR: join(directory, "temp"), + TMP: join(directory, "temp"), + TEMP: join(directory, "temp"), + DIFFEX_CODING_AGENT_DIR: join(directory, "agent"), + DIFFEX_NO_LOCAL_LLM: "1", + DIFFEX_OFFLINE: "1", + TERM: "xterm-256color", + }; +} + +export function installConsumer( + directory, + manager, + selected, + records, + tarballDirectory, + cacheDirectory = join(directory, "cache"), +) { + directory = resolveArtifactDirectory(directory); + cacheDirectory = resolveArtifactDirectory(cacheDirectory); + tarballDirectory = resolveArtifactDirectory(tarballDirectory); + assert(manager === "npm" || manager === "bun", "Unsupported package manager"); + const command = manager === "npm" ? "npm" : "bun"; + assert(selected.every((pkg) => typeof pkg.name === "string" && /^(?:@[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*$/.test(pkg.name)), "Invalid package name"); + mkdirSync(directory, { recursive: true }); + const env = isolatedEnvironment(directory); + mkdirSync(cacheDirectory, { recursive: true }); + env.npm_config_cache = join(cacheDirectory, "npm"); + env.BUN_INSTALL_CACHE_DIR = join(cacheDirectory, "bun"); + writeFileSync(join(directory, ".npmrc"), "@diffexai:registry=http://127.0.0.1:9/\n"); + const dependencies = Object.fromEntries( + selected.map((pkg) => [ + pkg.name, + `file:${join(tarballDirectory, records.find((record) => record.name === pkg.name).filename)}`, + ]), + ); + writeFileSync( + join(directory, "package.json"), + JSON.stringify({ private: true, type: "module", dependencies, overrides: dependencies }), + ); + execFileSync( + command, + manager === "npm" + ? ["install", "--omit=dev", "--ignore-scripts", "--no-audit", "--no-fund"] + : ["install", "--production", "--ignore-scripts", "--network-concurrency=8"], + { cwd: directory, env, stdio: "inherit", timeout: 600_000 }, + ); + for (const pkg of selected) { + const installed = join(directory, "node_modules", pkg.name); + assertContained(installed, directory); + const info = readJson(join(installed, "package.json")); + assert.equal(info.name, pkg.name); + assert.equal(info.version, pkg.version); + const record = records.find((entry) => entry.name === pkg.name); + const inventory = JSON.parse( + execFileSync("tar", ["-xOf", join(tarballDirectory, record.filename), "package/distribution-files.json"], { + encoding: "utf8", + maxBuffer: 10 * 1024 * 1024, + }), + ); + assert.deepEqual( + readJson(join(installed, "distribution-files.json")), + inventory, + "Installed inventory differs from the checked tarball", + ); + for (const file of inventory.files) { + assert(typeof file.path === "string" && !isAbsolute(file.path) && !file.path.split(/[\\/]/).includes(".."), "Invalid inventory path"); + assertContained(join(installed, file.path), installed); + assert.equal( + digest(readFileSync(join(installed, file.path))), + file.sha256, + `Installed bytes differ: ${pkg.name}/${file.path}`, + ); + } + } + return env; +} diff --git a/scripts/artifact-installation.test.mjs b/scripts/artifact-installation.test.mjs new file mode 100644 index 000000000..5f51d416b --- /dev/null +++ b/scripts/artifact-installation.test.mjs @@ -0,0 +1,58 @@ +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import test from "node:test"; +import { installConsumer, isolatedEnvironment, resolveArtifactDirectory, verifyTarballs } from "./artifact-installation.mjs"; + +test("artifact directories stay within trusted roots and reject symlink components", () => { + const root = mkdtempSync(join(tmpdir(), "diffex-artifact-paths-")); + try { + assert.equal(resolveArtifactDirectory(join(root, "new", "consumer")), join(root, "new", "consumer")); + assert.throws(() => resolveArtifactDirectory("/etc/diffex-artifacts"), /workspace, home, or temporary/); + assert.throws(() => resolveArtifactDirectory(""), /Invalid artifact/); + assert.throws(() => resolveArtifactDirectory("bad\0path"), /Invalid artifact/); + const target = join(root, "target"); + mkdirSync(target); + symlinkSync(target, join(root, "linked")); + assert.throws(() => resolveArtifactDirectory(join(root, "linked", "consumer")), /symlinks/); + symlinkSync(join(root, "missing"), join(root, "dangling")); + assert.throws(() => resolveArtifactDirectory(join(root, "dangling", "consumer")), /symlinks/); + writeFileSync(join(root, "file"), "not a directory"); + assert.throws(() => resolveArtifactDirectory(join(root, "file", "consumer")), /non-directory/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("isolated installation environment copies literal non-secret allowlisted fields only", () => { + const root = mkdtempSync(join(tmpdir(), "diffex-artifact-env-")); + const original = { ...process.env }; + try { + process.env.OPENAI_API_KEY = "synthetic-private-openai-value"; + process.env.DAYTONA_API_KEY = "synthetic-private-daytona-value"; + process.env.NODE_OPTIONS = "synthetic-private-node-options"; + const environment = isolatedEnvironment(root); + assert.equal(environment.PATH, original.PATH); + assert.equal(environment.OPENAI_API_KEY, undefined); + assert.equal(environment.DAYTONA_API_KEY, undefined); + assert.equal(environment.NODE_OPTIONS, undefined); + assert.equal(environment.HOME, join(root, "home")); + } finally { + process.env = original; + rmSync(root, { recursive: true, force: true }); + } +}); + +test("package manager selection is a fixed allowlist and invalid package paths fail before execution", () => { + const root = mkdtempSync(join(tmpdir(), "diffex-artifact-install-")); + try { + assert.throws(() => installConsumer(join(root, "consumer"), "node --eval malicious", [], [], root), /Unsupported package manager/); + assert.throws(() => installConsumer(join(root, "consumer"), "npm", [{ name: "../../outside" }], [], root), /Invalid package name/); + const pkg = { name: "@diffexai/example", version: "1.0.0" }; + assert.throws(() => verifyTarballs(root, [{ ...pkg, filename: "../outside.tgz" }], [pkg]), /Invalid tarball filename/); + assert.equal(resolveArtifactDirectory(resolve(root, "consumer")), join(root, "consumer")); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/scripts/daytona-behaviors-workflow.test.mjs b/scripts/daytona-behaviors-workflow.test.mjs new file mode 100644 index 000000000..9283ccf3a --- /dev/null +++ b/scripts/daytona-behaviors-workflow.test.mjs @@ -0,0 +1,239 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { parse } from "yaml"; + +const workflow = parse(readFileSync(new URL("../.github/workflows/daytona-behaviors.yml", import.meta.url), "utf8")); +const gate = "github.event_name == 'workflow_dispatch' || vars.DAYTONA_BEHAVIORS_QUALIFIED == 'true'"; + +test("Daytona triggers every main push, allows qualification dispatch and never cancels overlapping runs", () => { + assert.deepEqual(workflow.on.push, { branches: ["main"] }); + assert.ok(Object.hasOwn(workflow.on, "workflow_dispatch")); + assert.equal(workflow.concurrency, undefined); + assert.equal(workflow.jobs.prepare.if, "${{ " + gate + " }}"); + assert.match(workflow.jobs["daytona-behaviors"].if, /always\(\)/); + assert.ok(workflow.jobs["daytona-behaviors"].if.includes(gate)); + assert.equal(workflow.env.DIFFEX_SOURCE_COMMIT, "${{ github.sha }}"); + for (const job of Object.values(workflow.jobs)) { + assert.equal(job.concurrency, undefined); + for (const step of job.steps) { + if (step.uses?.startsWith("actions/checkout@")) { + assert.equal(step.with.ref, "${{ github.sha }}"); + assert.equal(step.with["persist-credentials"], false); + } + if (step.uses) assert.match(step.uses, /@[a-f0-9]{40}$/); + if (step.uses?.startsWith("actions/upload-artifact@")) { + assert.equal(step.with["retention-days"], 30); + assert.equal(step.with["if-no-files-found"], "error"); + } + if (step.run) { + const parsed = spawnSync("bash", ["-n"], { input: step.run, encoding: "utf8" }); + assert.equal(parsed.status, 0, `${step.name}: ${parsed.stderr}`); + } + } + } +}); + +test("one 35-minute preparation freezes, builds offline, prepares and verifies the reused artifact bytes", () => { + const prepare = workflow.jobs.prepare; + assert.equal(prepare["timeout-minutes"], 35); + const script = prepare.steps.map((step) => step.run ?? "").join("\n"); + const commands = [ + "npm run hydrate:model-data", + "node scripts/release-artifacts.mjs freeze .artifacts/release", + "npm run build:offline", + "node scripts/release-artifacts.mjs prepare .artifacts/release", + "node scripts/release-artifacts.mjs verify .artifacts/release", + ]; + let previous = -1; + for (const command of commands) { + const position = script.indexOf(command); + assert.ok(position > previous, `Missing or reordered command: ${command}`); + previous = position; + } + assert.ok(script.includes("npm ci --ignore-scripts")); + assert.ok(script.includes("sha256sum .artifacts/daytona-bundle/bundle.tar.gz")); + assert.match(script, /-m unittest discover -s scripts\/daytona-behaviors/); + assert.match(script, /-m unittest discover -s scripts -p 'test_verify_candidate_artifacts\.py'/); + assert.match(script, /node --test scripts\/artifact-installation\.test\.mjs/); + assert.match(script, /node --test .*daytona-behaviors\/fixture\.test\.mjs/); + for (const step of prepare.steps) { + assert.equal(step.env?.DAYTONA_API_KEY, undefined); + assert.equal(step.env?.OPENAI_API_KEY, undefined); + } + for (const mode of ["deterministic", "live"]) { + const job = workflow.jobs[mode]; + assert.equal(job.needs, "prepare"); + assert.equal(job.environment, "daytona-behaviors"); + const commands = job.steps.map((step) => step.run ?? "").join("\n"); + assert.doesNotMatch(commands, /npm run (build|hydrate)|release-artifacts\.mjs (prepare|freeze)/); + assert.match(commands, /release-artifacts\.mjs verify/); + assert.match(commands, /sha256sum -c/); + assert.ok(job.steps.some((step) => step.with?.name === "daytona-release-${{ github.sha }}-${{ github.run_attempt }}")); + assert.ok(job.env.DAYTONA_BEHAVIOR_RUN_ID.endsWith(`-${mode}`)); + const run = job.steps.find((step) => step.id === "run"); + const recover = job.steps.find((step) => step.id === "recover"); + assert.ok(run.run.includes(`--mode ${mode}`)); + assert.ok(run.run.includes('--snapshot "$DAYTONA_SNAPSHOT"')); + assert.ok(recover.if.includes("always()")); + assert.ok(recover.if.includes("steps.run.outcome != 'skipped'")); + assert.match(recover.run, /daytona_runner\.py recover/); + assert.doesNotMatch(recover.run, /--mode|--artifacts/); + assert.equal(recover.env.OPENAI_API_KEY, undefined); + assert.equal(run.env.OPENAI_API_KEY !== undefined, mode === "live"); + assert.equal(job.steps.at(-1).if, "${{ always() && steps.sanitize.outputs.safe_to_upload == 'true' }}"); + } + assert.deepEqual(workflow.jobs["daytona-behaviors"].needs, ["prepare", "deterministic", "live"]); +}); + +test("only a completed all-files credential guard permits sanitized evidence publication", () => { + assert.equal(workflow.env.DAYTONA_API_KEY, undefined); + assert.equal(workflow.env.OPENAI_API_KEY, undefined); + for (const mode of ["deterministic", "live"]) { + const job = workflow.jobs[mode]; + const guard = job.steps.find((step) => step.id === "sanitize"); + const guardIndex = job.steps.indexOf(guard); + const recordIndex = job.steps.findIndex((step) => step.name === "Record execution and cleanup outcomes"); + assert.equal(guardIndex, recordIndex + 1); + assert.equal(guard.if, "always()"); + assert.equal(guard["continue-on-error"], undefined); + assert.deepEqual(guard.env, mode === "live" + ? { DAYTONA_API_KEY: "${{ secrets.DAYTONA_API_KEY }}", OPENAI_API_KEY: "${{ secrets.OPENAI_API_KEY }}" } + : { DAYTONA_API_KEY: "${{ secrets.DAYTONA_API_KEY }}" }); + assert.match(guard.run, /python3 scripts\/daytona-behaviors\/sanitize_evidence\.py/); + assert.ok(guard.run.includes(`--root .artifacts/daytona-${mode} --output .artifacts/daytona-${mode}-sanitized`)); + assert.ok(guard.run.includes(`--mode ${mode}`)); + assert.doesNotMatch(guard.run, /\$.*(?:DAYTONA_API_KEY|OPENAI_API_KEY)|continue-on-error|\|\| true/); + assert.equal(job.env.DAYTONA_API_KEY, undefined); + assert.equal(job.env.OPENAI_API_KEY, undefined); + const uploads = job.steps.filter((step) => step.uses?.startsWith("actions/upload-artifact@")); + assert.equal(uploads.length, 1); + const upload = uploads[0]; + assert.equal(job.steps.indexOf(upload), guardIndex + 1); + assert.equal(upload.if, "${{ always() && steps.sanitize.outputs.safe_to_upload == 'true' }}"); + assert.equal(upload.with.path, `.artifacts/daytona-${mode}-sanitized/`); + assert.equal(upload.with["include-hidden-files"], true); + } + const aggregate = workflow.jobs["daytona-behaviors"]; + assert.equal(aggregate.env.DETERMINISTIC_RESULT, "${{ needs.deterministic.result }}"); + assert.equal(aggregate.env.LIVE_RESULT, "${{ needs.live.result }}"); + assert.equal(aggregate.env.DAYTONA_API_KEY, undefined); + assert.equal(aggregate.env.OPENAI_API_KEY, undefined); + assert.equal(aggregate.steps.at(-1).with["include-hidden-files"], true); + for (const job of Object.values(workflow.jobs)) { + for (const step of job.steps) assert.doesNotMatch(step.name, /private/i); + } +}); + +for (const scenario of ["pass", "missing-artifact", "missing-result", "invalid-result", "missing-xml", "failed-job", "sanitizer-failed", "missing-safety-report", "unsafe-report", "credential-found", "cleanup-failed", "wrong-commit", "wrong-run", "orchestration-failed", "cleanup-unconfirmed", "partial-scenarios", "wrong-result-commit", "different-tarballs"]) { + test(`aggregate fails closed: ${scenario}`, () => { + const root = mkdtempSync(join(tmpdir(), "daytona workflow ")); + try { + const commit = "a".repeat(40); + const fixtures = join(root, "fixtures"); + for (const mode of ["deterministic", "live"]) { + const directory = join(fixtures, mode); + mkdirSync(join(directory, "evidence"), { recursive: true }); + writeFileSync(join(directory, "publication-safety.json"), JSON.stringify({ + schemaVersion: 1, mode, safe_to_upload: true, outcome: "clean", credential_matches: 0, + })); + writeFileSync(join(directory, ".orchestration-exit"), "0\n"); + writeFileSync(join(directory, "workflow-job.json"), JSON.stringify({ + commit, mode, runId: `123-2-${mode}`, execution: "success", recovery: "success", + })); + writeFileSync(join(directory, "orchestration.json"), JSON.stringify({ + commit, mode, run_id: `123-2-${mode}`, outcome: "passed", execution: "passed", + exit_code: 0, installed_outcome: "passed", reports_complete: true, cleanup: "confirmed", + })); + const identities = mode === "live" + ? ["artifact-integrity", "npm-package-integrity", "npm-live-openai"] + : ["artifact-integrity", "npm-package-integrity", "bun-package-integrity", ...["npm", "bun"].flatMap((manager) => + ["installation", "print", "builtin-auth-transport", "terminal-tools-cancel-resume", "artifact-terminal"].map((name) => `${manager}-${name}`))]; + writeFileSync(join(directory, "evidence/results.json"), JSON.stringify({ + schemaVersion: 1, commit, mode, outcome: "passed", artifactHashes: { "diffex.tgz": "c".repeat(64) }, + scenarios: identities.map((id) => ({ id, commit, outcome: "passed" })), + })); + for (const file of ["orchestration.xml", "evidence/results.xml"]) writeFileSync(join(directory, file), ""); + } + const live = join(fixtures, "live"); + if (scenario === "missing-artifact") rmSync(live, { recursive: true }); + if (scenario === "missing-safety-report") rmSync(join(live, "publication-safety.json")); + if (["unsafe-report", "credential-found"].includes(scenario)) { + const report = JSON.parse(readFileSync(join(live, "publication-safety.json"), "utf8")); + if (scenario === "unsafe-report") report.safe_to_upload = false; + if (scenario === "credential-found") { + report.outcome = "credential-found"; + report.credential_matches = 1; + } + writeFileSync(join(live, "publication-safety.json"), JSON.stringify(report)); + } + if (scenario === "missing-result") rmSync(join(live, "evidence/results.json")); + if (scenario === "invalid-result") writeFileSync(join(live, "evidence/results.json"), "[]"); + if (scenario === "missing-xml") rmSync(join(live, "orchestration.xml")); + if (["orchestration-failed", "cleanup-unconfirmed"].includes(scenario)) { + const report = JSON.parse(readFileSync(join(live, "orchestration.json"), "utf8")); + if (scenario === "orchestration-failed") report.outcome = "failed"; + if (scenario === "cleanup-unconfirmed") report.cleanup = "unconfirmed"; + writeFileSync(join(live, "orchestration.json"), JSON.stringify(report)); + } + if (["partial-scenarios", "wrong-result-commit", "different-tarballs"].includes(scenario)) { + const report = JSON.parse(readFileSync(join(live, "evidence/results.json"), "utf8")); + if (scenario === "partial-scenarios") report.scenarios.pop(); + if (scenario === "wrong-result-commit") report.commit = "b".repeat(40); + if (scenario === "different-tarballs") report.artifactHashes["diffex.tgz"] = "d".repeat(64); + writeFileSync(join(live, "evidence/results.json"), JSON.stringify(report)); + } + if (["cleanup-failed", "wrong-commit", "wrong-run"].includes(scenario)) { + const marker = JSON.parse(readFileSync(join(live, "workflow-job.json"), "utf8")); + if (scenario === "cleanup-failed") marker.recovery = "failure"; + if (scenario === "wrong-commit") marker.commit = "b".repeat(40); + if (scenario === "wrong-run") marker.runId = "123-1-live"; + writeFileSync(join(live, "workflow-job.json"), JSON.stringify(marker)); + } + mkdirSync(join(root, "bin")); + writeFileSync(join(root, "bin/gh"), `#!/usr/bin/env node +const { cpSync, existsSync } = require("node:fs"); +const { join } = require("node:path"); +const args = process.argv.slice(2); +const name = args[args.indexOf("--name") + 1]; +const mode = name.includes("-deterministic-") ? "deterministic" : "live"; +const source = join(process.env.DAYTONA_FIXTURES, mode); +if (!existsSync(source)) process.exit(1); +cpSync(source, args[args.indexOf("--dir") + 1], { recursive: true }); +`, { mode: 0o755 }); + const aggregate = workflow.jobs["daytona-behaviors"].steps[0].run; + const result = spawnSync("bash", ["-e", "-o", "pipefail", "-c", aggregate], { + cwd: root, + encoding: "utf8", + env: { + ...process.env, + PATH: `${join(root, "bin")}:${process.env.PATH}`, + DAYTONA_FIXTURES: fixtures, + DIFFEX_SOURCE_COMMIT: commit, + GITHUB_RUN_ID: "123", + GITHUB_RUN_ATTEMPT: "2", + GITHUB_REPOSITORY: "diffexai/diffex", + GITHUB_STEP_SUMMARY: join(root, "summary.md"), + PREPARE_RESULT: "success", + DETERMINISTIC_RESULT: "success", + LIVE_RESULT: ["failed-job", "sanitizer-failed"].includes(scenario) ? "failure" : "success", + }, + }); + assert.equal(result.status, scenario === "pass" ? 0 : 1, `${result.stdout}\n${result.stderr}`); + const summary = JSON.parse(readFileSync(join(root, ".artifacts/daytona-summary/summary.json"), "utf8")); + assert.equal(summary.passed, scenario === "pass"); + // No evidence JSON (including provider response fields) is echoed into job logs. + assert.doesNotMatch(result.stdout, /artifactHashes|diffex\.tgz|credential_matches|schemaVersion/); + // Downloaded evidence, including hidden digest members, survives failed cases. + if (scenario !== "missing-artifact") { + assert.ok(readFileSync(join(root, ".artifacts/daytona-summary/live/workflow-job.json"))); + assert.equal(readFileSync(join(root, ".artifacts/daytona-summary/live/.orchestration-exit"), "utf8"), "0\n"); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +} diff --git a/scripts/daytona-behaviors/Dockerfile b/scripts/daytona-behaviors/Dockerfile new file mode 100644 index 000000000..30db33690 --- /dev/null +++ b/scripts/daytona-behaviors/Dockerfile @@ -0,0 +1,22 @@ +# Toolchain only: no repository COPY, release artifacts, credentials, or agent config. +FROM node:22.23.2-bookworm-slim + +ARG BUN_VERSION=1.3.11 +ARG NPM_VERSION=11.16.0 +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl unzip python3 tmux ripgrep fd-find git \ + && rm -rf /var/lib/apt/lists/* \ + && ln -s /usr/bin/fdfind /usr/local/bin/fd \ + && npm install --global --ignore-scripts npm@${NPM_VERSION} \ + && test "$(node --version)" = "v22.23.2" \ + && test "$(npm --version)" = "${NPM_VERSION}" +RUN test "$(dpkg --print-architecture)" = amd64 \ + && curl --fail --silent --show-error --location \ + "https://github.com/oven-sh/bun/releases/download/bun-v${BUN_VERSION}/bun-linux-x64.zip" -o /tmp/bun.zip \ + && unzip /tmp/bun.zip -d /tmp/bun \ + && install /tmp/bun/bun-linux-x64/bun /usr/local/bin/bun \ + && rm -rf /tmp/bun /tmp/bun.zip \ + && test "$(bun --version)" = "${BUN_VERSION}" \ + && python3 --version && tmux -V && rg --version && fd --version +WORKDIR /tmp diff --git a/scripts/daytona-behaviors/baselines/startup-120x40.txt b/scripts/daytona-behaviors/baselines/startup-120x40.txt new file mode 100644 index 000000000..13db84d0a --- /dev/null +++ b/scripts/daytona-behaviors/baselines/startup-120x40.txt @@ -0,0 +1,40 @@ + + /----\_/----\ + | \|/ | + \---\ /---/ Diffex v + X + /---/ \---\ + | /|\ | + \____/^\____/ +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +› Ask Diffex to do something for you. +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +fixture · thinking off · context 0.0% / 131k · harness default · ctrl+p model · /hotkeys + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/scripts/daytona-behaviors/baselines/startup-80x24.txt b/scripts/daytona-behaviors/baselines/startup-80x24.txt new file mode 100644 index 000000000..157153cd5 --- /dev/null +++ b/scripts/daytona-behaviors/baselines/startup-80x24.txt @@ -0,0 +1,24 @@ + + /----\_/----\ + | \|/ | + \---\ /---/ Diffex v + X + /---/ \---\ + | /|\ | + \____/^\____/ +──────────────────────────────────────────────────────────────────────────────── + +──────────────────────────────────────────────────────────────────────────────── +› Ask Diffex to do something for you. +──────────────────────────────────────────────────────────────────────────────── +fixture · thinking off · context 0.0% / 131k · harness default · /hotkeys + + + + + + + + + + diff --git a/scripts/daytona-behaviors/baselines/startup-resize-80x24.txt b/scripts/daytona-behaviors/baselines/startup-resize-80x24.txt new file mode 100644 index 000000000..157153cd5 --- /dev/null +++ b/scripts/daytona-behaviors/baselines/startup-resize-80x24.txt @@ -0,0 +1,24 @@ + + /----\_/----\ + | \|/ | + \---\ /---/ Diffex v + X + /---/ \---\ + | /|\ | + \____/^\____/ +──────────────────────────────────────────────────────────────────────────────── + +──────────────────────────────────────────────────────────────────────────────── +› Ask Diffex to do something for you. +──────────────────────────────────────────────────────────────────────────────── +fixture · thinking off · context 0.0% / 131k · harness default · /hotkeys + + + + + + + + + + diff --git a/scripts/daytona-behaviors/daytona_runner.py b/scripts/daytona-behaviors/daytona_runner.py new file mode 100644 index 000000000..38555cf6b --- /dev/null +++ b/scripts/daytona-behaviors/daytona_runner.py @@ -0,0 +1,644 @@ +#!/usr/bin/env python3 +"""Independent Daytona behavior orchestration (Python 3.11+, no Harbor). + +Workflow contract:: + + python daytona_runner.py prepare-snapshot --name diffex-behaviors-toolchain --output snapshot-record + python daytona_runner.py run --run-id "$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" \ + --artifacts retained-release --bundle behavior-bundle.tar.gz \ + --snapshot diffex-behaviors-toolchain --output daytona-results \ + --mode deterministic --commit "$GITHUB_SHA" + python daytona_runner.py recover --output daytona-results + +Always retain/upload the entire output directory, including state.json. Recovery never +creates a replacement or replays a possibly-started command. OPENAI_API_KEY is read only +for live mode and passed through process.exec(env=...), never sandbox env or metadata. +DAYTONA_API_KEY is used by the local SDK only. Installed.py owns artifact installation. + +SDK 0.226.1 API and lifecycle semantics inspected at: +https://www.daytona.io/docs/en/sandboxes/#wall-clock-ttl +https://www.daytona.io/docs/en/python-sdk/sync/daytona/ +`ttl_minutes=45` destroys in every state; auto_delete_interval alone is NOT a TTL. +Snapshots inherit resources, so both snapshot and sandbox resources are checked. +""" +from __future__ import annotations + +import argparse +from contextlib import contextmanager +from dataclasses import asdict, dataclass, field +import hashlib +from importlib.metadata import version +import json +import os +from pathlib import Path, PurePosixPath +import re +import shlex +import signal +import tarfile +import tempfile +import time +from typing import Iterator +import uuid +import xml.etree.ElementTree as ET + +from daytona import ( + CreateSandboxFromSnapshotParams, CreateSnapshotParams, Daytona, + DaytonaNotFoundError, Image, ListSandboxesQuery, Resources, Sandbox, +) + +SDK_VERSION = "0.226.1" +TTL_MINUTES = 45 +RESOURCES = {"cpu": 2, "memory": 4, "disk": 8} +REMOTE_BUNDLE = "/tmp/diffex-behaviors" +REMOTE_RELEASE = "/tmp/diffex-release" +REMOTE_EVIDENCE = "/tmp/diffex-evidence" +INSTALLED = f"{REMOTE_BUNDLE}/scripts/daytona-behaviors/installed.py" +MAX_FILE_BYTES = 64 * 1024 * 1024 +MAX_EVIDENCE_BYTES = 512 * 1024 * 1024 + +# Uploaded scripts contain no credentials, even for live mode. +EXTRACT_SCRIPT = '''import hashlib, os, pathlib, shutil, stat, sys, tarfile +# Verify BOTH uploads before extracting either input. +for archive, expected in zip(("/tmp/diffex-release.tar.gz", "/tmp/diffex-bundle.tar.gz"), sys.argv[1:], strict=True): + with open(archive, "rb") as stream: + actual = hashlib.file_digest(stream, "sha256").hexdigest() + if actual != expected: + raise ValueError("uploaded input SHA256 mismatch") +for archive, destination in (("/tmp/diffex-release.tar.gz", "/tmp/diffex-release"), + ("/tmp/diffex-bundle.tar.gz", "/tmp/diffex-behaviors")): + root = pathlib.Path(destination) + root.mkdir(parents=True, exist_ok=True) + with tarfile.open(archive) as source: + for member in source: + path = pathlib.PurePosixPath(member.name) + if path.is_absolute() or ".." in path.parts or not (member.isdir() or member.isfile()): + raise ValueError("unsafe archive member") + target = root.joinpath(*path.parts) + target.parent.mkdir(parents=True, exist_ok=True) + if member.isdir(): + target.mkdir(exist_ok=True) + else: + with source.extractfile(member) as src, target.open("wb") as dst: + shutil.copyfileobj(src, dst) + target.chmod(member.mode & 0o777) +''' +MANIFEST_SCRIPT = '''import hashlib, json, os, pathlib, stat +root = pathlib.Path("/tmp/diffex-evidence") +files = [] +for directory, dirs, names in os.walk(root, followlinks=False): + dirs[:] = sorted(d for d in dirs if not pathlib.Path(directory, d).is_symlink()) + for name in sorted(names): + path = pathlib.Path(directory, name) + info = path.lstat() + if not stat.S_ISREG(info.st_mode): + raise ValueError("nonregular evidence") + if info.st_size > 67108864: + raise ValueError("evidence file too large") + with path.open("rb") as stream: + digest = hashlib.file_digest(stream, "sha256").hexdigest() + files.append({"path": path.relative_to(root).as_posix(), "size": info.st_size, "sha256": digest}) +print(json.dumps(files)) +''' + + +class OrchestrationError(Exception): + """Safe, locally constructed diagnostic (never includes credentials).""" + + +def atomic_json(path: Path, value: object) -> None: + temporary = path.with_name(path.name + ".tmp") + with temporary.open("w", encoding="utf-8") as stream: + json.dump(value, stream, indent=2, sort_keys=True) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + temporary.replace(path) + + +def sha256(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def safe_relative(name: str) -> PurePosixPath: + path = PurePosixPath(name) + if not name or not path.parts or path.is_absolute() or ".." in path.parts or "\\" in name: + raise OrchestrationError("unsafe artifact path") + return path + + +def archive_directory(source: Path, destination: Path) -> None: + """Archive only explicit inputs; never follow symlinks or read ambient config.""" + if not source.is_dir() or source.is_symlink(): + raise OrchestrationError("input must be a regular directory") + with tarfile.open(destination, "w:gz") as archive: + for path in sorted(source.rglob("*")): + name = path.relative_to(source).as_posix() + safe_relative(name) + if path.is_symlink() or not (path.is_file() or path.is_dir()): + raise OrchestrationError("input contains links or special files") + if path.name == ".env" or path.name.startswith(".env.") or path.name in (".npmrc", ".netrc"): + raise OrchestrationError("input contains credential configuration") + archive.add(path, arcname=name, recursive=False) + + +def validate_archive(path: Path) -> None: + with tarfile.open(path) as archive: + for member in archive: + safe_relative(member.name) + if not (member.isfile() or member.isdir()): + raise OrchestrationError("bundle contains links or special files") + name = PurePosixPath(member.name).name + if name == ".env" or name.startswith(".env.") or name in (".npmrc", ".netrc"): + raise OrchestrationError("bundle contains credential configuration") + + +@dataclass +class State: + run_id: str + name: str + labels: dict[str, str] + commit: str + mode: str + snapshot: str + created_at: float + deadline: float + sandbox_id: str | None = None + create_attempted: bool = False + create_returned: bool = False + launch_attempted: bool = False + execution: str = "pending" + cleanup: str = "pending" + exit_code: int | None = None + errors: list[str] = field(default_factory=list) + downloaded: dict[str, str] = field(default_factory=dict) + inputs: dict[str, str] = field(default_factory=dict) + evidence_manifest: dict[str, str] | None = None + + def save(self, output: Path) -> None: + atomic_json(output / "state.json", asdict(self)) + + def error(self, stage: str, error: Exception) -> None: + # SDK error bodies can contain HTTP request details. Do not serialize them. + detail = str(error) if isinstance(error, OrchestrationError) else type(error).__name__ + self.errors.append(f"{stage}: {detail}") + + +@contextmanager +def termination_handlers() -> Iterator[None]: + def terminate(signum: int, _frame: object) -> None: + # A second TERM must not interrupt the independent cleanup/final report. + signal.signal(signal.SIGTERM, signal.SIG_IGN) + signal.signal(signal.SIGINT, signal.SIG_IGN) + raise OrchestrationError(f"interrupted by signal {signum}") + previous = {sig: signal.signal(sig, terminate) for sig in (signal.SIGTERM, signal.SIGINT)} + try: + yield + finally: + for sig, handler in previous.items(): + signal.signal(sig, handler) + + +class Runner: + def __init__(self, client: Daytona, output: Path, state: State, poll_seconds: float = 5): + self.client = client + self.output = output.resolve() + self.state = state + self.poll_seconds = poll_seconds + + def owned(self, sandbox: Sandbox) -> Sandbox: + if sandbox.name != self.state.name or any( + sandbox.labels.get(key) != value for key, value in self.state.labels.items() + ): + raise OrchestrationError("sandbox identity/ownership mismatch; refusing access or deletion") + if self.state.sandbox_id and sandbox.id != self.state.sandbox_id: + raise OrchestrationError("sandbox ID changed; refusing access or deletion") + return sandbox + + def lookup(self) -> Sandbox: + return self.owned(self.client.get(self.state.sandbox_id or self.state.name, request_timeout=10)) + + def create(self) -> Sandbox: + snapshot = self.client.snapshot.get(self.state.snapshot) + self.check_resources(snapshot) + if snapshot.state != "active": + raise OrchestrationError("snapshot must already be active") + self.state.create_attempted = True + self.state.save(self.output) # durable name + labels BEFORE the only create request + sandbox = self.client.create(CreateSandboxFromSnapshotParams( + name=self.state.name, snapshot=self.state.snapshot, labels=dict(self.state.labels), + public=False, auto_stop_interval=0, auto_pause_interval=0, + auto_delete_interval=0, ttl_minutes=TTL_MINUTES, + ), timeout=180) + self.owned(sandbox) + self.state.sandbox_id = sandbox.id + self.state.create_returned = True + self.state.save(self.output) + self.check_resources(sandbox) + return sandbox + + @staticmethod + def check_resources(resource: object) -> None: + # Snapshot DTO calls memory `mem`; Sandbox exposes `memory`. + actual = {"cpu": getattr(resource, "cpu", None), + "memory": getattr(resource, "memory", getattr(resource, "mem", None)), + "disk": getattr(resource, "disk", None)} + if actual != RESOURCES: + raise OrchestrationError(f"expected resources {RESOURCES}, got {actual}") + + @staticmethod + def execute(sandbox: Sandbox, command: str, env: dict[str, str] | None = None) -> str: + result = sandbox.process.exec(command, env=env, timeout=30) + if result.exit_code != 0: + raise OrchestrationError("remote orchestration command failed") + return result.result + + def stage(self, sandbox: Sandbox, artifacts: Path, bundle: Path) -> None: + with tempfile.TemporaryDirectory(prefix="diffex-daytona-") as directory: + temp = Path(directory) + release_archive = temp / "release.tar.gz" + archive_directory(artifacts, release_archive) + bundle_archive = bundle + if bundle.is_dir(): + bundle_archive = temp / "bundle.tar.gz" + archive_directory(bundle, bundle_archive) + validate_archive(bundle_archive) + self.state.inputs = {"release_sha256": sha256(release_archive), + "bundle_sha256": sha256(bundle_archive)} + self.state.save(self.output) + sandbox.fs.upload_file(str(release_archive), "/tmp/diffex-release.tar.gz", timeout=120) + sandbox.fs.upload_file(str(bundle_archive), "/tmp/diffex-bundle.tar.gz", timeout=120) + sandbox.fs.upload_file(EXTRACT_SCRIPT.encode(), "/tmp/diffex-extract.py", timeout=30) + self.execute(sandbox, "python3 /tmp/diffex-extract.py " + + self.state.inputs["release_sha256"] + " " + self.state.inputs["bundle_sha256"]) + self.execute(sandbox, f"test -f {INSTALLED} && mkdir -p {REMOTE_EVIDENCE}") + + def launch(self, sandbox: Sandbox, live_key: str | None) -> None: + argv = ["python3", INSTALLED, "--artifacts", REMOTE_RELEASE, + "--output", REMOTE_EVIDENCE, "--mode", self.state.mode, + "--commit", self.state.commit] + script = ( + "import json, os, pathlib, subprocess\n" + f"root = pathlib.Path({REMOTE_EVIDENCE!r})\n" + "with (root / 'installed.log').open('wb') as log:\n" + f" result = subprocess.run({argv!r}, cwd={REMOTE_BUNDLE!r}, stdout=log, stderr=subprocess.STDOUT)\n" + "temporary = root / '.orchestration-exit.tmp'\n" + "temporary.write_text(json.dumps({'exit_code': result.returncode}))\n" + "temporary.replace(root / '.orchestration-exit.json')\n" + ) + sandbox.fs.upload_file(script.encode(), "/tmp/diffex-launch.py", timeout=30) + self.state.launch_attempted = True + self.state.execution = "running" + self.state.save(self.output) # ambiguous exec is never replayed by recover + env = {"OPENAI_API_KEY": live_key} if self.state.mode == "live" and live_key else {} + self.execute(sandbox, + "nohup python3 /tmp/diffex-launch.py {REMOTE_EVIDENCE}/launcher.log 2>&1 &", env=env) + + def collect(self, sandbox: Sandbox, *, final: bool = False) -> None: + # Only a complete, quiescent final collection can qualify retained evidence. + self.state.evidence_manifest = None + manifest = json.loads(self.execute(sandbox, "python3 -c " + shlex.quote(MANIFEST_SCRIPT))) + if not isinstance(manifest, list): + raise OrchestrationError("invalid evidence manifest") + total = 0 + evidence = self.output / "evidence" + evidence.mkdir(exist_ok=True) + manifest_hashes: dict[str, str] = {} + for entry in manifest: + if not isinstance(entry, dict): + raise OrchestrationError("invalid evidence manifest entry") + name, size, digest = entry.get("path"), entry.get("size"), entry.get("sha256") + if (not isinstance(name, str) or type(size) is not int + or not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest)): + raise OrchestrationError("invalid evidence manifest entry") + relative = safe_relative(name) + canonical = relative.as_posix() + if canonical in manifest_hashes: + raise OrchestrationError("duplicate evidence manifest path") + manifest_hashes[canonical] = digest + total += size + if size < 0 or size > MAX_FILE_BYTES or total > MAX_EVIDENCE_BYTES: + raise OrchestrationError("evidence exceeds download limits") + target = evidence.joinpath(*relative.parts) + # Do not follow pre-existing local symlinks, including parent directories. + if any(path.is_symlink() for path in [target, *target.parents]): + raise OrchestrationError("local evidence symlink") + if self.state.downloaded.get(name) == digest and target.is_file() and sha256(target) == digest: + continue + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_name(target.name + ".download") + received = 0 + try: + with temporary.open("wb") as stream: + # SDK 0.226.1 rejects an otherwise valid empty-file download. + # Materialize it locally; the same size/SHA256 checks still apply. + if size: + for chunk in sandbox.fs.download_file_stream(f"{REMOTE_EVIDENCE}/{name}", timeout=30): + received += len(chunk) + if received > MAX_FILE_BYTES: + raise OrchestrationError("evidence grew beyond download limit") + stream.write(chunk) + if received != size or sha256(temporary) != digest: + # Polling tolerates evolving logs, but final evidence must be complete. + if final: + raise OrchestrationError("final evidence checksum or size mismatch") + continue + temporary.replace(target) + self.state.downloaded[name] = digest + self.state.save(self.output) + finally: + temporary.unlink(missing_ok=True) + if final: + self.state.evidence_manifest = manifest_hashes + self.state.save(self.output) + + def poll(self, sandbox: Sandbox, deadline: float) -> None: + while True: + try: + self.collect(sandbox) + exit_file = self.output / "evidence" / ".orchestration-exit.json" + if exit_file.is_file(): + code = json.loads(exit_file.read_text())["exit_code"] + if not isinstance(code, int): + raise OrchestrationError("invalid installed runner exit code") + self.state.exit_code = code + self.state.execution = "passed" if code == 0 else "failed" + self.state.save(self.output) + return + except Exception as error: + self.state.error("evidence", error) + self.state.save(self.output) + if time.time() >= deadline: + raise OrchestrationError("evidence/execution deadline exceeded") + time.sleep(min(self.poll_seconds, max(0, deadline - time.time()))) + + def cleanup(self, attempts: int = 6) -> None: + if not self.state.create_attempted: + self.state.cleanup = "not-needed" + self.state.save(self.output) + return + for attempt in range(attempts): + try: + sandbox = self.lookup() + self.state.sandbox_id = sandbox.id + self.state.save(self.output) + try: + self.client.delete(sandbox, timeout=15, wait=False) + except Exception as error: + self.state.error("delete-request", error) + except DaytonaNotFoundError: + pass + except Exception as error: + self.state.error("cleanup-lookup", error) + # Independent fresh reads, even when deletion raised or claimed success. + try: + identifiers = {self.state.name} + if self.state.sandbox_id: + identifiers.add(self.state.sandbox_id) + for identifier in identifiers: + try: + found = self.client.get(identifier, request_timeout=10) + except DaytonaNotFoundError: + continue + self.owned(found) + raise OrchestrationError("sandbox still visible after deletion") + remaining = list(self.client.list(ListSandboxesQuery(labels=self.state.labels), request_timeout=10)) + if remaining: + raise OrchestrationError("sandbox labels still visible after deletion") + # A 404 immediately after an ambiguous POST is not proof it cannot appear later. + if not self.state.sandbox_id: + raise OrchestrationError("creation unresolved; recover by saved name/labels; TTL bounds lifetime") + self.state.cleanup = "confirmed" + self.state.save(self.output) + return + except Exception as error: + self.state.error("cleanup-confirm", error) + self.state.cleanup = "unconfirmed" + self.state.save(self.output) + if attempt + 1 < attempts: + time.sleep(self.poll_seconds) + + def validate_reports(self, installed: dict[str, object]) -> None: + evidence = self.output / "evidence" + manifest = self.state.evidence_manifest + if manifest is None or not {"results.json", "results.xml", ".orchestration-exit.json"} <= manifest.keys(): + raise OrchestrationError("missing complete final evidence manifest") + for name, digest in manifest.items(): + target = evidence.joinpath(*safe_relative(name).parts) + if (any(path.is_symlink() for path in [target, *target.parents]) + or not target.is_file() or sha256(target) != digest): + raise OrchestrationError("retained evidence does not match final manifest") + hashes, scenarios = installed.get("artifactHashes"), installed.get("scenarios") + if (installed.get("schemaVersion") != 1 or installed.get("commit") != self.state.commit + or installed.get("mode") != self.state.mode + or installed.get("outcome") not in ("passed", "failed") + or not isinstance(hashes, dict) or not hashes + or not all(isinstance(value, str) and re.fullmatch(r"[0-9a-f]{64}", value) + for value in hashes.values()) + or not isinstance(scenarios, list) or not scenarios): + raise OrchestrationError("invalid installed report identity or inventory") + scenario_outcomes: dict[str, str] = {} + for scenario in scenarios: + if not isinstance(scenario, dict): + raise OrchestrationError("invalid installed scenario") + identity, outcome, references = scenario.get("id"), scenario.get("outcome"), scenario.get("evidence") + if (not isinstance(identity, str) or not identity or identity in scenario_outcomes + or outcome not in ("passed", "failed") or scenario.get("commit") != self.state.commit + or scenario.get("artifactHashes") != hashes + or not isinstance(references, list) or not references): + raise OrchestrationError("invalid installed scenario identity or evidence references") + scenario_outcomes[identity] = outcome + for reference in references: + if not isinstance(reference, str): + raise OrchestrationError("invalid installed evidence reference") + relative = safe_relative(reference) + prefix = relative.as_posix() + target = evidence.joinpath(*relative.parts) + if (any(path.is_symlink() for path in [target, *target.parents]) + or not target.exists() or not any( + name == prefix or (target.is_dir() and name.startswith(prefix + "/")) + for name in manifest)): + raise OrchestrationError("missing installed scenario evidence") + expected_outcome = "passed" if all(value == "passed" for value in scenario_outcomes.values()) else "failed" + if installed["outcome"] != expected_outcome: + raise OrchestrationError("installed report outcome disagrees with scenarios") + suite = ET.parse(evidence / "results.xml").getroot() + cases = suite.findall("testcase") + failures = sum(value != "passed" for value in scenario_outcomes.values()) + if (suite.tag != "testsuite" or int(suite.get("tests", "-1")) != len(scenario_outcomes) + or int(suite.get("failures", "-1")) != failures or int(suite.get("errors", "0")) != 0 + or int(suite.get("skipped", "0")) != 0 or len(cases) != len(scenario_outcomes) + or {case.get("name") for case in cases} != scenario_outcomes.keys()): + raise OrchestrationError("installed JUnit inventory disagrees with JSON report") + for case in cases: + expected_failures = int(scenario_outcomes[case.get("name")] != "passed") + if (len(case.findall("failure")) != expected_failures or case.find("error") is not None + or case.find("skipped") is not None): + raise OrchestrationError("installed JUnit outcome disagrees with JSON report") + + def report(self) -> int: + installed: dict[str, object] = {} + reports_complete = False + try: + value = json.loads((self.output / "evidence" / "results.json").read_text()) + if not isinstance(value, dict): + raise OrchestrationError("invalid installed results.json") + installed = value + self.validate_reports(installed) + reports_complete = True + except (OrchestrationError, ValueError, OSError, TypeError, KeyError, ET.ParseError) as error: + self.state.error("report", error) + success = (self.state.execution == "passed" and self.state.exit_code == 0 + and installed.get("outcome") == "passed" and reports_complete + and self.state.cleanup in ("confirmed", "not-needed")) + report = {"outcome": "passed" if success else "failed", "category": "daytona", + "input_hashes": self.state.inputs, "run_id": self.state.run_id, + "commit": self.state.commit, "mode": self.state.mode, "name": self.state.name, + "sandbox_id": self.state.sandbox_id, "execution": self.state.execution, + "exit_code": self.state.exit_code, "installed_outcome": installed.get("outcome"), + "reports_complete": reports_complete, "cleanup": self.state.cleanup, + "duration_seconds": max(0, time.time() - self.state.created_at), + "artifact_hashes": self.state.downloaded, "evidence_manifest": self.state.evidence_manifest, + "errors": self.state.errors} + atomic_json(self.output / "orchestration.json", report) + suite = ET.Element("testsuite", name="daytona-orchestration", tests="3", failures=str(sum(( + self.state.execution != "passed", not reports_complete or installed.get("outcome") != "passed", + self.state.cleanup not in ("confirmed", "not-needed"))))) + for name, passed, detail in ( + ("execution", self.state.execution == "passed", self.state.execution), + ("installed-evidence", reports_complete and installed.get("outcome") == "passed", str(installed.get("outcome"))), + ("cleanup", self.state.cleanup in ("confirmed", "not-needed"), self.state.cleanup), + ): + case = ET.SubElement(suite, "testcase", name=name, classname="daytona") + if not passed: + ET.SubElement(case, "failure", message=detail).text = "\n".join(self.state.errors) + ET.ElementTree(suite).write(self.output / "orchestration.xml", encoding="utf-8", xml_declaration=True) + self.state.save(self.output) + return 0 if success else 1 + + def run(self, artifacts: Path, bundle: Path, live_key: str | None) -> int: + try: + sandbox = self.create() + self.stage(sandbox, artifacts, bundle) + self.launch(sandbox, live_key) + self.poll(sandbox, self.state.deadline) + except Exception as error: + self.state.execution = "failed" + self.state.error("run", error) + finally: + if self.state.launch_attempted: + try: + self.collect(self.lookup(), final=True) + except Exception as error: + self.state.error("final-evidence", error) + try: + self.cleanup() + except Exception as error: + self.state.cleanup = "unconfirmed" + self.state.error("cleanup", error) + self.state.save(self.output) + return self.report() + + def recover(self, wait_seconds: float) -> int: + try: + if self.state.create_attempted and self.state.cleanup != "confirmed": + sandbox = self.lookup() + if self.state.launch_attempted: + self.poll(sandbox, min(self.state.deadline, time.time() + wait_seconds)) + except DaytonaNotFoundError: + pass + except Exception as error: + self.state.error("recover-evidence", error) + finally: + if self.state.launch_attempted and self.state.cleanup != "confirmed": + try: + self.collect(self.lookup(), final=True) + except Exception as error: + self.state.error("recover-final-evidence", error) + try: + self.cleanup() + except Exception as error: + self.state.cleanup = "unconfirmed" + self.state.error("recover-cleanup", error) + return self.report() + + +def prepare_snapshot(client: Daytona, name: str, output: Path) -> int: + dockerfile = Path(__file__).with_name("Dockerfile") + record: dict[str, object] = {"name": name, "dockerfile_sha256": sha256(dockerfile), + "resources": RESOURCES, "status": "requested"} + output.mkdir(parents=True, exist_ok=False) + atomic_json(output / "snapshot.json", record) + try: + # No replacement or delete: an existing name or ambiguous POST requires inspection. + snapshot = client.snapshot.create(CreateSnapshotParams( + name=name, image=Image.from_dockerfile(dockerfile, strict_context=True), + resources=Resources(**RESOURCES), + ), timeout=1200) + Runner.check_resources(snapshot) + record.update(status="active", id=snapshot.id) + return 0 + except Exception as error: + record.update(status="failed-or-ambiguous", error=type(error).__name__) + return 1 + finally: + atomic_json(output / "snapshot.json", record) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + commands = parser.add_subparsers(dest="command", required=True) + run = commands.add_parser("run") + run.add_argument("--artifacts", type=Path, required=True) + run.add_argument("--bundle", type=Path, required=True) + run.add_argument("--snapshot", required=True) + run.add_argument("--run-id", required=True) + run.add_argument("--commit", required=True) + run.add_argument("--mode", choices=("deterministic", "live"), required=True) + run.add_argument("--timeout-seconds", type=int, default=2100) + run.add_argument("--live-key-env", choices=("OPENAI_API_KEY",), default="OPENAI_API_KEY") + recover = commands.add_parser("recover") + recover.add_argument("--wait-seconds", type=int, default=0) + snapshot = commands.add_parser("prepare-snapshot") + snapshot.add_argument("--name", required=True) + for command in (run, recover, snapshot): + command.add_argument("--output", type=Path, required=True) + args = parser.parse_args(argv) + if version("daytona") != SDK_VERSION: + parser.error(f"install requirements.txt: daytona=={SDK_VERSION} required") + if args.command == "run": + if not re.fullmatch(r"[0-9a-fA-F]{40}", args.commit): + parser.error("--commit must be a full 40-character commit SHA") + if not re.fullmatch(r"[A-Za-z0-9_-]{1,80}", args.run_id): + parser.error("--run-id must be 1-80 alphanumeric, dash or underscore characters") + if not 1 <= args.timeout_seconds <= 2400: + parser.error("--timeout-seconds must be 1..2400, below the 45-minute wall-clock TTL") + if not args.artifacts.is_dir() or not args.bundle.exists(): + parser.error("artifacts directory and bundle must exist") + live_key = os.environ.get(args.live_key_env) if args.mode == "live" else None + if args.mode == "live" and not live_key: + parser.error("live mode requires OPENAI_API_KEY") + args.output.mkdir(parents=True, exist_ok=False) + token = uuid.uuid4().hex + now = time.time() + state = State(run_id=args.run_id, name=f"diffex-behaviors-{args.run_id[:32]}-{token[:12]}", + labels={"diffex-purpose": "installed-behaviors", "diffex-run": token, + "diffex-run-id": args.run_id, "diffex-commit": args.commit}, + commit=args.commit, mode=args.mode, snapshot=args.snapshot, + created_at=now, deadline=now + args.timeout_seconds) + state.save(args.output) + with termination_handlers(): + return Runner(Daytona(), args.output, state).run(args.artifacts, args.bundle, live_key) + if args.command == "recover": + if args.wait_seconds < 0 or args.wait_seconds > 2400: + parser.error("--wait-seconds must be 0..2400") + state = State(**json.loads((args.output / "state.json").read_text())) + with termination_handlers(): + return Runner(Daytona(), args.output, state).recover(args.wait_seconds) + with termination_handlers(): + return prepare_snapshot(Daytona(), args.name, args.output) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/daytona-behaviors/fixture.mjs b/scripts/daytona-behaviors/fixture.mjs new file mode 100644 index 000000000..dd7c4da57 --- /dev/null +++ b/scripts/daytona-behaviors/fixture.mjs @@ -0,0 +1,288 @@ +// Installed acceptance extension: only provider inference/HTTP are synthetic. +// Sessions, built-in tools, auth resolution, agent orchestration and UI remain real. +import { appendFileSync, closeSync, constants, fstatSync, lstatSync, openSync, realpathSync } from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; +import { Socket } from "node:net"; +import { basename, dirname, resolve, sep } from "node:path"; +import { createAssistantMessageEventStream } from "@diffexai/diffex-ai"; + +const toolSteps = [ + { name: "read", arguments: { path: "input.txt" } }, + { name: "write", arguments: { path: "output.txt", content: "behavior-written\n" } }, + { + name: "edit", + arguments: { path: "output.txt", edits: [{ oldText: "written", newText: "edited" }] }, + }, + { name: "bash", arguments: { command: "cat output.txt > bash-result.txt" } }, +]; + +function textOf(message) { + return typeof message.content === "string" + ? message.content + : message.content.filter((block) => block.type === "text").map((block) => block.text).join("\n"); +} + +function checkedResults(messages, promptIndex) { + const following = messages.slice(promptIndex + 1); + const calls = following.flatMap((message) => + message.role === "assistant" ? message.content.filter((block) => block.type === "toolCall") : [], + ); + const results = following.filter((message) => message.role === "toolResult"); + if (results.length > toolSteps.length || calls.length !== results.length) { + throw new Error("Behavior tools context has missing or extra tool results"); + } + for (const [index, result] of results.entries()) { + const step = toolSteps[index]; + const call = calls[index]; + if ( + result.isError || + result.toolName !== step.name || + call.name !== step.name || + call.id !== result.toolCallId || + JSON.stringify(call.arguments) !== JSON.stringify(step.arguments) + ) { + throw new Error(`Behavior ${step.name} tool result/context mismatch`); + } + } + return results; +} + +function authResponse() { + const text = "behavior-auth-ok"; + const item = { + type: "message", + id: "msg_behavior_auth", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text, annotations: [] }], + }; + const events = [ + { type: "response.created", response: { id: "resp_behavior_auth", status: "in_progress" } }, + { type: "response.output_item.added", output_index: 0, item: { ...item, status: "in_progress", content: [] } }, + { type: "response.content_part.added", output_index: 0, content_index: 0, item_id: item.id, part: { type: "output_text", text: "", annotations: [] } }, + { type: "response.output_text.delta", output_index: 0, content_index: 0, item_id: item.id, delta: text }, + { type: "response.output_text.done", output_index: 0, content_index: 0, item_id: item.id, text }, + { type: "response.output_item.done", output_index: 0, item }, + { + type: "response.completed", + response: { + id: "resp_behavior_auth", + object: "response", + model: "gpt-4.1-mini", + status: "completed", + output: [item], + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2, input_tokens_details: { cached_tokens: 0 }, output_tokens_details: { reasoning_tokens: 0 } }, + }, + }, + ]; + const encoder = new TextEncoder(); + return new Response(new ReadableStream({ + start(controller) { + for (const [sequence_number, event] of events.entries()) { + controller.enqueue(encoder.encode(`event: ${event.type}\ndata: ${JSON.stringify({ ...event, sequence_number })}\n\n`)); + } + controller.close(); + }, + }), { headers: { "content-type": "text/event-stream", "x-request-id": "req_behavior_auth" } }); +} + +// Kept self-contained: installed.py copies this extension beside installed node_modules. +function openRecording() { + const requested = process.env.DIFFEX_BEHAVIOR_RECORD; + if (!requested) throw new Error("DIFFEX_BEHAVIOR_RECORD is required"); + if (requested.includes("\0") || requested.split(/[\\/]/).includes("..")) { + throw new Error("Invalid behavior recording path"); + } + const absolute = resolve(requested); + // These are filesystem boundaries, not env-controlled TMPDIR or record dirname. + const roots = [realpathSync(process.cwd()), realpathSync("/tmp")]; + const lexicalRoots = [...roots, process.cwd(), "/tmp"]; + if (process.platform === "darwin") { + roots.push(realpathSync("/private/var/folders")); + lexicalRoots.push("/var/folders", "/private/var/folders"); + } + const lexicalRoot = lexicalRoots.find((entry) => absolute.startsWith(`${entry}${sep}`)); + if (!lexicalRoot || !absolute.startsWith(`${lexicalRoot}${sep}`)) { + throw new Error("Unauthorized behavior recording destination"); + } + const parent = realpathSync(dirname(absolute)); + const recordPath = resolve(parent, basename(absolute)); + const root = roots.find((entry) => recordPath.startsWith(`${entry}${sep}`)); + if (!root || !recordPath.startsWith(`${root}${sep}`)) { + throw new Error("Unauthorized behavior recording destination"); + } + const uid = process.getuid(); + for (let directory = parent; ; directory = dirname(directory)) { + const info = lstatSync(directory); + // Permit OS-owned sticky temp roots, but not writable/shared intermediate dirs. + if (!info.isDirectory() || (info.uid !== uid && info.uid !== 0) || + ((info.mode & 0o022) !== 0 && !(info.uid === 0 && (info.mode & 0o1000) !== 0))) { + throw new Error("Unsafe behavior recording directory"); + } + if (directory === root) break; + } + // Open without truncation before inspecting the file; never trust a path precheck. + const fd = openSync(recordPath, constants.O_RDWR | constants.O_APPEND | constants.O_CREAT | constants.O_NOFOLLOW | constants.O_NONBLOCK, 0o600); + try { + const opened = fstatSync(fd); + const current = lstatSync(recordPath); + if ( + !opened.isFile() || opened.nlink !== 1 || opened.uid !== uid || (opened.mode & 0o022) !== 0 || !current.isFile() || + opened.dev !== current.dev || opened.ino !== current.ino || + realpathSync(parent) !== parent + ) throw new Error("Behavior recording changed while opening"); + return fd; + } catch (error) { + closeSync(fd); + throw error; + } +} + +export default function (diffex) { + const recordFd = openRecording(); + diffex.on("session_shutdown", () => closeSync(recordFd)); + const record = (value) => appendFileSync(recordFd, `${JSON.stringify(value)}\n`); + diffex.on("agent_settled", () => { record({ type: "settled" }); }); + diffex.on("tool_result", (event) => { record({ ...event, type: "tool_result" }); }); + const transport = process.env.DIFFEX_BEHAVIOR_TRANSPORT === "1"; + const blocked = (details) => { + record({ type: "blocked-network", ...details }); + throw new Error("Network disabled by behavior acceptance fixture"); + }; + globalThis.fetch = async (input, init) => { + const url = input instanceof Request ? input.url : String(input); + const method = init?.method ?? (input instanceof Request ? input.method : "GET"); + record({ type: "request", url, method }); + if (url === "https://api.diffex.ai/api/latest-version" && method === "GET") { + return Response.json({ error: "no_published_release" }, { status: 404 }); + } + if (!transport || url !== "https://api.openai.com/v1/responses" || method !== "POST") { + return blocked({ transport: "fetch", url, method }); + } + const headers = new Headers(input instanceof Request ? input.headers : undefined); + new Headers(init?.headers).forEach((value, name) => headers.set(name, value)); + const allowedHeaders = new Set([ + "accept", "authorization", "content-type", "user-agent", "session_id", "x-client-request-id", + "x-stainless-arch", "x-stainless-lang", "x-stainless-os", "x-stainless-package-version", + "x-stainless-runtime", "x-stainless-runtime-version", "x-stainless-retry-count", "x-stainless-timeout", + ]); + for (const name of headers.keys()) { + if (!allowedHeaders.has(name)) return blocked({ transport: "fetch", url, header: name }); + } + if (process.env.OPENAI_API_KEY !== "fake" || headers.get("authorization") !== "Bearer fake" || headers.get("content-type") !== "application/json") { + return blocked({ transport: "fetch", url, reason: "Unexpected auth or content-type" }); + } + const body = init?.body ?? (input instanceof Request ? await input.text() : undefined); + if (typeof body !== "string") return blocked({ transport: "fetch", url, reason: "Expected JSON body" }); + const payload = JSON.parse(body); + if (payload.model !== "gpt-4.1-mini" || payload.stream !== true || !Array.isArray(payload.input)) { + return blocked({ transport: "fetch", url, reason: "Unexpected model or streaming payload" }); + } + record({ type: "inference", provider: "openai", model: payload.model, transport: "fetch", credentialVerified: true }); + record({ type: "context", transport: "fetch", input: payload.input }); + record({ type: "auth-request", url, credentialVerified: true, headers: { ...Object.fromEntries(headers), authorization: "Bearer [fixture]" }, payload }); + return authResponse(); + }; + Socket.prototype.connect = function () { + return blocked({ transport: "socket" }); + }; + syncBuiltinESMExports(); + // Crucially, transport mode does not register/override any provider or auth. + if (transport) return; + + diffex.registerProvider("behavior", { + baseUrl: "https://behavior.invalid", + apiKey: "local-fixture", + api: "behavior", + models: [{ + id: "fixture", + name: "Behavior fixture", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 131072, + maxTokens: 1024, + }], + streamSimple(model, context, options = {}) { + const stream = createAssistantMessageEventStream(); + const message = { + role: "assistant", content: [], api: model.api, provider: model.provider, model: model.id, + usage: { input: 1, output: 1, cacheRead: 0, cacheWrite: 0, totalTokens: 2, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, + stopReason: "pending", timestamp: Date.now(), + }; + queueMicrotask(async () => { + try { + const messages = context.messages; + const promptIndex = messages.findLastIndex((entry) => entry.role === "user"); + const prompt = promptIndex < 0 ? "" : textOf(messages[promptIndex]); + const modes = prompt.match(/\bBEHAVIOR_(?:PRINT|TOOLS|CANCEL|RECOVER|RESUME)\b/g) ?? []; + if (new Set(modes).size !== 1) throw new Error("Expected exactly one BEHAVIOR mode in latest user prompt"); + const mode = modes[0]; + record({ type: "inference", provider: model.provider, model: model.id, mode, prompt }); + record({ type: "context", mode, messages, tools: context.tools?.map((tool) => tool.name) ?? [] }); + for (const result of messages.filter((entry) => entry.role === "toolResult")) { + record({ type: "tool-result", mode, ...result }); + } + await options.onPayload?.({ model: model.id, messages }, model); + await options.onResponse?.({ status: 200, headers: {} }, model); + if (options.signal?.aborted) throw new Error("Request was aborted"); + stream.push({ type: "start", partial: message }); + let text; + if (mode === "BEHAVIOR_TOOLS") { + const results = checkedResults(messages, promptIndex); + if (results.length < toolSteps.length) { + const step = toolSteps[results.length]; + if (!context.tools?.some((tool) => tool.name === step.name)) throw new Error(`Missing built-in ${step.name} tool`); + const call = { type: "toolCall", id: `behavior_${promptIndex}_${step.name}`, ...step }; + message.content.push(call); + stream.push({ type: "toolcall_start", contentIndex: 0, partial: message }); + stream.push({ type: "toolcall_delta", contentIndex: 0, delta: JSON.stringify(step.arguments), partial: message }); + stream.push({ type: "toolcall_end", contentIndex: 0, toolCall: call, partial: message }); + message.stopReason = "toolUse"; + stream.push({ type: "done", reason: "toolUse", message }); + stream.end(message); + return; + } + text = "behavior-tools-ok"; + } else if (mode === "BEHAVIOR_RESUME") { + const earlierIndex = messages.findLastIndex((entry, index) => index < promptIndex && entry.role === "user" && /\bBEHAVIOR_TOOLS\b/.test(textOf(entry))); + if (earlierIndex < 0 || checkedResults(messages.slice(0, promptIndex), earlierIndex).length !== toolSteps.length || !messages.slice(earlierIndex + 1, promptIndex).some((entry) => entry.role === "assistant" && textOf(entry) === "behavior-tools-ok")) { + throw new Error("Resumed context is missing the earlier BEHAVIOR_TOOLS prompt, results or final answer"); + } + record({ type: "resume-verified", prompt: textOf(messages[earlierIndex]), toolNames: toolSteps.map((step) => step.name) }); + text = "behavior-resumed-ok"; + } else { + text = { BEHAVIOR_PRINT: "behavior-print-ok", BEHAVIOR_RECOVER: "behavior-recovered-ok", BEHAVIOR_CANCEL: "behavior-cancel-started" }[mode]; + } + message.content.push({ type: "text", text }); + stream.push({ type: "text_start", contentIndex: 0, partial: message }); + stream.push({ type: "text_delta", contentIndex: 0, delta: text, partial: message }); + if (mode === "BEHAVIOR_CANCEL") { + if (!options.signal) throw new Error("Cancellation fixture requires an AbortSignal"); + record({ type: "cancel-started" }); + await new Promise((resolve) => { + // Keep print/RPC processes alive as a real pending HTTP request would. + const keepAlive = setInterval(() => {}, 1000); + const abort = () => { clearInterval(keepAlive); resolve(); }; + options.signal.addEventListener("abort", abort, { once: true }); + if (options.signal.aborted) { options.signal.removeEventListener("abort", abort); abort(); } + }); + record({ type: "abort", mode, aborted: options.signal.aborted }); + throw new Error("Behavior fixture request aborted"); + } + stream.push({ type: "text_end", contentIndex: 0, content: text, partial: message }); + message.stopReason = "stop"; + stream.push({ type: "done", reason: "stop", message }); + stream.end(message); + } catch (error) { + message.stopReason = options.signal?.aborted ? "aborted" : "error"; + message.errorMessage = error instanceof Error ? error.message : String(error); + record({ type: "stream-error", reason: message.stopReason, message: message.errorMessage }); + stream.push({ type: "error", reason: message.stopReason, error: message }); + stream.end(message); + } + }); + return stream; + }, + }); +} diff --git a/scripts/daytona-behaviors/fixture.test.mjs b/scripts/daytona-behaviors/fixture.test.mjs new file mode 100644 index 000000000..08833a2d1 --- /dev/null +++ b/scripts/daytona-behaviors/fixture.test.mjs @@ -0,0 +1,263 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { chmodSync, linkSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { Socket } from "node:net"; +import { tmpdir } from "node:os"; +import { join, relative, resolve } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; +import fixture from "./fixture.mjs"; +import guard from "./live-guard.mjs"; + +const cli = fileURLToPath(new URL("../../packages/coding-agent/dist/cli.js", import.meta.url)); +const extension = fileURLToPath(new URL("./fixture.mjs", import.meta.url)); + +function records(path) { + return readFileSync(path, "utf8").trim().split("\n").map((line) => JSON.parse(line)); +} + +function runCli(directory, record, args, env = {}) { + const result = spawnSync(process.execPath, [cli, "--no-extensions", "-e", extension, ...args], { + cwd: directory, + env: { + ...process.env, + DIFFEX_CODING_AGENT_DIR: join(directory, "config"), + DIFFEX_BEHAVIOR_RECORD: record, + DIFFEX_BEHAVIOR_TRANSPORT: "0", + ...env, + }, + encoding: "utf8", + timeout: 30000, + }); + assert.equal(result.error, undefined); + assert.equal(result.status, 0, result.stderr || result.stdout); + return result.stdout; +} + +test("installed extension drives real print, built-in tools and persisted resume", () => { + const directory = mkdtempSync(join(tmpdir(), "behavior-fixture-")); + try { + const record = join(directory, "record.jsonl"); + const session = join(directory, "session.jsonl"); + writeFileSync(join(directory, "input.txt"), "behavior-input\n"); + assert.match(runCli(directory, record, ["--model", "behavior/fixture", "--no-session", "-p", "BEHAVIOR_PRINT"]), /behavior-print-ok/); + assert.match(runCli(directory, record, ["--model", "behavior/fixture", "--session", session, "-p", "BEHAVIOR_TOOLS"]), /behavior-tools-ok/); + assert.equal(readFileSync(join(directory, "output.txt"), "utf8"), "behavior-edited\n"); + assert.equal(readFileSync(join(directory, "bash-result.txt"), "utf8"), "behavior-edited\n"); + assert.match(runCli(directory, record, ["--model", "behavior/fixture", "--session", session, "-p", "BEHAVIOR_RESUME"]), /behavior-resumed-ok/); + const events = records(record); + assert.equal(events.filter((event) => event.type === "settled").length, 3); + assert.deepEqual(events.filter((event) => event.type === "tool_result").map((event) => event.toolName), ["read", "write", "edit", "bash"]); + assert.equal(events.filter((event) => event.type === "resume-verified").length, 1); + assert.equal(events.some((event) => event.type === "blocked-network"), false); + const resumed = events.find((event) => event.type === "context" && event.mode === "BEHAVIOR_RESUME"); + assert.equal(resumed.messages.filter((message) => message.role === "toolResult").length, 4); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("built-in OpenAI provider resolves fake environment credentials and consumes Responses SSE", () => { + const directory = mkdtempSync(join(tmpdir(), "behavior-auth-")); + try { + const record = join(directory, "record.jsonl"); + assert.match(runCli(directory, record, ["--model", "openai/gpt-4.1-mini", "--no-session", "-p", "BEHAVIOR_AUTH"], { + DIFFEX_BEHAVIOR_TRANSPORT: "1", OPENAI_API_KEY: "fake", + }), /behavior-auth-ok/); + const events = records(record); + assert(events.some((event) => event.type === "auth-request" && event.credentialVerified)); + assert.equal(events.filter((event) => event.type === "settled").length, 1); + assert.equal(events.some((event) => event.type === "blocked-network"), false); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("cancellation stays pending until AbortSignal, then recovery and missing-resume checks work", async () => { + const directory = mkdtempSync(join(tmpdir(), "behavior-cancel-")); + const originalFetch = globalThis.fetch; + const originalConnect = Socket.prototype.connect; + const originalEnv = { ...process.env }; + try { + const record = join(directory, "record.jsonl"); + process.env.DIFFEX_BEHAVIOR_RECORD = record; + process.env.DIFFEX_BEHAVIOR_TRANSPORT = "0"; + let provider; + fixture({ on() {}, registerProvider(_name, config) { provider = config; } }); + const model = { api: "behavior", provider: "behavior", id: "fixture" }; + const controller = new AbortController(); + const stream = provider.streamSimple(model, { messages: [{ role: "user", content: "BEHAVIOR_CANCEL", timestamp: 1 }] }, { signal: controller.signal }); + let settled = false; + stream.result().then(() => { settled = true; }); + for await (const event of stream) { + if (event.type === "text_delta") { + assert.equal(event.delta, "behavior-cancel-started"); + await new Promise((resolve) => setTimeout(resolve, 20)); + assert.equal(settled, false); + assert.equal(records(record).some((entry) => entry.type === "abort"), false); + controller.abort(); + } + } + assert.equal((await stream.result()).stopReason, "aborted"); + assert(records(record).some((entry) => entry.type === "abort" && entry.aborted)); + const recovered = await provider.streamSimple(model, { messages: [{ role: "user", content: "BEHAVIOR_RECOVER", timestamp: 2 }] }).result(); + assert.equal(recovered.content[0].text, "behavior-recovered-ok"); + const invalidResume = await provider.streamSimple(model, { messages: [{ role: "user", content: "BEHAVIOR_RESUME", timestamp: 3 }] }).result(); + assert.equal(invalidResume.stopReason, "error"); + assert.match(invalidResume.errorMessage, /missing the earlier/); + await assert.rejects(fetch("https://example.com/"), /Network disabled/); + assert.throws(() => new Socket().connect(443, "example.com"), /Network disabled/); + assert.equal((await fetch("https://api.diffex.ai/api/latest-version")).status, 404); + } finally { + globalThis.fetch = originalFetch; + Socket.prototype.connect = originalConnect; + process.env = originalEnv; + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("transport rejects unexpected hosts, credentials and custom headers without registering providers", async () => { + const directory = mkdtempSync(join(tmpdir(), "behavior-network-")); + const originalFetch = globalThis.fetch; + const originalConnect = Socket.prototype.connect; + const originalEnv = { ...process.env }; + try { + process.env.DIFFEX_BEHAVIOR_RECORD = join(directory, "record.jsonl"); + process.env.DIFFEX_BEHAVIOR_TRANSPORT = "1"; + process.env.OPENAI_API_KEY = "fake"; + fixture({ on() {}, registerProvider() { assert.fail("Transport fixture must not register a provider"); } }); + const request = { + method: "POST", + headers: { authorization: "Bearer fake", "content-type": "application/json" }, + body: JSON.stringify({ model: "gpt-4.1-mini", stream: true, input: [] }), + }; + await assert.rejects(fetch("https://unexpected.example/v1/responses", request), /Network disabled/); + await assert.rejects(fetch("https://api.openai.com/v1/responses", { ...request, headers: { ...request.headers, authorization: "Bearer real-secret" } }), /Network disabled/); + await assert.rejects(fetch("https://api.openai.com/v1/responses", { ...request, headers: { ...request.headers, "x-api-key": "unexpected" } }), /Network disabled/); + assert.equal(records(process.env.DIFFEX_BEHAVIOR_RECORD).filter((event) => event.type === "blocked-network").length, 3); + assert.equal(readFileSync(process.env.DIFFEX_BEHAVIOR_RECORD, "utf8").includes("real-secret"), false); + } finally { + globalThis.fetch = originalFetch; + Socket.prototype.connect = originalConnect; + process.env = originalEnv; + rmSync(directory, { recursive: true, force: true }); + } +}); + +for (const [name, extension] of [["fixture", fixture], ["live guard", guard]]) { + test(`${name} rejects traversal, unauthorized paths, symlinks, hardlinks and nonregular recordings`, () => { + const directory = mkdtempSync(join(tmpdir(), "behavior-record-boundary-")); + const outside = mkdtempSync("/var/tmp/behavior-record-outside-"); + const originalEnv = { ...process.env }; + try { + const target = join(directory, "untouched.jsonl"); + writeFileSync(target, "fixture-owned marker\n"); + const symlink = join(directory, "symlink.jsonl"); + symlinkSync(target, symlink); + const hardlink = join(directory, "hardlink.jsonl"); + linkSync(target, hardlink); + const nonregular = join(directory, "directory.jsonl"); + mkdirSync(nonregular); + const fifo = join(directory, "fifo.jsonl"); + assert.equal(spawnSync("mkfifo", [fifo]).status, 0); + const escaped = join(directory, "escaped"); + symlinkSync(outside, escaped); + writeFileSync(join(outside, "events.jsonl"), "fixture-owned marker\n"); + const shared = join(directory, "shared"); + mkdirSync(shared); + chmodSync(shared, 0o777); + const writable = join(directory, "shared.jsonl"); + writeFileSync(writable, "fixture-owned marker\n"); + chmodSync(writable, 0o666); + for (const path of [ + `${directory}/../escape.jsonl`, + "/not-a-behavior-record-root/events.jsonl", + `${process.cwd()}-not-the-workspace/events.jsonl`, + `${directory}/nul\0.jsonl`, + symlink, hardlink, nonregular, fifo, writable, join(shared, "events.jsonl"), join(escaped, "events.jsonl"), + ]) { + process.env.DIFFEX_BEHAVIOR_RECORD = path; + // Neither TMPDIR nor isolated config vars may grant arbitrary filesystem access. + process.env.TMPDIR = "/not-a-behavior-record-root"; + process.env.DIFFEX_CODING_AGENT_DIR = "/not-a-behavior-record-root/agent"; + assert.throws(() => extension({ on() { assert.fail("Unsafe recording accepted"); } }), /recording (path|destination|directory|must|changed)|ELOOP|EISDIR/); + } + delete process.env.DIFFEX_BEHAVIOR_RECORD; + assert.throws(() => extension({}), /DIFFEX_BEHAVIOR_RECORD is required/); + assert.equal(readFileSync(target, "utf8"), "fixture-owned marker\n"); + assert.equal(readFileSync(join(outside, "events.jsonl"), "utf8"), "fixture-owned marker\n"); + } finally { + process.env = originalEnv; + rmSync(directory, { recursive: true, force: true }); + rmSync(outside, { recursive: true, force: true }); + } + }); + + test(`${name} pins the validated file descriptor across path and environment replacement`, () => { + const directory = mkdtempSync(join(tmpdir(), "behavior-record-fd-")); + const originalEnv = { ...process.env }; + const originalFetch = globalThis.fetch; + const originalConnect = Socket.prototype.connect; + const callbacks = new Map(); + try { + const path = join(directory, "events.jsonl"); + const pinned = join(directory, "pinned.jsonl"); + const target = join(directory, "untouched.jsonl"); + writeFileSync(target, "fixture-owned marker\n"); + process.env.DIFFEX_BEHAVIOR_RECORD = path; + process.env.DIFFEX_BEHAVIOR_TRANSPORT = "0"; + extension({ on(name, callback) { callbacks.set(name, callback); }, registerProvider() {} }); + renameSync(path, pinned); + symlinkSync(target, path); + process.env.DIFFEX_BEHAVIOR_RECORD = target; + callbacks.get("agent_settled")(); + assert.deepEqual(records(pinned), [{ type: "settled" }]); + assert.equal(readFileSync(target, "utf8"), "fixture-owned marker\n"); + } finally { + callbacks.get("session_shutdown")?.(); + globalThis.fetch = originalFetch; + Socket.prototype.connect = originalConnect; + process.env = originalEnv; + rmSync(directory, { recursive: true, force: true }); + } + }); + + test(`${name} permits cwd-relative and isolated sandbox recordings with variable filenames`, () => { + const directory = mkdtempSync(join(tmpdir(), "behavior-record-local-")); + const local = mkdtempSync(join(process.cwd(), ".behavior-record-local-")); + const originalEnv = { ...process.env }; + const originalCwd = process.cwd(); + const originalFetch = globalThis.fetch; + const originalConnect = Socket.prototype.connect; + try { + for (const name of ["project", "agent", "home", "temp"]) mkdirSync(join(directory, name)); + process.env.HOME = join(directory, "home"); + process.env.TMPDIR = join(directory, "temp"); + process.env.DIFFEX_CODING_AGENT_DIR = join(directory, "agent"); + process.env.DIFFEX_BEHAVIOR_TRANSPORT = "0"; + for (const [cwd, path] of [ + [originalCwd, relative(originalCwd, join(local, "local-events.jsonl"))], + [join(directory, "project"), join(directory, "different-name.jsonl")], + ]) { + process.chdir(cwd); + process.env.DIFFEX_BEHAVIOR_RECORD = path; + const callbacks = new Map(); + try { + extension({ on(name, callback) { callbacks.set(name, callback); }, registerProvider() {} }); + callbacks.get("agent_settled")(); + assert.deepEqual(records(path), [{ type: "settled" }]); + } finally { + callbacks.get("session_shutdown")?.(); + rmSync(resolve(path), { force: true }); + globalThis.fetch = originalFetch; + Socket.prototype.connect = originalConnect; + } + } + } finally { + process.chdir(originalCwd); + process.env = originalEnv; + rmSync(directory, { recursive: true, force: true }); + rmSync(local, { recursive: true, force: true }); + } + }); +} diff --git a/scripts/daytona-behaviors/install.mjs b/scripts/daytona-behaviors/install.mjs new file mode 100644 index 000000000..16f96b728 --- /dev/null +++ b/scripts/daytona-behaviors/install.mjs @@ -0,0 +1,24 @@ +import assert from "node:assert/strict"; +import { readFileSync, realpathSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { installConsumer, resolveArtifactDirectory, verifyTarballs } from "../artifact-installation.mjs"; + +const [artifacts, consumer, manager, commit] = process.argv.slice(2); +assert(artifacts && consumer && ["npm", "bun"].includes(manager) && /^[a-f0-9]{40}$/.test(commit), + "Usage: node install.mjs ARTIFACTS CONSUMER npm|bun COMMIT"); +const artifactDirectory = resolveArtifactDirectory(artifacts); +const consumerDirectory = resolveArtifactDirectory(consumer); +const manifest = JSON.parse(readFileSync(join(artifactDirectory, "manifest.json"), "utf8")); +assert.equal(manifest.source.commit, commit, "Retained source does not match triggering commit"); +assert.equal(manifest.schemaVersion, 1); +const packages = manifest.packages; +assert.equal(packages.length, 1, "Expected the standalone CLI distribution"); +assert.equal(packages[0].name, "@diffexai/diffex"); +verifyTarballs(artifactDirectory, packages, packages); +installConsumer(consumerDirectory, manager, packages, packages, artifactDirectory); +const executable = join(consumerDirectory, "node_modules/.bin/diffex"); +assert(realpathSync(executable).startsWith(`${consumerDirectory}/node_modules/`)); +writeFileSync(join(consumerDirectory, "installation.json"), `${JSON.stringify({ + manager, executable, resolved: realpathSync(executable), commit, + artifacts: packages.map(({ name, sha256 }) => ({ name, sha256 })), integrity: "passed", +}, null, 2)}\n`); diff --git a/scripts/daytona-behaviors/installed.py b/scripts/daytona-behaviors/installed.py new file mode 100644 index 000000000..d322db508 --- /dev/null +++ b/scripts/daytona-behaviors/installed.py @@ -0,0 +1,441 @@ +"""Installed-package acceptance. No repository imports, provider keys in evidence, or UI mocks.""" +import argparse +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import re +import shutil +import shlex +import subprocess +import sys +import tempfile +import time +import uuid +import xml.etree.ElementTree as ET + +HERE = Path(__file__).resolve().parent + + +def read_events(path): + return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else [] + + +def write_json(path, value): + path.write_text(json.dumps(value, indent=2) + "\n") + + +class Terminal: + """Real tmux PTY, isolated server, bounded state waits, and incremental capture.""" + def __init__(self, executable, args, env, project, evidence, deadline): + self.socket = "diffex-behavior-" + uuid.uuid4().hex + self.evidence, self.deadline = evidence, deadline + self.env = env + (evidence / "tmux.conf").write_text("set -g extended-keys on\nset -gq extended-keys-format csi-u\n") + # Some sandbox init processes reap tmux's pane status. Record the actual + # application child's status independently, while retaining its real PTY. + launcher = evidence / "terminal-launch.py" + launcher.write_text( + "import json, pathlib, subprocess\n" + f"result = subprocess.run({[executable, *args]!r})\n" + f"temporary = pathlib.Path({str(evidence / '.terminal-exit.tmp')!r})\n" + "temporary.write_text(json.dumps({'exitCode': result.returncode}))\n" + f"temporary.replace({str(evidence / 'terminal-exit.json')!r})\n" + "raise SystemExit(0 if result.returncode == 0 else 1)\n" + ) + self.command("new-session", "-d", "-s", "behavior", "-x", "80", "-y", "24", + "-c", str(project), + " ".join(shlex.quote(a) for a in [sys.executable, str(launcher)])) + self.command("set-option", "-g", "remain-on-exit", "on") + + def command(self, *args): + result = subprocess.run(["tmux", "-f", str(self.evidence / "tmux.conf"), "-L", self.socket, *args], capture_output=True, text=True, + timeout=10, env=self.env) + if result.returncode: + raise RuntimeError("tmux: " + result.stderr) + return result.stdout + + def send(self, text): + self.command("send-keys", "-l", text) + self.command("send-keys", "Enter") + + def capture(self): + screen = self.command("capture-pane", "-p", "-S", "-2000") + (self.evidence / "terminal.txt").write_text(screen.replace(self.env.get("OPENAI_API_KEY", "\0"), "[REDACTED]")) + return screen + + def wait(self, text, seconds=30): + end = min(self.deadline, time.monotonic() + seconds) + while time.monotonic() < end: + screen = self.capture() + if text in screen: + return screen + time.sleep(.1) + raise AssertionError(f"Terminal did not reach {text!r} within bounded wait") + + def settled(self, record, previous, seconds=30): + end = min(self.deadline, time.monotonic() + seconds) + while time.monotonic() < end: + self.capture() + if sum(e["type"] == "settled" for e in read_events(record)) > previous: + return + time.sleep(.1) + raise AssertionError("Agent did not settle before deadline") + + def screen(self, name, size, baseline): + width, height = size + self.command("resize-window", "-x", str(width), "-y", str(height)) + # Wait for a stable rendered layout, not an arbitrary startup sleep. + end = min(self.deadline, time.monotonic() + 5) + previous, stable = None, 0 + while time.monotonic() < end: + screen = self.command("capture-pane", "-p") + stable = stable + 1 if screen == previous else 0 + previous = screen + if stable >= 3: + break + time.sleep(.1) + assert stable >= 3, "Terminal layout never stabilized" + (self.evidence / (name + ".txt")).write_text(screen) + normalized = normalize_screen(screen, self.env) + (self.evidence / (name + ".normalized.txt")).write_text(normalized) + expected = HERE / "baselines" / (name + ".txt") + if baseline: + expected.parent.mkdir(exist_ok=True) + expected.write_text(normalized) + assert expected.exists(), f"Missing reviewed baseline: {expected.name}" + assert normalized == expected.read_text(), f"Layout differs: {expected.name} (see captures)" + assert len(screen.splitlines()) == height, "Incorrect terminal height" + assert all(len(line) <= width for line in screen.splitlines()), "Terminal overflow" + + def exit(self): + self.send("/exit") + end = min(self.deadline, time.monotonic() + 10) + while time.monotonic() < end: + self.capture() + status_file = self.evidence / "terminal-exit.json" + if status_file.exists(): + status = json.loads(status_file.read_text())["exitCode"] + assert status == 0, f"Unclean terminal exit: {status}" + dead = self.command("display-message", "-p", "#{pane_dead}").strip() + if dead == "1": + return + time.sleep(.1) + raise AssertionError("Terminal failed to exit") + + def close(self): + try: + self.capture() + finally: + self.command("kill-server") + + +def normalize_screen(screen, env): + # Only machine-specific identity/timing is normalized; whitespace and geometry remain exact. + screen = screen.replace(str(Path(env["DIFFEX_PROJECT"]).resolve()), "") + screen = screen.replace(env["HOME"], "").replace(env["DIFFEX_PROJECT"], "") + screen = re.sub(r"Diffex v[\w.\-]+", "Diffex v", screen) + screen = re.sub(r"\b\d+(?:\.\d+)?[km]?/\d+(?:\.\d+)?[km]?", "", screen) + screen = re.sub(r"\b\d+(?:\.\d+)?(?:ms|s)\b", "